[SIMPL-30451] Distributed execution reference service and container-cluster configuration guide #1

Merged
j.r merged 23 commits from feature/SIMPL-30451-distributed-execution into develop 2026-09-02 17:38:23 +00:00
Showing only changes of commit b47c530d46 - Show all commits

View File

@@ -4,6 +4,13 @@
# registry whose auth behaviour was checked, but they have not been applied to a # registry whose auth behaviour was checked, but they have not been applied to a
# Simpl cluster. Confirm and amend after the first deployment. # Simpl cluster. Confirm and amend after the first deployment.
# #
# VERSION SKEW, unresolved. These images carry dagster 1.13.19 (requirements.txt
# pins no upper bound, so the build took whatever was current). The sandbox
# control plane is 1.12.8. A code server is a gRPC server the webserver and
# daemon call into, so the two versions have to be compatible; a minor-version
# gap is not a supported configuration. Pin dagster to the control plane version
# and rebuild before reading a failure here as a bug in this service.
#
# Use these instead of the code.europa.eu references in # Use these instead of the code.europa.eu references in
# yaml/values-dagster-distributed-execution.yaml and # yaml/values-dagster-distributed-execution.yaml and
# yaml/loosely-coupled/values-pipes-payload.yaml when deploying to the # yaml/loosely-coupled/values-pipes-payload.yaml when deploying to the
@@ -26,17 +33,33 @@
# The payload namespace needs it too: PipesK8sClient creates that Job, and a # The payload namespace needs it too: PipesK8sClient creates that Job, and a
# missing pull secret there leaves the op waiting on a pod that never starts, # missing pull secret there leaves the op waiting on a pod that never starts,
# which surfaces as pod_wait_timeout rather than as an image error. # which surfaces as pod_wait_timeout rather than as an image error.
#
# PULL SECRET PLACEMENT. Checked against the chart templates in dagster 1.12.8,
# which is what the sandbox runs. There is no per-deployment imagePullSecrets
# field - deployment-user.yaml reads the SUBCHART value and the run launcher
# helper reads the PARENT value - so a pull secret nested under a deployments[]
# entry is silently discarded, and the pod fails with ImagePullBackOff against a
# values file that looks correct.
#
# Both keys below therefore apply to every code location in the release, not
# just this one. Extra pull secrets are ignored for registries they do not match,
# so the existing code locations are unaffected.
dagster: dagster:
# Run pods, webserver and daemon.
imagePullSecrets:
- name: gitea-registry
dagster-user-deployments: dagster-user-deployments:
# Code server pods.
imagePullSecrets:
- name: gitea-registry
deployments: deployments:
- name: distributed-execution - name: distributed-execution
image: image:
repository: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution repository: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution
tag: 5122da4691f9 tag: 5122da4691f9
pullPolicy: IfNotPresent pullPolicy: IfNotPresent
imagePullSecrets:
- name: gitea-registry
env: env:
- name: PIPES_PAYLOAD_IMAGE - name: PIPES_PAYLOAD_IMAGE
value: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution-payload:5122da4691f9 value: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution-payload:5122da4691f9
@@ -46,8 +69,6 @@ dagster:
runLauncher: runLauncher:
config: config:
k8sRunLauncher: k8sRunLauncher:
imagePullSecrets:
- name: gitea-registry
runK8sConfig: runK8sConfig:
containerConfig: containerConfig:
env: env: