[SIMPL-30451] Add distributed-execution service with guide and reference implementations

Canonical location for documentation, example workflows and reference service implementations covering distributed execution patterns. Covers AC1-AC4: execution-target selection, decision support, readiness checks and code-level linkage. Tightly coupled jobs and the loosely coupled subprocess transport are verified by the test suite; the Kubernetes pipes transport is implemented but not yet cluster-run and is marked as such in the guide.

Changelog: added

yaml/loosely-coupled/rbac-pipes-dispatch.yaml

@@ -0,0 +1,41 @@
+# Kubernetes RBAC for the loosely coupled execution target.
+#
+# Granted to the service account running the Dagster RUN pods - the pod that
+# calls PipesK8sClient.run() is the one that creates and watches the payload Job.
+#
+# Note what is NOT here: the payload itself needs no RBAC, no database
+# credentials and no Vault role. Its only channel back to the control plane is
+# the pod log stream, which the dispatching pod reads.
+#
+# If the payload runs in a different namespace from the dispatcher, apply this
+# Role in the PAYLOAD namespace and keep the RoleBinding subject pointing at the
+# dispatcher's service account.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: distributed-execution-pipes-dispatcher
+  namespace: dagster
+rules:
+  - apiGroups: ["batch"]
+    resources: ["jobs", "jobs/status"]
+    verbs: ["create", "get", "list", "watch", "delete"]
+  # pods/log is the message channel for PipesK8sPodLogsMessageReader.
+  # Without it the payload runs but reports nothing.
+  - apiGroups: [""]
+    resources: ["pods", "pods/log", "pods/status"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: distributed-execution-pipes-dispatcher
+  namespace: dagster
+subjects:
+  - kind: ServiceAccount
+    name: dagster-dev
+    namespace: dagster
+roleRef:
+  kind: Role
+  name: distributed-execution-pipes-dispatcher
+  apiGroup: rbac.authorization.k8s.io

yaml/loosely-coupled/values-pipes-payload.yaml

@@ -0,0 +1,32 @@
+# Loosely coupled execution target - code location overrides.
+#
+# Merge into the dagster-user-deployments entry for this code location. These
+# values point the pipes client at the payload image and target namespace.
+#
+# Contrast with yaml/tightly-coupled/values-run-launcher.yaml: there, every
+# execution pod needed S3 and Vault credentials. Here the payload needs none -
+# only the dispatching pod does.
+
+dagster:
+  dagster-user-deployments:
+    deployments:
+      - name: distributed-execution
+        envSecrets: []
+        env:
+          - name: PIPES_PAYLOAD_IMAGE
+            value: code.europa.eu:4567/simpl/simpl-open/data/supporting-data-services/distributed-execution/payload:0.1.0
+          - name: PIPES_PAYLOAD_NAMESPACE
+            value: dagster
+
+  # The dispatching RUN pod creates the payload Job, so the image reference and
+  # namespace must also be visible to run pods, not just the code server.
+  runLauncher:
+    config:
+      k8sRunLauncher:
+        runK8sConfig:
+          containerConfig:
+            env:
+              - name: PIPES_PAYLOAD_IMAGE
+                value: code.europa.eu:4567/simpl/simpl-open/data/supporting-data-services/distributed-execution/payload:0.1.0
+              - name: PIPES_PAYLOAD_NAMESPACE
+                value: dagster

yaml/tightly-coupled/rbac-step-executor.yaml

@@ -0,0 +1,34 @@
+# Kubernetes RBAC required by the k8s_job_executor.
+#
+# The in_process and multiprocess executors need NONE of this - the run worker
+# does all the work itself. Only tightly_coupled_k8s_job, which creates one
+# Kubernetes Job per step, needs permission to manage Jobs and read their pods.
+#
+# Bind to the service account used by the Dagster run pods (dagster.serviceAccount.name).
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: distributed-execution-step-runner
+  namespace: dagster
+rules:
+  - apiGroups: ["batch"]
+    resources: ["jobs", "jobs/status"]
+    verbs: ["create", "get", "list", "watch", "delete"]
+  - apiGroups: [""]
+    resources: ["pods", "pods/log", "pods/status"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: distributed-execution-step-runner
+  namespace: dagster
+subjects:
+  - kind: ServiceAccount
+    name: dagster-dev
+    namespace: dagster
+roleRef:
+  kind: Role
+  name: distributed-execution-step-runner
+  apiGroup: rbac.authorization.k8s.io

yaml/tightly-coupled/values-run-launcher.yaml

@@ -0,0 +1,54 @@
+# Tightly coupled execution target - instance-level run launcher.
+#
+# Merge into the Dagster chart values. This is the half of the execution-target
+# binding that lives OUTSIDE workflow code: it decides where the run worker for
+# an entire run is placed, and what that pod can reach.
+#
+# Companion to src/distributed_execution/tightly_coupled/jobs.py, which decides
+# how the steps INSIDE that run worker execute.
+
+dagster:
+  runLauncher:
+    type: K8sRunLauncher
+    config:
+      k8sRunLauncher:
+        # Namespace the run pods land in. Must be a namespace whose NetworkPolicy
+        # permits egress to Postgres, object storage and Vault - see AC1
+        # prerequisites in the user guide.
+        jobNamespace: dagster
+
+        # Surfaces step failures as pod failures so kubectl and Dagster agree.
+        failPodOnRunFailure: true
+
+        runK8sConfig:
+          podTemplateSpecMetadata:
+            annotations:
+              vault.security.banzaicloud.io/vault-inject: "true"
+              vault.security.banzaicloud.io/vault-addr: https://secrets.common01.dev.simpl-europe.eu
+              vault.security.banzaicloud.io/vault-role: dev-role
+              vault.security.banzaicloud.io/vault-skip-verify: "true"
+              vault.security.banzaicloud.io/vault-path: kubernetes
+
+          containerConfig:
+            resources:
+              requests:
+                cpu: 100m
+                memory: 128Mi
+              limits:
+                cpu: 500m
+                memory: 512Mi
+
+            # Everything below is what "tightly coupled" costs you: each run pod
+            # needs credentials for, and network reachability to, the full
+            # orchestration runtime. A loosely coupled target needs none of it.
+            env:
+              - name: DAGSTER_TELEMETRY_ENABLED
+                value: "false"
+              - name: TOKEN
+                value: "vault:dev/data/dagster/dagster-workflow-vault-secret#VAULT_ACCESS_TOKEN"
+              - name: S3_ENDPOINT_URL
+                value: "https://s3.dev.simpl-europe.eu"
+              - name: S3_ACCESS_KEY
+                value: "vault:dev/data/dev-orchestration-platform#S3_ACCESS_KEY"
+              - name: S3_SECRET_KEY
+                value: "vault:dev/data/dev-orchestration-platform#S3_SECRET_KEY"

yaml/values-dagster-distributed-execution.yaml

@@ -0,0 +1,54 @@
+dagster:
+  # ===========================================================================
+  # CODE LOCATION - distributed-execution
+  # ===========================================================================
+  # Deploys this repository as a Dagster code location so the reference jobs
+  # appear in the UI alongside production workflows.
+  # MANAGED BY GITLAB CI/CD - DO NOT CHANGE MANUALLY
+  dagster-user-deployments:
+    deployments:
+      - name: distributed-execution
+        location_name: distributed-execution
+        enabled: true
+        enableSubchart: true
+
+        codeServerArgs:
+          - "--host"
+          - "0.0.0.0"
+          - "--port"
+          - "4000"
+          - "--python-file"
+          - "src/distributed_execution/repository.py"
+
+        envFrom:
+          - secretRef:
+              name: distributed-execution-dagster-db-secret
+          - configMapRef:
+              name: distributed-execution-dagster-db-config
+
+        image:
+          pullPolicy: IfNotPresent
+          repository: code.europa.eu:4567/simpl/simpl-open/data/supporting-data-services/distributed-execution
+          tag: 0.0.0 # DO NOT CHANGE - replaced automatically by the pipeline before releasing
+
+        port: 4000
+
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: 500m
+            memory: 512Mi
+
+        livenessProbe:
+          enabled: true
+          periodSeconds: 30
+          timeoutSeconds: 5
+          failureThreshold: 3
+
+        readinessProbe:
+          enabled: true
+          periodSeconds: 10
+          timeoutSeconds: 3
+          failureThreshold: 3