From f139a0dd8612d179b1fab3e9f29895bf2b9cd53a Mon Sep 17 00:00:00 2001 From: ILay Date: Tue, 1 Sep 2026 19:14:22 +0200 Subject: [PATCH 1/2] [SIMPL-30451] Restore sandbox image workflow Keep the Gitea-only image publishing workflow on the sandbox mirror while building the shared-storage step pod fix. Changelog: other --- .gitea/workflows/docker-publish.yml | 151 ++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 .gitea/workflows/docker-publish.yml diff --git a/.gitea/workflows/docker-publish.yml b/.gitea/workflows/docker-publish.yml new file mode 100644 index 0000000..a21b82e --- /dev/null +++ b/.gitea/workflows/docker-publish.yml @@ -0,0 +1,151 @@ +name: Build and Push Docker Images + +on: + push: + branches: + - main + workflow_dispatch: + +env: + BASE_DOMAIN: dataprovider01.sandbox-cat-dat.simpl-europe.eu + OWNER: j.r + +jobs: + build-and-push: + runs-on: orchestration-platform + defaults: + run: + shell: sh + env: + REGISTRY: gitea.${{ env.BASE_DOMAIN }} + IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution + PAYLOAD_IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution-payload + REPO_DIR: repo + REPO_CLONE_URL: https://gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution.git + steps: + - name: Checkout repository (shell) + run: | + CLONE_USER="${{ secrets.REGISTRY_USERNAME }}" + CLONE_PASS="${{ secrets.REGISTRY_PASSWORD }}" + REF_NAME="${GITHUB_REF_NAME}" + if [ -z "${REF_NAME}" ]; then + REF_NAME="${GITHUB_REF#refs/heads/}" + fi + + if [ -z "${CLONE_USER}" ] || [ -z "${CLONE_PASS}" ]; then + echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret" + exit 1 + fi + + rm -rf "${REPO_DIR}" + AUTH_HEADER="$(printf '%s:%s' "${CLONE_USER}" "${CLONE_PASS}" | base64 | tr -d '\n')" + git clone --depth 1 --branch "${REF_NAME}" \ + -c "http.extraHeader=Authorization: Basic ${AUTH_HEADER}" \ + "${REPO_CLONE_URL}" \ + "${REPO_DIR}" + + if [ ! -f "${REPO_DIR}/Dockerfile" ]; then + echo "Code location Dockerfile not found after clone" + exit 1 + fi + + if [ ! -f "${REPO_DIR}/payload/Dockerfile" ]; then + echo "Payload Dockerfile not found after clone" + exit 1 + fi + + - name: Validate registry secrets + run: | + if [ -z "${{ secrets.REGISTRY_USERNAME }}" ] || [ -z "${{ secrets.REGISTRY_PASSWORD }}" ]; then + echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret" + exit 1 + fi + + - name: Login to registry + run: | + echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \ + -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin + + # Both images take the same SHA tag: that is what keeps the code location and + # the payload it dispatches on the same version. + - name: Build code location image + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + cd "${REPO_DIR}" + docker build \ + -t "${IMAGE_REPO}:latest" \ + -t "${IMAGE_REPO}:${SHORT_SHA}" \ + . + + - name: Build payload image + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + cd "${REPO_DIR}" + docker build \ + -f payload/Dockerfile \ + -t "${PAYLOAD_IMAGE_REPO}:latest" \ + -t "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" \ + payload/ + + - name: Validate code location image + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \ + dagster definitions validate -f src/distributed_execution/repository.py + docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \ + test -f /app/payload/work.py + + # Readiness checklist L11: the payload image must not carry the orchestration + # runtime, otherwise the isolation argument for the loosely coupled target is void. + - name: Validate payload image isolation + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + docker run --rm "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" python -c " + import importlib.util + assert importlib.util.find_spec('dagster_pipes') is not None, 'dagster_pipes missing from payload image' + assert importlib.util.find_spec('dagster') is None, 'payload image must not contain the dagster package' + print('payload isolation OK') + " + + - name: Push code location image tags + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + docker push "${IMAGE_REPO}:latest" + docker push "${IMAGE_REPO}:${SHORT_SHA}" + + - name: Push payload image tags + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + docker push "${PAYLOAD_IMAGE_REPO}:latest" + docker push "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" + + - name: Report image references + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + echo "Code location image: ${IMAGE_REPO}:${SHORT_SHA}" + echo "Payload image: ${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" + echo "" + echo "Set PIPES_PAYLOAD_IMAGE on both the code location and the run pods:" + echo " PIPES_PAYLOAD_IMAGE=${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" + echo "See yaml/loosely-coupled/values-pipes-payload.yaml." + + # The automated update of the deployment requires a technical user with + # their kube config in the secrets. See the template repository's user manual. + # - name: Update Dagster user deployment image + # run: | + # COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + # SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + # kubectl patch deployment "${DEPLOYMENT_NAME}" \ + # -n "${K8S_NAMESPACE}" \ + # --type='strategic' \ + # -p="{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"dagster-user-deployments\",\"image\":\"${IMAGE_REPO}:${SHORT_SHA}\",\"env\":[{\"name\":\"DAGSTER_CURRENT_IMAGE\",\"value\":\"${IMAGE_REPO}:${SHORT_SHA}\"},{\"name\":\"PIPES_PAYLOAD_IMAGE\",\"value\":\"${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}\"}]}]}}}}" + # kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \ + # -n "${K8S_NAMESPACE}" \ + # --timeout=5m From 90a0c07351acf2d0d08de7709218ba3c536146c2 Mon Sep 17 00:00:00 2001 From: ILay Date: Tue, 1 Sep 2026 20:03:23 +0200 Subject: [PATCH 2/2] fix(tightly-coupled): pin k8s step IO manager to shared storage --- src/distributed_execution/tightly_coupled/jobs.py | 15 ++++++++------- tests/test_tightly_coupled.py | 9 ++++++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/distributed_execution/tightly_coupled/jobs.py b/src/distributed_execution/tightly_coupled/jobs.py index cfa88da..dc49278 100644 --- a/src/distributed_execution/tightly_coupled/jobs.py +++ b/src/distributed_execution/tightly_coupled/jobs.py @@ -12,7 +12,7 @@ is the instance-level run launcher (see ``yaml/tightly-coupled/``). from __future__ import annotations -from dagster import graph, in_process_executor, multiprocess_executor +from dagster import fs_io_manager, graph, in_process_executor, multiprocess_executor from dagster_k8s import k8s_job_executor from distributed_execution.ops import ( @@ -28,15 +28,14 @@ COMMON_TAGS = { "business_operation": "DISTRIBUTED_EXECUTION_REFERENCE", } +# RWX PVC mounted into the run worker and every step pod. Set as the I/O manager +# base_dir rather than via DAGSTER_HOME: the Dagster chart already injects +# DAGSTER_HOME=/tmp/dagster and the earlier duplicate wins in the step pod. +SHARED_IO_BASE_DIR = "/dagster/shared/distributed-execution/storage" + # Per-step pod shape. Only honoured by k8s_job_executor. STEP_K8S_CONFIG = { "container_config": { - "env": [ - { - "name": "DAGSTER_HOME", - "value": "/dagster/shared/distributed-execution", - } - ], "resources": { "requests": {"cpu": "100m", "memory": "128Mi"}, "limits": {"cpu": "500m", "memory": "512Mi"}, @@ -104,6 +103,8 @@ tightly_coupled_k8s_job = distributed_execution_reference.to_job( # config `execution:` block is accepted and then silently discarded - which # also makes job_image and job_namespace impossible to supply at launch time. executor_def=k8s_job_executor, + # Steps run in separate pods, so step outputs have to land on shared storage. + resource_defs={"io_manager": fs_io_manager.configured({"base_dir": SHARED_IO_BASE_DIR})}, config={ "execution": { "config": { diff --git a/tests/test_tightly_coupled.py b/tests/test_tightly_coupled.py index e3571a1..2a4b921 100644 --- a/tests/test_tightly_coupled.py +++ b/tests/test_tightly_coupled.py @@ -5,6 +5,7 @@ import pytest from distributed_execution.preflight import check_env_vars, describe_pod_identity from distributed_execution.repository import defs from distributed_execution.tightly_coupled.jobs import ( + SHARED_IO_BASE_DIR, STEP_K8S_CONFIG, tightly_coupled_in_process_job, tightly_coupled_k8s_job, @@ -40,12 +41,18 @@ def test_k8s_step_pods_mount_shared_dagster_storage(): container_config = STEP_K8S_CONFIG["container_config"] pod_spec_config = STEP_K8S_CONFIG["pod_spec_config"] - assert {"name": "DAGSTER_HOME", "value": "/dagster/shared/distributed-execution"} in container_config["env"] assert {"name": "dagster-shared-storage", "mount_path": "/dagster/shared"} in container_config["volume_mounts"] assert { "name": "dagster-shared-storage", "persistent_volume_claim": {"claim_name": "dagster-shared-pvc"}, } in pod_spec_config["volumes"] + assert SHARED_IO_BASE_DIR.startswith("/dagster/shared/") + + +def test_k8s_job_pins_io_manager_to_shared_storage(): + io_manager = tightly_coupled_k8s_job.resource_defs["io_manager"] + resolved = io_manager.apply_config_mapping({}).value + assert resolved["config"]["base_dir"] == SHARED_IO_BASE_DIR def test_in_process_job_runs_end_to_end():