Merge branch 'main' into develop

The feature branch is shared with GitLab and deliberately carries no .gitea/ directory, so merging it removed docker-publish.yml from develop and would have stopped the sandbox image builds. Restore it unchanged from 2320b41.

Changelog: other

.gitea/workflows/docker-publish.yml

@@ -0,0 +1,151 @@
+name: Build and Push Docker Images
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  BASE_DOMAIN: dataprovider01.sandbox-cat-dat.simpl-europe.eu
+  OWNER: j.r
+
+jobs:
+  build-and-push:
+    runs-on: orchestration-platform
+    defaults:
+      run:
+        shell: sh
+    env:
+      REGISTRY: gitea.${{ env.BASE_DOMAIN }}
+      IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution
+      PAYLOAD_IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution-payload
+      REPO_DIR: repo
+      REPO_CLONE_URL: https://gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution.git
+    steps:
+      - name: Checkout repository (shell)
+        run: |
+          CLONE_USER="${{ secrets.REGISTRY_USERNAME }}"
+          CLONE_PASS="${{ secrets.REGISTRY_PASSWORD }}"
+          REF_NAME="${GITHUB_REF_NAME}"
+          if [ -z "${REF_NAME}" ]; then
+            REF_NAME="${GITHUB_REF#refs/heads/}"
+          fi
+
+          if [ -z "${CLONE_USER}" ] || [ -z "${CLONE_PASS}" ]; then
+            echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret"
+            exit 1
+          fi
+
+          rm -rf "${REPO_DIR}"
+          AUTH_HEADER="$(printf '%s:%s' "${CLONE_USER}" "${CLONE_PASS}" | base64 | tr -d '\n')"
+          git clone --depth 1 --branch "${REF_NAME}" \
+            -c "http.extraHeader=Authorization: Basic ${AUTH_HEADER}" \
+            "${REPO_CLONE_URL}" \
+            "${REPO_DIR}"
+
+          if [ ! -f "${REPO_DIR}/Dockerfile" ]; then
+            echo "Code location Dockerfile not found after clone"
+            exit 1
+          fi
+
+          if [ ! -f "${REPO_DIR}/payload/Dockerfile" ]; then
+            echo "Payload Dockerfile not found after clone"
+            exit 1
+          fi
+
+      - name: Validate registry secrets
+        run: |
+          if [ -z "${{ secrets.REGISTRY_USERNAME }}" ] || [ -z "${{ secrets.REGISTRY_PASSWORD }}" ]; then
+            echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret"
+            exit 1
+          fi
+
+      - name: Login to registry
+        run: |
+          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
+            -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin
+
+      # Both images take the same SHA tag: that is what keeps the code location and
+      # the payload it dispatches on the same version.
+      - name: Build code location image
+        run: |
+          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
+          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
+          cd "${REPO_DIR}"
+          docker build \
+            -t "${IMAGE_REPO}:latest" \
+            -t "${IMAGE_REPO}:${SHORT_SHA}" \
+            .
+
+      - name: Build payload image
+        run: |
+          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
+          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
+          cd "${REPO_DIR}"
+          docker build \
+            -f payload/Dockerfile \
+            -t "${PAYLOAD_IMAGE_REPO}:latest" \
+            -t "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" \
+            payload/
+
+      - name: Validate code location image
+        run: |
+          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
+          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
+          docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \
+            dagster definitions validate -f src/distributed_execution/repository.py
+          docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \
+            test -f /app/payload/work.py
+
+      # Readiness checklist L11: the payload image must not carry the orchestration
+      # runtime, otherwise the isolation argument for the loosely coupled target is void.
+      - name: Validate payload image isolation
+        run: |
+          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
+          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
+          docker run --rm "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" python -c "
+          import importlib.util
+          assert importlib.util.find_spec('dagster_pipes') is not None, 'dagster_pipes missing from payload image'
+          assert importlib.util.find_spec('dagster') is None, 'payload image must not contain the dagster package'
+          print('payload isolation OK')
+          "
+
+      - name: Push code location image tags
+        run: |
+          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
+          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
+          docker push "${IMAGE_REPO}:latest"
+          docker push "${IMAGE_REPO}:${SHORT_SHA}"
+
+      - name: Push payload image tags
+        run: |
+          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
+          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
+          docker push "${PAYLOAD_IMAGE_REPO}:latest"
+          docker push "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}"
+
+      - name: Report image references
+        run: |
+          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
+          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
+          echo "Code location image: ${IMAGE_REPO}:${SHORT_SHA}"
+          echo "Payload image:       ${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}"
+          echo ""
+          echo "Set PIPES_PAYLOAD_IMAGE on both the code location and the run pods:"
+          echo "  PIPES_PAYLOAD_IMAGE=${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}"
+          echo "See yaml/loosely-coupled/values-pipes-payload.yaml."
+
+      # The automated update of the deployment requires a technical user with
+      # their kube config in the secrets. See the template repository's user manual.
+      # - name: Update Dagster user deployment image
+      #   run: |
+      #     COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
+      #     SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
+      #     kubectl patch deployment "${DEPLOYMENT_NAME}" \
+      #       -n "${K8S_NAMESPACE}" \
+      #       --type='strategic' \
+      #       -p="{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"dagster-user-deployments\",\"image\":\"${IMAGE_REPO}:${SHORT_SHA}\",\"env\":[{\"name\":\"DAGSTER_CURRENT_IMAGE\",\"value\":\"${IMAGE_REPO}:${SHORT_SHA}\"},{\"name\":\"PIPES_PAYLOAD_IMAGE\",\"value\":\"${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}\"}]}]}}}}"
+      #     kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \
+      #       -n "${K8S_NAMESPACE}" \
+      #       --timeout=5m