.gitea/workflows/docker-publish.yml

@@ -1,151 +0,0 @@
-name: Build and Push Docker Images
-
-on:
-  push:
-    branches:
-      - main
-  workflow_dispatch:
-
-env:
-  BASE_DOMAIN: dataprovider01.sandbox-cat-dat.simpl-europe.eu
-  OWNER: ${{ github.repository_owner }}
-
-jobs:
-  build-and-push:
-    runs-on: orchestration-platform
-    defaults:
-      run:
-        shell: sh
-    env:
-      REGISTRY: gitea.${{ env.BASE_DOMAIN }}
-      IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution
-      PAYLOAD_IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution-payload
-      REPO_DIR: repo
-      REPO_CLONE_URL: https://gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution.git
-    steps:
-      - name: Checkout repository (shell)
-        run: |
-          CLONE_USER="${{ secrets.REGISTRY_USERNAME }}"
-          CLONE_PASS="${{ secrets.REGISTRY_PASSWORD }}"
-          REF_NAME="${GITHUB_REF_NAME}"
-          if [ -z "${REF_NAME}" ]; then
-            REF_NAME="${GITHUB_REF#refs/heads/}"
-          fi
-
-          if [ -z "${CLONE_USER}" ] || [ -z "${CLONE_PASS}" ]; then
-            echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret"
-            exit 1
-          fi
-
-          rm -rf "${REPO_DIR}"
-          AUTH_HEADER="$(printf '%s:%s' "${CLONE_USER}" "${CLONE_PASS}" | base64 | tr -d '\n')"
-          git clone --depth 1 --branch "${REF_NAME}" \
-            -c "http.extraHeader=Authorization: Basic ${AUTH_HEADER}" \
-            "${REPO_CLONE_URL}" \
-            "${REPO_DIR}"
-
-          if [ ! -f "${REPO_DIR}/Dockerfile" ]; then
-            echo "Code location Dockerfile not found after clone"
-            exit 1
-          fi
-
-          if [ ! -f "${REPO_DIR}/payload/Dockerfile" ]; then
-            echo "Payload Dockerfile not found after clone"
-            exit 1
-          fi
-
-      - name: Validate registry secrets
-        run: |
-          if [ -z "${{ secrets.REGISTRY_USERNAME }}" ] || [ -z "${{ secrets.REGISTRY_PASSWORD }}" ]; then
-            echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret"
-            exit 1
-          fi
-
-      - name: Login to registry
-        run: |
-          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
-            -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin
-
-      # Both images take the same SHA tag: that is what keeps the code location and
-      # the payload it dispatches on the same version.
-      - name: Build code location image
-        run: |
-          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
-          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
-          cd "${REPO_DIR}"
-          docker build \
-            -t "${IMAGE_REPO}:latest" \
-            -t "${IMAGE_REPO}:${SHORT_SHA}" \
-            .
-
-      - name: Build payload image
-        run: |
-          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
-          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
-          cd "${REPO_DIR}"
-          docker build \
-            -f payload/Dockerfile \
-            -t "${PAYLOAD_IMAGE_REPO}:latest" \
-            -t "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" \
-            payload/
-
-      - name: Validate code location image
-        run: |
-          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
-          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
-          docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \
-            dagster definitions validate -f src/distributed_execution/repository.py
-          docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \
-            test -f /app/payload/work.py
-
-      # Readiness checklist L11: the payload image must not carry the orchestration
-      # runtime, otherwise the isolation argument for the loosely coupled target is void.
-      - name: Validate payload image isolation
-        run: |
-          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
-          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
-          docker run --rm "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" python -c "
-          import importlib.util
-          assert importlib.util.find_spec('dagster_pipes') is not None, 'dagster_pipes missing from payload image'
-          assert importlib.util.find_spec('dagster') is None, 'payload image must not contain the dagster package'
-          print('payload isolation OK')
-          "
-
-      - name: Push code location image tags
-        run: |
-          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
-          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
-          docker push "${IMAGE_REPO}:latest"
-          docker push "${IMAGE_REPO}:${SHORT_SHA}"
-
-      - name: Push payload image tags
-        run: |
-          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
-          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
-          docker push "${PAYLOAD_IMAGE_REPO}:latest"
-          docker push "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}"
-
-      - name: Report image references
-        run: |
-          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
-          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
-          echo "Code location image: ${IMAGE_REPO}:${SHORT_SHA}"
-          echo "Payload image:       ${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}"
-          echo ""
-          echo "Set PIPES_PAYLOAD_IMAGE on both the code location and the run pods:"
-          echo "  PIPES_PAYLOAD_IMAGE=${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}"
-          echo "See yaml/loosely-coupled/values-pipes-payload.yaml."
-
-      # The automated update of the deployment requires a technical user with
-      # their kube config in the secrets. See the template repository's user manual.
-      # - name: Update Dagster user deployment image
-      #   run: |
-      #     COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
-      #     SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
-      #     kubectl patch deployment "${DEPLOYMENT_NAME}" \
-      #       -n "${K8S_NAMESPACE}" \
-      #       --type='strategic' \
-      #       -p="{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"dagster-user-deployments\",\"image\":\"${IMAGE_REPO}:${SHORT_SHA}\",\"env\":[{\"name\":\"DAGSTER_CURRENT_IMAGE\",\"value\":\"${IMAGE_REPO}:${SHORT_SHA}\"},{\"name\":\"PIPES_PAYLOAD_IMAGE\",\"value\":\"${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}\"}]}]}}}}"
-      #     kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \
-      #       -n "${K8S_NAMESPACE}" \
-      #       --timeout=5m

README.md

@@ -81,10 +81,10 @@ Both images build and have been smoke tested locally: the code location image
 loads its definitions, and the payload image contains `dagster_pipes` without
 `dagster` — check L11 in the readiness checklist.
 
-[.gitea/workflows/docker-publish.yml](.gitea/workflows/docker-publish.yml) runs
+Both must be tagged from the same commit. That shared tag is what keeps a code
-the same two builds on every push to `main`, applies those checks as gates, and
+location and the payload it dispatches on the same version, and nothing at
-tags both images with the same short commit SHA — that shared tag is what keeps a
+runtime checks the pairing — see the *Outstanding work* section for what the
-code location and the payload it dispatches on the same version.
+pipeline still has to learn.
 
 ## Status
 

yaml/loosely-coupled/values-pipes-payload.yaml

@@ -17,8 +17,7 @@
 # For values that work today, see yaml/sandbox/values-sandbox-gitea.yaml. The
 # shared GitLab pipeline (ds.gitlab-ci.yml) builds exactly one image per project -
 # $CI_REGISTRY_IMAGE from the root Dockerfile - so the payload reference below
-# resolves only once that pipeline learns to build a second image. The Gitea
+# resolves only once that pipeline learns to build a second image.
-# workflow in .gitea/workflows/docker-publish.yml already builds both.
 
 dagster:
   dagster-user-deployments:

yaml/sandbox/values-sandbox-gitea.yaml

@@ -7,8 +7,8 @@
 # Use these instead of the code.europa.eu references in
 # yaml/values-dagster-distributed-execution.yaml and
 # yaml/loosely-coupled/values-pipes-payload.yaml when deploying to the
-# dataprovider01 sandbox. Both images are published by
+# dataprovider01 sandbox. Both images were built and published by a Gitea Actions
-# .gitea/workflows/docker-publish.yml.
+# workflow held in the sandbox Gitea mirror of this repository, not here.
 #
 # The tag below is a short commit SHA and is the same for both images. That is
 # the version lock: the code location and the payload it dispatches must come