distributed-execution/.gitea/workflows/docker-publish.yml

name: Build and Push Docker Images

on:
  push:
    branches:
      - main
  workflow_dispatch:

env:
  BASE_DOMAIN: dataprovider01.sandbox-cat-dat.simpl-europe.eu
  OWNER: j.r

jobs:
  build-and-push:
    runs-on: orchestration-platform
    defaults:
      run:
        shell: sh
    env:
      REGISTRY: gitea.${{ env.BASE_DOMAIN }}
      IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution
      PAYLOAD_IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution-payload
      REPO_DIR: repo
      REPO_CLONE_URL: https://gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution.git
    steps:
      - name: Checkout repository (shell)
        run: |
          CLONE_USER="${{ secrets.REGISTRY_USERNAME }}"
          CLONE_PASS="${{ secrets.REGISTRY_PASSWORD }}"
          REF_NAME="${GITHUB_REF_NAME}"
          if [ -z "${REF_NAME}" ]; then
            REF_NAME="${GITHUB_REF#refs/heads/}"
          fi

          if [ -z "${CLONE_USER}" ] || [ -z "${CLONE_PASS}" ]; then
            echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret"
            exit 1
          fi

          rm -rf "${REPO_DIR}"
          AUTH_HEADER="$(printf '%s:%s' "${CLONE_USER}" "${CLONE_PASS}" | base64 | tr -d '\n')"
          git clone --depth 1 --branch "${REF_NAME}" \
            -c "http.extraHeader=Authorization: Basic ${AUTH_HEADER}" \
            "${REPO_CLONE_URL}" \
            "${REPO_DIR}"

          if [ ! -f "${REPO_DIR}/Dockerfile" ]; then
            echo "Code location Dockerfile not found after clone"
            exit 1
          fi

          if [ ! -f "${REPO_DIR}/payload/Dockerfile" ]; then
            echo "Payload Dockerfile not found after clone"
            exit 1
          fi

      - name: Validate registry secrets
        run: |
          if [ -z "${{ secrets.REGISTRY_USERNAME }}" ] || [ -z "${{ secrets.REGISTRY_PASSWORD }}" ]; then
            echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret"
            exit 1
          fi

      - name: Login to registry
        run: |
          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
            -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin

      # Both images take the same SHA tag: that is what keeps the code location and
      # the payload it dispatches on the same version.
      - name: Build code location image
        run: |
          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
          cd "${REPO_DIR}"
          docker build \
            -t "${IMAGE_REPO}:latest" \
            -t "${IMAGE_REPO}:${SHORT_SHA}" \
            .

      - name: Build payload image
        run: |
          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
          cd "${REPO_DIR}"
          docker build \
            -f payload/Dockerfile \
            -t "${PAYLOAD_IMAGE_REPO}:latest" \
            -t "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" \
            payload/

      - name: Validate code location image
        run: |
          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
          docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \
            dagster definitions validate -f src/distributed_execution/repository.py
          docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \
            test -f /app/payload/work.py

      # Readiness checklist L11: the payload image must not carry the orchestration
      # runtime, otherwise the isolation argument for the loosely coupled target is void.
      - name: Validate payload image isolation
        run: |
          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
          docker run --rm "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" python -c "
          import importlib.util
          assert importlib.util.find_spec('dagster_pipes') is not None, 'dagster_pipes missing from payload image'
          assert importlib.util.find_spec('dagster') is None, 'payload image must not contain the dagster package'
          print('payload isolation OK')
          "

      - name: Push code location image tags
        run: |
          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
          docker push "${IMAGE_REPO}:latest"
          docker push "${IMAGE_REPO}:${SHORT_SHA}"

      - name: Push payload image tags
        run: |
          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
          docker push "${PAYLOAD_IMAGE_REPO}:latest"
          docker push "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}"

      - name: Report image references
        run: |
          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
          echo "Code location image: ${IMAGE_REPO}:${SHORT_SHA}"
          echo "Payload image:       ${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}"
          echo ""
          echo "Set PIPES_PAYLOAD_IMAGE on both the code location and the run pods:"
          echo "  PIPES_PAYLOAD_IMAGE=${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}"
          echo "See yaml/loosely-coupled/values-pipes-payload.yaml."

      # The automated update of the deployment requires a technical user with
      # their kube config in the secrets. See the template repository's user manual.
      # - name: Update Dagster user deployment image
      #   run: |
      #     COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
      #     SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
      #     kubectl patch deployment "${DEPLOYMENT_NAME}" \
      #       -n "${K8S_NAMESPACE}" \
      #       --type='strategic' \
      #       -p="{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"dagster-user-deployments\",\"image\":\"${IMAGE_REPO}:${SHORT_SHA}\",\"env\":[{\"name\":\"DAGSTER_CURRENT_IMAGE\",\"value\":\"${IMAGE_REPO}:${SHORT_SHA}\"},{\"name\":\"PIPES_PAYLOAD_IMAGE\",\"value\":\"${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}\"}]}]}}}}"
      #     kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \
      #       -n "${K8S_NAMESPACE}" \
      #       --timeout=5m