From 5122da4691f990823f404bf997a9a0aeb4351dfc Mon Sep 17 00:00:00 2001 From: ILay Date: Thu, 27 Aug 2026 18:58:38 +0200 Subject: [PATCH] [SIMPL-30451] Add Gitea Actions workflow that builds and publishes both images Adapted from template-code-location's docker-publish.yml. The repository ships two images, so the workflow builds the code location from the root Dockerfile and the payload from payload/Dockerfile, and tags both with the same short SHA - that shared tag is what keeps a code location and the payload it dispatches on the same version. Each image gets its own gate before it is pushed. The code location image must load its Dagster definitions and carry payload/work.py for the subprocess transport. The payload image must contain dagster_pipes and must not contain dagster, which is readiness checklist L11; running that assertion against the code location image fails as expected, so the check discriminates rather than passing vacuously. Changelog: added --- .gitea/workflows/docker-publish.yml | 151 ++++++++++++++++++ .../user-guide/distributed-execution-guide.md | 2 +- 2 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 .gitea/workflows/docker-publish.yml diff --git a/.gitea/workflows/docker-publish.yml b/.gitea/workflows/docker-publish.yml new file mode 100644 index 0000000..a21b82e --- /dev/null +++ b/.gitea/workflows/docker-publish.yml @@ -0,0 +1,151 @@ +name: Build and Push Docker Images + +on: + push: + branches: + - main + workflow_dispatch: + +env: + BASE_DOMAIN: dataprovider01.sandbox-cat-dat.simpl-europe.eu + OWNER: j.r + +jobs: + build-and-push: + runs-on: orchestration-platform + defaults: + run: + shell: sh + env: + REGISTRY: gitea.${{ env.BASE_DOMAIN }} + IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution + PAYLOAD_IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution-payload + REPO_DIR: repo + REPO_CLONE_URL: https://gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution.git + steps: + - name: Checkout repository (shell) + run: | + CLONE_USER="${{ secrets.REGISTRY_USERNAME }}" + CLONE_PASS="${{ secrets.REGISTRY_PASSWORD }}" + REF_NAME="${GITHUB_REF_NAME}" + if [ -z "${REF_NAME}" ]; then + REF_NAME="${GITHUB_REF#refs/heads/}" + fi + + if [ -z "${CLONE_USER}" ] || [ -z "${CLONE_PASS}" ]; then + echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret" + exit 1 + fi + + rm -rf "${REPO_DIR}" + AUTH_HEADER="$(printf '%s:%s' "${CLONE_USER}" "${CLONE_PASS}" | base64 | tr -d '\n')" + git clone --depth 1 --branch "${REF_NAME}" \ + -c "http.extraHeader=Authorization: Basic ${AUTH_HEADER}" \ + "${REPO_CLONE_URL}" \ + "${REPO_DIR}" + + if [ ! -f "${REPO_DIR}/Dockerfile" ]; then + echo "Code location Dockerfile not found after clone" + exit 1 + fi + + if [ ! -f "${REPO_DIR}/payload/Dockerfile" ]; then + echo "Payload Dockerfile not found after clone" + exit 1 + fi + + - name: Validate registry secrets + run: | + if [ -z "${{ secrets.REGISTRY_USERNAME }}" ] || [ -z "${{ secrets.REGISTRY_PASSWORD }}" ]; then + echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret" + exit 1 + fi + + - name: Login to registry + run: | + echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \ + -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin + + # Both images take the same SHA tag: that is what keeps the code location and + # the payload it dispatches on the same version. + - name: Build code location image + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + cd "${REPO_DIR}" + docker build \ + -t "${IMAGE_REPO}:latest" \ + -t "${IMAGE_REPO}:${SHORT_SHA}" \ + . + + - name: Build payload image + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + cd "${REPO_DIR}" + docker build \ + -f payload/Dockerfile \ + -t "${PAYLOAD_IMAGE_REPO}:latest" \ + -t "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" \ + payload/ + + - name: Validate code location image + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \ + dagster definitions validate -f src/distributed_execution/repository.py + docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \ + test -f /app/payload/work.py + + # Readiness checklist L11: the payload image must not carry the orchestration + # runtime, otherwise the isolation argument for the loosely coupled target is void. + - name: Validate payload image isolation + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + docker run --rm "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" python -c " + import importlib.util + assert importlib.util.find_spec('dagster_pipes') is not None, 'dagster_pipes missing from payload image' + assert importlib.util.find_spec('dagster') is None, 'payload image must not contain the dagster package' + print('payload isolation OK') + " + + - name: Push code location image tags + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + docker push "${IMAGE_REPO}:latest" + docker push "${IMAGE_REPO}:${SHORT_SHA}" + + - name: Push payload image tags + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + docker push "${PAYLOAD_IMAGE_REPO}:latest" + docker push "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" + + - name: Report image references + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + echo "Code location image: ${IMAGE_REPO}:${SHORT_SHA}" + echo "Payload image: ${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" + echo "" + echo "Set PIPES_PAYLOAD_IMAGE on both the code location and the run pods:" + echo " PIPES_PAYLOAD_IMAGE=${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" + echo "See yaml/loosely-coupled/values-pipes-payload.yaml." + + # The automated update of the deployment requires a technical user with + # their kube config in the secrets. See the template repository's user manual. + # - name: Update Dagster user deployment image + # run: | + # COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + # SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + # kubectl patch deployment "${DEPLOYMENT_NAME}" \ + # -n "${K8S_NAMESPACE}" \ + # --type='strategic' \ + # -p="{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"dagster-user-deployments\",\"image\":\"${IMAGE_REPO}:${SHORT_SHA}\",\"env\":[{\"name\":\"DAGSTER_CURRENT_IMAGE\",\"value\":\"${IMAGE_REPO}:${SHORT_SHA}\"},{\"name\":\"PIPES_PAYLOAD_IMAGE\",\"value\":\"${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}\"}]}]}}}}" + # kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \ + # -n "${K8S_NAMESPACE}" \ + # --timeout=5m diff --git a/documents/user-guide/distributed-execution-guide.md b/documents/user-guide/distributed-execution-guide.md index 89d264b..ac0cfb4 100644 --- a/documents/user-guide/distributed-execution-guide.md +++ b/documents/user-guide/distributed-execution-guide.md @@ -514,7 +514,7 @@ Tracked under SIMPL-30451. | Runnable loosely coupled reference implementation | AC4 | Complete (subprocess verified, K8s not cluster-run) | | Both images build; payload image passes the isolation check (L11) | Tech details | Complete | | Locally runnable jobs launched from the Dagster UI, evidence recorded in section 5.6 | Tech details | Complete | -| Payload image published to the container registry | Tech details | **Pending** — needs the CI build job | +| Payload image published to the container registry | Tech details | **Pending** — build workflow exists, awaiting registry secrets and a first run | | End-to-end run of `loosely_coupled_k8s_job` on a cluster | Tech details | **Pending** | | Screenshots of UI surfaces | Tech details | **Pending** — needs a deployed platform instance, not a local dev server | | Platform architecture document update | Tech details | **Pending** |