From f139a0dd8612d179b1fab3e9f29895bf2b9cd53a Mon Sep 17 00:00:00 2001 From: ILay Date: Tue, 1 Sep 2026 19:14:22 +0200 Subject: [PATCH] [SIMPL-30451] Restore sandbox image workflow Keep the Gitea-only image publishing workflow on the sandbox mirror while building the shared-storage step pod fix. Changelog: other --- .gitea/workflows/docker-publish.yml | 151 ++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 .gitea/workflows/docker-publish.yml diff --git a/.gitea/workflows/docker-publish.yml b/.gitea/workflows/docker-publish.yml new file mode 100644 index 0000000..a21b82e --- /dev/null +++ b/.gitea/workflows/docker-publish.yml @@ -0,0 +1,151 @@ +name: Build and Push Docker Images + +on: + push: + branches: + - main + workflow_dispatch: + +env: + BASE_DOMAIN: dataprovider01.sandbox-cat-dat.simpl-europe.eu + OWNER: j.r + +jobs: + build-and-push: + runs-on: orchestration-platform + defaults: + run: + shell: sh + env: + REGISTRY: gitea.${{ env.BASE_DOMAIN }} + IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution + PAYLOAD_IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution-payload + REPO_DIR: repo + REPO_CLONE_URL: https://gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution.git + steps: + - name: Checkout repository (shell) + run: | + CLONE_USER="${{ secrets.REGISTRY_USERNAME }}" + CLONE_PASS="${{ secrets.REGISTRY_PASSWORD }}" + REF_NAME="${GITHUB_REF_NAME}" + if [ -z "${REF_NAME}" ]; then + REF_NAME="${GITHUB_REF#refs/heads/}" + fi + + if [ -z "${CLONE_USER}" ] || [ -z "${CLONE_PASS}" ]; then + echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret" + exit 1 + fi + + rm -rf "${REPO_DIR}" + AUTH_HEADER="$(printf '%s:%s' "${CLONE_USER}" "${CLONE_PASS}" | base64 | tr -d '\n')" + git clone --depth 1 --branch "${REF_NAME}" \ + -c "http.extraHeader=Authorization: Basic ${AUTH_HEADER}" \ + "${REPO_CLONE_URL}" \ + "${REPO_DIR}" + + if [ ! -f "${REPO_DIR}/Dockerfile" ]; then + echo "Code location Dockerfile not found after clone" + exit 1 + fi + + if [ ! -f "${REPO_DIR}/payload/Dockerfile" ]; then + echo "Payload Dockerfile not found after clone" + exit 1 + fi + + - name: Validate registry secrets + run: | + if [ -z "${{ secrets.REGISTRY_USERNAME }}" ] || [ -z "${{ secrets.REGISTRY_PASSWORD }}" ]; then + echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret" + exit 1 + fi + + - name: Login to registry + run: | + echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \ + -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin + + # Both images take the same SHA tag: that is what keeps the code location and + # the payload it dispatches on the same version. + - name: Build code location image + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + cd "${REPO_DIR}" + docker build \ + -t "${IMAGE_REPO}:latest" \ + -t "${IMAGE_REPO}:${SHORT_SHA}" \ + . + + - name: Build payload image + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + cd "${REPO_DIR}" + docker build \ + -f payload/Dockerfile \ + -t "${PAYLOAD_IMAGE_REPO}:latest" \ + -t "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" \ + payload/ + + - name: Validate code location image + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \ + dagster definitions validate -f src/distributed_execution/repository.py + docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \ + test -f /app/payload/work.py + + # Readiness checklist L11: the payload image must not carry the orchestration + # runtime, otherwise the isolation argument for the loosely coupled target is void. + - name: Validate payload image isolation + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + docker run --rm "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" python -c " + import importlib.util + assert importlib.util.find_spec('dagster_pipes') is not None, 'dagster_pipes missing from payload image' + assert importlib.util.find_spec('dagster') is None, 'payload image must not contain the dagster package' + print('payload isolation OK') + " + + - name: Push code location image tags + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + docker push "${IMAGE_REPO}:latest" + docker push "${IMAGE_REPO}:${SHORT_SHA}" + + - name: Push payload image tags + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + docker push "${PAYLOAD_IMAGE_REPO}:latest" + docker push "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" + + - name: Report image references + run: | + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + echo "Code location image: ${IMAGE_REPO}:${SHORT_SHA}" + echo "Payload image: ${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" + echo "" + echo "Set PIPES_PAYLOAD_IMAGE on both the code location and the run pods:" + echo " PIPES_PAYLOAD_IMAGE=${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" + echo "See yaml/loosely-coupled/values-pipes-payload.yaml." + + # The automated update of the deployment requires a technical user with + # their kube config in the secrets. See the template repository's user manual. + # - name: Update Dagster user deployment image + # run: | + # COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + # SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + # kubectl patch deployment "${DEPLOYMENT_NAME}" \ + # -n "${K8S_NAMESPACE}" \ + # --type='strategic' \ + # -p="{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"dagster-user-deployments\",\"image\":\"${IMAGE_REPO}:${SHORT_SHA}\",\"env\":[{\"name\":\"DAGSTER_CURRENT_IMAGE\",\"value\":\"${IMAGE_REPO}:${SHORT_SHA}\"},{\"name\":\"PIPES_PAYLOAD_IMAGE\",\"value\":\"${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}\"}]}]}}}}" + # kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \ + # -n "${K8S_NAMESPACE}" \ + # --timeout=5m