diff --git a/.gitea/workflows/docker-publish.yml b/.gitea/workflows/docker-publish.yml deleted file mode 100644 index 61a0443..0000000 --- a/.gitea/workflows/docker-publish.yml +++ /dev/null @@ -1,151 +0,0 @@ -name: Build and Push Docker Images - -on: - push: - branches: - - main - workflow_dispatch: - -env: - BASE_DOMAIN: dataprovider01.sandbox-cat-dat.simpl-europe.eu - OWNER: ${{ github.repository_owner }} - -jobs: - build-and-push: - runs-on: orchestration-platform - defaults: - run: - shell: sh - env: - REGISTRY: gitea.${{ env.BASE_DOMAIN }} - IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution - PAYLOAD_IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution-payload - REPO_DIR: repo - REPO_CLONE_URL: https://gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution.git - steps: - - name: Checkout repository (shell) - run: | - CLONE_USER="${{ secrets.REGISTRY_USERNAME }}" - CLONE_PASS="${{ secrets.REGISTRY_PASSWORD }}" - REF_NAME="${GITHUB_REF_NAME}" - if [ -z "${REF_NAME}" ]; then - REF_NAME="${GITHUB_REF#refs/heads/}" - fi - - if [ -z "${CLONE_USER}" ] || [ -z "${CLONE_PASS}" ]; then - echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret" - exit 1 - fi - - rm -rf "${REPO_DIR}" - AUTH_HEADER="$(printf '%s:%s' "${CLONE_USER}" "${CLONE_PASS}" | base64 | tr -d '\n')" - git clone --depth 1 --branch "${REF_NAME}" \ - -c "http.extraHeader=Authorization: Basic ${AUTH_HEADER}" \ - "${REPO_CLONE_URL}" \ - "${REPO_DIR}" - - if [ ! -f "${REPO_DIR}/Dockerfile" ]; then - echo "Code location Dockerfile not found after clone" - exit 1 - fi - - if [ ! -f "${REPO_DIR}/payload/Dockerfile" ]; then - echo "Payload Dockerfile not found after clone" - exit 1 - fi - - - name: Validate registry secrets - run: | - if [ -z "${{ secrets.REGISTRY_USERNAME }}" ] || [ -z "${{ secrets.REGISTRY_PASSWORD }}" ]; then - echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret" - exit 1 - fi - - - name: Login to registry - run: | - echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \ - -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin - - # Both images take the same SHA tag: that is what keeps the code location and - # the payload it dispatches on the same version. - - name: Build code location image - run: | - COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" - SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" - cd "${REPO_DIR}" - docker build \ - -t "${IMAGE_REPO}:latest" \ - -t "${IMAGE_REPO}:${SHORT_SHA}" \ - . - - - name: Build payload image - run: | - COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" - SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" - cd "${REPO_DIR}" - docker build \ - -f payload/Dockerfile \ - -t "${PAYLOAD_IMAGE_REPO}:latest" \ - -t "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" \ - payload/ - - - name: Validate code location image - run: | - COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" - SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" - docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \ - dagster definitions validate -f src/distributed_execution/repository.py - docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \ - test -f /app/payload/work.py - - # Readiness checklist L11: the payload image must not carry the orchestration - # runtime, otherwise the isolation argument for the loosely coupled target is void. - - name: Validate payload image isolation - run: | - COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" - SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" - docker run --rm "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" python -c " - import importlib.util - assert importlib.util.find_spec('dagster_pipes') is not None, 'dagster_pipes missing from payload image' - assert importlib.util.find_spec('dagster') is None, 'payload image must not contain the dagster package' - print('payload isolation OK') - " - - - name: Push code location image tags - run: | - COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" - SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" - docker push "${IMAGE_REPO}:latest" - docker push "${IMAGE_REPO}:${SHORT_SHA}" - - - name: Push payload image tags - run: | - COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" - SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" - docker push "${PAYLOAD_IMAGE_REPO}:latest" - docker push "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" - - - name: Report image references - run: | - COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" - SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" - echo "Code location image: ${IMAGE_REPO}:${SHORT_SHA}" - echo "Payload image: ${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" - echo "" - echo "Set PIPES_PAYLOAD_IMAGE on both the code location and the run pods:" - echo " PIPES_PAYLOAD_IMAGE=${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" - echo "See yaml/loosely-coupled/values-pipes-payload.yaml." - - # The automated update of the deployment requires a technical user with - # their kube config in the secrets. See the template repository's user manual. - # - name: Update Dagster user deployment image - # run: | - # COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" - # SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" - # kubectl patch deployment "${DEPLOYMENT_NAME}" \ - # -n "${K8S_NAMESPACE}" \ - # --type='strategic' \ - # -p="{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"dagster-user-deployments\",\"image\":\"${IMAGE_REPO}:${SHORT_SHA}\",\"env\":[{\"name\":\"DAGSTER_CURRENT_IMAGE\",\"value\":\"${IMAGE_REPO}:${SHORT_SHA}\"},{\"name\":\"PIPES_PAYLOAD_IMAGE\",\"value\":\"${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}\"}]}]}}}}" - # kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \ - # -n "${K8S_NAMESPACE}" \ - # --timeout=5m diff --git a/README.md b/README.md index 14ed312..3241130 100644 --- a/README.md +++ b/README.md @@ -81,10 +81,10 @@ Both images build and have been smoke tested locally: the code location image loads its definitions, and the payload image contains `dagster_pipes` without `dagster` — check L11 in the readiness checklist. -[.gitea/workflows/docker-publish.yml](.gitea/workflows/docker-publish.yml) runs -the same two builds on every push to `main`, applies those checks as gates, and -tags both images with the same short commit SHA — that shared tag is what keeps a -code location and the payload it dispatches on the same version. +Both must be tagged from the same commit. That shared tag is what keeps a code +location and the payload it dispatches on the same version, and nothing at +runtime checks the pairing — see the *Outstanding work* section for what the +pipeline still has to learn. ## Status diff --git a/yaml/loosely-coupled/values-pipes-payload.yaml b/yaml/loosely-coupled/values-pipes-payload.yaml index b529d82..b49640b 100644 --- a/yaml/loosely-coupled/values-pipes-payload.yaml +++ b/yaml/loosely-coupled/values-pipes-payload.yaml @@ -17,8 +17,7 @@ # For values that work today, see yaml/sandbox/values-sandbox-gitea.yaml. The # shared GitLab pipeline (ds.gitlab-ci.yml) builds exactly one image per project - # $CI_REGISTRY_IMAGE from the root Dockerfile - so the payload reference below -# resolves only once that pipeline learns to build a second image. The Gitea -# workflow in .gitea/workflows/docker-publish.yml already builds both. +# resolves only once that pipeline learns to build a second image. dagster: dagster-user-deployments: diff --git a/yaml/sandbox/values-sandbox-gitea.yaml b/yaml/sandbox/values-sandbox-gitea.yaml index 883ee37..949cc63 100644 --- a/yaml/sandbox/values-sandbox-gitea.yaml +++ b/yaml/sandbox/values-sandbox-gitea.yaml @@ -7,8 +7,8 @@ # Use these instead of the code.europa.eu references in # yaml/values-dagster-distributed-execution.yaml and # yaml/loosely-coupled/values-pipes-payload.yaml when deploying to the -# dataprovider01 sandbox. Both images are published by -# .gitea/workflows/docker-publish.yml. +# dataprovider01 sandbox. Both images were built and published by a Gitea Actions +# workflow held in the sandbox Gitea mirror of this repository, not here. # # The tag below is a short commit SHA and is the same for both images. That is # the version lock: the code location and the payload it dispatches must come