name: Build and Push Docker Images on: push: branches: - main workflow_dispatch: env: BASE_DOMAIN: dataprovider01.sandbox-cat-dat.simpl-europe.eu OWNER: ${{ github.repository_owner }} jobs: build-and-push: runs-on: orchestration-platform defaults: run: shell: sh env: REGISTRY: gitea.${{ env.BASE_DOMAIN }} IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution PAYLOAD_IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution-payload REPO_DIR: repo REPO_CLONE_URL: https://gitea.${{ env.BASE_DOMAIN }}/${{ env.OWNER }}/distributed-execution.git steps: - name: Checkout repository (shell) run: | CLONE_USER="${{ secrets.REGISTRY_USERNAME }}" CLONE_PASS="${{ secrets.REGISTRY_PASSWORD }}" REF_NAME="${GITHUB_REF_NAME}" if [ -z "${REF_NAME}" ]; then REF_NAME="${GITHUB_REF#refs/heads/}" fi if [ -z "${CLONE_USER}" ] || [ -z "${CLONE_PASS}" ]; then echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret" exit 1 fi rm -rf "${REPO_DIR}" AUTH_HEADER="$(printf '%s:%s' "${CLONE_USER}" "${CLONE_PASS}" | base64 | tr -d '\n')" git clone --depth 1 --branch "${REF_NAME}" \ -c "http.extraHeader=Authorization: Basic ${AUTH_HEADER}" \ "${REPO_CLONE_URL}" \ "${REPO_DIR}" if [ ! -f "${REPO_DIR}/Dockerfile" ]; then echo "Code location Dockerfile not found after clone" exit 1 fi if [ ! -f "${REPO_DIR}/payload/Dockerfile" ]; then echo "Payload Dockerfile not found after clone" exit 1 fi - name: Validate registry secrets run: | if [ -z "${{ secrets.REGISTRY_USERNAME }}" ] || [ -z "${{ secrets.REGISTRY_PASSWORD }}" ]; then echo "Missing REGISTRY_USERNAME or REGISTRY_PASSWORD secret" exit 1 fi - name: Login to registry run: | echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \ -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin # Both images take the same SHA tag: that is what keeps the code location and # the payload it dispatches on the same version. - name: Build code location image run: | COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" cd "${REPO_DIR}" docker build \ -t "${IMAGE_REPO}:latest" \ -t "${IMAGE_REPO}:${SHORT_SHA}" \ . - name: Build payload image run: | COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" cd "${REPO_DIR}" docker build \ -f payload/Dockerfile \ -t "${PAYLOAD_IMAGE_REPO}:latest" \ -t "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" \ payload/ - name: Validate code location image run: | COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \ dagster definitions validate -f src/distributed_execution/repository.py docker run --rm "${IMAGE_REPO}:${SHORT_SHA}" \ test -f /app/payload/work.py # Readiness checklist L11: the payload image must not carry the orchestration # runtime, otherwise the isolation argument for the loosely coupled target is void. - name: Validate payload image isolation run: | COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" docker run --rm "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" python -c " import importlib.util assert importlib.util.find_spec('dagster_pipes') is not None, 'dagster_pipes missing from payload image' assert importlib.util.find_spec('dagster') is None, 'payload image must not contain the dagster package' print('payload isolation OK') " - name: Push code location image tags run: | COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" docker push "${IMAGE_REPO}:latest" docker push "${IMAGE_REPO}:${SHORT_SHA}" - name: Push payload image tags run: | COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" docker push "${PAYLOAD_IMAGE_REPO}:latest" docker push "${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" - name: Report image references run: | COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" echo "Code location image: ${IMAGE_REPO}:${SHORT_SHA}" echo "Payload image: ${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" echo "" echo "Set PIPES_PAYLOAD_IMAGE on both the code location and the run pods:" echo " PIPES_PAYLOAD_IMAGE=${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}" echo "See yaml/loosely-coupled/values-pipes-payload.yaml." # The automated update of the deployment requires a technical user with # their kube config in the secrets. See the template repository's user manual. # - name: Update Dagster user deployment image # run: | # COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" # SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" # kubectl patch deployment "${DEPLOYMENT_NAME}" \ # -n "${K8S_NAMESPACE}" \ # --type='strategic' \ # -p="{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"dagster-user-deployments\",\"image\":\"${IMAGE_REPO}:${SHORT_SHA}\",\"env\":[{\"name\":\"DAGSTER_CURRENT_IMAGE\",\"value\":\"${IMAGE_REPO}:${SHORT_SHA}\"},{\"name\":\"PIPES_PAYLOAD_IMAGE\",\"value\":\"${PAYLOAD_IMAGE_REPO}:${SHORT_SHA}\"}]}]}}}}" # kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \ # -n "${K8S_NAMESPACE}" \ # --timeout=5m