Merge pull request 'develop' (#13) from develop into main

Reviewed-on: #13
Reviewed-by: Mary Brown <m.b@email.com>

.gitea/workflows/check-deleted-workflows.yml

@@ -11,6 +11,9 @@ on:
       - ready_for_review
   workflow_dispatch:
 
+env:
+  BASE_DOMAIN: dataprovider01.sandbox-cat-dat.simpl-europe.eu
+
 jobs:
   check-deleted-workflows:
     runs-on: orchestration-platform
@@ -19,6 +22,9 @@ jobs:
     defaults:
       run:
         shell: bash
+    env:
+      REPO_DIR: repo
+      REPO_CLONE_URL: https://gitea.${{ env.BASE_DOMAIN }}/dataprovider01/template-code-location.git
     steps:
       - name: Install git
         run: |
@@ -27,8 +33,6 @@ jobs:
 
       - name: Checkout repository
         run: |
-          REPO_DIR="repo"
-          REPO_CLONE_URL="https://gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/dataprovider01/template-code-location.git"
           CLONE_USER="${{ secrets.REGISTRY_USERNAME }}"
           CLONE_PASS="${{ secrets.REGISTRY_PASSWORD }}"
 
@@ -55,12 +59,12 @@ jobs:
 
       - name: Install project dependencies
         run: |
-          cd repo
+          cd "${REPO_DIR}"
           uv sync --frozen --no-dev --no-install-package torch
 
       - name: Compute deleted workflows/jobs against main
         run: |
-          cd repo
+          cd "${REPO_DIR}"
           PATH="$PWD/.venv/bin:$PATH" \
           BASE_REF="${GITHUB_BASE_REF:-main}" \
           HEAD_REF="${GITHUB_HEAD_REF:-HEAD}" \
@@ -71,7 +75,7 @@ jobs:
 
       - name: Compute active workflows list
         run: |
-          cd repo
+          cd "${REPO_DIR}"
           PATH="$PWD/.venv/bin:$PATH" \
           ONLY_ACTIVE="true" \
           REGISTRY_USERNAME="${{ secrets.REGISTRY_USERNAME }}" \
@@ -82,7 +86,7 @@ jobs:
 
       - name: Fail only on overlap with active workflows
         run: |
-          cd repo
+          cd "${REPO_DIR}"
 
           echo "--- Deleted workflows ---"
           cat deleted_workflows.txt 2>/dev/null || echo "(none)"

.gitea/workflows/docker-publish.yml

@@ -6,6 +6,9 @@ on:
       - main
   workflow_dispatch:
 
+env:
+  BASE_DOMAIN: dataprovider01.sandbox-cat-dat.simpl-europe.eu
+
 jobs:
   build-and-push:
     runs-on: orchestration-platform
@@ -13,16 +16,13 @@ jobs:
       run:
         shell: sh
     env:
-      REGISTRY: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu
-      IMAGE_REPO: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/dataprovider01/template-code-location
-      K8S_NAMESPACE: dataprovider01
-      HELM_RELEASE: dataprovider01-dataprovider-orchestration-platform
-      DAGSTER_CHART_VERSION: 0.2.0
+      REGISTRY: gitea.${{ env.BASE_DOMAIN }}
+      IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/dataprovider01/template-code-location
+      REPO_DIR: repo
+      REPO_CLONE_URL: https://gitea.${{ env.BASE_DOMAIN }}/dataprovider01/template-code-location.git
     steps:
       - name: Checkout repository (shell)
         run: |
-          REPO_DIR="repo"
-          REPO_CLONE_URL="https://gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/dataprovider01/template-code-location.git"
           CLONE_USER="${{ secrets.REGISTRY_USERNAME }}"
           CLONE_PASS="${{ secrets.REGISTRY_PASSWORD }}"
           REF_NAME="${GITHUB_REF_NAME}"
@@ -63,7 +63,7 @@ jobs:
         run: |
           COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
           SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
-          cd repo
+          cd "${REPO_DIR}"
           docker build \
             -t "${IMAGE_REPO}:latest" \
             -t "${IMAGE_REPO}:${SHORT_SHA}" \
@@ -83,30 +83,30 @@ jobs:
           docker push "${IMAGE_REPO}:latest"
           docker push "${IMAGE_REPO}:${SHORT_SHA}"
 
-      - name: Install kubectl
-        run: |
-          apk add --no-cache kubectl
+      # The automated update of the deployment requires a technical user with
+      # their kube config in the secrets. See the user manual.
+      # - name: Install kubectl
+      #   run: |
+      #     apk add --no-cache kubectl
 
-      - name: Setup kubectl
-        run: |
-          mkdir -p "${HOME}/.kube"
-          echo "${{ secrets.KUBE_CONFIG_B64 }}" | base64 -d > "${HOME}/.kube/config"
-          chmod 600 "${HOME}/.kube/config"
+      # - name: Setup kubectl
+      #   run: |
+      #     mkdir -p "${KUBE_CONFIG_DIR}"
+      #     echo "${{ secrets.KUBE_CONFIG_B64 }}" | base64 -d > "${KUBE_CONFIG_FILE}"
+      #     chmod 600 "${KUBE_CONFIG_FILE}"
 
-      - name: Update Dagster user deployment image
-        run: |
-          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
-          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
+      # - name: Update Dagster user deployment image
+      #   run: |
+      #     COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
+      #     SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
 
-          DEPLOYMENT_NAME="dataprovider01-dataprovider-orchestration-platform-dagster-user-template-code-location"
+      #     # Keep both the code server image and DAGSTER_CURRENT_IMAGE in sync.
+      #     kubectl patch deployment "${DEPLOYMENT_NAME}" \
+      #       -n "${K8S_NAMESPACE}" \
+      #       --type='strategic' \
+      #       -p="{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"dagster-user-deployments\",\"image\":\"${IMAGE_REPO}:${SHORT_SHA}\",\"env\":[{\"name\":\"DAGSTER_CURRENT_IMAGE\",\"value\":\"${IMAGE_REPO}:${SHORT_SHA}\"}]}]}}}}"
 
-          # Keep both the code server image and DAGSTER_CURRENT_IMAGE in sync.
-          kubectl patch deployment "${DEPLOYMENT_NAME}" \
-            -n "${K8S_NAMESPACE}" \
-            --type='strategic' \
-            -p="{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"dagster-user-deployments\",\"image\":\"${IMAGE_REPO}:${SHORT_SHA}\",\"env\":[{\"name\":\"DAGSTER_CURRENT_IMAGE\",\"value\":\"${IMAGE_REPO}:${SHORT_SHA}\"}]}]}}}}"
-
-          # Wait for rollout to complete
-          kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \
-            -n "${K8S_NAMESPACE}" \
-            --timeout=5m
+      #     # Wait for rollout to complete
+      #     kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \
+      #       -n "${K8S_NAMESPACE}" \
+      #       --timeout=5m

src/template_code_location/repository.py

@@ -13,7 +13,7 @@ from util_services.resources import s3_resource
 defs = Definitions(
     jobs=[
         data_processing_job,
-        example_dataframe_demo_job,
+        example_job,
         k_anonymity_job_s3,
         l_diversity_job_s3,
         t_closeness_job_s3,