Merge pull request 'develop' (#12) from develop into main

Reviewed-on: #12
Reviewed-by: Jessica Roberts <j.r@email.com>

.gitea/workflows/check-deleted-workflows.yml

@@ -11,9 +11,6 @@ on:
       - ready_for_review
   workflow_dispatch:
 
-env:
-  BASE_DOMAIN: dataprovider01.sandbox-cat-dat.simpl-europe.eu
-
 jobs:
   check-deleted-workflows:
     runs-on: orchestration-platform
@@ -22,9 +19,6 @@ jobs:
     defaults:
       run:
         shell: bash
-    env:
-      REPO_DIR: repo
-      REPO_CLONE_URL: https://gitea.${{ env.BASE_DOMAIN }}/dataprovider01/template-code-location.git
     steps:
       - name: Install git
         run: |
@@ -33,6 +27,8 @@ jobs:
 
       - name: Checkout repository
         run: |
+          REPO_DIR="repo"
+          REPO_CLONE_URL="https://gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/dataprovider01/template-code-location.git"
           CLONE_USER="${{ secrets.REGISTRY_USERNAME }}"
           CLONE_PASS="${{ secrets.REGISTRY_PASSWORD }}"
 
@@ -59,12 +55,12 @@ jobs:
 
       - name: Install project dependencies
         run: |
-          cd "${REPO_DIR}"
+          cd repo
           uv sync --frozen --no-dev --no-install-package torch
 
       - name: Compute deleted workflows/jobs against main
         run: |
-          cd "${REPO_DIR}"
+          cd repo
           PATH="$PWD/.venv/bin:$PATH" \
           BASE_REF="${GITHUB_BASE_REF:-main}" \
           HEAD_REF="${GITHUB_HEAD_REF:-HEAD}" \
@@ -75,7 +71,7 @@ jobs:
 
       - name: Compute active workflows list
         run: |
-          cd "${REPO_DIR}"
+          cd repo
           PATH="$PWD/.venv/bin:$PATH" \
           ONLY_ACTIVE="true" \
           REGISTRY_USERNAME="${{ secrets.REGISTRY_USERNAME }}" \
@@ -86,7 +82,7 @@ jobs:
 
       - name: Fail only on overlap with active workflows
         run: |
-          cd "${REPO_DIR}"
+          cd repo
 
           echo "--- Deleted workflows ---"
           cat deleted_workflows.txt 2>/dev/null || echo "(none)"

.gitea/workflows/docker-publish.yml

@@ -6,9 +6,6 @@ on:
       - main
   workflow_dispatch:
 
-env:
-  BASE_DOMAIN: dataprovider01.sandbox-cat-dat.simpl-europe.eu
-
 jobs:
   build-and-push:
     runs-on: orchestration-platform
@@ -16,13 +13,16 @@ jobs:
       run:
         shell: sh
     env:
-      REGISTRY: gitea.${{ env.BASE_DOMAIN }}
+      REGISTRY: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu
-      IMAGE_REPO: gitea.${{ env.BASE_DOMAIN }}/dataprovider01/template-code-location
+      IMAGE_REPO: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/dataprovider01/template-code-location
-      REPO_DIR: repo
+      K8S_NAMESPACE: dataprovider01
-      REPO_CLONE_URL: https://gitea.${{ env.BASE_DOMAIN }}/dataprovider01/template-code-location.git
+      HELM_RELEASE: dataprovider01-dataprovider-orchestration-platform
+      DAGSTER_CHART_VERSION: 0.2.0
     steps:
       - name: Checkout repository (shell)
         run: |
+          REPO_DIR="repo"
+          REPO_CLONE_URL="https://gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/dataprovider01/template-code-location.git"
           CLONE_USER="${{ secrets.REGISTRY_USERNAME }}"
           CLONE_PASS="${{ secrets.REGISTRY_PASSWORD }}"
           REF_NAME="${GITHUB_REF_NAME}"
@@ -63,7 +63,7 @@ jobs:
         run: |
           COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
           SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
-          cd "${REPO_DIR}"
+          cd repo
           docker build \
             -t "${IMAGE_REPO}:latest" \
             -t "${IMAGE_REPO}:${SHORT_SHA}" \
@@ -83,30 +83,30 @@ jobs:
           docker push "${IMAGE_REPO}:latest"
           docker push "${IMAGE_REPO}:${SHORT_SHA}"
 
-      # The automated update of the deployment requires a technical user with
+      - name: Install kubectl
-      # their kube config in the secrets. See the user manual.
+        run: |
-      # - name: Install kubectl
+          apk add --no-cache kubectl
-      #   run: |
-      #     apk add --no-cache kubectl
 
-      # - name: Setup kubectl
+      - name: Setup kubectl
-      #   run: |
+        run: |
-      #     mkdir -p "${KUBE_CONFIG_DIR}"
+          mkdir -p "${HOME}/.kube"
-      #     echo "${{ secrets.KUBE_CONFIG_B64 }}" | base64 -d > "${KUBE_CONFIG_FILE}"
+          echo "${{ secrets.KUBE_CONFIG_B64 }}" | base64 -d > "${HOME}/.kube/config"
-      #     chmod 600 "${KUBE_CONFIG_FILE}"
+          chmod 600 "${HOME}/.kube/config"
 
-      # - name: Update Dagster user deployment image
+      - name: Update Dagster user deployment image
-      #   run: |
+        run: |
-      #     COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
+          COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}"
-      #     SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
+          SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)"
 
-      #     # Keep both the code server image and DAGSTER_CURRENT_IMAGE in sync.
+          DEPLOYMENT_NAME="dataprovider01-dataprovider-orchestration-platform-dagster-user-template-code-location"
-      #     kubectl patch deployment "${DEPLOYMENT_NAME}" \
-      #       -n "${K8S_NAMESPACE}" \
-      #       --type='strategic' \
-      #       -p="{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"dagster-user-deployments\",\"image\":\"${IMAGE_REPO}:${SHORT_SHA}\",\"env\":[{\"name\":\"DAGSTER_CURRENT_IMAGE\",\"value\":\"${IMAGE_REPO}:${SHORT_SHA}\"}]}]}}}}"
 
-      #     # Wait for rollout to complete
+          # Keep both the code server image and DAGSTER_CURRENT_IMAGE in sync.
-      #     kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \
+          kubectl patch deployment "${DEPLOYMENT_NAME}" \
-      #       -n "${K8S_NAMESPACE}" \
+            -n "${K8S_NAMESPACE}" \
-      #       --timeout=5m
+            --type='strategic' \
+            -p="{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"dagster-user-deployments\",\"image\":\"${IMAGE_REPO}:${SHORT_SHA}\",\"env\":[{\"name\":\"DAGSTER_CURRENT_IMAGE\",\"value\":\"${IMAGE_REPO}:${SHORT_SHA}\"}]}]}}}}"
+
+          # Wait for rollout to complete
+          kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \
+            -n "${K8S_NAMESPACE}" \
+            --timeout=5m

src/template_code_location/repository.py

@@ -13,7 +13,7 @@ from util_services.resources import s3_resource
 defs = Definitions(
     jobs=[
         data_processing_job,
-        example_job,
+        example_dataframe_demo_job,
         k_anonymity_job_s3,
         l_diversity_job_s3,
         t_closeness_job_s3,