diff --git a/.gitea/workflows/docker-publish.yml b/.gitea/workflows/docker-publish.yml index adb281c..066febb 100644 --- a/.gitea/workflows/docker-publish.yml +++ b/.gitea/workflows/docker-publish.yml @@ -15,6 +15,9 @@ jobs: env: REGISTRY: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu IMAGE_REPO: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/template-code-location + K8S_NAMESPACE: dataprovider01 + HELM_RELEASE: dataprovider01-dataprovider-orchestration-platform + DAGSTER_CHART_VERSION: 0.2.0 steps: - name: Checkout repository (shell) run: | @@ -72,3 +75,60 @@ jobs: SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" docker push "${IMAGE_REPO}:latest" docker push "${IMAGE_REPO}:${SHORT_SHA}" + + - name: Install helm and kubectl + run: | + ARCH="$(uname -m)" + if [ "${ARCH}" = "x86_64" ]; then + ARCH="amd64" + elif [ "${ARCH}" = "aarch64" ]; then + ARCH="arm64" + fi + + if ! command -v helm >/dev/null 2>&1; then + apk add --no-cache curl tar gzip ca-certificates + HELM_VERSION="v3.16.4" + curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-${ARCH}.tar.gz" -o /tmp/helm.tgz + tar -xzf /tmp/helm.tgz -C /tmp + mv "/tmp/linux-${ARCH}/helm" /usr/local/bin/helm + chmod +x /usr/local/bin/helm + fi + + if ! command -v kubectl >/dev/null 2>&1; then + KUBECTL_VERSION="v1.31.2" + curl -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl" -o /usr/local/bin/kubectl + chmod +x /usr/local/bin/kubectl + fi + + helm version + kubectl version --client + + - name: Deploy to Dagster with Helm + run: | + if [ -z "${{ secrets.KUBE_CONFIG_B64 }}" ]; then + echo "Missing KUBE_CONFIG_B64 secret" + exit 1 + fi + + COMMIT_SHA="${GITHUB_SHA:-$GITEA_SHA}" + SHORT_SHA="$(echo "${COMMIT_SHA}" | cut -c1-12)" + + mkdir -p "${HOME}/.kube" + echo "${{ secrets.KUBE_CONFIG_B64 }}" | base64 -d > "${HOME}/.kube/config" + export KUBECONFIG="${HOME}/.kube/config" + + helm upgrade --install "${HELM_RELEASE}" dagster \ + --repo "https://code.europa.eu/api/v4/projects/1304/packages/helm/stable" \ + --version "${DAGSTER_CHART_VERSION}" \ + --namespace "${K8S_NAMESPACE}" \ + --reuse-values \ + --set-string "dagster.dagster-user-deployments.deployments[0].image.repository=${IMAGE_REPO}" \ + --set-string "dagster.dagster-user-deployments.deployments[0].image.tag=${SHORT_SHA}" + + DEPLOYMENT_NAME="$(kubectl -n "${K8S_NAMESPACE}" get deployment -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep 'template-code-location' | head -n 1)" + if [ -n "${DEPLOYMENT_NAME}" ]; then + kubectl -n "${K8S_NAMESPACE}" rollout restart "deployment/${DEPLOYMENT_NAME}" + kubectl -n "${K8S_NAMESPACE}" rollout status "deployment/${DEPLOYMENT_NAME}" --timeout=300s + else + echo "No deployment name matched template-code-location; Helm upgrade completed without explicit rollout restart." + fi diff --git a/.gitea/workflows/gitearunnerstateful.yml b/.gitea/workflows/gitearunnerstateful.yml deleted file mode 100644 index 9b2823d..0000000 --- a/.gitea/workflows/gitearunnerstateful.yml +++ /dev/null @@ -1,480 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - annotations: - checksum/config: 168300d8582bd312023d57c317895b11631c866e2aa01f7c6b276a97d1f490e7 - cni.projectcalico.org/containerID: c6194494339ec0e482dacc13b1893533beaccf7a78fc51e0aa50ca9b8681d180 - cni.projectcalico.org/podIP: 10.2.2.122/32 - cni.projectcalico.org/podIPs: 10.2.2.122/32 - creationTimestamp: '2026-06-11T13:50:27Z' - generateName: gitea-act-runner- - labels: - app: actions-act-runner - app.kubernetes.io/instance: dataprovider01-dataprovider-orchestration-platform - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: actions-act-runner - app.kubernetes.io/version: 0.261.3 - apps.kubernetes.io/pod-index: '0' - controller-revision-hash: gitea-act-runner-766f4d69b7 - helm.sh/chart: actions-0.0.3 - statefulset.kubernetes.io/pod-name: gitea-act-runner-0 - version: 0.261.3 - managedFields: - - apiVersion: v1 - fieldsType: FieldsV1 - fieldsV1: - f:metadata: - f:annotations: - .: {} - f:checksum/config: {} - f:generateName: {} - f:labels: - .: {} - f:app: {} - f:app.kubernetes.io/instance: {} - f:app.kubernetes.io/managed-by: {} - f:app.kubernetes.io/name: {} - f:app.kubernetes.io/version: {} - f:apps.kubernetes.io/pod-index: {} - f:controller-revision-hash: {} - f:helm.sh/chart: {} - f:statefulset.kubernetes.io/pod-name: {} - f:version: {} - f:ownerReferences: - .: {} - k:{"uid":"e8adf92d-6e2d-4662-b09b-490a495cde2b"}: {} - f:spec: - f:containers: - k:{"name":"act-runner"}: - .: {} - f:env: - .: {} - k:{"name":"CONFIG_FILE"}: - .: {} - f:name: {} - f:value: {} - k:{"name":"GITEA_INSTANCE_URL"}: - .: {} - f:name: {} - f:value: {} - k:{"name":"GITEA_RUNNER_REGISTRATION_TOKEN"}: - .: {} - f:name: {} - f:valueFrom: - .: {} - f:secretKeyRef: {} - k:{"name":"TZ"}: - .: {} - f:name: {} - f:value: {} - f:image: {} - f:imagePullPolicy: {} - f:name: {} - f:resources: {} - f:terminationMessagePath: {} - f:terminationMessagePolicy: {} - f:volumeMounts: - .: {} - k:{"mountPath":"/actrunner/config.yaml"}: - .: {} - f:mountPath: {} - f:name: {} - f:subPath: {} - k:{"mountPath":"/data"}: - .: {} - f:mountPath: {} - f:name: {} - k:{"mountPath":"/var/run/docker.sock"}: - .: {} - f:mountPath: {} - f:name: {} - f:subPath: {} - f:workingDir: {} - f:dnsPolicy: {} - f:enableServiceLinks: {} - f:hostname: {} - f:initContainers: - .: {} - k:{"name":"dind"}: - .: {} - f:image: {} - f:imagePullPolicy: {} - f:livenessProbe: - .: {} - f:exec: - .: {} - f:command: {} - f:failureThreshold: {} - f:periodSeconds: {} - f:successThreshold: {} - f:timeoutSeconds: {} - f:name: {} - f:resources: {} - f:restartPolicy: {} - f:securityContext: - .: {} - f:privileged: {} - f:startupProbe: - .: {} - f:exec: - .: {} - f:command: {} - f:failureThreshold: {} - f:periodSeconds: {} - f:successThreshold: {} - f:timeoutSeconds: {} - f:terminationMessagePath: {} - f:terminationMessagePolicy: {} - f:volumeMounts: - .: {} - k:{"mountPath":"/var/run/"}: - .: {} - f:mountPath: {} - f:name: {} - k:{"name":"init-gitea"}: - .: {} - f:command: {} - f:image: {} - f:imagePullPolicy: {} - f:name: {} - f:resources: {} - f:terminationMessagePath: {} - f:terminationMessagePolicy: {} - f:restartPolicy: {} - f:schedulerName: {} - f:securityContext: {} - f:terminationGracePeriodSeconds: {} - f:volumes: - .: {} - k:{"name":"act-runner-config"}: - .: {} - f:configMap: - .: {} - f:defaultMode: {} - f:name: {} - f:name: {} - k:{"name":"data-act-runner"}: - .: {} - f:name: {} - f:persistentVolumeClaim: - .: {} - f:claimName: {} - k:{"name":"docker-socket"}: - .: {} - f:emptyDir: {} - f:name: {} - manager: kube-controller-manager - operation: Update - time: '2026-06-11T13:50:27Z' - - apiVersion: v1 - fieldsType: FieldsV1 - fieldsV1: - f:metadata: - f:annotations: - f:cni.projectcalico.org/containerID: {} - f:cni.projectcalico.org/podIP: {} - f:cni.projectcalico.org/podIPs: {} - manager: calico - operation: Update - subresource: status - time: '2026-06-11T13:50:36Z' - - apiVersion: v1 - fieldsType: FieldsV1 - fieldsV1: - f:status: - f:conditions: - k:{"type":"ContainersReady"}: - .: {} - f:lastProbeTime: {} - f:lastTransitionTime: {} - f:status: {} - f:type: {} - k:{"type":"Initialized"}: - .: {} - f:lastProbeTime: {} - f:lastTransitionTime: {} - f:status: {} - f:type: {} - k:{"type":"PodReadyToStartContainers"}: - .: {} - f:lastProbeTime: {} - f:lastTransitionTime: {} - f:status: {} - f:type: {} - k:{"type":"Ready"}: - .: {} - f:lastProbeTime: {} - f:lastTransitionTime: {} - f:status: {} - f:type: {} - f:containerStatuses: {} - f:hostIP: {} - f:hostIPs: {} - f:initContainerStatuses: {} - f:phase: {} - f:podIP: {} - f:podIPs: - .: {} - k:{"ip":"10.2.2.122"}: - .: {} - f:ip: {} - f:startTime: {} - manager: kubelet - operation: Update - subresource: status - time: '2026-06-11T13:50:51Z' - name: gitea-act-runner-0 - namespace: dataprovider01 - ownerReferences: - - apiVersion: apps/v1 - blockOwnerDeletion: true - controller: true - kind: StatefulSet - name: gitea-act-runner - uid: e8adf92d-6e2d-4662-b09b-490a495cde2b - resourceVersion: '11641437698' - uid: a99c0e20-45e5-4641-9d3a-646324a72c9a -spec: - containers: - - env: - - name: GITEA_RUNNER_REGISTRATION_TOKEN - valueFrom: - secretKeyRef: - key: token - name: gitea-runner-token - - name: GITEA_INSTANCE_URL - value: >- - http://dataprovider01-dataprovider-orchestration-platform-gitea-http.dataprovider01.svc.cluster.local:3000 - - name: CONFIG_FILE - value: /actrunner/config.yaml - - name: TZ - value: Etc/UTC - image: docker.gitea.com/act_runner:0.2.13 - imagePullPolicy: Always - name: act-runner - resources: {} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /actrunner/config.yaml - name: act-runner-config - subPath: config.yaml - - mountPath: /var/run/docker.sock - name: docker-socket - subPath: docker.sock - - mountPath: /data - name: data-act-runner - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - name: kube-api-access-gff5j - readOnly: true - workingDir: /data - dnsPolicy: ClusterFirst - enableServiceLinks: true - hostname: gitea-act-runner-0 - initContainers: - - command: - - sh - - '-c' - - > - echo 'Trying to reach Gitea on - http://dataprovider01-dataprovider-orchestration-platform-gitea-http.dataprovider01.svc.cluster.local:3000' - - until timeout 10 wget --no-check-certificate --spider - http://dataprovider01-dataprovider-orchestration-platform-gitea-http.dataprovider01.svc.cluster.local:3000; - do - sleep 3 - echo "Trying again in 3 seconds..." - done - - echo "Gitea has been reached!" - image: busybox:1.37.0 - imagePullPolicy: Always - name: init-gitea - resources: {} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - name: kube-api-access-gff5j - readOnly: true - - image: docker:28.3.3-dind - imagePullPolicy: Always - livenessProbe: - exec: - command: - - /usr/bin/test - - '-S' - - /var/run/docker.sock - failureThreshold: 3 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: dind - resources: {} - restartPolicy: Always - securityContext: - privileged: true - startupProbe: - exec: - command: - - /usr/bin/test - - '-S' - - /var/run/docker.sock - failureThreshold: 3 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /var/run/ - name: docker-socket - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - name: kube-api-access-gff5j - readOnly: true - nodeName: dev-dev-nodepool-node-548bf4 - preemptionPolicy: PreemptLowerPriority - priority: 0 - restartPolicy: Always - schedulerName: default-scheduler - securityContext: {} - serviceAccount: default - serviceAccountName: default - terminationGracePeriodSeconds: 30 - tolerations: - - effect: NoExecute - key: node.kubernetes.io/not-ready - operator: Exists - tolerationSeconds: 300 - - effect: NoExecute - key: node.kubernetes.io/unreachable - operator: Exists - tolerationSeconds: 300 - volumes: - - name: data-act-runner - persistentVolumeClaim: - claimName: data-act-runner-gitea-act-runner-0 - - configMap: - defaultMode: 420 - name: gitea-act-runner-config - name: act-runner-config - - emptyDir: {} - name: docker-socket - - name: kube-api-access-gff5j - projected: - defaultMode: 420 - sources: - - serviceAccountToken: - expirationSeconds: 3607 - path: token - - configMap: - items: - - key: ca.crt - path: ca.crt - name: kube-root-ca.crt - - downwardAPI: - items: - - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - path: namespace -status: - conditions: - - lastProbeTime: null - lastTransitionTime: '2026-06-11T13:50:37Z' - status: 'True' - type: PodReadyToStartContainers - - lastProbeTime: null - lastTransitionTime: '2026-06-11T13:50:46Z' - status: 'True' - type: Initialized - - lastProbeTime: null - lastTransitionTime: '2026-06-11T13:50:51Z' - status: 'True' - type: Ready - - lastProbeTime: null - lastTransitionTime: '2026-06-11T13:50:51Z' - status: 'True' - type: ContainersReady - - lastProbeTime: null - lastTransitionTime: '2026-06-11T13:50:27Z' - status: 'True' - type: PodScheduled - containerStatuses: - - containerID: >- - containerd://1da759e9fb5fdb80460199ea4f1b73c5c349db183bd2556dc15a7c2d822e049b - image: docker.gitea.com/act_runner:0.2.13 - imageID: >- - docker.gitea.com/act_runner@sha256:8477d5b61b655caad4449888bae39f1f34bebd27db56cb15a62dccb3dcf3a944 - lastState: {} - name: act-runner - ready: true - restartCount: 0 - started: true - state: - running: - startedAt: '2026-06-11T13:50:50Z' - volumeMounts: - - mountPath: /actrunner/config.yaml - name: act-runner-config - - mountPath: /var/run/docker.sock - name: docker-socket - - mountPath: /data - name: data-act-runner - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - name: kube-api-access-gff5j - readOnly: true - recursiveReadOnly: Disabled - hostIP: 10.1.0.160 - hostIPs: - - ip: 10.1.0.160 - initContainerStatuses: - - containerID: >- - containerd://8e372247be3a64f4d98d2bf78173c127000fb7204758b9cd7119431482701d98 - image: docker.io/library/busybox:1.37.0 - imageID: >- - docker.io/library/busybox@sha256:9532d8c39891ca2ecde4d30d7710e01fb739c87a8b9299685c63704296b16028 - lastState: {} - name: init-gitea - ready: true - restartCount: 0 - started: false - state: - terminated: - containerID: >- - containerd://8e372247be3a64f4d98d2bf78173c127000fb7204758b9cd7119431482701d98 - exitCode: 0 - finishedAt: '2026-06-11T13:50:36Z' - reason: Completed - startedAt: '2026-06-11T13:50:36Z' - volumeMounts: - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - name: kube-api-access-gff5j - readOnly: true - recursiveReadOnly: Disabled - - containerID: >- - containerd://349aa1cb4245a87a10a22b58087367bdefc46b2c9ff7bd199abf62df5370a8f2 - image: docker.io/library/docker:28.3.3-dind - imageID: >- - docker.io/library/docker@sha256:a56b3bdde89315ed2cc0e4906e582b5033d93bf20d9cb9510c2cdd4e7f7690b1 - lastState: {} - name: dind - ready: true - restartCount: 0 - started: true - state: - running: - startedAt: '2026-06-11T13:50:37Z' - volumeMounts: - - mountPath: /var/run/ - name: docker-socket - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - name: kube-api-access-gff5j - readOnly: true - recursiveReadOnly: Disabled - phase: Running - podIP: 10.2.2.122 - podIPs: - - ip: 10.2.2.122 - qosClass: BestEffort - startTime: '2026-06-11T13:50:27Z' diff --git a/src/template_code_location/repository.py b/src/template_code_location/repository.py index 1a9839c..cc62377 100644 --- a/src/template_code_location/repository.py +++ b/src/template_code_location/repository.py @@ -6,7 +6,6 @@ from template_code_location.jobs.jobs import data_processing_job defs = Definitions( jobs=[ - data_processing_job ], sensors=[], resources={"s3": s3_resource.configured({"resource_name": "selfS3"})},