# Tightly coupled probe for sandbox-cat-dat - the part that is safe to run. # # Covers T4 and proves the run pod can reach the metadata database, using a # LOCAL SQLite instance. It deliberately does NOT write to the platform's # Postgres. See "What this cannot cover" below. # # Runs three things in one pod: # 1. A TCP probe to Postgres using this repository's own check_tcp_reachable. # 2. tightly_coupled_in_process_job - single process, no fan-out. # 3. tightly_coupled_local_job - multiprocess fan-out inside one pod (T4). # # --- What this cannot cover ------------------------------------------------ # # T6 and T7 need `tightly_coupled_k8s_job`, and that cannot use a local SQLite # instance: k8s_job_executor puts every step in its own pod, and each step pod # opens the metadata database itself. That is the definition of tightly coupled. # The step pods must therefore share instance storage, which means Postgres. # # The only Postgres available is `dataprovider01_dagster`, the platform's live # metadata database, and these images carry dagster 1.13.19 against a control # plane on 1.12.8. Pointing a newer Dagster at an older schema is not a safe # operation on a shared database: at best it refuses to start with a schema # mismatch, and it is not worth finding out the worse case on someone else's # instance. Pin dagster to the control plane version and rebuild before trying. # # T1 is only partly covered. Its stated evidence expects DAGSTER_POSTGRES_HOST, # DAGSTER_POSTGRES_USER and DAGSTER_POSTGRES_DB in the pod environment; the # platform sets none of them - it passes DAGSTER_PG_PASSWORD and bakes the rest # into dagster.yaml. Left unset here deliberately, so the run reports the real # deployment's state rather than a flattering one. # # --- Running --------------------------------------------------------------- # # kubectl -n dataprovider01 apply -f yaml/sandbox/probe-tightly-coupled-sandbox.yaml # kubectl -n dataprovider01 logs -f job/distexec-tc-probe # kubectl -n dataprovider01 delete job distexec-tc-probe apiVersion: batch/v1 kind: Job metadata: name: distexec-tc-probe namespace: dataprovider01 labels: app.kubernetes.io/name: distributed-execution app.kubernetes.io/component: probe spec: backoffLimit: 0 activeDeadlineSeconds: 1800 template: metadata: labels: app.kubernetes.io/name: distributed-execution app.kubernetes.io/component: probe spec: restartPolicy: Never serviceAccountName: dagster-svc-account automountServiceAccountToken: true securityContext: runAsNonRoot: true runAsUser: 1000 runAsGroup: 1000 fsGroup: 1000 seccompProfile: type: RuntimeDefault containers: - name: runner image: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution:5122da4691f9 imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL env: - name: DAGSTER_HOME value: /dagster-home command: - sh - -c - | echo "===== T1: can this pod reach the metadata database? =====" python -c "from distributed_execution.preflight import check_tcp_reachable; print(check_tcp_reachable('pg-cluster.common01.svc.cluster.local', 5432))" echo "===== tightly_coupled_in_process_job =====" dagster job execute -f src/distributed_execution/repository.py -j tightly_coupled_in_process_job echo "===== T4: tightly_coupled_local_job =====" dagster job execute -f src/distributed_execution/repository.py -j tightly_coupled_local_job volumeMounts: - name: dagster-home mountPath: /dagster-home resources: requests: cpu: 200m memory: 512Mi limits: cpu: "1" memory: 1Gi volumes: - name: dagster-home emptyDir: {}