distributed-execution/yaml/sandbox/probe-k8s-executor-sandbox.yaml

# T6/T7 probe - tightly coupled via k8s_job_executor, one pod per step.
#
# This is the invasive one. Unlike the pipes probe it writes to the platform's
# live metadata database, because that is what tightly coupled means: every step
# pod opens the database itself, so they must share instance storage. Safe only
# because the images are pinned to dagster 1.12.8, exactly matching the control
# plane, so no schema migration can be triggered.
#
# Two things the loosely coupled probe did not need:
#
#   1. Shared instance storage. DAGSTER_HOME is copied from the platform's
#      `dagster-instance` ConfigMap into a writable volume, so run and event
#      storage are the same Postgres the platform uses. Step pods get the same
#      config via the run launcher's `instance_config_map`.
#
#   2. A shared I/O manager. The default writes step outputs to pod-local disk,
#      which under this executor means a downstream step cannot read an upstream
#      output. base_dir is redirected to the RWX `dagster-shared-pvc` the
#      launcher already declares.
#
# The run config lives in its own ConfigMap rather than a heredoc inside the
# container command. A heredoc nested in a YAML block scalar silently produced an
# empty file on the first attempt, and an empty run config is valid YAML, so the
# run started and only failed later with "No image included in either executor
# config or the job" - a confusing distance from the actual cause. The command
# echoes the config before using it so that failure mode cannot recur silently.
#
# --- Running ---------------------------------------------------------------
#
#   kubectl -n dataprovider01 apply -f yaml/sandbox/probe-k8s-executor-sandbox.yaml
#   kubectl -n dataprovider01 logs -f job/distexec-k8sexec-probe
#   kubectl -n dataprovider01 get jobs -l dagster/job=tightly_coupled_k8s_job   # T6
#
# --- Teardown --------------------------------------------------------------
#
#   kubectl -n dataprovider01 delete job distexec-k8sexec-probe
#   kubectl -n dataprovider01 delete jobs -l dagster/job=tightly_coupled_k8s_job
#   kubectl -n dataprovider01 delete cm distexec-probe-run-config
#
# The run itself stays in the platform's Dagster database and will be visible in
# its UI. That is not reversible with kubectl.

apiVersion: v1
kind: ConfigMap
metadata:
  name: distexec-probe-run-config
  namespace: dataprovider01
  labels:
    app.kubernetes.io/name: distributed-execution
    app.kubernetes.io/component: probe
data:
  run-config.yaml: |
    execution:
      config:
        job_namespace: dataprovider01
        job_image: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution:3f8175a17f0a
        service_account_name: dagster-svc-account
        image_pull_policy: IfNotPresent
        max_concurrent: 2
        # Step pods inherit DAGSTER_HOME from the launching process otherwise,
        # which points at a volume only that pod has.
        env_vars:
          - DAGSTER_HOME=/dagster/shared/distexec-probe/home
        volumes:
          - name: dagster-shared-storage
            persistent_volume_claim:
              claim_name: dagster-shared-pvc
        volume_mounts:
          - name: dagster-shared-storage
            mount_path: /dagster/shared
        step_k8s_config:
          pod_spec_config:
            automount_service_account_token: true
---
apiVersion: batch/v1
kind: Job
metadata:
  name: distexec-k8sexec-probe
  namespace: dataprovider01
  labels:
    app.kubernetes.io/name: distributed-execution
    app.kubernetes.io/component: probe
spec:
  backoffLimit: 0
  activeDeadlineSeconds: 1800
  template:
    metadata:
      labels:
        app.kubernetes.io/name: distributed-execution
        app.kubernetes.io/component: probe
    spec:
      restartPolicy: Never
      serviceAccountName: dagster-svc-account
      automountServiceAccountToken: true
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        runAsGroup: 1000
        fsGroup: 1000
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: runner
          image: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution:3f8175a17f0a
          imagePullPolicy: IfNotPresent
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
                - ALL
          env:
            # On the RWX volume, not a pod-local one: the default I/O manager writes
            # step outputs under $DAGSTER_HOME/storage, and every step pod has to
            # read what the previous one wrote.
            - name: DAGSTER_HOME
              value: /dagster/shared/distexec-probe/home
            # How the chart tells a code location which image its steps run as.
            - name: DAGSTER_CURRENT_IMAGE
              value: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution:3f8175a17f0a
            - name: DAGSTER_PG_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: dagster-postgresql-secret
                  key: postgresql-password
          command:
            - sh
            - -c
            - |
              set -e
              mkdir -p "$DAGSTER_HOME"
              cp /instance-cm/dagster.yaml "$DAGSTER_HOME/dagster.yaml"
              echo "===== run config in use ====="
              cat /run-config/run-config.yaml
              echo "===== T6/T7: tightly_coupled_k8s_job ====="
              dagster job execute -f src/distributed_execution/repository.py -j tightly_coupled_k8s_job -c /run-config/run-config.yaml
          volumeMounts:
            - name: instance-cm
              mountPath: /instance-cm
              readOnly: true
            - name: run-config
              mountPath: /run-config
              readOnly: true
            - name: dagster-shared-storage
              mountPath: /dagster/shared
          resources:
            requests:
              cpu: 200m
              memory: 512Mi
            limits:
              cpu: "1"
              memory: 1Gi
      volumes:
        - name: instance-cm
          configMap:
            name: dagster-instance
        - name: run-config
          configMap:
            name: distexec-probe-run-config
        - name: dagster-shared-storage
          persistentVolumeClaim:
            claimName: dagster-shared-pvc