distributed-execution/yaml/loosely-coupled/probe-pipes-k8s.yaml

# Cluster probe for the loosely coupled Kubernetes transport.
#
# Runs `loosely_coupled_k8s_job` from a single throwaway pod. The point is to
# prove the part of this service that no test can reach: that the dispatching
# process creates payload Jobs and that messages come back over the pod log
# stream. It clears readiness checks L4-L9 in one run.
#
# Deliberately NOT a Dagster deployment. No webserver, no daemon, no database,
# no code location registration. That sidesteps the control-plane version skew -
# the pod is its own control plane, so the 1.13.19 images here have nothing to be
# compatible with. Everything lives in one namespace you delete afterwards.
#
# --- Before applying -------------------------------------------------------
#
# 1. Pull secret. The registry rejects anonymous pulls, so both this pod and the
#    payload Jobs it creates need credentials. Use a token with read:package.
#
#      kubectl -n distexec-probe create secret docker-registry gitea-registry \
#        --docker-server=gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu \
#        --docker-username=<gitea-user> \
#        --docker-password=<gitea-token>
#
# 2. Payload pods inherit the pull secret from the default service account.
#    PipesK8sClient does not set serviceAccountName on the Jobs it creates, so
#    they run as `default`, and this service does not yet pass imagePullSecrets
#    through base_pod_spec:
#
#      kubectl -n distexec-probe patch serviceaccount default \
#        -p '{"imagePullSecrets":[{"name":"gitea-registry"}]}'
#
#    Skip this and the payload pod never starts. That surfaces as the op waiting
#    until pod_wait_timeout, which reads like a hung workload rather than a
#    missing credential.
#
# Order does not matter much: apply this file first and the dispatcher pod sits
# in ImagePullBackOff until the secret appears, then pulls on the kubelet's next
# retry. The default service account only has to be patched before the first
# payload Job is created, which is after the dispatcher starts.
#
#      kubectl apply -f yaml/loosely-coupled/probe-pipes-k8s.yaml
#
# `--dry-run=server` reports "namespaces distexec-probe not found" for the four
# namespaced objects. That is the dry run declining to create the namespace it
# would need, not a fault in the manifests; `--dry-run=client` passes clean.
#
# --- Reading the result ----------------------------------------------------
#
#   kubectl -n distexec-probe logs -f job/pipes-probe
#   kubectl -n distexec-probe get jobs -l app.kubernetes.io/name=distributed-execution-payload
#
# Success is RUN_SUCCESS plus four payload Jobs, one per work unit. The
# summarise_results output carries contributing_workers; on a cluster those
# should be four distinct pod hostnames, which is the claim the guide makes and
# a laptop cannot demonstrate.
#
# A run that reaches RUN_FAILURE with "no pipes messages received" means the
# payload ran but its log stream never came back - check the pods/log rule
# below before suspecting the payload.
#
# --- Teardown --------------------------------------------------------------
#
#   kubectl delete namespace distexec-probe
#
# NOT CLUSTER-VERIFIED. Written against the published images and the RBAC the
# pipes client is documented to need, but not yet applied to a cluster.

apiVersion: v1
kind: Namespace
metadata:
  name: distexec-probe
  labels:
    app.kubernetes.io/name: distributed-execution
    app.kubernetes.io/component: probe
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: pipes-dispatcher
  namespace: distexec-probe
imagePullSecrets:
  - name: gitea-registry
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: pipes-dispatcher
  namespace: distexec-probe
rules:
  - apiGroups: ["batch"]
    resources: ["jobs", "jobs/status"]
    verbs: ["create", "get", "list", "watch", "delete"]
  # pods/log is the message channel for PipesK8sPodLogsMessageReader, not just
  # an observability convenience. Without it the payload runs to completion and
  # reports nothing, and the op fails on an empty message list.
  - apiGroups: [""]
    resources: ["pods", "pods/log", "pods/status"]
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: pipes-dispatcher
  namespace: distexec-probe
subjects:
  - kind: ServiceAccount
    name: pipes-dispatcher
    namespace: distexec-probe
roleRef:
  kind: Role
  name: pipes-dispatcher
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: batch/v1
kind: Job
metadata:
  name: pipes-probe
  namespace: distexec-probe
spec:
  # One shot. A retry would obscure which attempt produced which payload Jobs.
  backoffLimit: 0
  activeDeadlineSeconds: 1800
  template:
    metadata:
      labels:
        app.kubernetes.io/name: distributed-execution
        app.kubernetes.io/component: probe
    spec:
      restartPolicy: Never
      serviceAccountName: pipes-dispatcher
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        runAsGroup: 1000
        # Makes the emptyDir writable by UID 1000, which DAGSTER_HOME needs.
        fsGroup: 1000
      containers:
        - name: dispatcher
          image: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution:5122da4691f9
          imagePullPolicy: IfNotPresent
          command:
            - dagster
            - job
            - execute
            - -f
            - src/distributed_execution/repository.py
            - -j
            - loosely_coupled_k8s_job
          env:
            - name: DAGSTER_HOME
              value: /dagster-home
            # Must be the same commit as the image above.
            - name: PIPES_PAYLOAD_IMAGE
              value: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution-payload:5122da4691f9
            - name: PIPES_PAYLOAD_NAMESPACE
              value: distexec-probe
          volumeMounts:
            - name: dagster-home
              mountPath: /dagster-home
          resources:
            requests:
              cpu: 200m
              memory: 512Mi
            limits:
              cpu: "1"
              memory: 1Gi
      volumes:
        # The job runs on multiprocess_executor, so the instance has to be a real
        # sqlite instance on disk rather than an ephemeral one.
        - name: dagster-home
          emptyDir: {}