[SIMPL-30451] Execute T6 and T7 on the sandbox
Adds the k8s_job_executor probe and records the result. Six step Jobs were created, one per step, and four steps executed and succeeded against the platform's live Postgres. The run then failed before summarise_results: run monitoring polls for a dagster-run-<id> Job that a hand-rolled probe never creates, and with no launcher-supplied container context it looked in the launcher's default namespace, where dagster-svc-account is denied jobs/status. Two step pods had already started and logged 'Skipping step execution' before exiting 0, which is why all six Jobs read Complete while only four steps ran. Changelog: added
This commit is contained in:
ILay
161
yaml/sandbox/probe-k8s-executor-sandbox.yaml
Normal file
161
yaml/sandbox/probe-k8s-executor-sandbox.yaml
Normal file
@@ -0,0 +1,161 @@
|
||||
# T6/T7 probe - tightly coupled via k8s_job_executor, one pod per step.
|
||||
#
|
||||
# This is the invasive one. Unlike the pipes probe it writes to the platform's
|
||||
# live metadata database, because that is what tightly coupled means: every step
|
||||
# pod opens the database itself, so they must share instance storage. Safe only
|
||||
# because the images are pinned to dagster 1.12.8, exactly matching the control
|
||||
# plane, so no schema migration can be triggered.
|
||||
#
|
||||
# Two things the loosely coupled probe did not need:
|
||||
#
|
||||
# 1. Shared instance storage. DAGSTER_HOME is copied from the platform's
|
||||
# `dagster-instance` ConfigMap into a writable volume, so run and event
|
||||
# storage are the same Postgres the platform uses. Step pods get the same
|
||||
# config via the run launcher's `instance_config_map`.
|
||||
#
|
||||
# 2. A shared I/O manager. The default writes step outputs to pod-local disk,
|
||||
# which under this executor means a downstream step cannot read an upstream
|
||||
# output. base_dir is redirected to the RWX `dagster-shared-pvc` the
|
||||
# launcher already declares.
|
||||
#
|
||||
# The run config lives in its own ConfigMap rather than a heredoc inside the
|
||||
# container command. A heredoc nested in a YAML block scalar silently produced an
|
||||
# empty file on the first attempt, and an empty run config is valid YAML, so the
|
||||
# run started and only failed later with "No image included in either executor
|
||||
# config or the job" - a confusing distance from the actual cause. The command
|
||||
# echoes the config before using it so that failure mode cannot recur silently.
|
||||
#
|
||||
# --- Running ---------------------------------------------------------------
|
||||
#
|
||||
# kubectl -n dataprovider01 apply -f yaml/sandbox/probe-k8s-executor-sandbox.yaml
|
||||
# kubectl -n dataprovider01 logs -f job/distexec-k8sexec-probe
|
||||
# kubectl -n dataprovider01 get jobs -l dagster/job=tightly_coupled_k8s_job # T6
|
||||
#
|
||||
# --- Teardown --------------------------------------------------------------
|
||||
#
|
||||
# kubectl -n dataprovider01 delete job distexec-k8sexec-probe
|
||||
# kubectl -n dataprovider01 delete jobs -l dagster/job=tightly_coupled_k8s_job
|
||||
# kubectl -n dataprovider01 delete cm distexec-probe-run-config
|
||||
#
|
||||
# The run itself stays in the platform's Dagster database and will be visible in
|
||||
# its UI. That is not reversible with kubectl.
|
||||
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: distexec-probe-run-config
|
||||
namespace: dataprovider01
|
||||
labels:
|
||||
app.kubernetes.io/name: distributed-execution
|
||||
app.kubernetes.io/component: probe
|
||||
data:
|
||||
run-config.yaml: |
|
||||
execution:
|
||||
config:
|
||||
job_namespace: dataprovider01
|
||||
job_image: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution:3f8175a17f0a
|
||||
service_account_name: dagster-svc-account
|
||||
image_pull_policy: IfNotPresent
|
||||
max_concurrent: 2
|
||||
# Step pods inherit DAGSTER_HOME from the launching process otherwise,
|
||||
# which points at a volume only that pod has.
|
||||
env_vars:
|
||||
- DAGSTER_HOME=/dagster/shared/distexec-probe/home
|
||||
volumes:
|
||||
- name: dagster-shared-storage
|
||||
persistent_volume_claim:
|
||||
claim_name: dagster-shared-pvc
|
||||
volume_mounts:
|
||||
- name: dagster-shared-storage
|
||||
mount_path: /dagster/shared
|
||||
step_k8s_config:
|
||||
pod_spec_config:
|
||||
automount_service_account_token: true
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: distexec-k8sexec-probe
|
||||
namespace: dataprovider01
|
||||
labels:
|
||||
app.kubernetes.io/name: distributed-execution
|
||||
app.kubernetes.io/component: probe
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
activeDeadlineSeconds: 1800
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: distributed-execution
|
||||
app.kubernetes.io/component: probe
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
serviceAccountName: dagster-svc-account
|
||||
automountServiceAccountToken: true
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: runner
|
||||
image: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution:3f8175a17f0a
|
||||
imagePullPolicy: IfNotPresent
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
env:
|
||||
# On the RWX volume, not a pod-local one: the default I/O manager writes
|
||||
# step outputs under $DAGSTER_HOME/storage, and every step pod has to
|
||||
# read what the previous one wrote.
|
||||
- name: DAGSTER_HOME
|
||||
value: /dagster/shared/distexec-probe/home
|
||||
# How the chart tells a code location which image its steps run as.
|
||||
- name: DAGSTER_CURRENT_IMAGE
|
||||
value: gitea.dataprovider01.sandbox-cat-dat.simpl-europe.eu/j.r/distributed-execution:3f8175a17f0a
|
||||
- name: DAGSTER_PG_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: dagster-postgresql-secret
|
||||
key: postgresql-password
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
mkdir -p "$DAGSTER_HOME"
|
||||
cp /instance-cm/dagster.yaml "$DAGSTER_HOME/dagster.yaml"
|
||||
echo "===== run config in use ====="
|
||||
cat /run-config/run-config.yaml
|
||||
echo "===== T6/T7: tightly_coupled_k8s_job ====="
|
||||
dagster job execute -f src/distributed_execution/repository.py -j tightly_coupled_k8s_job -c /run-config/run-config.yaml
|
||||
volumeMounts:
|
||||
- name: instance-cm
|
||||
mountPath: /instance-cm
|
||||
readOnly: true
|
||||
- name: run-config
|
||||
mountPath: /run-config
|
||||
readOnly: true
|
||||
- name: dagster-shared-storage
|
||||
mountPath: /dagster/shared
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
volumes:
|
||||
- name: instance-cm
|
||||
configMap:
|
||||
name: dagster-instance
|
||||
- name: run-config
|
||||
configMap:
|
||||
name: distexec-probe-run-config
|
||||
- name: dagster-shared-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: dagster-shared-pvc
|
||||
Reference in New Issue
Block a user