43dbb81a95
Canonical location for documentation, example workflows and reference service implementations covering distributed execution patterns. Covers AC1-AC4: execution-target selection, decision support, readiness checks and code-level linkage. Tightly coupled jobs and the loosely coupled subprocess transport are verified by the test suite; the Kubernetes pipes transport is implemented but not yet cluster-run and is marked as such in the guide. Changelog: added
55 lines
2.2 KiB
YAML
55 lines
2.2 KiB
YAML
# Tightly coupled execution target - instance-level run launcher.
|
|
#
|
|
# Merge into the Dagster chart values. This is the half of the execution-target
|
|
# binding that lives OUTSIDE workflow code: it decides where the run worker for
|
|
# an entire run is placed, and what that pod can reach.
|
|
#
|
|
# Companion to src/distributed_execution/tightly_coupled/jobs.py, which decides
|
|
# how the steps INSIDE that run worker execute.
|
|
|
|
dagster:
|
|
runLauncher:
|
|
type: K8sRunLauncher
|
|
config:
|
|
k8sRunLauncher:
|
|
# Namespace the run pods land in. Must be a namespace whose NetworkPolicy
|
|
# permits egress to Postgres, object storage and Vault - see AC1
|
|
# prerequisites in the user guide.
|
|
jobNamespace: dagster
|
|
|
|
# Surfaces step failures as pod failures so kubectl and Dagster agree.
|
|
failPodOnRunFailure: true
|
|
|
|
runK8sConfig:
|
|
podTemplateSpecMetadata:
|
|
annotations:
|
|
vault.security.banzaicloud.io/vault-inject: "true"
|
|
vault.security.banzaicloud.io/vault-addr: https://secrets.common01.dev.simpl-europe.eu
|
|
vault.security.banzaicloud.io/vault-role: dev-role
|
|
vault.security.banzaicloud.io/vault-skip-verify: "true"
|
|
vault.security.banzaicloud.io/vault-path: kubernetes
|
|
|
|
containerConfig:
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 512Mi
|
|
|
|
# Everything below is what "tightly coupled" costs you: each run pod
|
|
# needs credentials for, and network reachability to, the full
|
|
# orchestration runtime. A loosely coupled target needs none of it.
|
|
env:
|
|
- name: DAGSTER_TELEMETRY_ENABLED
|
|
value: "false"
|
|
- name: TOKEN
|
|
value: "vault:dev/data/dagster/dagster-workflow-vault-secret#VAULT_ACCESS_TOKEN"
|
|
- name: S3_ENDPOINT_URL
|
|
value: "https://s3.dev.simpl-europe.eu"
|
|
- name: S3_ACCESS_KEY
|
|
value: "vault:dev/data/dev-orchestration-platform#S3_ACCESS_KEY"
|
|
- name: S3_SECRET_KEY
|
|
value: "vault:dev/data/dev-orchestration-platform#S3_SECRET_KEY"
|