[SIMPL-30451] Add distributed-execution service with guide and reference implementations
Canonical location for documentation, example workflows and reference service implementations covering distributed execution patterns. Covers AC1-AC4: execution-target selection, decision support, readiness checks and code-level linkage. Tightly coupled jobs and the loosely coupled subprocess transport are verified by the test suite; the Kubernetes pipes transport is implemented but not yet cluster-run and is marked as such in the guide. Changelog: added
This commit is contained in:
54
yaml/tightly-coupled/values-run-launcher.yaml
Normal file
54
yaml/tightly-coupled/values-run-launcher.yaml
Normal file
@@ -0,0 +1,54 @@
|
||||
# Tightly coupled execution target - instance-level run launcher.
|
||||
#
|
||||
# Merge into the Dagster chart values. This is the half of the execution-target
|
||||
# binding that lives OUTSIDE workflow code: it decides where the run worker for
|
||||
# an entire run is placed, and what that pod can reach.
|
||||
#
|
||||
# Companion to src/distributed_execution/tightly_coupled/jobs.py, which decides
|
||||
# how the steps INSIDE that run worker execute.
|
||||
|
||||
dagster:
|
||||
runLauncher:
|
||||
type: K8sRunLauncher
|
||||
config:
|
||||
k8sRunLauncher:
|
||||
# Namespace the run pods land in. Must be a namespace whose NetworkPolicy
|
||||
# permits egress to Postgres, object storage and Vault - see AC1
|
||||
# prerequisites in the user guide.
|
||||
jobNamespace: dagster
|
||||
|
||||
# Surfaces step failures as pod failures so kubectl and Dagster agree.
|
||||
failPodOnRunFailure: true
|
||||
|
||||
runK8sConfig:
|
||||
podTemplateSpecMetadata:
|
||||
annotations:
|
||||
vault.security.banzaicloud.io/vault-inject: "true"
|
||||
vault.security.banzaicloud.io/vault-addr: https://secrets.common01.dev.simpl-europe.eu
|
||||
vault.security.banzaicloud.io/vault-role: dev-role
|
||||
vault.security.banzaicloud.io/vault-skip-verify: "true"
|
||||
vault.security.banzaicloud.io/vault-path: kubernetes
|
||||
|
||||
containerConfig:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
# Everything below is what "tightly coupled" costs you: each run pod
|
||||
# needs credentials for, and network reachability to, the full
|
||||
# orchestration runtime. A loosely coupled target needs none of it.
|
||||
env:
|
||||
- name: DAGSTER_TELEMETRY_ENABLED
|
||||
value: "false"
|
||||
- name: TOKEN
|
||||
value: "vault:dev/data/dagster/dagster-workflow-vault-secret#VAULT_ACCESS_TOKEN"
|
||||
- name: S3_ENDPOINT_URL
|
||||
value: "https://s3.dev.simpl-europe.eu"
|
||||
- name: S3_ACCESS_KEY
|
||||
value: "vault:dev/data/dev-orchestration-platform#S3_ACCESS_KEY"
|
||||
- name: S3_SECRET_KEY
|
||||
value: "vault:dev/data/dev-orchestration-platform#S3_SECRET_KEY"
|
||||
Reference in New Issue
Block a user