[SIMPL-30451] Add distributed-execution service with guide and reference implementations

Canonical location for documentation, example workflows and reference service implementations covering distributed execution patterns. Covers AC1-AC4: execution-target selection, decision support, readiness checks and code-level linkage. Tightly coupled jobs and the loosely coupled subprocess transport are verified by the test suite; the Kubernetes pipes transport is implemented but not yet cluster-run and is marked as such in the guide.

Changelog: added
This commit is contained in:
ILay
2026-08-26 18:07:06 +02:00
commit 43dbb81a95
32 changed files with 4071 additions and 0 deletions

View File

@@ -0,0 +1,61 @@
"""Readiness checks backing the pre-run validation checklist.
Each function returns a plain dict so the same result can be logged as Dagster
metadata (evidence for the checklist) or asserted in a test.
"""
from __future__ import annotations
import os
import socket
from urllib.parse import urlparse
# Runtime dependencies a tightly coupled execution pod must be able to resolve.
TIGHTLY_COUPLED_ENV_VARS = (
"DAGSTER_POSTGRES_HOST",
"DAGSTER_POSTGRES_USER",
"DAGSTER_POSTGRES_DB",
)
def check_env_vars(names: tuple[str, ...] | list[str]) -> dict:
"""Report which of ``names`` are present, without echoing their values."""
present = [name for name in names if os.environ.get(name)]
missing = [name for name in names if not os.environ.get(name)]
return {
"check": "env_vars",
"passed": not missing,
"present": present,
"missing": missing,
}
def check_tcp_reachable(host: str, port: int, timeout: float = 3.0) -> dict:
"""Open a TCP connection to prove the execution pod can reach a dependency."""
result = {"check": "tcp_reachable", "target": f"{host}:{port}", "passed": False, "error": None}
try:
with socket.create_connection((host, port), timeout=timeout):
result["passed"] = True
except OSError as exc:
result["error"] = str(exc)
return result
def check_endpoint_reachable(url: str, timeout: float = 3.0) -> dict:
"""TCP-level reachability for an endpoint expressed as a URL."""
parsed = urlparse(url)
if not parsed.hostname:
return {"check": "tcp_reachable", "target": url, "passed": False, "error": "no hostname in URL"}
port = parsed.port or (443 if parsed.scheme == "https" else 80)
return check_tcp_reachable(parsed.hostname, port, timeout=timeout)
def describe_pod_identity() -> dict:
"""Where this process is actually running - the primary execution-target evidence."""
return {
"hostname": socket.gethostname(),
"pid": os.getpid(),
"namespace": os.environ.get("DAGSTER_K8S_PIPELINE_RUN_NAMESPACE", "<not-in-kubernetes>"),
"run_id": os.environ.get("DAGSTER_RUN_ID", "<unset>"),
"image": os.environ.get("DAGSTER_K8S_PIPELINE_RUN_IMAGE", "<unset>"),
}