[SIMPL-30451] Add distributed-execution service with guide and reference implementations
Canonical location for documentation, example workflows and reference service implementations covering distributed execution patterns. Covers AC1-AC4: execution-target selection, decision support, readiness checks and code-level linkage. Tightly coupled jobs and the loosely coupled subprocess transport are verified by the test suite; the Kubernetes pipes transport is implemented but not yet cluster-run and is marked as such in the guide. Changelog: added
This commit is contained in:
61
src/distributed_execution/preflight.py
Normal file
61
src/distributed_execution/preflight.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""Readiness checks backing the pre-run validation checklist.
|
||||
|
||||
Each function returns a plain dict so the same result can be logged as Dagster
|
||||
metadata (evidence for the checklist) or asserted in a test.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Runtime dependencies a tightly coupled execution pod must be able to resolve.
|
||||
TIGHTLY_COUPLED_ENV_VARS = (
|
||||
"DAGSTER_POSTGRES_HOST",
|
||||
"DAGSTER_POSTGRES_USER",
|
||||
"DAGSTER_POSTGRES_DB",
|
||||
)
|
||||
|
||||
|
||||
def check_env_vars(names: tuple[str, ...] | list[str]) -> dict:
|
||||
"""Report which of ``names`` are present, without echoing their values."""
|
||||
present = [name for name in names if os.environ.get(name)]
|
||||
missing = [name for name in names if not os.environ.get(name)]
|
||||
return {
|
||||
"check": "env_vars",
|
||||
"passed": not missing,
|
||||
"present": present,
|
||||
"missing": missing,
|
||||
}
|
||||
|
||||
|
||||
def check_tcp_reachable(host: str, port: int, timeout: float = 3.0) -> dict:
|
||||
"""Open a TCP connection to prove the execution pod can reach a dependency."""
|
||||
result = {"check": "tcp_reachable", "target": f"{host}:{port}", "passed": False, "error": None}
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=timeout):
|
||||
result["passed"] = True
|
||||
except OSError as exc:
|
||||
result["error"] = str(exc)
|
||||
return result
|
||||
|
||||
|
||||
def check_endpoint_reachable(url: str, timeout: float = 3.0) -> dict:
|
||||
"""TCP-level reachability for an endpoint expressed as a URL."""
|
||||
parsed = urlparse(url)
|
||||
if not parsed.hostname:
|
||||
return {"check": "tcp_reachable", "target": url, "passed": False, "error": "no hostname in URL"}
|
||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||
return check_tcp_reachable(parsed.hostname, port, timeout=timeout)
|
||||
|
||||
|
||||
def describe_pod_identity() -> dict:
|
||||
"""Where this process is actually running - the primary execution-target evidence."""
|
||||
return {
|
||||
"hostname": socket.gethostname(),
|
||||
"pid": os.getpid(),
|
||||
"namespace": os.environ.get("DAGSTER_K8S_PIPELINE_RUN_NAMESPACE", "<not-in-kubernetes>"),
|
||||
"run_id": os.environ.get("DAGSTER_RUN_ID", "<unset>"),
|
||||
"image": os.environ.get("DAGSTER_K8S_PIPELINE_RUN_IMAGE", "<unset>"),
|
||||
}
|
||||
Reference in New Issue
Block a user