43dbb81a95
Canonical location for documentation, example workflows and reference service implementations covering distributed execution patterns. Covers AC1-AC4: execution-target selection, decision support, readiness checks and code-level linkage. Tightly coupled jobs and the loosely coupled subprocess transport are verified by the test suite; the Kubernetes pipes transport is implemented but not yet cluster-run and is marked as such in the guide. Changelog: added
56 lines
1.7 KiB
Python
56 lines
1.7 KiB
Python
"""External payload for the loosely coupled execution target.
|
|
|
|
This script is deliberately NOT a Dagster code location. It depends only on
|
|
``dagster-pipes``, never on ``dagster``, and it never opens a connection to the
|
|
orchestration metadata database. Everything it reports reaches the control plane
|
|
through the pipes message channel.
|
|
|
|
Run by ``distributed_execution.loosely_coupled.jobs`` via either
|
|
``PipesSubprocessClient`` (local) or ``PipesK8sClient`` (cluster).
|
|
"""
|
|
|
|
import os
|
|
import socket
|
|
|
|
from dagster_pipes import open_dagster_pipes
|
|
|
|
# Presence of any of these would mean the payload was granted orchestration
|
|
# runtime credentials it has no business holding.
|
|
ORCHESTRATION_ENV_VARS = (
|
|
"DAGSTER_POSTGRES_HOST",
|
|
"DAGSTER_POSTGRES_USER",
|
|
"DAGSTER_POSTGRES_DB",
|
|
)
|
|
|
|
|
|
def main() -> None:
|
|
with open_dagster_pipes() as pipes:
|
|
units = pipes.get_extra("units")
|
|
host = socket.gethostname()
|
|
|
|
pipes.log.info(f"External payload started on {host} with {len(units)} work units")
|
|
|
|
results = [{"unit": unit, "squared": unit * unit, "host": host} for unit in units]
|
|
|
|
leaked = [name for name in ORCHESTRATION_ENV_VARS if os.environ.get(name)]
|
|
if leaked:
|
|
pipes.log.warning(
|
|
"Orchestration runtime credentials are visible to this payload: "
|
|
f"{', '.join(leaked)}. A loosely coupled target should not have them."
|
|
)
|
|
|
|
# The only channel back to the control plane.
|
|
pipes.report_custom_message(
|
|
{
|
|
"results": results,
|
|
"host": host,
|
|
"orchestration_env_visible": leaked,
|
|
}
|
|
)
|
|
|
|
pipes.log.info(f"External payload finished on {host}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|