[SIMPL-30451] Fan out work units so the execution target is observable

The reference graph processed every work unit inside a single op, so exactly one worker was ever reported regardless of executor. That made the guide's claim that k8s_job_executor yields several distinct contributing_hosts false, and left the reference implementations unable to demonstrate the executor choice at all. generate_work_units is now a DynamicOut and both graphs map over it, so one step is created per unit and the loosely coupled pattern dispatches one external workload per unit.

Evidence is split into contributing_workers (host and pid, differs per process) and contributing_hosts (differs only across machines), because the previous single field could not distinguish multiprocess fan-out from no fan-out. Tests now assert the mapped step keys rather than a host count, since execute_in_process ignores executor_def and cannot prove executor behaviour on its own. Adds a Windows note: multiprocess_executor did not complete during authoring and left orphaned processes.

Changelog: fixed

src/distributed_execution/loosely_coupled/jobs.py

@@ -14,7 +14,16 @@ import os
 import sys
 from pathlib import Path
 
-from dagster import Failure, OpExecutionContext, Out, PipesSubprocessClient, graph, op
+from dagster import (
+    Failure,
+    OpExecutionContext,
+    Out,
+    PipesSubprocessClient,
+    graph,
+    in_process_executor,
+    multiprocess_executor,
+    op,
+)
 from dagster_k8s import PipesK8sClient
 
 from distributed_execution.ops import (
@@ -39,8 +48,8 @@ COMMON_TAGS = {
 }
 
 
-def _results_from_pipes(context: OpExecutionContext, completed) -> list:
-    """Read the payload's results off the message channel.
+def _result_from_pipes(context: OpExecutionContext, completed) -> dict:
+    """Read one unit's result off the message channel.
 
     A broken message path is the defining failure mode of this pattern: the
     external workload can exit 0 while reporting nothing, so silence is treated
@@ -66,42 +75,43 @@ def _results_from_pipes(context: OpExecutionContext, completed) -> list:
             ", ".join(leaked),
         )
 
-    context.log.info("Received %s results from external host %s", len(payload["results"]), payload["host"])
-    return payload["results"]
+    row = payload["results"][0]
+    context.log.info("Unit %s computed by external worker %s", row["unit"], row["worker"])
+    return row
 
 
 @op(
-    description="Dispatches the external payload as a local subprocess and listens on the pipes channel.",
-    out=Out(list),
+    description="Dispatches one external payload as a local subprocess and listens on the pipes channel.",
+    out=Out(dict),
 )
 def dispatch_external_work_subprocess(
     context: OpExecutionContext,
-    units: list,
+    unit: int,
     pipes_subprocess_client: PipesSubprocessClient,
-) -> list:
+) -> dict:
     completed = pipes_subprocess_client.run(
         context=context,
         command=[sys.executable, PAYLOAD_SCRIPT],
-        extras={"units": units},
+        extras={"units": [unit]},
     )
-    return _results_from_pipes(context, completed)
+    return _result_from_pipes(context, completed)
 
 
 @op(
-    description="Dispatches the external payload as a Kubernetes Job and listens on the pod log stream.",
-    out=Out(list),
+    description="Dispatches one external payload as a Kubernetes Job and listens on the pod log stream.",
+    out=Out(dict),
 )
 def dispatch_external_work_k8s(
     context: OpExecutionContext,
-    units: list,
+    unit: int,
     pipes_k8s_client: PipesK8sClient,
-) -> list:
+) -> dict:
     completed = pipes_k8s_client.run(
         context=context,
         image=PAYLOAD_IMAGE,
         command=["python", "/app/work.py"],
         namespace=PAYLOAD_NAMESPACE,
-        extras={"units": units},
+        extras={"units": [unit]},
         base_pod_meta={
             "labels": {
                 "app.kubernetes.io/name": "distributed-execution-payload",
@@ -109,14 +119,14 @@ def dispatch_external_work_k8s(
             }
         },
     )
-    return _results_from_pipes(context, completed)
+    return _result_from_pipes(context, completed)
 
 
 @graph
 def loosely_coupled_subprocess_reference():
     target_report = report_execution_target()
     units = generate_work_units()
-    results = dispatch_external_work_subprocess(units)
+    results = units.map(dispatch_external_work_subprocess).collect()
     return summarise_results(results, target_report)
 
 
@@ -124,7 +134,7 @@ def loosely_coupled_subprocess_reference():
 def loosely_coupled_k8s_reference():
     target_report = report_execution_target()
     units = generate_work_units()
-    results = dispatch_external_work_k8s(units)
+    results = units.map(dispatch_external_work_k8s).collect()
     return summarise_results(results, target_report)
 
 
@@ -134,6 +144,9 @@ loosely_coupled_subprocess_job = loosely_coupled_subprocess_reference.to_job(
         "Loosely coupled via subprocess transport. Demonstrates the pipes contract end to end "
         "on a laptop, with no cluster and no metadata database access from the payload."
     ),
+    # Dispatch is I/O-bound and the real work happens in the payload, so orchestrating
+    # in one process keeps the laptop demo free of platform-specific spawn behaviour.
+    executor_def=in_process_executor,
     resource_defs={"pipes_subprocess_client": PipesSubprocessClient()},
     tags={**COMMON_TAGS, "transport": "subprocess"},
 )
@@ -144,6 +157,7 @@ loosely_coupled_k8s_job = loosely_coupled_k8s_reference.to_job(
         "Loosely coupled via Kubernetes Job dispatch. Messages return over the pod log stream. "
         "Requires a cluster and RBAC to create Jobs in the target namespace."
     ),
+    executor_def=multiprocess_executor.configured({"max_concurrent": 2}),
     resource_defs={"pipes_k8s_client": PipesK8sClient()},
     tags={**COMMON_TAGS, "transport": "k8s_pod_logs"},
 )

src/distributed_execution/ops.py

@@ -6,7 +6,7 @@ happens and *how* that fact reaches the control plane.
 
 # No `from __future__ import annotations`: it stringifies the `context` hint and
 # Dagster's op context validation then rejects it.
-from dagster import Config, MetadataValue, OpExecutionContext, Out, op
+from dagster import Config, DynamicOut, DynamicOutput, MetadataValue, OpExecutionContext, Out, op
 
 from distributed_execution.preflight import (
     TIGHTLY_COUPLED_ENV_VARS,
@@ -48,34 +48,47 @@ def report_execution_target(context: OpExecutionContext) -> dict:
     return {"identity": identity, "env_check": env_check}
 
 
-@op(description="Produces the work units that later steps fan out over.", out=Out(list))
-def generate_work_units(context: OpExecutionContext, config: WorkUnitsConfig) -> list[int]:
-    units = list(range(config.count))
-    context.log.info("Generated %s work units", len(units))
-    return units
+@op(
+    description="Fans out one dynamic output per work unit, so each unit becomes its own step.",
+    out=DynamicOut(int),
+)
+def generate_work_units(context: OpExecutionContext, config: WorkUnitsConfig):
+    context.log.info("Fanning out %s work units", config.count)
+    for unit in range(config.count):
+        yield DynamicOutput(unit, mapping_key=f"unit_{unit}")
 
 
-@op(description="Performs one unit of work per element; runs once per step-execution slot.", out=Out(list))
-def process_work_units(context: OpExecutionContext, units: list[int]) -> list[dict]:
+@op(
+    description="Performs one unit of work. One step per unit, so one process or pod per unit.",
+    out=Out(dict),
+)
+def process_work_unit(context: OpExecutionContext, unit: int) -> dict:
     identity = describe_pod_identity()
-    results = [{"unit": unit, "squared": unit * unit, "host": identity["hostname"]} for unit in units]
-    context.log.info("Processed %s units on %s", len(results), identity["hostname"])
-    return results
+    context.log.info("Processing unit %s on %s", unit, identity["worker"])
+    return {
+        "unit": unit,
+        "squared": unit * unit,
+        "host": identity["hostname"],
+        "worker": identity["worker"],
+    }
 
 
-@op(description="Aggregates results and attaches the distinct hosts that contributed.")
-def summarise_results(context: OpExecutionContext, results: list[dict], target_report: dict) -> dict:
+@op(description="Aggregates results and records which workers actually contributed.")
+def summarise_results(context: OpExecutionContext, results: list, target_report: dict) -> dict:
     hosts = sorted({row["host"] for row in results})
+    workers = sorted({row["worker"] for row in results})
     summary = {
         "units": len(results),
         "total": sum(row["squared"] for row in results),
         "contributing_hosts": hosts,
+        "contributing_workers": workers,
         "launcher_namespace": target_report["identity"]["namespace"],
     }
     context.add_output_metadata(
         {
             "units": MetadataValue.int(summary["units"]),
             "contributing_hosts": MetadataValue.json(hosts),
+            "contributing_workers": MetadataValue.json(workers),
             "launcher_namespace": MetadataValue.text(summary["launcher_namespace"]),
         }
     )

src/distributed_execution/preflight.py

@@ -52,9 +52,12 @@ def check_endpoint_reachable(url: str, timeout: float = 3.0) -> dict:
 
 def describe_pod_identity() -> dict:
     """Where this process is actually running - the primary execution-target evidence."""
+    hostname = socket.gethostname()
     return {
-        "hostname": socket.gethostname(),
+        "hostname": hostname,
         "pid": os.getpid(),
+        # Distinguishes processes on one host, so multiprocess fan-out is visible locally.
+        "worker": f"{hostname}#{os.getpid()}",
         "namespace": os.environ.get("DAGSTER_K8S_PIPELINE_RUN_NAMESPACE", "<not-in-kubernetes>"),
         "run_id": os.environ.get("DAGSTER_RUN_ID", "<unset>"),
         "image": os.environ.get("DAGSTER_K8S_PIPELINE_RUN_IMAGE", "<unset>"),

src/distributed_execution/tightly_coupled/jobs.py

@@ -17,7 +17,7 @@ from dagster_k8s import k8s_job_executor
 
 from distributed_execution.ops import (
     generate_work_units,
-    process_work_units,
+    process_work_unit,
     report_execution_target,
     summarise_results,
 )
@@ -47,7 +47,7 @@ def distributed_execution_reference():
     """Shared topology, so the three jobs differ only by execution target."""
     target_report = report_execution_target()
     units = generate_work_units()
-    results = process_work_units(units)
+    results = units.map(process_work_unit).collect()
     return summarise_results(results, target_report)