[SIMPL-30451] Fan out work units so the execution target is observable

The reference graph processed every work unit inside a single op, so exactly one worker was ever reported regardless of executor. That made the guide's claim that k8s_job_executor yields several distinct contributing_hosts false, and left the reference implementations unable to demonstrate the executor choice at all. generate_work_units is now a DynamicOut and both graphs map over it, so one step is created per unit and the loosely coupled pattern dispatches one external workload per unit.

Evidence is split into contributing_workers (host and pid, differs per process) and contributing_hosts (differs only across machines), because the previous single field could not distinguish multiprocess fan-out from no fan-out. Tests now assert the mapped step keys rather than a host count, since execute_in_process ignores executor_def and cannot prove executor behaviour on its own. Adds a Windows note: multiprocess_executor did not complete during authoring and left orphaned processes.

Changelog: fixed

src/distributed_execution/loosely_coupled/jobs.py

@@ -14,7 +14,16 @@ import os
 import sys
 from pathlib import Path
 
-from dagster import Failure, OpExecutionContext, Out, PipesSubprocessClient, graph, op
+from dagster import (
+    Failure,
+    OpExecutionContext,
+    Out,
+    PipesSubprocessClient,
+    graph,
+    in_process_executor,
+    multiprocess_executor,
+    op,
+)
 from dagster_k8s import PipesK8sClient
 
 from distributed_execution.ops import (
@@ -39,8 +48,8 @@ COMMON_TAGS = {
 }
 
 
-def _results_from_pipes(context: OpExecutionContext, completed) -> list:
-    """Read the payload's results off the message channel.
+def _result_from_pipes(context: OpExecutionContext, completed) -> dict:
+    """Read one unit's result off the message channel.
 
     A broken message path is the defining failure mode of this pattern: the
     external workload can exit 0 while reporting nothing, so silence is treated
@@ -66,42 +75,43 @@ def _results_from_pipes(context: OpExecutionContext, completed) -> list:
             ", ".join(leaked),
         )
 
-    context.log.info("Received %s results from external host %s", len(payload["results"]), payload["host"])
-    return payload["results"]
+    row = payload["results"][0]
+    context.log.info("Unit %s computed by external worker %s", row["unit"], row["worker"])
+    return row
 
 
 @op(
-    description="Dispatches the external payload as a local subprocess and listens on the pipes channel.",
-    out=Out(list),
+    description="Dispatches one external payload as a local subprocess and listens on the pipes channel.",
+    out=Out(dict),
 )
 def dispatch_external_work_subprocess(
     context: OpExecutionContext,
-    units: list,
+    unit: int,
     pipes_subprocess_client: PipesSubprocessClient,
-) -> list:
+) -> dict:
     completed = pipes_subprocess_client.run(
         context=context,
         command=[sys.executable, PAYLOAD_SCRIPT],
-        extras={"units": units},
+        extras={"units": [unit]},
     )
-    return _results_from_pipes(context, completed)
+    return _result_from_pipes(context, completed)
 
 
 @op(
-    description="Dispatches the external payload as a Kubernetes Job and listens on the pod log stream.",
-    out=Out(list),
+    description="Dispatches one external payload as a Kubernetes Job and listens on the pod log stream.",
+    out=Out(dict),
 )
 def dispatch_external_work_k8s(
     context: OpExecutionContext,
-    units: list,
+    unit: int,
     pipes_k8s_client: PipesK8sClient,
-) -> list:
+) -> dict:
     completed = pipes_k8s_client.run(
         context=context,
         image=PAYLOAD_IMAGE,
         command=["python", "/app/work.py"],
         namespace=PAYLOAD_NAMESPACE,
-        extras={"units": units},
+        extras={"units": [unit]},
         base_pod_meta={
             "labels": {
                 "app.kubernetes.io/name": "distributed-execution-payload",
@@ -109,14 +119,14 @@ def dispatch_external_work_k8s(
             }
         },
     )
-    return _results_from_pipes(context, completed)
+    return _result_from_pipes(context, completed)
 
 
 @graph
 def loosely_coupled_subprocess_reference():
     target_report = report_execution_target()
     units = generate_work_units()
-    results = dispatch_external_work_subprocess(units)
+    results = units.map(dispatch_external_work_subprocess).collect()
     return summarise_results(results, target_report)
 
 
@@ -124,7 +134,7 @@ def loosely_coupled_subprocess_reference():
 def loosely_coupled_k8s_reference():
     target_report = report_execution_target()
     units = generate_work_units()
-    results = dispatch_external_work_k8s(units)
+    results = units.map(dispatch_external_work_k8s).collect()
     return summarise_results(results, target_report)
 
 
@@ -134,6 +144,9 @@ loosely_coupled_subprocess_job = loosely_coupled_subprocess_reference.to_job(
         "Loosely coupled via subprocess transport. Demonstrates the pipes contract end to end "
         "on a laptop, with no cluster and no metadata database access from the payload."
     ),
+    # Dispatch is I/O-bound and the real work happens in the payload, so orchestrating
+    # in one process keeps the laptop demo free of platform-specific spawn behaviour.
+    executor_def=in_process_executor,
     resource_defs={"pipes_subprocess_client": PipesSubprocessClient()},
     tags={**COMMON_TAGS, "transport": "subprocess"},
 )
@@ -144,6 +157,7 @@ loosely_coupled_k8s_job = loosely_coupled_k8s_reference.to_job(
         "Loosely coupled via Kubernetes Job dispatch. Messages return over the pod log stream. "
         "Requires a cluster and RBAC to create Jobs in the target namespace."
     ),
+    executor_def=multiprocess_executor.configured({"max_concurrent": 2}),
     resource_defs={"pipes_k8s_client": PipesK8sClient()},
     tags={**COMMON_TAGS, "transport": "k8s_pod_logs"},
 )