[SIMPL-30451] Fan out work units so the execution target is observable

The reference graph processed every work unit inside a single op, so exactly one worker was ever reported regardless of executor. That made the guide's claim that k8s_job_executor yields several distinct contributing_hosts false, and left the reference implementations unable to demonstrate the executor choice at all. generate_work_units is now a DynamicOut and both graphs map over it, so one step is created per unit and the loosely coupled pattern dispatches one external workload per unit.

Evidence is split into contributing_workers (host and pid, differs per process) and contributing_hosts (differs only across machines), because the previous single field could not distinguish multiprocess fan-out from no fan-out. Tests now assert the mapped step keys rather than a host count, since execute_in_process ignores executor_def and cannot prove executor behaviour on its own. Adds a Windows note: multiprocess_executor did not complete during authoring and left orphaned processes.

Changelog: fixed

tests/test_loosely_coupled.py

@@ -55,20 +55,29 @@ def test_subprocess_job_runs_end_to_end():
     assert summary["total"] == 14
 
 
+def test_each_unit_is_dispatched_to_its_own_external_worker():
+    result = loosely_coupled_subprocess_job.execute_in_process()
+
+    mapped = result.output_for_node("dispatch_external_work_subprocess")
+    assert set(mapped) == {"unit_0", "unit_1", "unit_2", "unit_3"}
+    # Each dispatch is its own OS process, so workers differ even on a single host.
+    assert len({row["worker"] for row in mapped.values()}) == 4
+
+
 def test_payload_reports_no_orchestration_credentials(monkeypatch):
     monkeypatch.delenv("DAGSTER_POSTGRES_HOST", raising=False)
     monkeypatch.delenv("DAGSTER_POSTGRES_USER", raising=False)
     monkeypatch.delenv("DAGSTER_POSTGRES_DB", raising=False)
 
     result = loosely_coupled_subprocess_job.execute_in_process()
-    results = result.output_for_node("dispatch_external_work_subprocess")
+    mapped = result.output_for_node("dispatch_external_work_subprocess")
 
-    assert len(results) == 4
-    assert all(row["host"] for row in results)
+    assert len(mapped) == 4
+    assert all(row["worker"] for row in mapped.values())
 
 
 def test_silent_message_path_is_treated_as_failure():
-    from distributed_execution.loosely_coupled.jobs import _results_from_pipes
+    from distributed_execution.loosely_coupled.jobs import _result_from_pipes
 
     class _Silent:
         def get_custom_messages(self):
@@ -83,4 +92,4 @@ def test_silent_message_path_is_treated_as_failure():
             def info(*_args, **_kwargs): ...
 
     with pytest.raises(Failure, match="No pipes messages received"):
-        _results_from_pipes(_Ctx(), _Silent())
+        _result_from_pipes(_Ctx(), _Silent())

tests/test_tightly_coupled.py

@@ -42,8 +42,16 @@ def test_in_process_job_runs_end_to_end():
     summary = result.output_for_node("summarise_results")
     assert summary["units"] == 4
     assert summary["total"] == 14  # 0 + 1 + 4 + 9
-    # In-process execution never fans out beyond the run worker.
-    assert len(summary["contributing_hosts"]) == 1
+    # In-process execution keeps every step in the run worker's own process.
+    assert len(summary["contributing_workers"]) == 1
+
+
+def test_work_units_fan_out_into_one_step_each():
+    result = tightly_coupled_in_process_job.execute_in_process()
+
+    mapped = result.output_for_node("process_work_unit")
+    # Without this the executor choice would be meaningless: one step cannot span pods.
+    assert set(mapped) == {"unit_0", "unit_1", "unit_2", "unit_3"}
 
 
 def test_env_var_check_reports_missing_names():