[SIMPL-30451] Add distributed-execution service with guide and reference implementations

Canonical location for documentation, example workflows and reference service implementations covering distributed execution patterns. Covers AC1-AC4: execution-target selection, decision support, readiness checks and code-level linkage. Tightly coupled jobs and the loosely coupled subprocess transport are verified by the test suite; the Kubernetes pipes transport is implemented but not yet cluster-run and is marked as such in the guide.

Changelog: added

tests/test_loosely_coupled.py

@@ -0,0 +1,86 @@
+"""Verifies the loosely coupled pipes reference end to end via the subprocess transport."""
+
+import pytest
+from dagster import DagsterInvariantViolationError, Failure
+
+from distributed_execution.loosely_coupled.jobs import (
+    LOOSELY_COUPLED_JOBS,
+    PAYLOAD_SCRIPT,
+    loosely_coupled_k8s_job,
+    loosely_coupled_subprocess_job,
+)
+from distributed_execution.repository import defs
+
+
+def test_payload_script_exists():
+    from pathlib import Path
+
+    assert Path(PAYLOAD_SCRIPT).is_file()
+
+
+def test_payload_does_not_import_dagster():
+    from pathlib import Path
+
+    source = Path(PAYLOAD_SCRIPT).read_text(encoding="utf-8")
+    assert "from dagster_pipes import" in source
+    assert "import dagster\n" not in source
+    assert "from dagster import" not in source
+
+
+def test_both_transports_are_registered():
+    names = {job.name for job in defs.jobs}
+    assert {"loosely_coupled_subprocess_job", "loosely_coupled_k8s_job"} <= names
+    assert len(LOOSELY_COUPLED_JOBS) == 2
+
+
+@pytest.mark.parametrize(
+    ("job", "transport"),
+    [
+        (loosely_coupled_subprocess_job, "subprocess"),
+        (loosely_coupled_k8s_job, "k8s_pod_logs"),
+    ],
+)
+def test_jobs_declare_their_execution_target(job, transport):
+    assert job.tags["execution_target"] == "loosely_coupled"
+    assert job.tags["transport"] == transport
+
+
+def test_subprocess_job_runs_end_to_end():
+    result = loosely_coupled_subprocess_job.execute_in_process()
+
+    assert result.success
+    summary = result.output_for_node("summarise_results")
+    # Same shape as the tightly coupled reference: only the middle node changed.
+    assert summary["units"] == 4
+    assert summary["total"] == 14
+
+
+def test_payload_reports_no_orchestration_credentials(monkeypatch):
+    monkeypatch.delenv("DAGSTER_POSTGRES_HOST", raising=False)
+    monkeypatch.delenv("DAGSTER_POSTGRES_USER", raising=False)
+    monkeypatch.delenv("DAGSTER_POSTGRES_DB", raising=False)
+
+    result = loosely_coupled_subprocess_job.execute_in_process()
+    results = result.output_for_node("dispatch_external_work_subprocess")
+
+    assert len(results) == 4
+    assert all(row["host"] for row in results)
+
+
+def test_silent_message_path_is_treated_as_failure():
+    from distributed_execution.loosely_coupled.jobs import _results_from_pipes
+
+    class _Silent:
+        def get_custom_messages(self):
+            return []
+
+    class _Ctx:
+        class log:  # noqa: N801
+            @staticmethod
+            def warning(*_args, **_kwargs): ...
+
+            @staticmethod
+            def info(*_args, **_kwargs): ...
+
+    with pytest.raises(Failure, match="No pipes messages received"):
+        _results_from_pipes(_Ctx(), _Silent())