[SIMPL-30451] Add distributed-execution service with guide and reference implementations

Canonical location for documentation, example workflows and reference service implementations covering distributed execution patterns. Covers AC1-AC4: execution-target selection, decision support, readiness checks and code-level linkage. Tightly coupled jobs and the loosely coupled subprocess transport are verified by the test suite; the Kubernetes pipes transport is implemented but not yet cluster-run and is marked as such in the guide.

Changelog: added

tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for the distributed-execution reference implementations."""

tests/test_loosely_coupled.py

@@ -0,0 +1,86 @@
+"""Verifies the loosely coupled pipes reference end to end via the subprocess transport."""
+
+import pytest
+from dagster import DagsterInvariantViolationError, Failure
+
+from distributed_execution.loosely_coupled.jobs import (
+    LOOSELY_COUPLED_JOBS,
+    PAYLOAD_SCRIPT,
+    loosely_coupled_k8s_job,
+    loosely_coupled_subprocess_job,
+)
+from distributed_execution.repository import defs
+
+
+def test_payload_script_exists():
+    from pathlib import Path
+
+    assert Path(PAYLOAD_SCRIPT).is_file()
+
+
+def test_payload_does_not_import_dagster():
+    from pathlib import Path
+
+    source = Path(PAYLOAD_SCRIPT).read_text(encoding="utf-8")
+    assert "from dagster_pipes import" in source
+    assert "import dagster\n" not in source
+    assert "from dagster import" not in source
+
+
+def test_both_transports_are_registered():
+    names = {job.name for job in defs.jobs}
+    assert {"loosely_coupled_subprocess_job", "loosely_coupled_k8s_job"} <= names
+    assert len(LOOSELY_COUPLED_JOBS) == 2
+
+
+@pytest.mark.parametrize(
+    ("job", "transport"),
+    [
+        (loosely_coupled_subprocess_job, "subprocess"),
+        (loosely_coupled_k8s_job, "k8s_pod_logs"),
+    ],
+)
+def test_jobs_declare_their_execution_target(job, transport):
+    assert job.tags["execution_target"] == "loosely_coupled"
+    assert job.tags["transport"] == transport
+
+
+def test_subprocess_job_runs_end_to_end():
+    result = loosely_coupled_subprocess_job.execute_in_process()
+
+    assert result.success
+    summary = result.output_for_node("summarise_results")
+    # Same shape as the tightly coupled reference: only the middle node changed.
+    assert summary["units"] == 4
+    assert summary["total"] == 14
+
+
+def test_payload_reports_no_orchestration_credentials(monkeypatch):
+    monkeypatch.delenv("DAGSTER_POSTGRES_HOST", raising=False)
+    monkeypatch.delenv("DAGSTER_POSTGRES_USER", raising=False)
+    monkeypatch.delenv("DAGSTER_POSTGRES_DB", raising=False)
+
+    result = loosely_coupled_subprocess_job.execute_in_process()
+    results = result.output_for_node("dispatch_external_work_subprocess")
+
+    assert len(results) == 4
+    assert all(row["host"] for row in results)
+
+
+def test_silent_message_path_is_treated_as_failure():
+    from distributed_execution.loosely_coupled.jobs import _results_from_pipes
+
+    class _Silent:
+        def get_custom_messages(self):
+            return []
+
+    class _Ctx:
+        class log:  # noqa: N801
+            @staticmethod
+            def warning(*_args, **_kwargs): ...
+
+            @staticmethod
+            def info(*_args, **_kwargs): ...
+
+    with pytest.raises(Failure, match="No pipes messages received"):
+        _results_from_pipes(_Ctx(), _Silent())

tests/test_tightly_coupled.py

@@ -0,0 +1,59 @@
+"""Verifies that execution-target linkage is what the guide claims it is."""
+
+import pytest
+
+from distributed_execution.preflight import check_env_vars, describe_pod_identity
+from distributed_execution.repository import defs
+from distributed_execution.tightly_coupled.jobs import (
+    tightly_coupled_in_process_job,
+    tightly_coupled_k8s_job,
+    tightly_coupled_local_job,
+)
+
+EXPECTED_JOBS = {
+    "tightly_coupled_in_process_job",
+    "tightly_coupled_local_job",
+    "tightly_coupled_k8s_job",
+}
+
+
+def test_repository_registers_tightly_coupled_jobs():
+    names = {job.name for job in defs.jobs}
+    assert EXPECTED_JOBS <= names
+
+
+@pytest.mark.parametrize(
+    ("job", "expected_executor"),
+    [
+        (tightly_coupled_in_process_job, "in_process"),
+        (tightly_coupled_local_job, "multiprocess"),
+        (tightly_coupled_k8s_job, "k8s_job"),
+    ],
+)
+def test_each_job_declares_its_execution_target(job, expected_executor):
+    assert job.tags["execution_target"] == "tightly_coupled"
+    assert job.tags["executor"] == expected_executor
+
+
+def test_in_process_job_runs_end_to_end():
+    result = tightly_coupled_in_process_job.execute_in_process()
+
+    assert result.success
+    summary = result.output_for_node("summarise_results")
+    assert summary["units"] == 4
+    assert summary["total"] == 14  # 0 + 1 + 4 + 9
+    # In-process execution never fans out beyond the run worker.
+    assert len(summary["contributing_hosts"]) == 1
+
+
+def test_env_var_check_reports_missing_names():
+    result = check_env_vars(("DEFINITELY_NOT_SET_12345",))
+
+    assert result["passed"] is False
+    assert result["missing"] == ["DEFINITELY_NOT_SET_12345"]
+
+
+def test_pod_identity_falls_back_outside_kubernetes(monkeypatch):
+    monkeypatch.delenv("DAGSTER_K8S_PIPELINE_RUN_NAMESPACE", raising=False)
+
+    assert describe_pod_identity()["namespace"] == "<not-in-kubernetes>"