43dbb81a95
Canonical location for documentation, example workflows and reference service implementations covering distributed execution patterns. Covers AC1-AC4: execution-target selection, decision support, readiness checks and code-level linkage. Tightly coupled jobs and the loosely coupled subprocess transport are verified by the test suite; the Kubernetes pipes transport is implemented but not yet cluster-run and is marked as such in the guide. Changelog: added
87 lines
2.6 KiB
Python
87 lines
2.6 KiB
Python
"""Verifies the loosely coupled pipes reference end to end via the subprocess transport."""
|
|
|
|
import pytest
|
|
from dagster import DagsterInvariantViolationError, Failure
|
|
|
|
from distributed_execution.loosely_coupled.jobs import (
|
|
LOOSELY_COUPLED_JOBS,
|
|
PAYLOAD_SCRIPT,
|
|
loosely_coupled_k8s_job,
|
|
loosely_coupled_subprocess_job,
|
|
)
|
|
from distributed_execution.repository import defs
|
|
|
|
|
|
def test_payload_script_exists():
|
|
from pathlib import Path
|
|
|
|
assert Path(PAYLOAD_SCRIPT).is_file()
|
|
|
|
|
|
def test_payload_does_not_import_dagster():
|
|
from pathlib import Path
|
|
|
|
source = Path(PAYLOAD_SCRIPT).read_text(encoding="utf-8")
|
|
assert "from dagster_pipes import" in source
|
|
assert "import dagster\n" not in source
|
|
assert "from dagster import" not in source
|
|
|
|
|
|
def test_both_transports_are_registered():
|
|
names = {job.name for job in defs.jobs}
|
|
assert {"loosely_coupled_subprocess_job", "loosely_coupled_k8s_job"} <= names
|
|
assert len(LOOSELY_COUPLED_JOBS) == 2
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("job", "transport"),
|
|
[
|
|
(loosely_coupled_subprocess_job, "subprocess"),
|
|
(loosely_coupled_k8s_job, "k8s_pod_logs"),
|
|
],
|
|
)
|
|
def test_jobs_declare_their_execution_target(job, transport):
|
|
assert job.tags["execution_target"] == "loosely_coupled"
|
|
assert job.tags["transport"] == transport
|
|
|
|
|
|
def test_subprocess_job_runs_end_to_end():
|
|
result = loosely_coupled_subprocess_job.execute_in_process()
|
|
|
|
assert result.success
|
|
summary = result.output_for_node("summarise_results")
|
|
# Same shape as the tightly coupled reference: only the middle node changed.
|
|
assert summary["units"] == 4
|
|
assert summary["total"] == 14
|
|
|
|
|
|
def test_payload_reports_no_orchestration_credentials(monkeypatch):
|
|
monkeypatch.delenv("DAGSTER_POSTGRES_HOST", raising=False)
|
|
monkeypatch.delenv("DAGSTER_POSTGRES_USER", raising=False)
|
|
monkeypatch.delenv("DAGSTER_POSTGRES_DB", raising=False)
|
|
|
|
result = loosely_coupled_subprocess_job.execute_in_process()
|
|
results = result.output_for_node("dispatch_external_work_subprocess")
|
|
|
|
assert len(results) == 4
|
|
assert all(row["host"] for row in results)
|
|
|
|
|
|
def test_silent_message_path_is_treated_as_failure():
|
|
from distributed_execution.loosely_coupled.jobs import _results_from_pipes
|
|
|
|
class _Silent:
|
|
def get_custom_messages(self):
|
|
return []
|
|
|
|
class _Ctx:
|
|
class log: # noqa: N801
|
|
@staticmethod
|
|
def warning(*_args, **_kwargs): ...
|
|
|
|
@staticmethod
|
|
def info(*_args, **_kwargs): ...
|
|
|
|
with pytest.raises(Failure, match="No pipes messages received"):
|
|
_results_from_pipes(_Ctx(), _Silent())
|