9b1b7056b0
Check the chart's actual DAGSTER_PG_PASSWORD injection in the tightly coupled preflight instead of DAGSTER_POSTGRES_*, which the Simpl chart never sets; the old check warned about a misconfiguration on a correctly configured run pod. Evidence reports the variable name only, never its value. Record the platform k8s_job_executor run as cluster-verified in the guide and readiness checklist, add the Dagster UI screenshots covering job list, graph, code-configured tags and a successful run. Changelog: fixed
94 lines
2.9 KiB
Python
94 lines
2.9 KiB
Python
"""Verifies the loosely coupled pipes reference end to end via the subprocess transport."""
|
|
|
|
import pytest
|
|
from dagster import DagsterInvariantViolationError, Failure
|
|
|
|
from distributed_execution.loosely_coupled.jobs import (
|
|
LOOSELY_COUPLED_JOBS,
|
|
PAYLOAD_SCRIPT,
|
|
loosely_coupled_k8s_job,
|
|
loosely_coupled_subprocess_job,
|
|
)
|
|
from distributed_execution.repository import defs
|
|
|
|
|
|
def test_payload_script_exists():
|
|
from pathlib import Path
|
|
|
|
assert Path(PAYLOAD_SCRIPT).is_file()
|
|
|
|
|
|
def test_payload_does_not_import_dagster():
|
|
from pathlib import Path
|
|
|
|
source = Path(PAYLOAD_SCRIPT).read_text(encoding="utf-8")
|
|
assert "from dagster_pipes import" in source
|
|
assert "import dagster\n" not in source
|
|
assert "from dagster import" not in source
|
|
|
|
|
|
def test_both_transports_are_registered():
|
|
names = {job.name for job in defs.jobs}
|
|
assert {"loosely_coupled_subprocess_job", "loosely_coupled_k8s_job"} <= names
|
|
assert len(LOOSELY_COUPLED_JOBS) == 2
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("job", "transport"),
|
|
[
|
|
(loosely_coupled_subprocess_job, "subprocess"),
|
|
(loosely_coupled_k8s_job, "k8s_pod_logs"),
|
|
],
|
|
)
|
|
def test_jobs_declare_their_execution_target(job, transport):
|
|
assert job.tags["execution_target"] == "loosely_coupled"
|
|
assert job.tags["transport"] == transport
|
|
|
|
|
|
def test_subprocess_job_runs_end_to_end():
|
|
result = loosely_coupled_subprocess_job.execute_in_process()
|
|
|
|
assert result.success
|
|
summary = result.output_for_node("summarise_results")
|
|
# Same shape as the tightly coupled reference: only the middle node changed.
|
|
assert summary["units"] == 4
|
|
assert summary["total"] == 14
|
|
|
|
|
|
def test_each_unit_is_dispatched_to_its_own_external_worker():
|
|
result = loosely_coupled_subprocess_job.execute_in_process()
|
|
|
|
mapped = result.output_for_node("dispatch_external_work_subprocess")
|
|
assert set(mapped) == {"unit_0", "unit_1", "unit_2", "unit_3"}
|
|
# Each dispatch is its own OS process, so workers differ even on a single host.
|
|
assert len({row["worker"] for row in mapped.values()}) == 4
|
|
|
|
|
|
def test_payload_reports_no_orchestration_credentials(monkeypatch):
|
|
monkeypatch.delenv("DAGSTER_PG_PASSWORD", raising=False)
|
|
|
|
result = loosely_coupled_subprocess_job.execute_in_process()
|
|
mapped = result.output_for_node("dispatch_external_work_subprocess")
|
|
|
|
assert len(mapped) == 4
|
|
assert all(row["worker"] for row in mapped.values())
|
|
|
|
|
|
def test_silent_message_path_is_treated_as_failure():
|
|
from distributed_execution.loosely_coupled.jobs import _result_from_pipes
|
|
|
|
class _Silent:
|
|
def get_custom_messages(self):
|
|
return []
|
|
|
|
class _Ctx:
|
|
class log: # noqa: N801
|
|
@staticmethod
|
|
def warning(*_args, **_kwargs): ...
|
|
|
|
@staticmethod
|
|
def info(*_args, **_kwargs): ...
|
|
|
|
with pytest.raises(Failure, match="No pipes messages received"):
|
|
_result_from_pipes(_Ctx(), _Silent())
|