tests/test_projects_yaml.py
Rama main · réplica de lectura del repositorio original.
"""Tests for projects.yaml defaults: load + persist round-trip.
Covers docs/projects-yaml.md §7 test list 1-6. The
``test_persist_projects_round_trip`` test is the critical regression for the
silent-defaults-erase bug: any change to ``_persist_projects`` MUST keep this
test green.
"""
from __future__ import annotations
from pathlib import Path, PurePosixPath
import pytest
import yaml
from jobd.broker.projects import _persist_projects
from jobd.config import (
ProjectDefaults,
ProjectEntry,
_path_is_within,
load_effective_projects,
load_projects,
project_from_cwd,
)
def _full_projects_yaml(tmp_path):
path = tmp_path / "projects.yaml"
path.write_text(
"""projects:
project-c:
priority: 55
defaults:
max_wall_s: 14400
idle_timeout_s: 1800
host_pin: desktop
requires:
gpu: true
needs: [cuda]
preemptible: true
escalate_to_arc: false
bare-project:
priority: 60
_default:
priority: 40
"""
)
return path
def test_load_projects_full_schema(tmp_path):
"""All keys under ``defaults`` parse into the ProjectDefaults dataclass."""
path = _full_projects_yaml(tmp_path)
projects = load_projects(path)
agri = projects["project-c"]
assert isinstance(agri, ProjectEntry)
assert agri.priority == 55
d = agri.defaults
assert d.max_wall_s == 14400
assert d.idle_timeout_s == 1800
assert d.host_pin == "desktop"
assert d.preemptible is True
assert d.escalate_to_arc is False
assert d.requires is not None
assert d.requires.gpu is True
assert d.requires.needs == ["cuda"]
def test_load_projects_missing_defaults_block(tmp_path):
"""Entry with only ``priority`` gets a zero-valued ProjectDefaults."""
path = _full_projects_yaml(tmp_path)
projects = load_projects(path)
bare = projects["bare-project"]
assert bare.priority == 60
assert bare.defaults == ProjectDefaults()
def test_load_projects_missing_file(tmp_path):
"""Missing projects.yaml returns ``{_default: ProjectEntry(priority=40)}``."""
path = tmp_path / "does-not-exist.yaml"
projects = load_projects(path)
assert "_default" in projects
assert projects["_default"].priority == 40
assert projects["_default"].defaults == ProjectDefaults()
def test_load_projects_malformed_yaml(tmp_path):
"""Corrupt YAML raises yaml.YAMLError, not silent empty dict."""
path = tmp_path / "broken.yaml"
path.write_text("projects:\n project-c:\n priority: [unclosed\n")
with pytest.raises(yaml.YAMLError):
load_projects(path)
def test_load_projects_unknown_keys_in_defaults(tmp_path):
"""Unrecognized keys under ``defaults`` are silently dropped."""
path = tmp_path / "projects.yaml"
path.write_text(
"""projects:
project-c:
priority: 55
defaults:
max_wall_s: 100
future_field_we_dont_know_yet: 42
another_unknown: hello
_default: { priority: 40 }
"""
)
projects = load_projects(path)
assert projects["project-c"].defaults.max_wall_s == 100
# No exception, no attribute on the dataclass; just dropped.
def _state(tmp_path, base_path):
"""Build a BrokerState the way build_app does: git baseline + writable overlay."""
overrides = tmp_path / "state" / "project-priorities.yaml"
projects, base_priorities = load_effective_projects(base_path, overrides)
return {
"projects": projects,
"base_priorities": base_priorities,
"paths": {"projects": base_path, "project_overrides": overrides},
}
def test_persist_writes_overlay_and_never_touches_git_config(tmp_path):
"""THE bug (audit 2026-07-12): _persist_projects used to rewrite
projects.yaml, which is bind-mounted :ro in the container — so every
`job projects set/nudge` raised OSError -> HTTP 500. The git baseline must
now be left byte-identical, with the delta landing in the overlay."""
base_path = _full_projects_yaml(tmp_path)
before = base_path.read_text()
state = _state(tmp_path, base_path)
state["projects"]["project-c"].priority = 65
_persist_projects(state)
# The git-owned baseline is untouched.
assert base_path.read_text() == before
# The delta is in the overlay.
overlay = yaml.safe_load(state["paths"]["project_overrides"].read_text())
assert overlay == {"priorities": {"project-c": 65}}
def test_persist_survives_a_read_only_config_dir(tmp_path):
"""Real-execution check of the production condition that caused the 500:
the config dir is genuinely read-only. Persist must still succeed."""
cfg_dir = tmp_path / "config"
cfg_dir.mkdir()
base_path = cfg_dir / "projects.yaml"
base_path.write_text("projects:\n project-c: { priority: 55 }\n _default: { priority: 40 }\n")
state = _state(tmp_path, base_path)
cfg_dir.chmod(0o555) # read-only, exactly like the :ro bind mount
try:
state["projects"]["project-c"].priority = 70
_persist_projects(state) # pre-fix: OSError: Read-only file system -> 500
reloaded, _ = load_effective_projects(base_path, state["paths"]["project_overrides"])
assert reloaded["project-c"].priority == 70
finally:
cfg_dir.chmod(0o755)
def test_persist_round_trip_preserves_defaults(tmp_path):
"""The original regression still holds — a nudge must not erase `defaults:`.
It is now structurally impossible: defaults live only in the git baseline,
which persist never rewrites."""
base_path = _full_projects_yaml(tmp_path)
state = _state(tmp_path, base_path)
state["projects"]["project-c"].priority = 65
_persist_projects(state)
reloaded, _ = load_effective_projects(base_path, state["paths"]["project_overrides"])
agri = reloaded["project-c"]
assert agri.priority == 65 # override applied
assert agri.defaults.max_wall_s == 14400 # whole defaults block survives
assert agri.defaults.idle_timeout_s == 1800
assert agri.defaults.host_pin == "desktop"
assert agri.defaults.preemptible is True
assert agri.defaults.requires is not None
assert agri.defaults.requires.gpu is True
assert agri.defaults.requires.needs == ["cuda"]
def test_overlay_holds_only_deltas_so_git_changes_still_flow(tmp_path):
"""Why the overlay stores diffs, not a full snapshot: a project nobody
nudged must still pick up a later config-as-code priority change, while a
nudged project keeps its runtime override."""
base_path = _full_projects_yaml(tmp_path)
state = _state(tmp_path, base_path)
state["projects"]["project-c"].priority = 65 # nudged at runtime
_persist_projects(state)
overlay = yaml.safe_load(state["paths"]["project_overrides"].read_text())
assert "bare-project" not in overlay["priorities"] # untouched -> not pinned
# Now `git pull` raises both priorities in the baseline.
base_path.write_text(
"projects:\n"
" project-c: { priority: 99 }\n"
" bare-project: { priority: 99 }\n"
" _default: { priority: 40 }\n"
)
reloaded, _ = load_effective_projects(base_path, state["paths"]["project_overrides"])
assert reloaded["bare-project"].priority == 99 # git change lands
assert reloaded["project-c"].priority == 65 # runtime override still wins
def test_overlay_can_materialize_a_project_absent_from_git(tmp_path):
"""`job projects set brand-new 70` on a project not in projects.yaml."""
base_path = _full_projects_yaml(tmp_path)
state = _state(tmp_path, base_path)
state["projects"]["brand-new"] = ProjectEntry(priority=70)
_persist_projects(state)
reloaded, _ = load_effective_projects(base_path, state["paths"]["project_overrides"])
assert reloaded["brand-new"].priority == 70
def test_priority_endpoints_work_against_a_read_only_config_dir(tmp_path):
"""End-to-end regression for the live 500 (audit 2026-07-12).
Reproduces production exactly: config/ is read-only (the `:ro` bind mount),
the DB lives in a writable data dir. Pre-fix, `POST /projects/{name}/nudge`
returned HTTP 500 (`OSError: Read-only file system`) — verified against the
running broker-host broker. Both mutation endpoints must now return 200 and survive
a reload.
"""
from fastapi.testclient import TestClient
from jobd.app import build_app
cfg = tmp_path / "config"
cfg.mkdir()
(cfg / "projects.yaml").write_text(
"projects:\n project-a: { priority: 80 }\n _default: { priority: 40 }\n"
)
(cfg / "profiles.yaml").write_text("profiles: {}\n")
(cfg / "classifier.yaml").write_text("rules: []\n")
data = tmp_path / "data"
data.mkdir()
app = build_app(
db_url=f"sqlite:///{data}/jobd.db",
projects_path=cfg / "projects.yaml",
profiles_path=cfg / "profiles.yaml",
classifier_path=cfg / "classifier.yaml",
logs_path=tmp_path / "logs",
)
client = TestClient(app)
cfg.chmod(0o555) # the :ro mount
try:
r = client.post("/projects/project-a/nudge", json={"delta": -10})
assert r.status_code == 200, r.text # was 500
assert r.json()["projects"]["project-a"]["priority"] == 70
r = client.post("/projects/project-d", json={"priority": 55})
assert r.status_code == 200, r.text
assert r.json()["projects"]["project-d"]["priority"] == 55
# Overrides survive a config reload (and the baseline is still readable).
assert client.post("/reload").status_code == 200
got = client.get("/projects").json()
assert got["project-a"]["priority"] == 70
assert got["project-d"]["priority"] == 55
finally:
cfg.chmod(0o755)
# --- defaults validation (audit 2026-07-15 F3) ------------------------------
#
# YAML values bypass Pydantic. An unvalidated `max_wall_s: 0` splits
# broker/worker semantics: the worker reads it as unset (`or`-falsy) and never
# enforces, the sweeper reads it as set (`is not None`) and orphans the healthy
# job as wall_clock_exceeded — which is not resurrectable. A STRING value
# TypeErrors the whole sweep pass every 30s. Both must die at parse time.
def _defaults_yaml(tmp_path, defaults_body: str):
path = tmp_path / "projects.yaml"
path.write_text(
f"""
projects:
p:
priority: 50
defaults:
{defaults_body}
_default:
priority: 40
"""
)
return load_projects(path)["p"].defaults
def test_zero_wall_clock_is_dropped_not_half_enforced(tmp_path, caplog):
d = _defaults_yaml(tmp_path, " max_wall_s: 0")
assert d.max_wall_s is None, (
"max_wall_s=0 must not reach Job rows: the worker treats 0 as unset but the "
"sweeper treats it as set, and orphans a healthy running job as wall_clock_exceeded"
)
assert any("max_wall_s" in r.message for r in caplog.records), "the drop must be loud"
def test_a_string_duration_is_dropped_before_it_can_kill_the_sweep(tmp_path):
d = _defaults_yaml(tmp_path, ' max_wall_s: "8h"')
assert d.max_wall_s is None, (
"a string max_wall_s flows into Job.max_wall_s and the sweeper's arithmetic "
"raises TypeError — aborting the ENTIRE sweep pass (timeouts, reclaims, "
"retention) every 30 seconds"
)
def test_out_of_bounds_values_are_dropped(tmp_path):
d = _defaults_yaml(
tmp_path,
" max_wall_s: 999999999\n idle_timeout_s: -5\n checkpoint_grace_s: 301\n priority: 150",
)
assert d.max_wall_s is None
assert d.idle_timeout_s is None
assert d.checkpoint_grace_s is None
assert d.priority is None
def test_a_boolean_is_not_an_int_here(tmp_path):
# YAML `max_wall_s: true` is a Python bool, which IS an int subclass —
# without the explicit bool check it would land as max_wall_s=1.
d = _defaults_yaml(tmp_path, " max_wall_s: true")
assert d.max_wall_s is None
def test_mistyped_flag_fields_are_dropped(tmp_path):
d = _defaults_yaml(tmp_path, ' preemptible: "yes please"\n host_pin: 7')
assert d.preemptible is None
assert d.host_pin is None
def test_valid_defaults_still_parse_bounds_inclusive(tmp_path):
d = _defaults_yaml(
tmp_path,
" max_wall_s: 604800\n idle_timeout_s: 1\n checkpoint_grace_s: 300\n priority: 0",
)
assert d.max_wall_s == 604800
assert d.idle_timeout_s == 1
assert d.checkpoint_grace_s == 300
assert d.priority == 0
def test_roots_are_parsed_and_normalized(tmp_path):
"""A trailing slash is a spelling of the same directory, not a different
root — normalizing at load means the matcher never has to care."""
p = tmp_path / "projects.yaml"
p.write_text(
"projects:\n"
" beta:\n"
" priority: 78\n"
" roots:\n"
" - /home/user/beta/\n"
" - /srv/beta\n"
)
projects = load_projects(p)
assert projects["beta"].roots == ["/home/user/beta", "/srv/beta"]
def test_a_project_without_roots_gets_an_empty_list(tmp_path):
"""Positive control: roots is optional, and its absence must not change
how an ordinary entry loads."""
p = tmp_path / "projects.yaml"
p.write_text("projects:\n plain: { priority: 60 }\n")
projects = load_projects(p)
assert projects["plain"].roots == []
assert projects["plain"].priority == 60
@pytest.mark.parametrize(
"bad, why",
[
("roots: [relative/path]", "not absolute"),
("roots: [17]", "not a string"),
("roots: /home/user/x", "not a list"),
("roots: ['/']", "filesystem root captures every job"),
("roots: ['/home/user/x/../y']", "parent-dir component"),
("roots: ['/home/user/..']", "parent-dir component"),
],
)
def test_a_bad_root_is_a_load_error_not_a_silent_drop(tmp_path, bad, why):
"""`defaults:` silently drops unknown keys by design. A dropped ROOT would
present to the operator as 'the feature just doesn't work on my machine',
so roots are validated loudly instead."""
p = tmp_path / "projects.yaml"
p.write_text(f"projects:\n beta:\n priority: 78\n {bad}\n")
with pytest.raises(ValueError, match="roots"):
load_projects(p)
# --- the config this repo actually ships ---------------------------------
#
# Nothing in the suite loaded `config/projects.yaml` itself, and `_parse_roots`
# RAISES on a malformed root by design. So a typo in that file aborts broker
# startup at boot while CI stays entirely green -- the failure lands on the
# operator, not on the change that caused it. One load closes that gap.
_REPO_ROOT = Path(__file__).resolve().parent.parent
def test_the_shipped_projects_yaml_parses():
projects = load_projects(_REPO_ROOT / "config" / "projects.yaml")
assert projects, "the shipped config must define at least one project"
for name, entry in projects.items():
for root in entry.roots:
assert root.startswith("/"), f"{name}: root {root!r} is not absolute"
assert ".." not in PurePosixPath(root).parts, f"{name}: root {root!r} has a '..'"
def test_the_shipped_roots_resolve_as_intended():
"""Behaviour, not an exact root list: editing the file must stay cheap.
Asserted here is the boundary property the whole feature rests on -- a
directory inside a shipped root takes that project's identity, and its
prefix-sharing sibling takes none.
"""
projects = load_projects(_REPO_ROOT / "config" / "projects.yaml")
hit = project_from_cwd(projects, "/home/user/beta/sweeps/a")
assert hit is not None, "a path under a shipped root must resolve"
name, root = hit
assert name == "beta"
assert _path_is_within("/home/user/beta/sweeps/a", root)
assert project_from_cwd(projects, "/home/user/beta2") is None
def test_local_overlay_replaces_roots_and_adds_projects(tmp_path):
"""`projects.local.yaml` beside projects.yaml overrides the tracked entry
wholesale and can add a project; names it does not declare are untouched,
and `_default` keeps the tracked priority."""
from jobd.config import load_effective_projects
(tmp_path / "projects.yaml").write_text(
"projects:\n _default: {priority: 40}\n"
" alpha: {priority: 80, roots: [/srv/example/alpha]}\n"
" beta: {priority: 70}\n"
)
(tmp_path / "projects.local.yaml").write_text(
"projects:\n alpha: {priority: 80, roots: [/home/me/alpha]}\n private-x: {priority: 55}\n"
)
eff, base = load_effective_projects(tmp_path / "projects.yaml", tmp_path / "overrides.yaml")
assert [str(r) for r in eff["alpha"].roots] == ["/home/me/alpha"]
assert eff["private-x"].priority == 55
assert eff["beta"].priority == 70
assert eff["_default"].priority == 40
assert base["private-x"] == 55
def test_no_local_overlay_is_a_noop(tmp_path):
"""Negative control for the test above: with no local file the effective
table is exactly the tracked one."""
from jobd.config import load_effective_projects, load_projects
(tmp_path / "projects.yaml").write_text(
"projects:\n alpha: {priority: 80, roots: [/srv/example/alpha]}\n"
)
eff, _ = load_effective_projects(tmp_path / "projects.yaml", tmp_path / "overrides.yaml")
tracked = load_projects(tmp_path / "projects.yaml")
assert set(eff) == set(tracked)
assert [str(r) for r in eff["alpha"].roots] == ["/srv/example/alpha"]
def test_roots_without_a_priority_is_a_load_error_not_a_silent_drop(tmp_path):
"""audit 2026-09-02 C-2: `load_projects` skipped any entry lacking
`priority` BEFORE `_parse_roots` ran, so a project declared as
`beta: {roots: [...]}` vanished with no log line -- exactly the
invisible removal the roots validation promises to prevent."""
p = tmp_path / "projects.yaml"
p.write_text("projects:\n beta:\n roots: ['/home/user/beta']\n")
with pytest.raises(ValueError, match="priority"):
load_projects(p)
def test_default_may_not_declare_roots(tmp_path):
"""`_default` is skipped by `project_from_cwd`, so a root on it was
validated, accepted, and then silently never consulted."""
p = tmp_path / "projects.yaml"
p.write_text("projects:\n _default:\n priority: 40\n roots: ['/home']\n")
with pytest.raises(ValueError, match="_default"):
load_projects(p)
def test_a_leading_double_slash_in_a_root_is_collapsed(tmp_path):
"""POSIX lets `//x` mean something other than `/x`, and `os.path.normpath`
preserves it, so `//home/x` was accepted as a root that could never match
a cwd of `/home/x/...` -- a silently dead root."""
p = tmp_path / "projects.yaml"
p.write_text("projects:\n beta:\n priority: 78\n roots: ['//home/user/x']\n")
assert load_projects(p)["beta"].roots == ["/home/user/x"]