tests/test_submit_depends_on_toctou.py

Rama main · réplica de lectura del repositorio original.

"""H-1 (audit 2026-07-10): submit depends_on TOCTOU → permanent QUEUED strand.

The failed-side reject in /submit is a point read of parent.state. Between that
read and the child INSERT+commit, a parent can reach a failed-side terminal and
run its cascade over the then-current QUEUED set — which does not yet include
the child. The child then commits QUEUED with a terminal-failed parent; the
parent never transitions again so its cascade never re-fires, and
_deps_satisfied needs the (never-reached) COMPLETED → the child strands QUEUED
forever.

The fix re-runs the cascade for each parent AFTER the child is committed and
visible, healing the strand. These tests exercise that post-commit sweep and
guard against false-positive cancellation of live/successful parents.

The race window is simulated deterministically by emptying
``jobd.broker.submit._FAILED_SIDE_TERMINAL`` for the child submit (so the point-read
reject is bypassed, exactly as it would be if the read observed the parent
mid-run) while the cascade in ``jobd.broker.state`` keeps the real constant.
"""

import json


def _submit(client, depends_on=None, any_exit=False):
    body = {"cmd": ["true"], "cwd": "/tmp", "project": "project-a"}
    if depends_on is not None:
        body["depends_on"] = depends_on
    if any_exit:
        body["depends_on_any_exit"] = True
    return client.post("/submit", json=body)


def _worker_body(host="w1"):
    return {
        "host": host,
        "free_vram_gb": 0,
        "unregistered_vram_gb": 0,
        "free_ram_gb": 8,
        "idle_cpus": 4,
        "arch": "x86_64",
        "os": "linux",
        "gpu": False,
        "tags": [],
        "host_aliases": [],
    }


def _claim_and_complete(client, job_id, final_state, exit_code):
    client.post("/heartbeat", json=_worker_body())
    client.post("/next-job", json=_worker_body())
    r = client.post(
        f"/jobs/{job_id}/complete",
        json={"exit_code": exit_code, "final_state": final_state},
    )
    assert r.status_code == 200, r.text


def _state(client, job_id):
    return client.get(f"/jobs/{job_id}").json()["state"]


def _events(tmp_path):
    p = tmp_path / "logs" / "events.jsonl"
    return [json.loads(x) for x in p.read_text().splitlines() if x.strip()] if p.exists() else []


def test_toctou_child_of_already_failed_parent_is_cancelled_not_stranded(
    client, tmp_path, monkeypatch
):
    parent = _submit(client).json()
    _claim_and_complete(client, parent["id"], "failed", 1)
    assert _state(client, parent["id"]) == "failed"

    # Simulate the race: the point-read reject observed the parent mid-run, so
    # it did not reject. (Empty set only for the app-level reject; the cascade
    # in jobd.broker.state keeps the real _FAILED_SIDE_TERMINAL.)
    monkeypatch.setattr("jobd.broker.submit._FAILED_SIDE_TERMINAL", frozenset())
    child = _submit(client, depends_on=[parent["id"]]).json()

    # Without the post-commit sweep this child would be QUEUED forever.
    assert _state(client, child["id"]) == "cancelled"

    cascade = [
        e
        for e in _events(tmp_path)
        if e["event"] == "job_cancelled"
        and e["payload"].get("by") == "cascade"
        and e["job_id"] == child["id"]
    ]
    assert len(cascade) == 1, cascade
    assert cascade[0]["payload"]["parent_job"] == parent["id"]
    assert cascade[0]["payload"]["parent_state"] == "failed"


def test_sweep_no_false_positive_for_running_parent(client):
    """A child of a still-RUNNING parent must stay QUEUED — the sweep only
    cancels children of failed-side-terminal parents."""
    parent = _submit(client).json()
    client.post("/heartbeat", json=_worker_body())
    client.post("/next-job", json=_worker_body())  # parent claimed (assigned)
    assert _state(client, parent["id"]) in ("assigned", "running")  # not terminal

    child = _submit(client, depends_on=[parent["id"]]).json()
    assert _state(client, child["id"]) == "queued"


def test_sweep_no_op_for_completed_parent(client):
    """A child of a COMPLETED parent must stay QUEUED (dispatchable) — COMPLETED
    is not failed-side, so the sweep is a no-op."""
    parent = _submit(client).json()
    _claim_and_complete(client, parent["id"], "completed", 0)
    assert _state(client, parent["id"]) == "completed"

    child = _submit(client, depends_on=[parent["id"]]).json()
    assert _state(client, child["id"]) == "queued"


def test_any_exit_child_of_failed_parent_not_cancelled(client, monkeypatch):
    """depends_on_any_exit children are satisfied by any terminal parent, so the
    sweep must not cancel them even when the parent failed."""
    parent = _submit(client).json()
    _claim_and_complete(client, parent["id"], "failed", 1)

    monkeypatch.setattr("jobd.broker.submit._FAILED_SIDE_TERMINAL", frozenset())
    child = _submit(client, depends_on=[parent["id"]], any_exit=True).json()
    assert _state(client, child["id"]) == "queued"