246 lines
8.3 KiB
Python
246 lines
8.3 KiB
Python
|
"""
|
||
|
|
Tests for scripts/validate_report.py
|
||
|
|
|
||
|
|
Wires the machine-report schema validator into pytest (Group 5 correctness
|
||
|
|
gate). Two layers:
|
||
|
|
|
||
|
|
1. End-to-end against the GOLDEN fixtures (Layer-2 reversion protection):
|
||
|
|
examples/golden/valid_report.json → PASS (exit 0),
|
||
|
|
examples/golden/invalid_report.json → FAIL (exit 1).
|
||
|
|
The golden files are human-gated; they are NOT mutated here.
|
||
|
|
|
||
|
|
2. Branch coverage via crafted dicts fed straight into ReportValidator —
|
||
|
|
the raw_tokens requirement, the op_type↔exact_edit biconditional
|
||
|
|
(invariant #11), and the safety_tier derivation rule (invariant #10).
|
||
|
|
|
||
|
|
sys.path is patched here (no shared conftest) so the test file is
|
||
|
|
self-contained.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import copy
|
||
|
|
import json
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
_SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
|
||
|
|
if str(_SCRIPTS_DIR) not in sys.path:
|
||
|
|
sys.path.insert(0, str(_SCRIPTS_DIR))
|
||
|
|
|
||
|
|
from validate_report import ReportValidator, derive_safety_tier # noqa: E402
|
||
|
|
|
||
|
|
_GOLDEN_DIR = Path(__file__).parent.parent / "examples" / "golden"
|
||
|
|
_VALIDATOR = _SCRIPTS_DIR / "validate_report.py"
|
||
|
|
|
||
|
|
|
||
|
|
def _validate(report: dict) -> list:
|
||
|
|
return ReportValidator(report).validate()
|
||
|
|
|
||
|
|
|
||
|
|
def _minimal_valid_report() -> dict:
|
||
|
|
"""A schema-valid single-entry report used as a base for mutation tests."""
|
||
|
|
return {
|
||
|
|
"schema_version": "1.0",
|
||
|
|
"tool_version": "0.1.0",
|
||
|
|
"generated_at": "2026-06-24T00:00:00Z",
|
||
|
|
"scan": {
|
||
|
|
"project_root": "/p",
|
||
|
|
"scope_globs": ["**/*.md"],
|
||
|
|
"excluded_dirs": ["build"],
|
||
|
|
"files_scanned": 1,
|
||
|
|
},
|
||
|
|
"shortlist": ["a.md"],
|
||
|
|
"entries": [
|
||
|
|
{
|
||
|
|
"path": "a.md",
|
||
|
|
"category": {"class": "bloat", "subtype": "distill"},
|
||
|
|
"signals": [],
|
||
|
|
"op": "Condense.",
|
||
|
|
"op_type": "generative",
|
||
|
|
"is_destructive": False,
|
||
|
|
"is_reversible": True,
|
||
|
|
"safety_tier": "confirm",
|
||
|
|
"token_estimate": {"raw_tokens": 100},
|
||
|
|
}
|
||
|
|
],
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
# ===========================================================================
|
||
|
|
# Layer 1 — golden fixtures end-to-end via the CLI (exit codes)
|
||
|
|
# ===========================================================================
|
||
|
|
|
||
|
|
class TestGoldenFixturesCLI:
|
||
|
|
|
||
|
|
def _run(self, fixture: Path):
|
||
|
|
return subprocess.run(
|
||
|
|
[sys.executable, str(_VALIDATOR), str(fixture)],
|
||
|
|
capture_output=True,
|
||
|
|
text=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
def test_valid_golden_passes(self):
|
||
|
|
result = self._run(_GOLDEN_DIR / "valid_report.json")
|
||
|
|
assert result.returncode == 0, result.stdout
|
||
|
|
payload = json.loads(result.stdout)
|
||
|
|
assert payload["valid"] is True
|
||
|
|
assert payload["violations"] == []
|
||
|
|
|
||
|
|
def test_invalid_golden_fails(self):
|
||
|
|
result = self._run(_GOLDEN_DIR / "invalid_report.json")
|
||
|
|
assert result.returncode == 1
|
||
|
|
payload = json.loads(result.stdout)
|
||
|
|
assert payload["valid"] is False
|
||
|
|
assert len(payload["violations"]) >= 1
|
||
|
|
|
||
|
|
def test_invalid_golden_flags_safety_tier_mismatch(self):
|
||
|
|
"""The golden invalid report has an auto tier on a destructive op."""
|
||
|
|
result = self._run(_GOLDEN_DIR / "invalid_report.json")
|
||
|
|
payload = json.loads(result.stdout)
|
||
|
|
fields = {v["field"] for v in payload["violations"]}
|
||
|
|
assert any("safety_tier" in f for f in fields)
|
||
|
|
|
||
|
|
def test_usage_error_exit_code(self):
|
||
|
|
result = subprocess.run(
|
||
|
|
[sys.executable, str(_VALIDATOR)],
|
||
|
|
capture_output=True,
|
||
|
|
text=True,
|
||
|
|
)
|
||
|
|
assert result.returncode == 2
|
||
|
|
|
||
|
|
|
||
|
|
# ===========================================================================
|
||
|
|
# Layer 2 — crafted-dict branch coverage
|
||
|
|
# ===========================================================================
|
||
|
|
|
||
|
|
class TestBaseFixtureIsValid:
|
||
|
|
|
||
|
|
def test_minimal_report_has_no_violations(self):
|
||
|
|
assert _validate(_minimal_valid_report()) == []
|
||
|
|
|
||
|
|
|
||
|
|
class TestRawTokensRequirement:
|
||
|
|
|
||
|
|
def test_missing_raw_tokens_is_rejected(self):
|
||
|
|
r = _minimal_valid_report()
|
||
|
|
r["entries"][0]["token_estimate"] = {"injection_frequency": None}
|
||
|
|
violations = _validate(r)
|
||
|
|
assert any(
|
||
|
|
v["field"].endswith("token_estimate.raw_tokens") for v in violations
|
||
|
|
)
|
||
|
|
|
||
|
|
def test_non_numeric_raw_tokens_is_rejected(self):
|
||
|
|
r = _minimal_valid_report()
|
||
|
|
r["entries"][0]["token_estimate"] = {"raw_tokens": "lots"}
|
||
|
|
violations = _validate(r)
|
||
|
|
assert any(
|
||
|
|
v["field"].endswith("token_estimate.raw_tokens") for v in violations
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
class TestOpTypeExactEditBiconditional:
|
||
|
|
"""Invariant #11: exact_edit present IFF op_type == deterministic."""
|
||
|
|
|
||
|
|
def test_generative_with_exact_edit_is_rejected(self):
|
||
|
|
r = _minimal_valid_report()
|
||
|
|
# base entry is generative; bolt on a forbidden exact_edit
|
||
|
|
r["entries"][0]["exact_edit"] = {
|
||
|
|
"kind": "insert-frontmatter",
|
||
|
|
"key": "hygiene",
|
||
|
|
"value": "frozen",
|
||
|
|
}
|
||
|
|
violations = _validate(r)
|
||
|
|
assert any(v["field"].endswith("exact_edit") for v in violations)
|
||
|
|
|
||
|
|
def test_deterministic_without_exact_edit_is_rejected(self):
|
||
|
|
r = _minimal_valid_report()
|
||
|
|
e = r["entries"][0]
|
||
|
|
e["op_type"] = "deterministic"
|
||
|
|
e["is_destructive"] = False
|
||
|
|
e["is_reversible"] = True
|
||
|
|
e["safety_tier"] = "auto" # derived for deterministic+nondestr+reversible
|
||
|
|
# no exact_edit present
|
||
|
|
violations = _validate(r)
|
||
|
|
assert any(
|
||
|
|
v["field"].endswith("exact_edit")
|
||
|
|
and "MUST carry" in v["message"]
|
||
|
|
for v in violations
|
||
|
|
)
|
||
|
|
|
||
|
|
def test_deterministic_with_valid_exact_edit_passes(self):
|
||
|
|
r = _minimal_valid_report()
|
||
|
|
e = r["entries"][0]
|
||
|
|
e["op_type"] = "deterministic"
|
||
|
|
e["is_destructive"] = False
|
||
|
|
e["is_reversible"] = True
|
||
|
|
e["safety_tier"] = "auto"
|
||
|
|
e["exact_edit"] = {
|
||
|
|
"kind": "insert-frontmatter",
|
||
|
|
"key": "hygiene",
|
||
|
|
"value": "frozen",
|
||
|
|
}
|
||
|
|
assert _validate(r) == []
|
||
|
|
|
||
|
|
|
||
|
|
class TestSafetyTierDerivation:
|
||
|
|
"""Invariant #10: recorded safety_tier must equal the derivation."""
|
||
|
|
|
||
|
|
def test_truth_table(self):
|
||
|
|
# generative ⇒ confirm regardless of booleans
|
||
|
|
assert derive_safety_tier("generative", False, True) == "confirm"
|
||
|
|
assert derive_safety_tier("generative", True, False) == "confirm"
|
||
|
|
# destructive ⇒ confirm
|
||
|
|
assert derive_safety_tier("deterministic", True, True) == "confirm"
|
||
|
|
# irreversible ⇒ confirm
|
||
|
|
assert derive_safety_tier("deterministic", False, False) == "confirm"
|
||
|
|
# deterministic + non-destructive + reversible ⇒ auto (only auto case)
|
||
|
|
assert derive_safety_tier("deterministic", False, True) == "auto"
|
||
|
|
|
||
|
|
def test_recorded_tier_mismatch_is_rejected(self):
|
||
|
|
"""generative op recorded as 'auto' must be rejected."""
|
||
|
|
r = _minimal_valid_report()
|
||
|
|
r["entries"][0]["safety_tier"] = "auto" # wrong: generative ⇒ confirm
|
||
|
|
violations = _validate(r)
|
||
|
|
assert any(
|
||
|
|
v["field"].endswith("safety_tier") and "mismatch" in v["message"]
|
||
|
|
for v in violations
|
||
|
|
)
|
||
|
|
|
||
|
|
def test_correct_tier_accepted(self):
|
||
|
|
r = _minimal_valid_report()
|
||
|
|
# base is generative with confirm — already correct
|
||
|
|
assert not any(
|
||
|
|
v["field"].endswith("safety_tier") for v in _validate(r)
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
class TestCategoryEnum:
|
||
|
|
|
||
|
|
def test_bad_subtype_for_class_rejected(self):
|
||
|
|
r = _minimal_valid_report()
|
||
|
|
r["entries"][0]["category"] = {"class": "stale", "subtype": "distill"}
|
||
|
|
violations = _validate(r)
|
||
|
|
assert any(v["field"].endswith("category.subtype") for v in violations)
|
||
|
|
|
||
|
|
|
||
|
|
class TestEnvelope:
|
||
|
|
|
||
|
|
def test_missing_top_level_field_rejected(self):
|
||
|
|
r = _minimal_valid_report()
|
||
|
|
del r["generated_at"]
|
||
|
|
violations = _validate(r)
|
||
|
|
assert any(v["field"] == "generated_at" for v in violations)
|
||
|
|
|
||
|
|
def test_entry_path_not_in_shortlist_rejected(self):
|
||
|
|
r = _minimal_valid_report()
|
||
|
|
r["entries"][0]["path"] = "ghost.md" # not in shortlist
|
||
|
|
violations = _validate(r)
|
||
|
|
assert any(
|
||
|
|
v["field"].endswith(".path") and "shortlist" in v["message"]
|
||
|
|
for v in violations
|
||
|
|
)
|