""" Hermetic classifier-golden regression harness (design D7 / add-check). These tests are the Layer-2 reversion-protection check for the classifier: a small static fixture doc-tree (`input/`) → an expected classification (`expected.json`). They are DISTINCT from the schema-shape fixtures (examples/golden/valid_report.json / invalid_report.json) that exercise only validate_report.py. HERMETIC by construction — NO live model / API call ever runs here. The model's *judgment* is captured once (committed in each case's expected.json, human-gated per the META-RULE). This suite asserts only the deterministic / stable parts: 1. scanner.py on input/ emits the EXPECTED signal(s) on the EXPECTED path(s). 2. validate_report.py accepts each expected.json (exit 0) — the goldens are themselves valid machine reports. 3. Internal consistency of every entry's stable classification fields: - category.class / subtype in the closed enum, - op_type valid, - exact_edit.kind == the case's intended kind, - safety_tier == derive_safety_tier(...) recomputed for the entry, - exact_edit.expected_sha256 == a fresh sha256 of the input/ file bytes (proves the golden's hashes track the fixtures). The LIVE model-classification regression (actually running /os-doc-hygiene:check against input/ and diffing the produced classification) lives OUTSIDE this suite — see examples/golden/classifier/README.md. sys.path is patched here (no shared conftest) to match the repo's test style. """ from __future__ import annotations import hashlib import json import sys from pathlib import Path import pytest # --------------------------------------------------------------------------- # Path bootstrap — inject scripts/ so we can import the deterministic seams. # --------------------------------------------------------------------------- _SCRIPTS_DIR = Path(__file__).parent.parent / "scripts" if str(_SCRIPTS_DIR) not in sys.path: sys.path.insert(0, str(_SCRIPTS_DIR)) from scanner import Scanner # noqa: E402 from validate_report import ( # noqa: E402 ReportValidator, derive_safety_tier, KIND_TABLE, STALE_SUBTYPES, BLOAT_SUBTYPES, ) _GOLDEN_DIR = Path(__file__).parent.parent / "examples" / "golden" / "classifier" # --------------------------------------------------------------------------- # Case table — the expected stable contract for each golden. # # Each case declares: # name : case directory name under examples/golden/classifier/ # expected_signals: { fixture-relative path -> set of signal names that MUST # appear on that path } # intended_kind : the exact_edit.kind the case is built to exercise, or None # for a generative case (no exact_edit). The distinct-kind # coverage of the suite is asserted from this column. # cls / subtype : the expected category for the single classified entry. # --------------------------------------------------------------------------- CASES = [ { "name": "1-orphaned", "expected_signals": {"docs/orphan.md": {"broken_reference"}}, "intended_kind": "delete-range", "cls": "stale", "subtype": "orphaned", }, { "name": "2-superseded", "expected_signals": {"docs/old-plan.md": {"stale_name_location"}}, "intended_kind": "move-to-archive", "cls": "stale", "subtype": "superseded", }, { "name": "3-completed-in-place", "expected_signals": { "docs/migration-checklist.md": {"archive_to_live_ratio"} }, "intended_kind": "insert-frontmatter", "cls": "stale", "subtype": "completed-in-place", }, { "name": "4-duplicated", "expected_signals": {"docs/setup-legacy.md": {"stale_name_location"}}, "intended_kind": "dedupe", "cls": "stale", "subtype": "duplicated", }, { "name": "5-distill", "expected_signals": {"docs/incident-log.md": {"archive_to_live_ratio"}}, "intended_kind": None, # generative — no exact_edit "cls": "bloat", "subtype": "distill", }, ] _CASE_IDS = [c["name"] for c in CASES] # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _case_dir(case: dict) -> Path: return _GOLDEN_DIR / case["name"] def _run_scanner(input_dir: Path) -> dict: """Run the scanner exactly as the CLI would, but pinned to input_dir. git_log_fn is disabled (None) so the suite is git-independent and the mtime-dependent edit_recency_vs_churn signal can never fire — keeping the run fully deterministic from the static fixture bytes. """ return Scanner( root=input_dir, git_log_fn=None, now_fn=lambda: 0.0, ).run() def _load_expected(case: dict) -> dict: return json.loads((_case_dir(case) / "expected.json").read_text(encoding="utf-8")) # --------------------------------------------------------------------------- # 1. Scanner emits the expected signals on the expected paths # --------------------------------------------------------------------------- class TestScannerSignalsOnFixtures: @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS) def test_expected_signals_present(self, case): input_dir = _case_dir(case) / "input" result = _run_scanner(input_dir) signals_map = result["signals"] for path, expected_names in case["expected_signals"].items(): assert path in result["shortlist"], ( f"{case['name']}: expected path '{path}' not in shortlist " f"{result['shortlist']}" ) got_names = {s["name"] for s in signals_map.get(path, [])} missing = expected_names - got_names assert not missing, ( f"{case['name']}: path '{path}' missing signals {missing}; " f"scanner emitted {got_names}" ) @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS) def test_classified_path_is_a_signal_bearing_candidate(self, case): """Every classified entry path must be one the scanner actually signalled (zero-signal files are presumptively cleared and never classified).""" input_dir = _case_dir(case) / "input" result = _run_scanner(input_dir) signal_bearing = set(result["signals"].keys()) report = _load_expected(case) for entry in report["entries"]: assert entry["path"] in signal_bearing, ( f"{case['name']}: classified '{entry['path']}' carries no scanner " f"signal (signal-bearing: {sorted(signal_bearing)})" ) # --------------------------------------------------------------------------- # 2. Each expected.json is itself a valid machine report # --------------------------------------------------------------------------- class TestGoldensAreValidReports: @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS) def test_expected_report_validates(self, case): report = _load_expected(case) violations = ReportValidator(report).validate() assert violations == [], ( f"{case['name']}: expected.json failed schema validation: {violations}" ) # --------------------------------------------------------------------------- # 3. Internal consistency of stable classification fields # --------------------------------------------------------------------------- class TestStableClassificationConsistency: @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS) def test_single_entry_matches_case_category(self, case): report = _load_expected(case) assert len(report["entries"]) == 1, ( f"{case['name']}: expected exactly one classified entry" ) entry = report["entries"][0] cat = entry["category"] assert cat["class"] == case["cls"] assert cat["subtype"] == case["subtype"] @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS) def test_category_in_closed_enum(self, case): report = _load_expected(case) for entry in report["entries"]: cls = entry["category"]["class"] subtype = entry["category"]["subtype"] assert cls in ("stale", "bloat") if cls == "stale": assert subtype in STALE_SUBTYPES, f"{subtype} not a stale subtype" else: assert subtype in BLOAT_SUBTYPES, f"{subtype} not a bloat subtype" @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS) def test_op_type_valid_and_exact_edit_biconditional(self, case): report = _load_expected(case) for entry in report["entries"]: op_type = entry["op_type"] assert op_type in ("deterministic", "generative") has_edit = "exact_edit" in entry assert has_edit == (op_type == "deterministic"), ( f"{case['name']}: exact_edit presence must iff op_type==" f"deterministic (op_type={op_type}, has_edit={has_edit})" ) @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS) def test_exact_edit_kind_matches_intended(self, case): report = _load_expected(case) entry = report["entries"][0] if case["intended_kind"] is None: assert "exact_edit" not in entry, ( f"{case['name']}: generative case must carry no exact_edit" ) else: assert entry["exact_edit"]["kind"] == case["intended_kind"], ( f"{case['name']}: exact_edit.kind " f"{entry['exact_edit'].get('kind')!r} != intended " f"{case['intended_kind']!r}" ) @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS) def test_safety_tier_is_derived(self, case): """The recorded safety_tier MUST equal the deterministic derivation for the entry's (op_type, is_destructive, is_reversible).""" report = _load_expected(case) for entry in report["entries"]: expected_tier = derive_safety_tier( entry["op_type"], entry["is_destructive"], entry["is_reversible"], ) assert entry["safety_tier"] == expected_tier, ( f"{case['name']}: safety_tier {entry['safety_tier']!r} != derived " f"{expected_tier!r}" ) @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS) def test_kind_table_booleans_match(self, case): """For deterministic entries, is_destructive/is_reversible must equal the kind table's inherent pair.""" report = _load_expected(case) for entry in report["entries"]: if entry["op_type"] != "deterministic": continue kind = entry["exact_edit"]["kind"] spec = KIND_TABLE[kind] assert entry["is_destructive"] == spec.is_destructive assert entry["is_reversible"] == spec.is_reversible # --------------------------------------------------------------------------- # 4. expected_sha256 tracks the fixture bytes # --------------------------------------------------------------------------- class TestExpectedShaTracksFixture: @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS) def test_expected_sha256_matches_input_bytes(self, case): """Re-verify expected_sha256 against a freshly-computed hash of the corresponding input/ file. Proves the golden's hashes track the fixtures (so a fixture edit without a regen is caught). Anchor-bearing kinds only carry expected_sha256; insert-frontmatter (has_anchor=False) and generative entries carry none — skip those. """ input_dir = _case_dir(case) / "input" report = _load_expected(case) checked = 0 for entry in report["entries"]: edit = entry.get("exact_edit") if not edit or "expected_sha256" not in edit: continue file_bytes = (input_dir / entry["path"]).read_bytes() fresh = hashlib.sha256(file_bytes).hexdigest() assert edit["expected_sha256"] == fresh, ( f"{case['name']}: expected_sha256 for {entry['path']} is stale " f"(golden={edit['expected_sha256']}, fixture={fresh})" ) checked += 1 # Sanity: the three anchor-bearing-kind cases MUST have a hash to check. intended = case["intended_kind"] if intended in ("delete-range", "move-to-archive", "dedupe", "replace-text"): assert checked >= 1, ( f"{case['name']}: anchor-bearing kind {intended} should carry an " f"expected_sha256 but none was checked" ) # --------------------------------------------------------------------------- # 5. Suite-level coverage: the intended kinds are distinct across cases # --------------------------------------------------------------------------- def test_intended_kinds_are_distinct(): """The goldens deliberately map each subtype to a DISTINCT exact_edit.kind (plus one generative case) so the suite covers the kind table + tier derivation. Guards against two goldens collapsing onto one kind.""" deterministic_kinds = [c["intended_kind"] for c in CASES if c["intended_kind"]] assert len(deterministic_kinds) == len(set(deterministic_kinds)), ( f"intended kinds must be distinct, got {deterministic_kinds}" ) # At least one generative case (intended_kind is None). assert any(c["intended_kind"] is None for c in CASES)