cc-os/plugins/os-doc-hygiene/tests/test_classifier_golden.py

"""
Hermetic classifier-golden regression harness (design D7 / add-check).

These tests are the Layer-2 reversion-protection check for the classifier: a
small static fixture doc-tree (`input/`) → an expected classification
(`expected.json`). They are DISTINCT from the schema-shape fixtures
(examples/golden/valid_report.json / invalid_report.json) that exercise only
validate_report.py.

HERMETIC by construction — NO live model / API call ever runs here. The model's
*judgment* is captured once (committed in each case's expected.json, human-gated
per the META-RULE). This suite asserts only the deterministic / stable parts:

  1. scanner.py on input/ emits the EXPECTED signal(s) on the EXPECTED path(s).
  2. validate_report.py accepts each expected.json (exit 0) — the goldens are
     themselves valid machine reports.
  3. Internal consistency of every entry's stable classification fields:
       - category.class / subtype in the closed enum,
       - op_type valid,
       - exact_edit.kind == the case's intended kind,
       - safety_tier == derive_safety_tier(...) recomputed for the entry,
       - exact_edit.expected_sha256 == a fresh sha256 of the input/ file bytes
         (proves the golden's hashes track the fixtures).

The LIVE model-classification regression (actually running /os-doc-hygiene:check
against input/ and diffing the produced classification) lives OUTSIDE this suite
— see examples/golden/classifier/README.md.

sys.path is patched here (no shared conftest) to match the repo's test style.
"""
from __future__ import annotations

import hashlib
import json
import sys
from pathlib import Path

import pytest

# ---------------------------------------------------------------------------
# Path bootstrap — inject scripts/ so we can import the deterministic seams.
# ---------------------------------------------------------------------------
_SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
if str(_SCRIPTS_DIR) not in sys.path:
    sys.path.insert(0, str(_SCRIPTS_DIR))

from scanner import Scanner  # noqa: E402
from validate_report import (  # noqa: E402
    ReportValidator,
    derive_safety_tier,
    KIND_TABLE,
    STALE_SUBTYPES,
    BLOAT_SUBTYPES,
)

_GOLDEN_DIR = Path(__file__).parent.parent / "examples" / "golden" / "classifier"


# ---------------------------------------------------------------------------
# Case table — the expected stable contract for each golden.
#
# Each case declares:
#   name            : case directory name under examples/golden/classifier/
#   expected_signals: { fixture-relative path -> set of signal names that MUST
#                       appear on that path }
#   intended_kind   : the exact_edit.kind the case is built to exercise, or None
#                     for a generative case (no exact_edit). The distinct-kind
#                     coverage of the suite is asserted from this column.
#   cls / subtype   : the expected category for the single classified entry.
# ---------------------------------------------------------------------------

CASES = [
    {
        "name": "1-orphaned",
        "expected_signals": {"docs/orphan.md": {"broken_reference"}},
        "intended_kind": "delete-range",
        "cls": "stale",
        "subtype": "orphaned",
    },
    {
        "name": "2-superseded",
        "expected_signals": {"docs/old-plan.md": {"stale_name_location"}},
        "intended_kind": "move-to-archive",
        "cls": "stale",
        "subtype": "superseded",
    },
    {
        "name": "3-completed-in-place",
        "expected_signals": {
            "docs/migration-checklist.md": {"archive_to_live_ratio"}
        },
        "intended_kind": "insert-frontmatter",
        "cls": "stale",
        "subtype": "completed-in-place",
    },
    {
        "name": "4-duplicated",
        "expected_signals": {"docs/setup-legacy.md": {"stale_name_location"}},
        "intended_kind": "dedupe",
        "cls": "stale",
        "subtype": "duplicated",
    },
    {
        "name": "5-distill",
        "expected_signals": {"docs/incident-log.md": {"archive_to_live_ratio"}},
        "intended_kind": None,  # generative — no exact_edit
        "cls": "bloat",
        "subtype": "distill",
    },
]

_CASE_IDS = [c["name"] for c in CASES]


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def _case_dir(case: dict) -> Path:
    return _GOLDEN_DIR / case["name"]


def _run_scanner(input_dir: Path) -> dict:
    """Run the scanner exactly as the CLI would, but pinned to input_dir.

    git_log_fn is disabled (None) so the suite is git-independent and the
    mtime-dependent edit_recency_vs_churn signal can never fire — keeping the
    run fully deterministic from the static fixture bytes.
    """
    return Scanner(
        root=input_dir,
        git_log_fn=None,
        now_fn=lambda: 0.0,
    ).run()


def _load_expected(case: dict) -> dict:
    return json.loads((_case_dir(case) / "expected.json").read_text(encoding="utf-8"))


# ---------------------------------------------------------------------------
# 1. Scanner emits the expected signals on the expected paths
# ---------------------------------------------------------------------------

class TestScannerSignalsOnFixtures:
    @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS)
    def test_expected_signals_present(self, case):
        input_dir = _case_dir(case) / "input"
        result = _run_scanner(input_dir)
        signals_map = result["signals"]

        for path, expected_names in case["expected_signals"].items():
            assert path in result["shortlist"], (
                f"{case['name']}: expected path '{path}' not in shortlist "
                f"{result['shortlist']}"
            )
            got_names = {s["name"] for s in signals_map.get(path, [])}
            missing = expected_names - got_names
            assert not missing, (
                f"{case['name']}: path '{path}' missing signals {missing}; "
                f"scanner emitted {got_names}"
            )

    @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS)
    def test_classified_path_is_a_signal_bearing_candidate(self, case):
        """Every classified entry path must be one the scanner actually signalled
        (zero-signal files are presumptively cleared and never classified)."""
        input_dir = _case_dir(case) / "input"
        result = _run_scanner(input_dir)
        signal_bearing = set(result["signals"].keys())

        report = _load_expected(case)
        for entry in report["entries"]:
            assert entry["path"] in signal_bearing, (
                f"{case['name']}: classified '{entry['path']}' carries no scanner "
                f"signal (signal-bearing: {sorted(signal_bearing)})"
            )


# ---------------------------------------------------------------------------
# 2. Each expected.json is itself a valid machine report
# ---------------------------------------------------------------------------

class TestGoldensAreValidReports:
    @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS)
    def test_expected_report_validates(self, case):
        report = _load_expected(case)
        violations = ReportValidator(report).validate()
        assert violations == [], (
            f"{case['name']}: expected.json failed schema validation: {violations}"
        )


# ---------------------------------------------------------------------------
# 3. Internal consistency of stable classification fields
# ---------------------------------------------------------------------------

class TestStableClassificationConsistency:
    @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS)
    def test_single_entry_matches_case_category(self, case):
        report = _load_expected(case)
        assert len(report["entries"]) == 1, (
            f"{case['name']}: expected exactly one classified entry"
        )
        entry = report["entries"][0]
        cat = entry["category"]
        assert cat["class"] == case["cls"]
        assert cat["subtype"] == case["subtype"]

    @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS)
    def test_category_in_closed_enum(self, case):
        report = _load_expected(case)
        for entry in report["entries"]:
            cls = entry["category"]["class"]
            subtype = entry["category"]["subtype"]
            assert cls in ("stale", "bloat")
            if cls == "stale":
                assert subtype in STALE_SUBTYPES, f"{subtype} not a stale subtype"
            else:
                assert subtype in BLOAT_SUBTYPES, f"{subtype} not a bloat subtype"

    @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS)
    def test_op_type_valid_and_exact_edit_biconditional(self, case):
        report = _load_expected(case)
        for entry in report["entries"]:
            op_type = entry["op_type"]
            assert op_type in ("deterministic", "generative")
            has_edit = "exact_edit" in entry
            assert has_edit == (op_type == "deterministic"), (
                f"{case['name']}: exact_edit presence must iff op_type=="
                f"deterministic (op_type={op_type}, has_edit={has_edit})"
            )

    @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS)
    def test_exact_edit_kind_matches_intended(self, case):
        report = _load_expected(case)
        entry = report["entries"][0]
        if case["intended_kind"] is None:
            assert "exact_edit" not in entry, (
                f"{case['name']}: generative case must carry no exact_edit"
            )
        else:
            assert entry["exact_edit"]["kind"] == case["intended_kind"], (
                f"{case['name']}: exact_edit.kind "
                f"{entry['exact_edit'].get('kind')!r} != intended "
                f"{case['intended_kind']!r}"
            )

    @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS)
    def test_safety_tier_is_derived(self, case):
        """The recorded safety_tier MUST equal the deterministic derivation for
        the entry's (op_type, is_destructive, is_reversible)."""
        report = _load_expected(case)
        for entry in report["entries"]:
            expected_tier = derive_safety_tier(
                entry["op_type"],
                entry["is_destructive"],
                entry["is_reversible"],
            )
            assert entry["safety_tier"] == expected_tier, (
                f"{case['name']}: safety_tier {entry['safety_tier']!r} != derived "
                f"{expected_tier!r}"
            )

    @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS)
    def test_kind_table_booleans_match(self, case):
        """For deterministic entries, is_destructive/is_reversible must equal the
        kind table's inherent pair."""
        report = _load_expected(case)
        for entry in report["entries"]:
            if entry["op_type"] != "deterministic":
                continue
            kind = entry["exact_edit"]["kind"]
            spec = KIND_TABLE[kind]
            assert entry["is_destructive"] == spec.is_destructive
            assert entry["is_reversible"] == spec.is_reversible


# ---------------------------------------------------------------------------
# 4. expected_sha256 tracks the fixture bytes
# ---------------------------------------------------------------------------

class TestExpectedShaTracksFixture:
    @pytest.mark.parametrize("case", CASES, ids=_CASE_IDS)
    def test_expected_sha256_matches_input_bytes(self, case):
        """Re-verify expected_sha256 against a freshly-computed hash of the
        corresponding input/ file. Proves the golden's hashes track the
        fixtures (so a fixture edit without a regen is caught).

        Anchor-bearing kinds only carry expected_sha256; insert-frontmatter
        (has_anchor=False) and generative entries carry none — skip those.
        """
        input_dir = _case_dir(case) / "input"
        report = _load_expected(case)
        checked = 0
        for entry in report["entries"]:
            edit = entry.get("exact_edit")
            if not edit or "expected_sha256" not in edit:
                continue
            file_bytes = (input_dir / entry["path"]).read_bytes()
            fresh = hashlib.sha256(file_bytes).hexdigest()
            assert edit["expected_sha256"] == fresh, (
                f"{case['name']}: expected_sha256 for {entry['path']} is stale "
                f"(golden={edit['expected_sha256']}, fixture={fresh})"
            )
            checked += 1

        # Sanity: the three anchor-bearing-kind cases MUST have a hash to check.
        intended = case["intended_kind"]
        if intended in ("delete-range", "move-to-archive", "dedupe", "replace-text"):
            assert checked >= 1, (
                f"{case['name']}: anchor-bearing kind {intended} should carry an "
                f"expected_sha256 but none was checked"
            )


# ---------------------------------------------------------------------------
# 5. Suite-level coverage: the intended kinds are distinct across cases
# ---------------------------------------------------------------------------

def test_intended_kinds_are_distinct():
    """The goldens deliberately map each subtype to a DISTINCT exact_edit.kind
    (plus one generative case) so the suite covers the kind table + tier
    derivation. Guards against two goldens collapsing onto one kind."""
    deterministic_kinds = [c["intended_kind"] for c in CASES if c["intended_kind"]]
    assert len(deterministic_kinds) == len(set(deterministic_kinds)), (
        f"intended kinds must be distinct, got {deterministic_kinds}"
    )
    # At least one generative case (intended_kind is None).
    assert any(c["intended_kind"] is None for c in CASES)