464 lines
19 KiB
Python
464 lines
19 KiB
Python
"""
|
|
Tests for scripts/report_builder.py — the model-free finalize pass.
|
|
|
|
Covers Phase 3 (add-check) report_builder:
|
|
- expected_sha256 = sha256 of the WHOLE fixture file bytes (hand-computed)
|
|
- (is_destructive, is_reversible) + derived safety_tier per kind, covering
|
|
delete-range→confirm, move-to-archive→auto, insert-frontmatter→auto,
|
|
dedupe→auto, replace-text→auto, generative→confirm
|
|
- raw_tokens populated by the estimator; weighting fields null
|
|
- per-entry exact_edit.generated_at = the injected hash instant (distinct
|
|
from the envelope generated_at — proven with a monotonic clock)
|
|
- tool_version read from .claude-plugin/plugin.json
|
|
- cleared (zero-signal / no-entry) shortlisted files yield no entries
|
|
- malformed proposals are rejected BEFORE any computation
|
|
- CRITICAL round-trip: assembled report passes validate_report.py
|
|
|
|
sys.path is patched here (no shared conftest) so the test file is
|
|
self-contained, matching the repo's test style.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Path bootstrap — inject scripts/ so we can import report_builder directly.
|
|
# ---------------------------------------------------------------------------
|
|
_SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
|
|
if str(_SCRIPTS_DIR) not in sys.path:
|
|
sys.path.insert(0, str(_SCRIPTS_DIR))
|
|
|
|
from report_builder import ( # noqa: E402
|
|
ReportBuilder,
|
|
MalformedProposalError,
|
|
)
|
|
from validate_report import ReportValidator # noqa: E402
|
|
from token_estimator import default_estimator # noqa: E402
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test doubles
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class MonotonicClock:
|
|
"""Returns a strictly increasing UTC datetime on each now() call.
|
|
|
|
This is what makes the generated_at test meaningful: a fixed clock would
|
|
make per-entry stamps vacuously equal to the envelope stamp.
|
|
"""
|
|
|
|
def __init__(self, start: datetime, step_seconds: int = 1) -> None:
|
|
self._next = start
|
|
self._step = timedelta(seconds=step_seconds)
|
|
self.calls: list[datetime] = []
|
|
|
|
def now(self) -> datetime:
|
|
value = self._next
|
|
self._next = self._next + self._step
|
|
self.calls.append(value)
|
|
return value
|
|
|
|
|
|
_T0 = datetime(2026, 6, 24, 12, 0, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures — a small doc tree + a scan artifact
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@pytest.fixture
|
|
def doc_tree(tmp_path: Path) -> Path:
|
|
"""Create a project root with several markdown files."""
|
|
(tmp_path / "docs").mkdir()
|
|
|
|
(tmp_path / "CLAUDE.md").write_text(
|
|
"\n".join(f"line {i}" for i in range(1, 21)) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "docs" / "old-plan.md").write_text(
|
|
"# Old plan\n\nbody\n", encoding="utf-8"
|
|
)
|
|
(tmp_path / "docs" / "setup.md").write_text(
|
|
"# Setup\n\nall done\n", encoding="utf-8"
|
|
)
|
|
(tmp_path / "docs" / "dupe.md").write_text(
|
|
"a\nb\nc\nd\ne\n", encoding="utf-8"
|
|
)
|
|
(tmp_path / "docs" / "links.md").write_text(
|
|
"see [x](missing.md)\nmore text\n", encoding="utf-8"
|
|
)
|
|
(tmp_path / "docs" / "big.md").write_text(
|
|
"\n".join(f"para {i}" for i in range(1, 51)) + "\n", encoding="utf-8"
|
|
)
|
|
(tmp_path / "docs" / "clean.md").write_text(
|
|
"# Clean\n\nnothing to fix\n", encoding="utf-8"
|
|
)
|
|
return tmp_path
|
|
|
|
|
|
@pytest.fixture
|
|
def scan_artifact(doc_tree: Path) -> dict:
|
|
return {
|
|
"project_root": str(doc_tree),
|
|
"scope_globs": ["**/*.md"],
|
|
"excluded_dirs": ["build", ".dochygiene"],
|
|
"files_scanned": 7,
|
|
"shortlist": [
|
|
"CLAUDE.md",
|
|
"docs/old-plan.md",
|
|
"docs/setup.md",
|
|
"docs/dupe.md",
|
|
"docs/links.md",
|
|
"docs/big.md",
|
|
"docs/clean.md",
|
|
],
|
|
"signals": {},
|
|
}
|
|
|
|
|
|
def _proposals() -> list:
|
|
"""One proposal per exact_edit kind + one generative + a gloss."""
|
|
return [
|
|
{ # delete-range → confirm
|
|
"path": "CLAUDE.md",
|
|
"category": {"class": "stale", "subtype": "contradicted"},
|
|
"signals": [{"name": "broken_reference", "detail": "links a dead path"}],
|
|
"op": "Delete contradicted block (lines 5-9).",
|
|
"op_type": "deterministic",
|
|
"confidence": 0.9,
|
|
"gloss": "block contradicts the live config",
|
|
"exact_edit": {
|
|
"kind": "delete-range",
|
|
"anchor": {"start_line": 5, "end_line": 9},
|
|
},
|
|
},
|
|
{ # move-to-archive → auto
|
|
"path": "docs/old-plan.md",
|
|
"category": {"class": "stale", "subtype": "superseded"},
|
|
"signals": [{"name": "stale_name_location", "detail": "named 'old'"}],
|
|
"op": "Move to archive/docs/old-plan.md.",
|
|
"op_type": "deterministic",
|
|
"confidence": 0.8,
|
|
"exact_edit": {
|
|
"kind": "move-to-archive",
|
|
"anchor": {"start_line": 1, "end_line": 3},
|
|
"dest_path": "archive/docs/old-plan.md",
|
|
},
|
|
},
|
|
{ # insert-frontmatter → auto (no anchor, no sha256/generated_at)
|
|
"path": "docs/setup.md",
|
|
"category": {"class": "stale", "subtype": "completed-in-place"},
|
|
"signals": [{"name": "archive_to_live_ratio", "detail": "all done"}],
|
|
"op": "Freeze with hygiene: frozen.",
|
|
"op_type": "deterministic",
|
|
"confidence": 0.85,
|
|
"exact_edit": {
|
|
"kind": "insert-frontmatter",
|
|
"key": "hygiene",
|
|
"value": "frozen",
|
|
},
|
|
},
|
|
{ # dedupe → auto
|
|
"path": "docs/dupe.md",
|
|
"category": {"class": "stale", "subtype": "duplicated"},
|
|
"signals": [{"name": "broken_reference", "detail": "dup span"}],
|
|
"op": "Remove duplicate span (lines 2-4); canonical elsewhere.",
|
|
"op_type": "deterministic",
|
|
"confidence": 0.7,
|
|
"exact_edit": {
|
|
"kind": "dedupe",
|
|
"anchor": {"start_line": 2, "end_line": 4},
|
|
"canonical_ref": "docs/canonical.md#section",
|
|
},
|
|
},
|
|
{ # replace-text → auto
|
|
"path": "docs/links.md",
|
|
"category": {"class": "stale", "subtype": "contradicted"},
|
|
"signals": [{"name": "broken_reference", "detail": "dead link"}],
|
|
"op": "Fix the dead link.",
|
|
"op_type": "deterministic",
|
|
"confidence": 0.9,
|
|
"exact_edit": {
|
|
"kind": "replace-text",
|
|
"anchor": {"start_line": 1, "end_line": 1},
|
|
"match": "missing.md",
|
|
"replacement": "found.md",
|
|
},
|
|
},
|
|
{ # generative → confirm (no exact_edit; reducible_range)
|
|
"path": "docs/big.md",
|
|
"category": {"class": "bloat", "subtype": "distill"},
|
|
"signals": [{"name": "archive_to_live_ratio", "detail": "long narrative"}],
|
|
"op": "Condense resolved-problem sections.",
|
|
"op_type": "generative",
|
|
"confidence": 0.6,
|
|
"reducible_range": {"start_line": 1, "end_line": 50},
|
|
},
|
|
]
|
|
|
|
|
|
def _build(doc_tree: Path, scan: dict, proposals: list, clock=None):
|
|
builder = ReportBuilder(
|
|
project_root=doc_tree,
|
|
clock=clock or MonotonicClock(_T0),
|
|
estimator=default_estimator(), # heuristic fallback; deterministic
|
|
)
|
|
return builder.build(scan, proposals)
|
|
|
|
|
|
# ===========================================================================
|
|
# expected_sha256 = whole-file hash
|
|
# ===========================================================================
|
|
|
|
class TestExpectedSha256:
|
|
|
|
def test_matches_handcomputed_whole_file_hash(self, doc_tree, scan_artifact):
|
|
result = _build(doc_tree, scan_artifact, _proposals())
|
|
entries = {e["path"]: e for e in result["machine_report"]["entries"]}
|
|
|
|
expected = hashlib.sha256(
|
|
(doc_tree / "CLAUDE.md").read_bytes()
|
|
).hexdigest()
|
|
assert entries["CLAUDE.md"]["exact_edit"]["expected_sha256"] == expected
|
|
|
|
def test_anchorless_insert_frontmatter_has_no_sha256(self, doc_tree, scan_artifact):
|
|
result = _build(doc_tree, scan_artifact, _proposals())
|
|
entries = {e["path"]: e for e in result["machine_report"]["entries"]}
|
|
ee = entries["docs/setup.md"]["exact_edit"]
|
|
assert "expected_sha256" not in ee
|
|
assert "generated_at" not in ee
|
|
|
|
def test_generative_has_no_exact_edit(self, doc_tree, scan_artifact):
|
|
result = _build(doc_tree, scan_artifact, _proposals())
|
|
entries = {e["path"]: e for e in result["machine_report"]["entries"]}
|
|
assert "exact_edit" not in entries["docs/big.md"]
|
|
|
|
|
|
# ===========================================================================
|
|
# (is_destructive, is_reversible) + derived safety_tier per kind
|
|
# ===========================================================================
|
|
|
|
class TestKindCharacterization:
|
|
|
|
@pytest.mark.parametrize(
|
|
"path,is_destructive,is_reversible,tier",
|
|
[
|
|
("CLAUDE.md", True, False, "confirm"), # delete-range
|
|
("docs/old-plan.md", False, True, "auto"), # move-to-archive
|
|
("docs/setup.md", False, True, "auto"), # insert-frontmatter
|
|
("docs/dupe.md", False, True, "auto"), # dedupe
|
|
("docs/links.md", False, True, "auto"), # replace-text
|
|
("docs/big.md", False, True, "confirm"), # generative
|
|
],
|
|
)
|
|
def test_booleans_and_tier(self, doc_tree, scan_artifact, path, is_destructive, is_reversible, tier):
|
|
result = _build(doc_tree, scan_artifact, _proposals())
|
|
entries = {e["path"]: e for e in result["machine_report"]["entries"]}
|
|
e = entries[path]
|
|
assert e["is_destructive"] is is_destructive
|
|
assert e["is_reversible"] is is_reversible
|
|
assert e["safety_tier"] == tier
|
|
|
|
def test_proposal_cannot_override_guardrail_fields(self, doc_tree, scan_artifact):
|
|
"""A lying proposal (is_destructive/safety_tier) is ignored; kind wins."""
|
|
proposals = _proposals()
|
|
# Inject bogus fields onto the delete-range proposal.
|
|
proposals[0]["is_destructive"] = False
|
|
proposals[0]["is_reversible"] = True
|
|
proposals[0]["safety_tier"] = "auto"
|
|
result = _build(doc_tree, scan_artifact, proposals)
|
|
e = {x["path"]: x for x in result["machine_report"]["entries"]}["CLAUDE.md"]
|
|
assert e["is_destructive"] is True
|
|
assert e["safety_tier"] == "confirm"
|
|
|
|
|
|
# ===========================================================================
|
|
# token_estimate
|
|
# ===========================================================================
|
|
|
|
class TestTokenEstimate:
|
|
|
|
def test_raw_tokens_populated_weighting_null(self, doc_tree, scan_artifact):
|
|
result = _build(doc_tree, scan_artifact, _proposals())
|
|
for e in result["machine_report"]["entries"]:
|
|
te = e["token_estimate"]
|
|
assert isinstance(te["raw_tokens"], int)
|
|
assert te["raw_tokens"] >= 0
|
|
assert te["injection_frequency"] is None
|
|
assert te["weighted_tokens"] is None
|
|
|
|
def test_raw_tokens_from_estimator_on_span(self, doc_tree, scan_artifact):
|
|
"""delete-range counts only the anchored span (lines 5-9)."""
|
|
result = _build(doc_tree, scan_artifact, _proposals())
|
|
e = {x["path"]: x for x in result["machine_report"]["entries"]}["CLAUDE.md"]
|
|
lines = (doc_tree / "CLAUDE.md").read_text().splitlines()
|
|
span = "\n".join(lines[4:9]) # lines 5..9 inclusive
|
|
assert e["token_estimate"]["raw_tokens"] == default_estimator().estimate(span)
|
|
|
|
|
|
# ===========================================================================
|
|
# per-entry generated_at = injected hash instant (distinct from envelope)
|
|
# ===========================================================================
|
|
|
|
class TestGeneratedAt:
|
|
|
|
def test_per_entry_generated_at_is_hash_instant(self, doc_tree, scan_artifact):
|
|
clock = MonotonicClock(_T0, step_seconds=1)
|
|
result = _build(doc_tree, scan_artifact, _proposals(), clock=clock)
|
|
machine = result["machine_report"]
|
|
|
|
# Each anchor-bearing entry's generated_at is one of the clock's instants
|
|
# and is NOT equal to the envelope generated_at (proves distinctness).
|
|
envelope = machine["generated_at"]
|
|
stamps = []
|
|
for e in machine["entries"]:
|
|
ee = e.get("exact_edit")
|
|
if ee and "generated_at" in ee:
|
|
stamps.append(ee["generated_at"])
|
|
assert ee["generated_at"] != envelope
|
|
|
|
# All hash instants are distinct from each other (monotonic clock).
|
|
assert len(stamps) == len(set(stamps))
|
|
# The envelope stamp is the last clock call (after all hashes).
|
|
assert envelope == max(clock.calls).isoformat()
|
|
|
|
|
|
# ===========================================================================
|
|
# tool_version from plugin.json
|
|
# ===========================================================================
|
|
|
|
class TestEnvelope:
|
|
|
|
def test_tool_version_from_plugin_json(self, doc_tree, scan_artifact):
|
|
plugin_json = _SCRIPTS_DIR.parent / ".claude-plugin" / "plugin.json"
|
|
expected = json.loads(plugin_json.read_text())["version"]
|
|
result = _build(doc_tree, scan_artifact, _proposals())
|
|
assert result["machine_report"]["tool_version"] == expected
|
|
|
|
def test_schema_version_and_scan_block(self, doc_tree, scan_artifact):
|
|
result = _build(doc_tree, scan_artifact, _proposals())
|
|
m = result["machine_report"]
|
|
assert m["schema_version"] == "1.0"
|
|
assert m["scan"]["files_scanned"] == 7
|
|
assert m["scan"]["project_root"] == str(doc_tree)
|
|
assert m["shortlist"] == scan_artifact["shortlist"] # verbatim
|
|
|
|
|
|
# ===========================================================================
|
|
# cleared files yield no entries
|
|
# ===========================================================================
|
|
|
|
class TestClearedFiles:
|
|
|
|
def test_shortlisted_but_unproposed_file_has_no_entry(self, doc_tree, scan_artifact):
|
|
result = _build(doc_tree, scan_artifact, _proposals())
|
|
paths = {e["path"] for e in result["machine_report"]["entries"]}
|
|
assert "docs/clean.md" not in paths
|
|
# but it stays in shortlist
|
|
assert "docs/clean.md" in result["machine_report"]["shortlist"]
|
|
|
|
def test_human_report_counts_cleared(self, doc_tree, scan_artifact):
|
|
result = _build(doc_tree, scan_artifact, _proposals())
|
|
md = result["human_report"]
|
|
# 7 shortlisted, 6 entries → 1 cleared
|
|
assert "candidates: 6" in md
|
|
assert "cleared: 1" in md
|
|
assert "docs/clean.md" in md
|
|
|
|
|
|
# ===========================================================================
|
|
# malformed proposals are rejected before compute
|
|
# ===========================================================================
|
|
|
|
class TestMalformedRejection:
|
|
|
|
def test_missing_path(self, doc_tree, scan_artifact):
|
|
bad = [{"category": {"class": "stale", "subtype": "orphaned"}, "op": "x", "op_type": "deterministic"}]
|
|
with pytest.raises(MalformedProposalError):
|
|
_build(doc_tree, scan_artifact, bad)
|
|
|
|
def test_unknown_kind(self, doc_tree, scan_artifact):
|
|
bad = [{
|
|
"path": "CLAUDE.md",
|
|
"category": {"class": "stale", "subtype": "orphaned"},
|
|
"op": "x", "op_type": "deterministic",
|
|
"exact_edit": {"kind": "nuke-it", "anchor": {"start_line": 1, "end_line": 2}},
|
|
}]
|
|
with pytest.raises(MalformedProposalError):
|
|
_build(doc_tree, scan_artifact, bad)
|
|
|
|
def test_deterministic_without_exact_edit(self, doc_tree, scan_artifact):
|
|
bad = [{
|
|
"path": "CLAUDE.md",
|
|
"category": {"class": "stale", "subtype": "orphaned"},
|
|
"op": "x", "op_type": "deterministic",
|
|
}]
|
|
with pytest.raises(MalformedProposalError):
|
|
_build(doc_tree, scan_artifact, bad)
|
|
|
|
def test_generative_with_exact_edit(self, doc_tree, scan_artifact):
|
|
bad = [{
|
|
"path": "docs/big.md",
|
|
"category": {"class": "bloat", "subtype": "distill"},
|
|
"op": "x", "op_type": "generative",
|
|
"reducible_range": {"start_line": 1, "end_line": 2},
|
|
"exact_edit": {"kind": "delete-range", "anchor": {"start_line": 1, "end_line": 2}},
|
|
}]
|
|
with pytest.raises(MalformedProposalError):
|
|
_build(doc_tree, scan_artifact, bad)
|
|
|
|
def test_missing_kind_specific_field(self, doc_tree, scan_artifact):
|
|
bad = [{
|
|
"path": "docs/old-plan.md",
|
|
"category": {"class": "stale", "subtype": "superseded"},
|
|
"op": "x", "op_type": "deterministic",
|
|
"exact_edit": {"kind": "move-to-archive", "anchor": {"start_line": 1, "end_line": 2}},
|
|
# dest_path missing
|
|
}]
|
|
with pytest.raises(MalformedProposalError):
|
|
_build(doc_tree, scan_artifact, bad)
|
|
|
|
def test_bad_subtype(self, doc_tree, scan_artifact):
|
|
bad = [{
|
|
"path": "CLAUDE.md",
|
|
"category": {"class": "stale", "subtype": "not-a-subtype"},
|
|
"op": "x", "op_type": "generative",
|
|
"reducible_range": {"start_line": 1, "end_line": 2},
|
|
}]
|
|
with pytest.raises(MalformedProposalError):
|
|
_build(doc_tree, scan_artifact, bad)
|
|
|
|
def test_rejected_before_any_file_read(self, tmp_path, scan_artifact):
|
|
"""A nonexistent file path must not be read — malformed structure fails first."""
|
|
# path missing op_type entirely → rejected before any IO on the file.
|
|
bad = [{"path": "CLAUDE.md", "category": {"class": "stale", "subtype": "orphaned"}, "op": "x"}]
|
|
builder = ReportBuilder(project_root=tmp_path, clock=MonotonicClock(_T0))
|
|
with pytest.raises(MalformedProposalError):
|
|
builder.build(scan_artifact, bad)
|
|
|
|
|
|
# ===========================================================================
|
|
# CRITICAL: round-trip through validate_report.py
|
|
# ===========================================================================
|
|
|
|
class TestValidatorRoundTrip:
|
|
|
|
def test_assembled_report_passes_validator(self, doc_tree, scan_artifact):
|
|
result = _build(doc_tree, scan_artifact, _proposals())
|
|
machine = result["machine_report"]
|
|
violations = ReportValidator(machine).validate()
|
|
assert violations == [], f"validator rejected report: {violations}"
|
|
|
|
def test_empty_proposals_still_valid(self, doc_tree, scan_artifact):
|
|
result = _build(doc_tree, scan_artifact, [])
|
|
machine = result["machine_report"]
|
|
assert machine["entries"] == []
|
|
violations = ReportValidator(machine).validate()
|
|
assert violations == []
|