263 lines
10 KiB
Python
263 lines
10 KiB
Python
|
"""
|
||
|
|
Tests for scripts/calibrate_helpers.py
|
||
|
|
|
||
|
|
Covers task 5.4 of the lifecycle-aware-doc-hygiene change:
|
||
|
|
- ClusterSampler: group unmatched paths by path-shape, capped sample per
|
||
|
|
cluster
|
||
|
|
- RuleReportBuilder: 5-element rule-report data (glob verbatim, matched
|
||
|
|
sample+total, near-miss boundary, lifetime+tier, why)
|
||
|
|
- RuleQualityChecker: class-not-path lint (instance-unique identifiers /
|
||
|
|
no-wildcard-single-match) and prefer-the-narrower-glob tie-break
|
||
|
|
|
||
|
|
sys.path is patched here (no shared conftest), matching the existing
|
||
|
|
tests/test_rulebook.py pattern.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import sys
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
_SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
|
||
|
|
if str(_SCRIPTS_DIR) not in sys.path:
|
||
|
|
sys.path.insert(0, str(_SCRIPTS_DIR))
|
||
|
|
|
||
|
|
from calibrate_helpers import ( # noqa: E402
|
||
|
|
ClusterSampler,
|
||
|
|
RuleQualityChecker,
|
||
|
|
RuleReportBuilder,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# ClusterSampler
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
class TestClusterSampler:
|
||
|
|
def test_groups_by_directory_and_filename_shape(self):
|
||
|
|
paths = [
|
||
|
|
"autoresearch/run-12345678/notes.md",
|
||
|
|
"autoresearch/run-87654321/notes.md",
|
||
|
|
"autoresearch/run-11112222/notes.md",
|
||
|
|
"docs/HANDOFF-2026-07-01.md",
|
||
|
|
"docs/HANDOFF-2026-07-02.md",
|
||
|
|
]
|
||
|
|
sampler = ClusterSampler()
|
||
|
|
clusters = sampler.cluster(paths)
|
||
|
|
|
||
|
|
# autoresearch/run-# dirs form one cluster (same dir-shape prefix
|
||
|
|
# pattern doesn't collapse dir segments, but filenames do) —
|
||
|
|
# here dir segments differ (run-12345678 vs run-87654321), so they
|
||
|
|
# are distinct directories; but the *filenames* are identical
|
||
|
|
# ("notes.md") which is itself a valid single-file-per-dir cluster
|
||
|
|
# test. Assert basic invariants instead of exact grouping count.
|
||
|
|
assert len(clusters) >= 1
|
||
|
|
total_paths = sum(c.total for c in [c for c in clusters])
|
||
|
|
# every path is accounted for
|
||
|
|
seen = set()
|
||
|
|
for c in clusters:
|
||
|
|
seen.update(c.paths)
|
||
|
|
assert seen == set(paths)
|
||
|
|
|
||
|
|
def test_same_dir_and_shape_groups_together(self):
|
||
|
|
paths = [
|
||
|
|
"logs/log-00000001.txt",
|
||
|
|
"logs/log-00000002.txt",
|
||
|
|
"logs/log-00000003.txt",
|
||
|
|
]
|
||
|
|
sampler = ClusterSampler()
|
||
|
|
clusters = sampler.cluster(paths)
|
||
|
|
assert len(clusters) == 1
|
||
|
|
assert clusters[0].total == 3
|
||
|
|
assert set(clusters[0].paths) == set(paths)
|
||
|
|
|
||
|
|
def test_sample_is_capped(self):
|
||
|
|
paths = [f"logs/log-{i:08d}.txt" for i in range(20)]
|
||
|
|
sampler = ClusterSampler(sample_cap=5)
|
||
|
|
clusters = sampler.cluster(paths)
|
||
|
|
assert len(clusters) == 1
|
||
|
|
assert clusters[0].total == 20
|
||
|
|
assert len(clusters[0].sample) == 5
|
||
|
|
|
||
|
|
def test_different_directories_are_different_clusters(self):
|
||
|
|
paths = ["a/file.md", "b/file.md"]
|
||
|
|
sampler = ClusterSampler()
|
||
|
|
clusters = sampler.cluster(paths)
|
||
|
|
assert len(clusters) == 2
|
||
|
|
|
||
|
|
def test_hex_run_collapses_to_shape_class(self):
|
||
|
|
paths = [
|
||
|
|
"cache/deadbeefcafe.json",
|
||
|
|
"cache/0123456789ab.json",
|
||
|
|
]
|
||
|
|
sampler = ClusterSampler()
|
||
|
|
clusters = sampler.cluster(paths)
|
||
|
|
# both filenames collapse to the same hex-shape class -> one cluster
|
||
|
|
assert len(clusters) == 1
|
||
|
|
assert clusters[0].total == 2
|
||
|
|
|
||
|
|
def test_to_dict_is_json_serializable(self):
|
||
|
|
import json
|
||
|
|
|
||
|
|
paths = ["docs/PRD.md"]
|
||
|
|
sampler = ClusterSampler()
|
||
|
|
dicts = sampler.cluster_to_dicts(paths)
|
||
|
|
json.dumps(dicts) # must not raise
|
||
|
|
assert dicts[0]["total"] == 1
|
||
|
|
assert dicts[0]["sample"] == ["docs/PRD.md"]
|
||
|
|
|
||
|
|
def test_empty_input_yields_no_clusters(self):
|
||
|
|
sampler = ClusterSampler()
|
||
|
|
assert sampler.cluster([]) == []
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# RuleReportBuilder
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
class TestRuleReportBuilder:
|
||
|
|
def test_glob_is_verbatim(self):
|
||
|
|
all_paths = ["docs/HANDOFF-1.md", "docs/HANDOFF-2.md"]
|
||
|
|
rule = {"glob": "docs/HANDOFF-*.md", "lifetime": "temporary", "tier": "auto"}
|
||
|
|
entry = RuleReportBuilder().build(rule, all_paths)
|
||
|
|
assert entry.glob == "docs/HANDOFF-*.md"
|
||
|
|
|
||
|
|
def test_matches_sample_and_total(self):
|
||
|
|
all_paths = [f"docs/HANDOFF-{i}.md" for i in range(10)]
|
||
|
|
rule = {"glob": "docs/HANDOFF-*.md", "lifetime": "temporary", "tier": "auto"}
|
||
|
|
entry = RuleReportBuilder(sample_cap=3).build(rule, all_paths)
|
||
|
|
assert entry.matched_total == 10
|
||
|
|
assert len(entry.matched_sample) == 3
|
||
|
|
|
||
|
|
def test_near_miss_boundary_detects_sibling_paths(self):
|
||
|
|
# A too-narrow glob silently misses a sibling path under the same
|
||
|
|
# parent directory — this is exactly the #45 boundary bug
|
||
|
|
# (autoresearch/classic-*/ missing autoresearch/improve-*/).
|
||
|
|
all_paths = [
|
||
|
|
"autoresearch/classic-260710-1057/report.md",
|
||
|
|
"autoresearch/improve-260710-1057/report.md",
|
||
|
|
]
|
||
|
|
rule = {"glob": "autoresearch/classic-*/report.md", "lifetime": "temporary", "tier": "confirm"}
|
||
|
|
entry = RuleReportBuilder().build(rule, all_paths)
|
||
|
|
assert entry.matched_total == 1
|
||
|
|
assert "autoresearch/improve-260710-1057/report.md" in entry.near_miss
|
||
|
|
|
||
|
|
def test_no_near_miss_when_glob_covers_all_similar_paths(self):
|
||
|
|
all_paths = [
|
||
|
|
"autoresearch/classic-1/report.md",
|
||
|
|
"autoresearch/classic-2/report.md",
|
||
|
|
]
|
||
|
|
rule = {"glob": "autoresearch/*/report.md", "lifetime": "temporary", "tier": "confirm"}
|
||
|
|
entry = RuleReportBuilder().build(rule, all_paths)
|
||
|
|
assert entry.matched_total == 2
|
||
|
|
assert entry.near_miss == []
|
||
|
|
|
||
|
|
def test_lifetime_and_tier_carried_through(self):
|
||
|
|
rule = {"glob": "x/*.md", "lifetime": "delete-once-served", "tier": "confirm"}
|
||
|
|
entry = RuleReportBuilder().build(rule, ["x/a.md"])
|
||
|
|
assert entry.lifetime == "delete-once-served"
|
||
|
|
assert entry.tier == "confirm"
|
||
|
|
|
||
|
|
def test_build_all_is_json_serializable(self):
|
||
|
|
import json
|
||
|
|
|
||
|
|
rules = [
|
||
|
|
{"glob": "docs/HANDOFF-*.md", "lifetime": "temporary", "tier": "auto"},
|
||
|
|
]
|
||
|
|
dicts = RuleReportBuilder().build_all(rules, ["docs/HANDOFF-1.md"])
|
||
|
|
json.dumps(dicts)
|
||
|
|
assert dicts[0]["matches"]["total"] == 1
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# RuleQualityChecker
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
class TestRuleQualityCheckerClassNotPath:
|
||
|
|
def test_convention_recurring_name_passes(self):
|
||
|
|
checker = RuleQualityChecker()
|
||
|
|
finding = checker.class_not_path("docs/HANDOFF-*.md")
|
||
|
|
assert finding.passed is True
|
||
|
|
|
||
|
|
def test_recurring_filename_glob_with_wildcard_passes(self):
|
||
|
|
# "**/PRD.md" is structurally able to match PRD.md in any directory
|
||
|
|
# -- the wildcard makes it a class, not a single-instance path, even
|
||
|
|
# though the literal basename "PRD.md" is itself a fixed name.
|
||
|
|
checker = RuleQualityChecker()
|
||
|
|
finding = checker.class_not_path(
|
||
|
|
"**/PRD.md", all_paths=["docs/PRD.md", "other/PRD.md"]
|
||
|
|
)
|
||
|
|
assert finding.passed is True
|
||
|
|
|
||
|
|
def test_long_digit_run_fails(self):
|
||
|
|
checker = RuleQualityChecker()
|
||
|
|
finding = checker.class_not_path("logs/log-20260714153045.txt")
|
||
|
|
assert finding.passed is False
|
||
|
|
assert "digit" in finding.reason.lower()
|
||
|
|
|
||
|
|
def test_hex_hash_fails(self):
|
||
|
|
checker = RuleQualityChecker()
|
||
|
|
finding = checker.class_not_path("cache/deadbeefcafe1234.bin")
|
||
|
|
assert finding.passed is False
|
||
|
|
assert "hash" in finding.reason.lower() or "hex" in finding.reason.lower()
|
||
|
|
|
||
|
|
def test_one_current_match_is_fine_if_glob_has_wildcard(self):
|
||
|
|
checker = RuleQualityChecker()
|
||
|
|
finding = checker.class_not_path("docs/HANDOFF-*.md", all_paths=["docs/HANDOFF-1.md"])
|
||
|
|
assert finding.passed is True
|
||
|
|
|
||
|
|
def test_wildcard_free_glob_matching_only_one_path_ever_fails(self):
|
||
|
|
checker = RuleQualityChecker()
|
||
|
|
finding = checker.class_not_path(
|
||
|
|
"docs/migration-report.md", all_paths=["docs/migration-report.md"]
|
||
|
|
)
|
||
|
|
assert finding.passed is False
|
||
|
|
assert "generalization" in finding.reason.lower()
|
||
|
|
|
||
|
|
def test_wildcard_free_glob_with_multiple_matches_passes(self):
|
||
|
|
# a bare name matching >1 existing path is a recurring convention
|
||
|
|
# even with no wildcard char (unusual but should not be flagged)
|
||
|
|
checker = RuleQualityChecker()
|
||
|
|
finding = checker.class_not_path(
|
||
|
|
"PRD.md", all_paths=["a/PRD.md"]
|
||
|
|
)
|
||
|
|
# single match with a bare literal glob (no wildcard) still fails,
|
||
|
|
# since "PRD.md" as an exact-path glob wouldn't even match "a/PRD.md"
|
||
|
|
# (fnmatch requires the full string) -- so total matches = 0 -> not
|
||
|
|
# flagged for "matches <=1 existing path" since it's 0 (never seen).
|
||
|
|
# This asserts the checker doesn't crash and returns a finding.
|
||
|
|
assert finding.check == "class_not_path"
|
||
|
|
|
||
|
|
|
||
|
|
class TestRuleQualityCheckerPreferNarrower:
|
||
|
|
def test_narrower_glob_by_match_count_wins(self):
|
||
|
|
checker = RuleQualityChecker()
|
||
|
|
all_paths = [
|
||
|
|
"autoresearch/classic-1/report.md",
|
||
|
|
"autoresearch/classic-2/report.md",
|
||
|
|
"autoresearch/improve-1/report.md",
|
||
|
|
]
|
||
|
|
result = checker.prefer_narrower(
|
||
|
|
"autoresearch/classic-*/report.md", "autoresearch/*/report.md", all_paths
|
||
|
|
)
|
||
|
|
assert result["narrower"] == "autoresearch/classic-*/report.md"
|
||
|
|
assert result["match_counts"]["autoresearch/classic-*/report.md"] == 2
|
||
|
|
assert result["match_counts"]["autoresearch/*/report.md"] == 3
|
||
|
|
|
||
|
|
def test_tie_break_prefers_longer_pattern(self):
|
||
|
|
checker = RuleQualityChecker()
|
||
|
|
all_paths = ["a/b.md"]
|
||
|
|
result = checker.prefer_narrower("a/*.md", "a/b.md", all_paths)
|
||
|
|
# both match exactly 1 path -> tie -> longer raw pattern wins
|
||
|
|
assert result["narrower"] == "a/b.md"
|
||
|
|
|
||
|
|
def test_result_is_json_serializable(self):
|
||
|
|
import json
|
||
|
|
|
||
|
|
checker = RuleQualityChecker()
|
||
|
|
result = checker.prefer_narrower("a/*.md", "a/*.txt", ["a/b.md"])
|
||
|
|
json.dumps(result)
|