From ef8b7e4a52b97c9a77b18a2990bc99acc77f7017 Mon Sep 17 00:00:00 2001 From: jared Date: Mon, 13 Jul 2026 08:06:48 -0400 Subject: [PATCH] os-doc-hygiene: deterministic file_length scanner signal (issue #25 part 1) Objective signal (line count + estimated tokens) fires when either threshold is exceeded; defaults 400 lines / 4000 tokens, configurable via constructor and --max-lines/--max-tokens. Injectable token estimator for tests. Suite: 286 passed. Parts 2-3 (implementation-status distill + index+PD convention ADR) remain human-gated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CLeMz48rvG3s9XpAsDxeho --- plugins/os-doc-hygiene/scripts/scanner.py | 79 +++++++++++ .../tests/test_scanner_file_length.py | 127 ++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 plugins/os-doc-hygiene/tests/test_scanner_file_length.py diff --git a/plugins/os-doc-hygiene/scripts/scanner.py b/plugins/os-doc-hygiene/scripts/scanner.py index 9791aaa..74edf9e 100644 --- a/plugins/os-doc-hygiene/scripts/scanner.py +++ b/plugins/os-doc-hygiene/scripts/scanner.py @@ -25,6 +25,8 @@ import sys from pathlib import Path from typing import Callable, Iterator, List, Optional +from token_estimator import TokenEstimator, default_estimator + # --------------------------------------------------------------------------- # Glob matching (Python 3.14-compatible) @@ -56,6 +58,8 @@ def _file_matches_glob(rel_path_str: str, glob_pattern: str) -> bool: # --------------------------------------------------------------------------- DEFAULT_SCOPE_GLOBS: List[str] = ["**/*.md"] +DEFAULT_MAX_LINES: int = 400 +DEFAULT_MAX_TOKENS: int = 4000 DEFAULT_EXCLUDED_DIRS: List[str] = [ "build", "vendor", @@ -222,6 +226,14 @@ class SignalComputer: tests pass a fake. Pass ``None`` to disable git-based signals. now_fn : callable() -> float Returns current time as a Unix timestamp. + max_lines : int + Line-count threshold for the ``file_length`` signal. + max_tokens : int + Estimated-token threshold for the ``file_length`` signal. + token_estimator : TokenEstimator, optional + Injectable estimator used for the ``file_length`` signal's token + count. Defaults to ``token_estimator.default_estimator()``. Tests + can inject a deterministic stub. """ def __init__( @@ -229,10 +241,16 @@ class SignalComputer: root: Path, git_log_fn: Optional[Callable[[Path], List[str]]] = None, now_fn: Optional[Callable[[], float]] = None, + max_lines: int = DEFAULT_MAX_LINES, + max_tokens: int = DEFAULT_MAX_TOKENS, + token_estimator: Optional[TokenEstimator] = None, ) -> None: self._root = root self._git_log_fn = git_log_fn self._now_fn = now_fn or (lambda: __import__("time").time()) + self._max_lines = max_lines + self._max_tokens = max_tokens + self._token_estimator = token_estimator or default_estimator() # ------------------------------------------------------------------ # Public @@ -249,6 +267,7 @@ class SignalComputer: signals.extend(self._location_signals(path, rel)) signals.extend(self._archive_to_live_ratio(path)) signals.extend(self._frontmatter_markers(path)) + signals.extend(self._file_length(path)) return signals @@ -476,6 +495,36 @@ class SignalComputer: ) return signals + def _file_length(self, path: Path) -> List[dict]: + """Flag files whose line count or estimated token count exceeds threshold. + + Objective fact only: reports the measured line count and estimated + token count against the configured thresholds. Fires if EITHER + threshold is exceeded. No classification. + """ + signals = [] + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + return signals + + line_count = len(text.splitlines()) + token_count = self._token_estimator.estimate(text) + + if line_count > self._max_lines or token_count > self._max_tokens: + token_count_k = token_count / 1000.0 + signals.append( + { + "name": "file_length", + "detail": ( + f"{line_count} lines (~{token_count_k:.1f}k tokens est), " + f"threshold {self._max_lines} lines / {self._max_tokens} tokens" + ), + } + ) + + return signals + # --------------------------------------------------------------------------- # Production git helper @@ -518,6 +567,13 @@ class Scanner: Injected git-log provider. Default: real git subprocess. now_fn : callable, optional Injected clock. Default: ``time.time``. + max_lines : int + Line-count threshold for the ``file_length`` signal. Default 400. + max_tokens : int + Estimated-token threshold for the ``file_length`` signal. Default 4000. + token_estimator : TokenEstimator, optional + Injectable estimator for the ``file_length`` signal. Default: + ``token_estimator.default_estimator()``. """ def __init__( @@ -527,6 +583,9 @@ class Scanner: excluded_dirs: Optional[List[str]] = None, git_log_fn: Optional[Callable[[Path], List[str]]] = None, now_fn: Optional[Callable[[], float]] = None, + max_lines: int = DEFAULT_MAX_LINES, + max_tokens: int = DEFAULT_MAX_TOKENS, + token_estimator: Optional[TokenEstimator] = None, ) -> None: self._root = root.resolve() self._scope_globs = scope_globs if scope_globs is not None else DEFAULT_SCOPE_GLOBS @@ -560,6 +619,9 @@ class Scanner: self._excluded_pairs.append((parent_name, child)) self._git_log_fn = git_log_fn self._now_fn = now_fn + self._max_lines = max_lines + self._max_tokens = max_tokens + self._token_estimator = token_estimator self._ignore_patterns: List[str] = _load_ignore_patterns(self._root) def _is_excluded_child(self, parent_dirpath: str, child_name: str) -> bool: @@ -587,6 +649,9 @@ class Scanner: root=self._root, git_log_fn=self._git_log_fn, now_fn=self._now_fn, + max_lines=self._max_lines, + max_tokens=self._max_tokens, + token_estimator=self._token_estimator, ) ignore_patterns = self._ignore_patterns @@ -685,6 +750,18 @@ def main(argv: Optional[List[str]] = None) -> int: default=None, help="Directory names to exclude", ) + parser.add_argument( + "--max-lines", + type=int, + default=DEFAULT_MAX_LINES, + help=f"Line-count threshold for the file_length signal (default: {DEFAULT_MAX_LINES})", + ) + parser.add_argument( + "--max-tokens", + type=int, + default=DEFAULT_MAX_TOKENS, + help=f"Estimated-token threshold for the file_length signal (default: {DEFAULT_MAX_TOKENS})", + ) args = parser.parse_args(argv) root = Path(args.root) if args.root else _resolve_project_root(Path.cwd()) @@ -694,6 +771,8 @@ def main(argv: Optional[List[str]] = None) -> int: scope_globs=args.globs, excluded_dirs=args.excluded_dirs, git_log_fn=_git_log_real, + max_lines=args.max_lines, + max_tokens=args.max_tokens, ) artifact = scanner.run() print(json.dumps(artifact, indent=2)) diff --git a/plugins/os-doc-hygiene/tests/test_scanner_file_length.py b/plugins/os-doc-hygiene/tests/test_scanner_file_length.py new file mode 100644 index 0000000..6872ddb --- /dev/null +++ b/plugins/os-doc-hygiene/tests/test_scanner_file_length.py @@ -0,0 +1,127 @@ +""" +Tests for the scanner's file_length signal (Forgejo issue #25, Part 1). + +Objective fact only: line count + estimated token count vs configurable +thresholds. No classification, no report-owned fields (token_estimate is +owned by report_builder.py, not the scanner). +""" + +import sys +from pathlib import Path + +_SCRIPTS_DIR = Path(__file__).parent.parent / "scripts" +if str(_SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPTS_DIR)) + +from scanner import Scanner, SignalComputer # noqa: E402 +from token_estimator import TokenEstimator # noqa: E402 + +from test_scanner_signals import ( # noqa: E402 + FORBIDDEN_SIGNAL_FIELDS, + _assert_no_classifier_fields, + _assert_signal_present, + _make_tree, +) + + +class _StubEstimator(TokenEstimator): + """Deterministic stub: 1 token per line, ignoring content.""" + + name = "stub-per-line" + + def estimate(self, text: str) -> int: + if not text: + return 0 + return len(text.splitlines()) + + +def _run(root: Path, **kwargs) -> dict: + return Scanner( + root=root, + git_log_fn=lambda p: [], + now_fn=lambda: 0.0, + token_estimator=kwargs.pop("token_estimator", _StubEstimator()), + **kwargs, + ).run() + + +class TestFileLengthSignal: + def test_short_file_no_signal(self, tmp_path): + _make_tree(tmp_path, {"doc.md": "\n".join(f"line {i}" for i in range(10))}) + result = _run(tmp_path) + sigs = result["signals"].get("doc.md", []) + names = [s["name"] for s in sigs] + assert "file_length" not in names + + def test_over_line_threshold_emits_signal(self, tmp_path): + content = "\n".join(f"line {i}" for i in range(500)) + _make_tree(tmp_path, {"doc.md": content}) + result = _run(tmp_path, max_lines=400, max_tokens=4000) + sigs = result["signals"].get("doc.md", []) + sig = _assert_signal_present(sigs, "file_length") + _assert_no_classifier_fields(sig) + assert "500" in sig["detail"] + assert "400" in sig["detail"] + + def test_over_token_threshold_emits_signal(self, tmp_path): + """Line count under threshold but stub estimator pushes tokens over.""" + content = "\n".join(f"line {i}" for i in range(50)) + _make_tree(tmp_path, {"doc.md": content}) + # Stub estimates 1 token/line -> 50 tokens; set max_tokens low so it fires + # even though line count (50) stays well under max_lines (400). + result = _run(tmp_path, max_lines=400, max_tokens=10) + sigs = result["signals"].get("doc.md", []) + sig = _assert_signal_present(sigs, "file_length") + _assert_no_classifier_fields(sig) + assert "threshold 400 lines / 10 tokens" in sig["detail"] + + def test_configurable_threshold_respected(self, tmp_path): + content = "\n".join(f"line {i}" for i in range(50)) + _make_tree(tmp_path, {"doc.md": content}) + # Below default threshold -> no signal + result_default = _run(tmp_path) + assert "file_length" not in [ + s["name"] for s in result_default["signals"].get("doc.md", []) + ] + # Lower the threshold below the actual line count -> signal fires + result_custom = _run(tmp_path, max_lines=10, max_tokens=4000) + sigs = result_custom["signals"].get("doc.md", []) + sig = _assert_signal_present(sigs, "file_length") + assert "threshold 10 lines" in sig["detail"] + + def test_silent_below_threshold_with_default_estimator(self, tmp_path): + """Uses the real default_estimator() (no injected stub) below threshold.""" + _make_tree(tmp_path, {"doc.md": "short content"}) + result = Scanner( + root=tmp_path, + git_log_fn=lambda p: [], + now_fn=lambda: 0.0, + ).run() + sigs = result["signals"].get("doc.md", []) + names = [s["name"] for s in sigs] + assert "file_length" not in names + + def test_signal_omits_forbidden_report_fields(self, tmp_path): + """file_length must not leak report-owned fields (e.g. token_estimate).""" + content = "\n".join(f"line {i}" for i in range(500)) + _make_tree(tmp_path, {"doc.md": content}) + result = _run(tmp_path, max_lines=400, max_tokens=4000) + sigs = result["signals"].get("doc.md", []) + sig = _assert_signal_present(sigs, "file_length") + for field in FORBIDDEN_SIGNAL_FIELDS: + assert field not in sig, f"forbidden field '{field}' found in file_length signal" + + def test_signal_computer_direct_default_thresholds(self, tmp_path): + """SignalComputer alone: constructor defaults are 400 lines / 4000 tokens.""" + content = "\n".join(f"line {i}" for i in range(500)) + _make_tree(tmp_path, {"doc.md": content}) + path = tmp_path / "doc.md" + computer = SignalComputer( + root=tmp_path, + git_log_fn=None, + now_fn=lambda: 0.0, + token_estimator=_StubEstimator(), + ) + sigs = computer._file_length(path) + sig = _assert_signal_present(sigs, "file_length") + assert "threshold 400 lines / 4000 tokens" in sig["detail"]