os-doc-hygiene: deterministic file_length scanner signal (issue #25 part 1)

Objective signal (line count + estimated tokens) fires when either
threshold is exceeded; defaults 400 lines / 4000 tokens, configurable via
constructor and --max-lines/--max-tokens. Injectable token estimator for
tests. Suite: 286 passed. Parts 2-3 (implementation-status distill +
index+PD convention ADR) remain human-gated.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CLeMz48rvG3s9XpAsDxeho

plugins/os-doc-hygiene/scripts/scanner.py

@@ -25,6 +25,8 @@ import sys
 from pathlib import Path
 from typing import Callable, Iterator, List, Optional
 
+from token_estimator import TokenEstimator, default_estimator
+
 
 # ---------------------------------------------------------------------------
 # Glob matching (Python 3.14-compatible)
@@ -56,6 +58,8 @@ def _file_matches_glob(rel_path_str: str, glob_pattern: str) -> bool:
 # ---------------------------------------------------------------------------
 
 DEFAULT_SCOPE_GLOBS: List[str] = ["**/*.md"]
+DEFAULT_MAX_LINES: int = 400
+DEFAULT_MAX_TOKENS: int = 4000
 DEFAULT_EXCLUDED_DIRS: List[str] = [
     "build",
     "vendor",
@@ -222,6 +226,14 @@ class SignalComputer:
         tests pass a fake.  Pass ``None`` to disable git-based signals.
     now_fn : callable() -> float
         Returns current time as a Unix timestamp.
+    max_lines : int
+        Line-count threshold for the ``file_length`` signal.
+    max_tokens : int
+        Estimated-token threshold for the ``file_length`` signal.
+    token_estimator : TokenEstimator, optional
+        Injectable estimator used for the ``file_length`` signal's token
+        count.  Defaults to ``token_estimator.default_estimator()``. Tests
+        can inject a deterministic stub.
     """
 
     def __init__(
@@ -229,10 +241,16 @@ class SignalComputer:
         root: Path,
         git_log_fn: Optional[Callable[[Path], List[str]]] = None,
         now_fn: Optional[Callable[[], float]] = None,
+        max_lines: int = DEFAULT_MAX_LINES,
+        max_tokens: int = DEFAULT_MAX_TOKENS,
+        token_estimator: Optional[TokenEstimator] = None,
     ) -> None:
         self._root = root
         self._git_log_fn = git_log_fn
         self._now_fn = now_fn or (lambda: __import__("time").time())
+        self._max_lines = max_lines
+        self._max_tokens = max_tokens
+        self._token_estimator = token_estimator or default_estimator()
 
     # ------------------------------------------------------------------
     # Public
@@ -249,6 +267,7 @@ class SignalComputer:
         signals.extend(self._location_signals(path, rel))
         signals.extend(self._archive_to_live_ratio(path))
         signals.extend(self._frontmatter_markers(path))
+        signals.extend(self._file_length(path))
 
         return signals
 
@@ -476,6 +495,36 @@ class SignalComputer:
                 )
         return signals
 
+    def _file_length(self, path: Path) -> List[dict]:
+        """Flag files whose line count or estimated token count exceeds threshold.
+
+        Objective fact only: reports the measured line count and estimated
+        token count against the configured thresholds. Fires if EITHER
+        threshold is exceeded. No classification.
+        """
+        signals = []
+        try:
+            text = path.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            return signals
+
+        line_count = len(text.splitlines())
+        token_count = self._token_estimator.estimate(text)
+
+        if line_count > self._max_lines or token_count > self._max_tokens:
+            token_count_k = token_count / 1000.0
+            signals.append(
+                {
+                    "name": "file_length",
+                    "detail": (
+                        f"{line_count} lines (~{token_count_k:.1f}k tokens est), "
+                        f"threshold {self._max_lines} lines / {self._max_tokens} tokens"
+                    ),
+                }
+            )
+
+        return signals
+
 
 # ---------------------------------------------------------------------------
 # Production git helper
@@ -518,6 +567,13 @@ class Scanner:
         Injected git-log provider.  Default: real git subprocess.
     now_fn : callable, optional
         Injected clock.  Default: ``time.time``.
+    max_lines : int
+        Line-count threshold for the ``file_length`` signal. Default 400.
+    max_tokens : int
+        Estimated-token threshold for the ``file_length`` signal. Default 4000.
+    token_estimator : TokenEstimator, optional
+        Injectable estimator for the ``file_length`` signal. Default:
+        ``token_estimator.default_estimator()``.
     """
 
     def __init__(
@@ -527,6 +583,9 @@ class Scanner:
         excluded_dirs: Optional[List[str]] = None,
         git_log_fn: Optional[Callable[[Path], List[str]]] = None,
         now_fn: Optional[Callable[[], float]] = None,
+        max_lines: int = DEFAULT_MAX_LINES,
+        max_tokens: int = DEFAULT_MAX_TOKENS,
+        token_estimator: Optional[TokenEstimator] = None,
     ) -> None:
         self._root = root.resolve()
         self._scope_globs = scope_globs if scope_globs is not None else DEFAULT_SCOPE_GLOBS
@@ -560,6 +619,9 @@ class Scanner:
                 self._excluded_pairs.append((parent_name, child))
         self._git_log_fn = git_log_fn
         self._now_fn = now_fn
+        self._max_lines = max_lines
+        self._max_tokens = max_tokens
+        self._token_estimator = token_estimator
         self._ignore_patterns: List[str] = _load_ignore_patterns(self._root)
 
     def _is_excluded_child(self, parent_dirpath: str, child_name: str) -> bool:
@@ -587,6 +649,9 @@ class Scanner:
             root=self._root,
             git_log_fn=self._git_log_fn,
             now_fn=self._now_fn,
+            max_lines=self._max_lines,
+            max_tokens=self._max_tokens,
+            token_estimator=self._token_estimator,
         )
         ignore_patterns = self._ignore_patterns
 
@@ -685,6 +750,18 @@ def main(argv: Optional[List[str]] = None) -> int:
         default=None,
         help="Directory names to exclude",
     )
+    parser.add_argument(
+        "--max-lines",
+        type=int,
+        default=DEFAULT_MAX_LINES,
+        help=f"Line-count threshold for the file_length signal (default: {DEFAULT_MAX_LINES})",
+    )
+    parser.add_argument(
+        "--max-tokens",
+        type=int,
+        default=DEFAULT_MAX_TOKENS,
+        help=f"Estimated-token threshold for the file_length signal (default: {DEFAULT_MAX_TOKENS})",
+    )
     args = parser.parse_args(argv)
 
     root = Path(args.root) if args.root else _resolve_project_root(Path.cwd())
@@ -694,6 +771,8 @@ def main(argv: Optional[List[str]] = None) -> int:
         scope_globs=args.globs,
         excluded_dirs=args.excluded_dirs,
         git_log_fn=_git_log_real,
+        max_lines=args.max_lines,
+        max_tokens=args.max_tokens,
     )
     artifact = scanner.run()
     print(json.dumps(artifact, indent=2))

plugins/os-doc-hygiene/tests/test_scanner_file_length.py

@@ -0,0 +1,127 @@
+"""
+Tests for the scanner's file_length signal (Forgejo issue #25, Part 1).
+
+Objective fact only: line count + estimated token count vs configurable
+thresholds. No classification, no report-owned fields (token_estimate is
+owned by report_builder.py, not the scanner).
+"""
+
+import sys
+from pathlib import Path
+
+_SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
+if str(_SCRIPTS_DIR) not in sys.path:
+    sys.path.insert(0, str(_SCRIPTS_DIR))
+
+from scanner import Scanner, SignalComputer  # noqa: E402
+from token_estimator import TokenEstimator  # noqa: E402
+
+from test_scanner_signals import (  # noqa: E402
+    FORBIDDEN_SIGNAL_FIELDS,
+    _assert_no_classifier_fields,
+    _assert_signal_present,
+    _make_tree,
+)
+
+
+class _StubEstimator(TokenEstimator):
+    """Deterministic stub: 1 token per line, ignoring content."""
+
+    name = "stub-per-line"
+
+    def estimate(self, text: str) -> int:
+        if not text:
+            return 0
+        return len(text.splitlines())
+
+
+def _run(root: Path, **kwargs) -> dict:
+    return Scanner(
+        root=root,
+        git_log_fn=lambda p: [],
+        now_fn=lambda: 0.0,
+        token_estimator=kwargs.pop("token_estimator", _StubEstimator()),
+        **kwargs,
+    ).run()
+
+
+class TestFileLengthSignal:
+    def test_short_file_no_signal(self, tmp_path):
+        _make_tree(tmp_path, {"doc.md": "\n".join(f"line {i}" for i in range(10))})
+        result = _run(tmp_path)
+        sigs = result["signals"].get("doc.md", [])
+        names = [s["name"] for s in sigs]
+        assert "file_length" not in names
+
+    def test_over_line_threshold_emits_signal(self, tmp_path):
+        content = "\n".join(f"line {i}" for i in range(500))
+        _make_tree(tmp_path, {"doc.md": content})
+        result = _run(tmp_path, max_lines=400, max_tokens=4000)
+        sigs = result["signals"].get("doc.md", [])
+        sig = _assert_signal_present(sigs, "file_length")
+        _assert_no_classifier_fields(sig)
+        assert "500" in sig["detail"]
+        assert "400" in sig["detail"]
+
+    def test_over_token_threshold_emits_signal(self, tmp_path):
+        """Line count under threshold but stub estimator pushes tokens over."""
+        content = "\n".join(f"line {i}" for i in range(50))
+        _make_tree(tmp_path, {"doc.md": content})
+        # Stub estimates 1 token/line -> 50 tokens; set max_tokens low so it fires
+        # even though line count (50) stays well under max_lines (400).
+        result = _run(tmp_path, max_lines=400, max_tokens=10)
+        sigs = result["signals"].get("doc.md", [])
+        sig = _assert_signal_present(sigs, "file_length")
+        _assert_no_classifier_fields(sig)
+        assert "threshold 400 lines / 10 tokens" in sig["detail"]
+
+    def test_configurable_threshold_respected(self, tmp_path):
+        content = "\n".join(f"line {i}" for i in range(50))
+        _make_tree(tmp_path, {"doc.md": content})
+        # Below default threshold -> no signal
+        result_default = _run(tmp_path)
+        assert "file_length" not in [
+            s["name"] for s in result_default["signals"].get("doc.md", [])
+        ]
+        # Lower the threshold below the actual line count -> signal fires
+        result_custom = _run(tmp_path, max_lines=10, max_tokens=4000)
+        sigs = result_custom["signals"].get("doc.md", [])
+        sig = _assert_signal_present(sigs, "file_length")
+        assert "threshold 10 lines" in sig["detail"]
+
+    def test_silent_below_threshold_with_default_estimator(self, tmp_path):
+        """Uses the real default_estimator() (no injected stub) below threshold."""
+        _make_tree(tmp_path, {"doc.md": "short content"})
+        result = Scanner(
+            root=tmp_path,
+            git_log_fn=lambda p: [],
+            now_fn=lambda: 0.0,
+        ).run()
+        sigs = result["signals"].get("doc.md", [])
+        names = [s["name"] for s in sigs]
+        assert "file_length" not in names
+
+    def test_signal_omits_forbidden_report_fields(self, tmp_path):
+        """file_length must not leak report-owned fields (e.g. token_estimate)."""
+        content = "\n".join(f"line {i}" for i in range(500))
+        _make_tree(tmp_path, {"doc.md": content})
+        result = _run(tmp_path, max_lines=400, max_tokens=4000)
+        sigs = result["signals"].get("doc.md", [])
+        sig = _assert_signal_present(sigs, "file_length")
+        for field in FORBIDDEN_SIGNAL_FIELDS:
+            assert field not in sig, f"forbidden field '{field}' found in file_length signal"
+
+    def test_signal_computer_direct_default_thresholds(self, tmp_path):
+        """SignalComputer alone: constructor defaults are 400 lines / 4000 tokens."""
+        content = "\n".join(f"line {i}" for i in range(500))
+        _make_tree(tmp_path, {"doc.md": content})
+        path = tmp_path / "doc.md"
+        computer = SignalComputer(
+            root=tmp_path,
+            git_log_fn=None,
+            now_fn=lambda: 0.0,
+            token_estimator=_StubEstimator(),
+        )
+        sigs = computer._file_length(path)
+        sig = _assert_signal_present(sigs, "file_length")
+        assert "threshold 400 lines / 4000 tokens" in sig["detail"]