cc-os/plugins/os-doc-hygiene/tests/test_scanner_exclusions.py

"""
Tests for scanner exclusion pipeline (task 2.5).

Verifies invariant #9: frozen/ignored/append-only files are present in the
tree but absent from the shortlist; .dochygiene/ and excluded dirs are never
scanned.

sys.path manipulation is intentional per task constraints (no shared conftest).
"""

import os
import shutil
import sys
import tempfile
from pathlib import Path

# Prepend scripts/ to sys.path so we can import scanner without a package
_SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
if str(_SCRIPTS_DIR) not in sys.path:
    sys.path.insert(0, str(_SCRIPTS_DIR))

from scanner import Scanner  # noqa: E402


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def _make_tree(tmp: Path, files: dict) -> None:
    """Create files in *tmp* from a dict of rel_path -> content."""
    for rel, content in files.items():
        p = tmp / rel
        p.parent.mkdir(parents=True, exist_ok=True)
        p.write_text(content, encoding="utf-8")


def _run(root: Path, **kwargs) -> dict:
    """Run scanner on *root* with no git and frozen clock."""
    return Scanner(
        root=root,
        git_log_fn=lambda p: [],
        now_fn=lambda: 0.0,
        **kwargs,
    ).run()


# ---------------------------------------------------------------------------
# 2.5a: hygiene: frozen file is excluded
# ---------------------------------------------------------------------------

class TestFrozenFrontmatter:
    def test_frozen_file_absent_from_shortlist(self, tmp_path):
        _make_tree(tmp_path, {
            "frozen_doc.md": "---\nhygiene: frozen\n---\n\n# Frozen\nContent.",
            "live_doc.md": "# Live\nThis is live content.",
        })
        result = _run(tmp_path)
        assert "frozen_doc.md" not in result["shortlist"], (
            "frozen_doc.md should be excluded by hygiene: frozen frontmatter"
        )

    def test_frozen_file_has_no_signals(self, tmp_path):
        _make_tree(tmp_path, {
            "frozen_doc.md": "---\nhygiene: frozen\n---\n\n# Frozen\nContent.",
        })
        result = _run(tmp_path)
        assert "frozen_doc.md" not in result["signals"], (
            "no signals should be computed for a frozen file"
        )

    def test_non_frozen_file_present_in_shortlist(self, tmp_path):
        _make_tree(tmp_path, {
            "frozen_doc.md": "---\nhygiene: frozen\n---\n\n# Frozen",
            "live_doc.md": "# Live\nNormal content.",
        })
        result = _run(tmp_path)
        assert "live_doc.md" in result["shortlist"], (
            "live_doc.md should be shortlisted"
        )

    def test_frozen_only_checked_at_frontmatter_not_content(self, tmp_path):
        """A file mentioning 'frozen' in body but not frontmatter is NOT excluded."""
        _make_tree(tmp_path, {
            "doc.md": "# Not frozen\nThis doc mentions hygiene: frozen in the body.",
        })
        result = _run(tmp_path)
        assert "doc.md" in result["shortlist"]

    def test_fixture_frozen_doc(self):
        """Smoke: use the checked-in fixture tree."""
        fixture_root = Path(__file__).parent / "fixtures" / "scanner" / "frozen"
        if not fixture_root.exists():
            return  # fixture not present, skip
        result = _run(fixture_root)
        assert "frozen_doc.md" not in result["shortlist"]
        assert "live_doc.md" in result["shortlist"]


# ---------------------------------------------------------------------------
# 2.5b: .dochygiene-ignore match is excluded
# ---------------------------------------------------------------------------

class TestIgnorePatterns:
    def test_ignored_file_absent_from_shortlist(self, tmp_path):
        _make_tree(tmp_path, {
            ".dochygiene-ignore": "ignored_doc.md\n",
            "ignored_doc.md": "# Ignored\nThis should be excluded.",
            "normal_doc.md": "# Normal\nThis should be shortlisted.",
        })
        result = _run(tmp_path)
        assert "ignored_doc.md" not in result["shortlist"]

    def test_ignored_file_has_no_signals(self, tmp_path):
        _make_tree(tmp_path, {
            ".dochygiene-ignore": "ignored_doc.md\n",
            "ignored_doc.md": "# Ignored",
        })
        result = _run(tmp_path)
        assert "ignored_doc.md" not in result["signals"]

    def test_non_ignored_file_present(self, tmp_path):
        _make_tree(tmp_path, {
            ".dochygiene-ignore": "ignored_doc.md\n",
            "normal_doc.md": "# Normal",
            "ignored_doc.md": "# Ignored",
        })
        result = _run(tmp_path)
        assert "normal_doc.md" in result["shortlist"]

    def test_glob_pattern_in_ignore(self, tmp_path):
        _make_tree(tmp_path, {
            ".dochygiene-ignore": "temp_*.md\n",
            "temp_notes.md": "# Temp",
            "design.md": "# Design",
        })
        result = _run(tmp_path)
        assert "temp_notes.md" not in result["shortlist"]
        assert "design.md" in result["shortlist"]

    def test_comment_lines_in_ignore_file_skipped(self, tmp_path):
        _make_tree(tmp_path, {
            ".dochygiene-ignore": "# This is a comment\nnotes.md\n",
            "notes.md": "# Notes",
            "design.md": "# Design",
        })
        result = _run(tmp_path)
        assert "notes.md" not in result["shortlist"]
        assert "design.md" in result["shortlist"]

    def test_no_ignore_file_means_nothing_excluded(self, tmp_path):
        _make_tree(tmp_path, {"doc.md": "# Doc"})
        result = _run(tmp_path)
        assert "doc.md" in result["shortlist"]

    def test_fixture_ignored_doc(self):
        """Smoke: use the checked-in fixture tree with a live .dochygiene-ignore."""
        fixture_root = Path(__file__).parent / "fixtures" / "scanner" / "ignored"
        if not fixture_root.exists():
            return
        # Write a .dochygiene-ignore into a temp copy so we don't modify the fixture dir
        import shutil, tempfile
        with tempfile.TemporaryDirectory() as tmp:
            tmp_path = Path(tmp)
            shutil.copytree(fixture_root, tmp_path / "tree")
            tree = tmp_path / "tree"
            (tree / ".dochygiene-ignore").write_text("ignored_doc.md\n")
            result = _run(tree)
            assert "ignored_doc.md" not in result["shortlist"]
            assert "normal_doc.md" in result["shortlist"]


# ---------------------------------------------------------------------------
# 2.5c: append-only log is excluded (positive + negative fixture)
# ---------------------------------------------------------------------------

class TestAppendOnlyExclusion:
    def test_append_only_explicit_marker_excluded(self, tmp_path):
        _make_tree(tmp_path, {
            "changelog.md": "<!-- append-only -->\n\n# Changelog\n\n- Entry 1\n",
            "normal.md": "# Normal doc\nContent here.",
        })
        result = _run(tmp_path)
        assert "changelog.md" not in result["shortlist"], (
            "file with explicit append-only marker should be excluded"
        )

    def test_append_only_explicit_marker_no_signals(self, tmp_path):
        _make_tree(tmp_path, {
            "changelog.md": "<!-- append-only -->\n# Log\n- Item\n",
        })
        result = _run(tmp_path)
        assert "changelog.md" not in result["signals"]

    def test_dated_sections_changelog_excluded(self, tmp_path):
        """Positive fixture: multi-entry dated changelog is append-only."""
        content = (
            "# Changelog\n\n"
            "## 2026-06-18\n\n- Feature added\n\n"
            "## 2026-06-17\n\n- Bug fixed\n\n"
            "## 2026-06-16\n\n- Project started\n"
        )
        _make_tree(tmp_path, {"CHANGELOG.md": content})
        result = _run(tmp_path)
        assert "CHANGELOG.md" not in result["shortlist"], (
            "multi-entry dated changelog should be detected as append-only"
        )

    def test_regular_doc_not_excluded(self, tmp_path):
        """Negative fixture: a regular architecture doc is NOT append-only."""
        content = (
            "# Architecture Overview\n\n"
            "## Components\n\nThe system has three components.\n\n"
            "## Rationale\n\nWe chose this design for clarity.\n\n"
            "## Trade-offs\n\nThe main trade-off is simplicity vs flexibility.\n"
        )
        _make_tree(tmp_path, {"ARCHITECTURE.md": content})
        result = _run(tmp_path)
        assert "ARCHITECTURE.md" in result["shortlist"], (
            "regular architecture doc (negative fixture) should NOT be excluded"
        )

    def test_fixture_positive_changelog(self):
        """Smoke: checked-in changelog fixture is excluded."""
        p = Path(__file__).parent / "fixtures" / "scanner" / "append_only" / "changelog_positive.md"
        if not p.exists():
            return
        import tempfile
        with tempfile.TemporaryDirectory() as tmp:
            tmp_path = Path(tmp)
            shutil.copy(p, tmp_path / p.name)
            result = _run(tmp_path)
            assert p.name not in result["shortlist"]

    def test_fixture_negative_regular_doc(self):
        """Smoke: checked-in regular doc fixture IS shortlisted."""
        p = Path(__file__).parent / "fixtures" / "scanner" / "append_only" / "regular_doc_negative.md"
        if not p.exists():
            return
        import tempfile
        with tempfile.TemporaryDirectory() as tmp:
            tmp_path = Path(tmp)
            shutil.copy(p, tmp_path / p.name)
            result = _run(tmp_path)
            assert p.name in result["shortlist"]


# ---------------------------------------------------------------------------
# 2.5d: excluded dirs are never descended
# ---------------------------------------------------------------------------

class TestExcludedDirs:
    def test_default_excluded_dirs_not_scanned(self, tmp_path):
        # Bare directory-name excludes (matched at any depth).  "golden" is NOT
        # here — it is a path-aware "examples/golden" pair, covered separately.
        bare_excludes = ["build", "vendor", "archive", "graphify-out", "fixtures"]
        for excluded_dir in bare_excludes:
            _make_tree(tmp_path, {
                f"{excluded_dir}/secret.md": f"# In {excluded_dir}\nShould not be scanned.",
            })
        _make_tree(tmp_path, {"live.md": "# Live"})
        result = _run(tmp_path)
        for excluded_dir in bare_excludes:
            for path in result["shortlist"]:
                assert not path.startswith(f"{excluded_dir}/"), (
                    f"file under {excluded_dir}/ should not appear in shortlist"
                )

    def test_dochygiene_self_excluded_always(self, tmp_path):
        """State dir is always self-excluded regardless of configuration."""
        _make_tree(tmp_path, {
            ".dochygiene/state.json": '{"last_check": null}',
            ".dochygiene/state_doc.md": "# This should never be scanned",
            "real_doc.md": "# Real doc",
        })
        result = _run(tmp_path)
        for path in result["shortlist"]:
            assert not path.startswith(".dochygiene/"), (
                ".dochygiene/ contents should never appear in shortlist"
            )
        assert "real_doc.md" in result["shortlist"]

    def test_dochygiene_excluded_even_if_not_in_config(self, tmp_path):
        """Even with an empty excluded_dirs config, .dochygiene/ is self-excluded."""
        _make_tree(tmp_path, {
            ".dochygiene/report.md": "# Should be excluded",
            "live.md": "# Live",
        })
        # Pass an empty excluded_dirs — .dochygiene/ must still be excluded
        result = Scanner(
            root=tmp_path,
            excluded_dirs=[],  # explicitly empty
            git_log_fn=lambda p: [],
            now_fn=lambda: 0.0,
        ).run()
        for path in result["shortlist"]:
            assert not path.startswith(".dochygiene/")
        assert "live.md" in result["shortlist"]

    def test_cc_os_self_excluded_always(self, tmp_path):
        """Canonical .cc-os/ state dir (ADR-027) is always self-excluded."""
        _make_tree(tmp_path, {
            ".cc-os/dochygiene/state.json": '{"last_check": null}',
            ".cc-os/dochygiene/state_doc.md": "# This should never be scanned",
            "real_doc.md": "# Real doc",
        })
        result = _run(tmp_path)
        for path in result["shortlist"]:
            assert not path.startswith(".cc-os/"), (
                ".cc-os/ contents should never appear in shortlist"
            )
        assert "real_doc.md" in result["shortlist"]

    def test_cc_os_excluded_even_if_not_in_config(self, tmp_path):
        """Even with an empty excluded_dirs config, .cc-os/ is self-excluded."""
        _make_tree(tmp_path, {
            ".cc-os/dochygiene/report.md": "# Should be excluded",
            "live.md": "# Live",
        })
        result = Scanner(
            root=tmp_path,
            excluded_dirs=[],  # explicitly empty
            git_log_fn=lambda p: [],
            now_fn=lambda: 0.0,
        ).run()
        for path in result["shortlist"]:
            assert not path.startswith(".cc-os/")
        assert "live.md" in result["shortlist"]

    def test_custom_excluded_dir_not_scanned(self, tmp_path):
        _make_tree(tmp_path, {
            "myexcluded/doc.md": "# Should not appear",
            "live.md": "# Live",
        })
        result = Scanner(
            root=tmp_path,
            excluded_dirs=["myexcluded"],
            git_log_fn=lambda p: [],
            now_fn=lambda: 0.0,
        ).run()
        for path in result["shortlist"]:
            assert not path.startswith("myexcluded/")
        assert "live.md" in result["shortlist"]

    def test_excluded_dir_files_cost_no_files_scanned_count(self, tmp_path):
        """Files inside excluded dirs must not increment files_scanned."""
        _make_tree(tmp_path, {
            "build/output.md": "# Build output",
            "live.md": "# Live",
        })
        result = _run(tmp_path)
        # Only live.md should have been scanned (build/ is pruned before walk)
        assert result["files_scanned"] == 1


# ---------------------------------------------------------------------------
# 2.5e: exclusion precedes signal computation
# ---------------------------------------------------------------------------

class TestExclusionPrecedesSignals:
    def test_frozen_file_with_broken_link_has_no_signals(self, tmp_path):
        """Frozen file with a broken reference should produce NO signals."""
        _make_tree(tmp_path, {
            "frozen.md": (
                "---\nhygiene: frozen\n---\n\n"
                "# Frozen\n[missing](./nonexistent.md)\n"
            ),
        })
        result = _run(tmp_path)
        assert "frozen.md" not in result["shortlist"]
        assert "frozen.md" not in result["signals"]

    def test_ignored_file_with_broken_link_has_no_signals(self, tmp_path):
        _make_tree(tmp_path, {
            ".dochygiene-ignore": "ignored.md\n",
            "ignored.md": "# Ignored\n[missing](./gone.md)\n",
        })
        result = _run(tmp_path)
        assert "ignored.md" not in result["signals"]


# ---------------------------------------------------------------------------
# 2.5f: output shape contains no classifier fields
# ---------------------------------------------------------------------------

class TestOutputShape:
    FORBIDDEN_FIELDS = {"entries", "category", "op", "op_type", "safety_tier",
                        "exact_edit", "token_estimate"}

    def test_no_classifier_fields_in_output(self, tmp_path):
        _make_tree(tmp_path, {"doc.md": "# Doc"})
        result = _run(tmp_path)
        for field in self.FORBIDDEN_FIELDS:
            assert field not in result, f"forbidden field '{field}' found in scanner output"

    def test_scan_metadata_fields_present(self, tmp_path):
        _make_tree(tmp_path, {"doc.md": "# Doc"})
        result = _run(tmp_path)
        assert "project_root" in result
        assert "scope_globs" in result
        assert "excluded_dirs" in result
        assert "files_scanned" in result
        assert "shortlist" in result
        assert "signals" in result

    def test_project_root_is_absolute(self, tmp_path):
        _make_tree(tmp_path, {"doc.md": "# Doc"})
        result = _run(tmp_path)
        assert os.path.isabs(result["project_root"])

    def test_files_scanned_count_is_correct(self, tmp_path):
        _make_tree(tmp_path, {
            "a.md": "# A",
            "b.md": "# B",
            "c.md": "# C",
        })
        result = _run(tmp_path)
        assert result["files_scanned"] == 3


# ---------------------------------------------------------------------------
# 2.5g: test-fixture and golden-example dirs are excluded by default
#
# These cover the real false-positive case: a project-level scan of a repo
# that contains test/golden input trees full of intentionally-stale docs.
#
# Two distinct semantics:
#   * "fixtures"        — bare directory-NAME match at any depth (broad).
#   * "examples/golden" — PATH-AWARE: a 'golden' dir is pruned only when its
#                         immediate parent is 'examples' (depth-independent), so
#                         unrelated projects' 'golden/' dirs are NOT skipped.
# ---------------------------------------------------------------------------

class TestFixtureAndGoldenExclusion:
    """'fixtures' (bare name) and 'examples/golden' (path-aware) are default-excluded."""

    def test_fixtures_subtree_not_scanned_by_default(self, tmp_path):
        """Files under any child dir named 'fixtures' are not scanned."""
        _make_tree(tmp_path, {
            "tests/fixtures/scanner/stale_fixture.md": "# Intentionally stale fixture",
            "tests/fixtures/scanner/another.md": "# Another scanner fixture",
            "live_doc.md": "# Real live doc",
        })
        result = _run(tmp_path)
        for path in result["shortlist"]:
            assert "fixtures/" not in path, (
                f"file under fixtures/ sub-dir should not appear in shortlist, got: {path}"
            )
        assert "live_doc.md" in result["shortlist"], (
            "live_doc.md at project root must still be shortlisted"
        )

    def test_examples_golden_subtree_not_scanned_by_default(self, tmp_path):
        """A 'golden' dir under an 'examples' parent is pruned (nested, monorepo-style)."""
        _make_tree(tmp_path, {
            # Mirrors the real monorepo path: examples/golden nested under a pkg dir.
            "doc-hygiene/examples/golden/classifier/1-orphaned/input/docs/orphan.md": (
                "# Orphaned\nThis deliberately-stale golden input must not be flagged."
            ),
            "live_doc.md": "# Real live doc",
        })
        result = _run(tmp_path)
        for path in result["shortlist"]:
            assert "examples/golden/" not in path, (
                f"file under examples/golden/ should not appear in shortlist, got: {path}"
            )
        assert "live_doc.md" in result["shortlist"]

    def test_bare_golden_dir_without_examples_parent_is_scanned(self, tmp_path):
        """A 'golden/' dir NOT under an 'examples' parent is legitimate and IS scanned.

        This is the false-negative the path-aware narrowing exists to prevent:
        an unrelated project with its own top-level golden/ dir must not be skipped.
        """
        _make_tree(tmp_path, {
            "golden/master_record.md": "# Golden source of truth\nLegit project content.",
            "data/golden/values.md": "# Golden values\nMore legit content.",
            "live.md": "# Live",
        })
        result = _run(tmp_path)
        assert "golden/master_record.md" in result["shortlist"], (
            "a top-level golden/ dir (parent != examples) must be scanned"
        )
        assert "data/golden/values.md" in result["shortlist"], (
            "a golden/ dir under 'data' (parent != examples) must be scanned"
        )

    def test_fixtures_files_cost_no_files_scanned_count(self, tmp_path):
        """Files inside fixtures/ dirs must not increment files_scanned (pruned at walk)."""
        _make_tree(tmp_path, {
            "tests/fixtures/scanner/input.md": "# fixture",
            "live.md": "# Live",
        })
        result = _run(tmp_path)
        # Only live.md should be scanned (fixtures/ is pruned before walk descends)
        assert result["files_scanned"] == 1, (
            f"expected 1 file scanned, got {result['files_scanned']}"
        )

    def test_examples_golden_files_cost_no_files_scanned_count(self, tmp_path):
        """Files inside examples/golden/ must not increment files_scanned (pruned at walk)."""
        _make_tree(tmp_path, {
            "examples/golden/1-case/input/docs/doc.md": "# golden input",
            "live.md": "# Live",
        })
        result = _run(tmp_path)
        assert result["files_scanned"] == 1, (
            f"expected 1 file scanned, got {result['files_scanned']}"
        )

    def test_explicit_excluded_dirs_override_does_not_auto_add_fixtures(self, tmp_path):
        """When excluded_dirs is passed explicitly, fixtures/examples-golden are NOT
        auto-added (they are defaults, not invariant-forced like .dochygiene/)."""
        _make_tree(tmp_path, {
            "fixtures/stale.md": "# stale fixture",
            "examples/golden/g.md": "# golden",
            "live.md": "# Live",
        })
        # Pass an explicit override list that does NOT include them
        result = Scanner(
            root=tmp_path,
            excluded_dirs=["build"],  # custom override — fixtures/golden not listed
            git_log_fn=lambda p: [],
            now_fn=lambda: 0.0,
        ).run()
        # Both should now be scanned (override didn't include them)
        assert "fixtures/stale.md" in result["shortlist"], (
            "with explicit excluded_dirs override omitting 'fixtures', "
            "fixtures/stale.md should be scanned"
        )
        assert "examples/golden/g.md" in result["shortlist"], (
            "with explicit excluded_dirs override omitting 'examples/golden', "
            "examples/golden/g.md should be scanned"
        )
        assert "live.md" in result["shortlist"]

    def test_path_aware_pair_passable_as_explicit_override(self, tmp_path):
        """A 'parent/child' entry passed explicitly applies the path-aware match."""
        _make_tree(tmp_path, {
            "examples/golden/g.md": "# excluded",
            "golden/top.md": "# scanned (no examples parent)",
            "live.md": "# Live",
        })
        result = Scanner(
            root=tmp_path,
            excluded_dirs=["examples/golden"],
            git_log_fn=lambda p: [],
            now_fn=lambda: 0.0,
        ).run()
        assert "examples/golden/g.md" not in result["shortlist"]
        assert "golden/top.md" in result["shortlist"]
        assert "live.md" in result["shortlist"]

    def test_golden_harness_root_scan_is_unaffected(self, tmp_path):
        """The golden test harness points the scanner at input/ as the root.

        os.walk never prunes the root itself (only child dirs), and the
        examples→golden parent/child pair lives in the ANCESTRY above the walk
        root — never re-encountered as a child below input/. So files under the
        harness root are fully scanned even with the path-aware exclude active.
        """
        # Simulate what test_classifier_golden.py does: root = .../1-orphaned/input/
        input_root = tmp_path / "examples" / "golden" / "1-orphaned" / "input"
        _make_tree(tmp_path, {
            "examples/golden/1-orphaned/input/docs/orphan.md": (
                "# Orphaned\n[missing](./gone.md)\n"
            ),
        })
        # Run scanner with root = input/ (exactly as the harness does)
        result = Scanner(
            root=input_root,
            git_log_fn=lambda p: [],
            now_fn=lambda: 0.0,
        ).run()
        # The file must still be scanned (root is never pruned; the examples→golden
        # pair is above the root, not a child below it).
        assert "docs/orphan.md" in result["shortlist"], (
            "golden harness: scanner rooted at input/ must still find docs/orphan.md"
        )