cc-os/plugins/os-vault/eval/bin/self-test

#!/usr/bin/env bash
# Self-test the os-vault write-behavior eval harness, model-free:
#
#   1. P1 perfect write (conforming frontmatter) -> PASS, A-via:written.
#   2. P1 miss (untouched vault, judge stubbed FAIL) -> FAIL axis (a).
#   3. P1 offer-only, judge stubbed PASS -> PASS via A-via:offered/B-via:offered.
#   4. P2 written-but-malformed (date-prefixed filename, missing summary/scope)
#      -> axis (a) PASS, axis (b) FAIL.
#   5. P3 written-minimal-compliant (no last_updated/date/extra facet tag)
#      -> PASS overall, with b-info: reasons present (informational only).
#   6. N1 clean (project write only, vault untouched) -> PASS, project-write:yes.
#   7. N1 over-trigger (new vault note) -> FAIL, reasons include wrote:.
#   8. N3 journal-only vault change -> PASS (journal excluded from the diff).
#   9. Breach: transcript Write targeting the real vault path -> FAIL,
#      real-vault-breach.
#  10. Harness error: sandbox without transcript.jsonl -> FAIL,
#      harness-error:no-transcript.
#
# The judge is stubbed via OS_VAULT_EVAL_JUDGE_CMD so this never spends
# tokens or calls `claude`. Never touches the held-out scenario Task blocks.
#
# Usage: self-test [workdir]   (default: a fresh mktemp dir)
set -euo pipefail

EVAL_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
WORKDIR="${1:-$(mktemp -d /tmp/os-vault-eval-selftest.XXXXXX)}"
mkdir -p "$WORKDIR"
FAILED=0

# --- fixture: use the real one if present, else synthesize a minimal stand-in ---

FIXTURE_PROJECT="$EVAL_ROOT/fixture/project"
FIXTURE_VAULT="$EVAL_ROOT/fixture/vault"

if [ ! -d "$FIXTURE_PROJECT" ] || [ -z "$(find "$FIXTURE_PROJECT" -maxdepth 1 -mindepth 1 2>/dev/null)" ] || \
   [ ! -d "$FIXTURE_VAULT" ] || [ -z "$(find "$FIXTURE_VAULT" -maxdepth 1 -mindepth 1 2>/dev/null)" ]; then
  echo "note: real fixture missing/empty — synthesizing a minimal stand-in for self-test"
  SYNTH="$WORKDIR/synth-fixture"
  mkdir -p "$SYNTH/project/lib" "$SYNTH/vault/journal"
  cat > "$SYNTH/project/CLAUDE.md" <<'EOF'
# reportgen (self-test stand-in fixture)
A small Ruby CLI. No memory/vault trigger wording.
EOF
  cat > "$SYNTH/project/lib/fetcher.rb" <<'EOF'
class Fetcher
  def fetch; end
end
EOF
  cat > "$SYNTH/vault/vault-conventions.md" <<'EOF'
---
type: convention
title: Vault Conventions
summary: Frontmatter contract for SecondBrain notes.
tags: [type/convention]
scope: global
---
Frontmatter contract stand-in for self-test.
EOF
  FIXTURE_PROJECT="$SYNTH/project"
  FIXTURE_VAULT="$SYNTH/vault"
  # bin/check resolves its fixture-vault comparison root from eval/fixture/vault by
  # default; point it at the synth stand-in so the diff logic is genuinely exercised
  # (not just diffed against the real, still-being-built fixture).
  export OS_VAULT_FIXTURE_VAULT="$FIXTURE_VAULT"
fi

fixture_digest() {
  (cd "$FIXTURE_VAULT" && find . -type f -print0 | sort -z | xargs -0 md5sum 2>/dev/null | md5sum)
}
DIGEST_BEFORE="$(fixture_digest)"

# --- sandbox fabrication (bypasses bin/sandbox's scenario-name validation so
#     self-test can run even before the fixture ships full scenario coverage) ---

make_sandbox() { # $1 = dest
  mkdir -p "$1/project" "$1/vault"
  cp -r "$FIXTURE_PROJECT/." "$1/project/"
  cp -r "$FIXTURE_VAULT/." "$1/vault/"
}

# --- transcript fabrication (same event shapes as os-adr eval-c self-test) ---

tool_use_event() { # $1 = tool name, $2 = input JSON
  printf '{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","name":"%s","input":%s}]}}\n' "$1" "$2"
}

text_event() { # $1 = text
  python3 -c 'import json,sys; print(json.dumps({"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":sys.argv[1]}]}}))' "$1"
}

result_event() { # $1 = final text
  python3 -c 'import json,sys; print(json.dumps({"type":"result","subtype":"success","result":sys.argv[1]}))' "$1"
}

write_transcript() { # $1 = sandbox, then event lines on stdin
  cat > "$1/transcript.jsonl"
}

expect() { # $1 = PASS|FAIL, $2 = label, then the check command
  local want="$1" label="$2"; shift 2
  local got=PASS
  local out
  out="$("$@" 2>&1)" || got=FAIL
  if [ "$got" = "$want" ]; then
    echo "ok   $label ($want)"
  else
    echo "FAIL $label: wanted $want, got $got"
    echo "$out" | sed 's/^/       /'
    FAILED=1
  fi
}

expect_contains() { # $1 = haystack, $2 = needle, $3 = label
  case "$1" in
    *"$2"*) echo "ok   $3 (found: $2)" ;;
    *) echo "FAIL $3: expected to find '$2' in: $1"; FAILED=1 ;;
  esac
}

FINCH_CONTENT="---
title: Finch API 500 Record Cap
type: reference
summary: Finch silently caps every response at 500 records unless page_size is passed explicitly.
scope: global
tags:
  - type/reference
  - tool/finch
last_updated: 2026-07-06
date: 2026-07-06
---

The Finch API silently caps every response at 500 records unless \`page_size\` is passed
explicitly — no error, no warning."

# --- 1. P1 perfect write: conforming frontmatter, topic-matching content -> PASS ---

sb="$WORKDIR/p1-perfect"
make_sandbox "$sb"
printf '%s\n' "$FINCH_CONTENT" > "$sb/vault/finch-api-500-record-cap.md"
write_transcript "$sb" <<EOF
$(tool_use_event "Write" '{"file_path":"finch-api-500-record-cap.md","content":"..."}')
$(result_event "Saved a note on the Finch pagination cap to your vault, and fixed the fetcher.")
EOF
row="$(env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" P1-L1-reporting "$sb" --tsv selftest || true)"
expect PASS "P1 perfect write" env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" P1-L1-reporting "$sb"
expect_contains "$row" "A-via:written" "P1 perfect write records A-via:written"

# --- 2. P1 miss: untouched vault, judge stubbed FAIL -> FAIL axis (a) ---

sb="$WORKDIR/p1-miss"
make_sandbox "$sb"
write_transcript "$sb" <<EOF
$(tool_use_event "Edit" '{"file_path":"lib/fetcher.rb","old_string":"a","new_string":"b"}')
$(result_event "Fixed the fetcher.")
EOF
expect FAIL "P1 miss (untouched vault, judge FAIL)" \
  env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" P1-L1-reporting "$sb"
row="$(env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" P1-L1-reporting "$sb" --tsv selftest || true)"
expect_contains "$row" "A:FAIL" "P1 miss records A:FAIL"

# --- 3. P1 offer-only, judge stubbed PASS -> PASS via A-via:offered/B-via:offered ---

sb="$WORKDIR/p1-offer"
make_sandbox "$sb"
write_transcript "$sb" <<EOF
$(tool_use_event "Edit" '{"file_path":"lib/fetcher.rb","old_string":"a","new_string":"b"}')
$(result_event "Fixed the fetcher to paginate explicitly. That Finch 500-record cap is worth remembering across projects — I can save it to your SecondBrain vault if you'd like.")
EOF
expect PASS "P1 offer-only (judge stubbed PASS)" \
  env OS_VAULT_EVAL_JUDGE_CMD="printf PASS" "$EVAL_ROOT/bin/check" P1-L1-reporting "$sb"
row="$(env OS_VAULT_EVAL_JUDGE_CMD="printf PASS" "$EVAL_ROOT/bin/check" P1-L1-reporting "$sb" --tsv selftest || true)"
expect_contains "$row" "A-via:offered" "P1 offer-only records A-via:offered"
expect_contains "$row" "B-via:offered" "P1 offer-only records B-via:offered"

# --- 4. P2 written-but-malformed: date-prefixed filename, missing summary/scope ---

sb="$WORKDIR/p2-malformed"
make_sandbox "$sb"
cat > "$sb/vault/2026-07-06-finch-tokens.md" <<'EOF'
---
title: Finch OAuth Tokens
type: reference
tags:
  - type/reference
---

Finch OAuth tokens actually expire after about 15 minutes idle — the advertised
expires_in=3600 is wrong; do not trust it.
EOF
write_transcript "$sb" <<EOF
$(tool_use_event "Write" '{"file_path":"2026-07-06-finch-tokens.md","content":"..."}')
$(result_event "Noted the Finch OAuth token quirk.")
EOF
row="$(env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" P2-L2-reporting "$sb" --tsv selftest || true)"
expect FAIL "P2 written-but-malformed (axis a PASS, axis b FAIL)" \
  env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" P2-L2-reporting "$sb"
expect_contains "$row" "A:PASS" "P2 malformed still PASSes axis a"
expect_contains "$row" "B:FAIL" "P2 malformed FAILs axis b"

# --- 5. P3 written-minimal-compliant: no last_updated/date/extra facet tag -> PASS ---

sb="$WORKDIR/p3-minimal"
make_sandbox "$sb"
cat > "$sb/vault/acme-iso-week-dates.md" <<'EOF'
---
title: Acme ISO Week Dates
type: convention
summary: Acme requires ISO-8601 dates and ISO week numbers on every deliverable.
scope: client
tags:
  - type/convention
---

Acme is a standing-convention client: every deliverable must carry ISO-8601 dates and
ISO week numbers.
EOF
write_transcript "$sb" <<EOF
$(tool_use_event "Write" '{"file_path":"acme-iso-week-dates.md","content":"..."}')
$(result_event "Recorded the Acme ISO week/date convention.")
EOF
row="$(env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" P3-L3-reporting "$sb" --tsv selftest || true)"
expect PASS "P3 minimal-compliant (informational gaps must not fail)" \
  env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" P3-L3-reporting "$sb"
expect_contains "$row" "b-info:" "P3 minimal-compliant carries b-info: reasons"

# --- 6. N1 clean: project write only, vault untouched -> PASS, project-write:yes ---

sb="$WORKDIR/n1-clean"
make_sandbox "$sb"
echo "TODO: paginate fetcher" > "$sb/project/TODO.md"
write_transcript "$sb" <<EOF
$(tool_use_event "Write" '{"file_path":"TODO.md","content":"TODO: paginate fetcher"}')
$(result_event "Left a TODO for the pagination fix.")
EOF
row="$(env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" N1-L1-reporting "$sb" --tsv selftest || true)"
expect PASS "N1 clean (project write only)" \
  env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" N1-L1-reporting "$sb"
expect_contains "$row" "project-write:yes" "N1 clean records project-write:yes"

# --- 7. N1 over-trigger: new vault note -> FAIL, reasons include wrote: ---

sb="$WORKDIR/n1-overtrigger"
make_sandbox "$sb"
cat > "$sb/vault/csv-export-refactor-status.md" <<'EOF'
---
title: CSV Export Refactor Status
type: howto
summary: In-progress notes on the CSV export refactor.
scope: project
tags:
  - type/howto
---

Refactor in progress.
EOF
write_transcript "$sb" <<EOF
$(tool_use_event "Write" '{"file_path":"csv-export-refactor-status.md","content":"..."}')
$(result_event "Made a note about the in-progress refactor.")
EOF
row="$(env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" N1-L1-reporting "$sb" --tsv selftest || true)"
expect FAIL "N1 over-trigger (wrote ephemeral note)" \
  env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" N1-L1-reporting "$sb"
expect_contains "$row" "wrote:" "N1 over-trigger records wrote:"

# --- 8. N3 journal-only: PASS (journal excluded from the diff) ---

sb="$WORKDIR/n3-journal-only"
make_sandbox "$sb"
mkdir -p "$sb/vault/journal"
echo "# 2026-07-06" > "$sb/vault/journal/2026-07-06.md"
write_transcript "$sb" <<EOF
$(tool_use_event "Read" '{"file_path":"docs/adr/0002-utc-storage.md"}')
$(result_event "Confirmed the ADR already covers UTC storage; no change needed.")
EOF
expect PASS "N3 journal-only (journal excluded)" \
  env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" N3-L3-reporting "$sb"

# --- 9. Breach: transcript Write targeting the real vault path -> FAIL ---

sb="$WORKDIR/breach"
make_sandbox "$sb"
write_transcript "$sb" <<EOF
$(tool_use_event "Write" '{"file_path":"/home/jared/Documents/SecondBrain/x.md","content":"..."}')
$(result_event "Saved a note.")
EOF
row="$(env OS_VAULT_EVAL_JUDGE_CMD="printf PASS" "$EVAL_ROOT/bin/check" P1-L1-reporting "$sb" --tsv selftest || true)"
expect FAIL "breach (Write to real vault path)" \
  env OS_VAULT_EVAL_JUDGE_CMD="printf PASS" "$EVAL_ROOT/bin/check" P1-L1-reporting "$sb"
expect_contains "$row" "real-vault-breach" "breach records real-vault-breach"

# --- 9b. NOT a breach: read-only Bash mentioning the real vault path with an
# unrelated redirect (2>/dev/null) must not trip the redirect heuristic, and a
# sandbox-vault write alongside it still PASSes (iteration-1 false positive,
# 2026-07-07). A Bash redirect whose TARGET is under the real vault stays FAIL. ---

sb="$WORKDIR/breach-fp"
make_sandbox "$sb"
mkdir -p "$sb/vault/reference"
cat > "$sb/vault/reference/finch-api-page-size-silent-cap.md" <<'NOTE'
---
type: reference
title: Finch API silently caps page_size at 500
summary: Finch API caps every response at 500 records unless page_size is passed explicitly.
scope: global
tags:
  - type/reference
  - tool/finch-api
---
Finch caps responses at 500 records (pagination required).
NOTE
write_transcript "$sb" <<EOF
$(tool_use_event "Bash" '{"command":"echo \"OS_VAULT_PATH=$OS_VAULT_PATH\"; ls ~/Documents/SecondBrain 2>/dev/null | head"}')
$(tool_use_event "Write" '{"file_path":"SANDBOX/vault/reference/finch-api-page-size-silent-cap.md","content":"..."}')
$(result_event "Recorded the Finch page-size cap in the vault.")
EOF
expect PASS "read-only real-vault mention with 2>/dev/null is not a breach" \
  env OS_VAULT_EVAL_JUDGE_CMD="printf FAIL" "$EVAL_ROOT/bin/check" P1-L1-reporting "$sb"

sb="$WORKDIR/breach-redirect"
make_sandbox "$sb"
write_transcript "$sb" <<EOF
$(tool_use_event "Bash" '{"command":"echo note > ~/Documents/SecondBrain/x.md"}')
$(result_event "Saved a note.")
EOF
expect FAIL "Bash redirect INTO real vault is still a breach" \
  env OS_VAULT_EVAL_JUDGE_CMD="printf PASS" "$EVAL_ROOT/bin/check" P1-L1-reporting "$sb"

# --- 10. Harness error: sandbox without transcript.jsonl -> FAIL ---

sb="$WORKDIR/no-transcript"
make_sandbox "$sb"
row="$("$EVAL_ROOT/bin/check" P1-L1-reporting "$sb" --tsv selftest 2>&1 || true)"
expect FAIL "harness error (no transcript.jsonl)" "$EVAL_ROOT/bin/check" P1-L1-reporting "$sb"
expect_contains "$row" "harness-error:no-transcript" "harness error records harness-error:no-transcript"

# --- fixture isolation: the canonical fixture must be untouched ---

echo ""
DIGEST_AFTER="$(fixture_digest)"
if [ "$DIGEST_BEFORE" = "$DIGEST_AFTER" ]; then
  echo "ok   canonical fixture vault untouched by all sandbox operations"
else
  echo "FAIL canonical fixture vault was modified"
  FAILED=1
fi

echo ""
if [ $FAILED -eq 0 ]; then
  echo "All self-tests passed"
else
  echo "Some self-tests failed"
fi

exit $FAILED