cc-os/plugins/os-adr/eval-c/bin/self-test

#!/usr/bin/env bash
# Self-test the Eval C harness in both directions, model-free:
#
#   1. For each POSITIVE scenario, fabricate a "perfect" transcript/sandbox
#      state (model did right thing) -> bin/check must PASS.
#   2. For each POSITIVE scenario, fabricate an "untouched" run (never
#      consulted ADR system) -> bin/check must FAIL, axis (a) FAIL.
#   3. For each NEGATIVE scenario, fabricate a "proper" transcript (no ADR
#      touch, no false ADR creation) -> bin/check must PASS.
#   4. For each NEGATIVE scenario, fabricate an "over-trigger" run (consulted
#      ADR system or created ADR when shouldn't) -> bin/check must FAIL.
#   5. Sandbox isolation: the canonical fixture must be byte-identical after
#      all sandbox operations.
#
# The judge is stubbed via ADR_EVAL_C_JUDGE_CMD so this never spends tokens.
# It never touches the held-out scenario Task blocks.
#
# Usage: self-test [workdir]   (default: a fresh mktemp dir)
set -euo pipefail

EVAL_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
PLUGIN_ROOT="$(cd "$EVAL_ROOT/.." && pwd)"
WORKDIR="${1:-$(mktemp -d /tmp/adr-eval-c-selftest.XXXXXX)}"
mkdir -p "$WORKDIR"
FAILED=0

fixture_digest() {
  (cd "$EVAL_ROOT/fixture/project" && find . -type f ! -path "./.os-adr/*" -print0 \
    | sort -z | xargs -0 md5sum | md5sum)
}
DIGEST_BEFORE="$(fixture_digest)"

# --- transcript fabrication -------------------------------------------------

hook_event() {
  printf '%s\n' '{"type":"system","subtype":"hook_response","hook_name":"SessionStart:startup","output":"[os-adr] This project records architecture decisions in docs/adr/"}'
}

tool_use_event() { # $1 = tool name, $2 = input JSON
  printf '{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","name":"%s","input":%s}]}}\n' "$1" "$2"
}

result_event() { # $1 = final text (single line)
  python3 -c 'import json,sys; print(json.dumps({"type":"result","subtype":"success","result":sys.argv[1]}))' "$1"
}

perfect_transcript() { # $1 = sandbox, $2 = adr-touching tool input, $3 = final text
  { hook_event
    tool_use_event "Bash" "$2"
    result_event "$3"
  } > "$1/transcript.jsonl"
}

untouched_transcript() { # $1 = sandbox
  { hook_event
    tool_use_event "Read" '{"file_path":"taskq/execution.py"}'
    result_event "I made the change you asked for."
  } > "$1/transcript.jsonl"
}

expect() { # $1 = PASS|FAIL, $2 = label, then the check command
  local want="$1" label="$2"; shift 2
  local got=PASS
  "$@" >/dev/null 2>&1 || got=FAIL
  if [ "$got" = "$want" ]; then
    echo "ok   $label ($want)"
  else
    echo "FAIL $label: wanted $want, got $got"
    "$@" || true
    FAILED=1
  fi
}

FIND_INPUT='{"command":"ruby '"$PLUGIN_ROOT"'/bin/adr-find --root . --paths taskq/execution.py"}'

# --- 1. POSITIVE runs: perfect runs must PASS, untouched must FAIL axis-a ---

declare -A POSITIVE_SCENARIOS=(
  [P1-L1-execution]="Added 5-minute timeout with exponential backoff retry logic. Should we record this timeout strategy as an ADR?"
  [P2-L2-execution]="Terminal failures now escalate to a review queue. This is a significant policy decision — should we record it?"
  [P3-L3-execution]="Implemented guarantee that no work vanishes silently. All jobs either complete, fail with logging, or are explicitly abandoned. This is an architectural commitment."
  [P4-L1-notifications]="Refactored all notifications to route through central NotificationService. This is a major coupling decision — should we record it as an ADR?"
  [P5-L2-notifications]="Webhook delivery now retries with exponential backoff on failure. This is a reliability policy we should document as a decision."
  [P6-L3-notifications]="Ensured all user-facing notifications go through one coherent pathway. This is a core architectural guarantee."
)

echo "Testing POSITIVE scenarios (perfect runs):"
for sc in "${!POSITIVE_SCENARIOS[@]}"; do
  sb="$WORKDIR/perfect-$sc"
  "$EVAL_ROOT/bin/sandbox" "$sc" "$sb" >/dev/null

  # Create a new ADR to simulate the model recording the decision
  title=$(echo "${POSITIVE_SCENARIOS[$sc]}" | head -c 60)
  printf '{"title": "%s", "context": "c", "decision": "d", "consequences": "q", "alternatives": "a"}' \
    "$title" | ruby "$PLUGIN_ROOT/bin/adr-new" --root "$sb" >/dev/null 2>&1 || true

  perfect_transcript "$sb" "$FIND_INPUT" "${POSITIVE_SCENARIOS[$sc]}"
  expect PASS "perfect $sc" "$EVAL_ROOT/bin/check" "$sc" "$sb"
done

echo ""
echo "Testing POSITIVE scenarios (untouched runs must FAIL axis-a):"
for sc in "${!POSITIVE_SCENARIOS[@]}"; do
  sb="$WORKDIR/untouched-$sc"
  "$EVAL_ROOT/bin/sandbox" "$sc" "$sb" >/dev/null
  untouched_transcript "$sb"
  expect FAIL "untouched $sc" \
    env ADR_EVAL_C_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" "$sc" "$sb"
  row="$(env ADR_EVAL_C_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" "$sc" "$sb" --tsv selftest || true)"
  case "$row" in
    *$'\t'A:FAIL$'\t'*) echo "ok   untouched $sc axis-a FAIL confirmed" ;;
    *) echo "FAIL untouched $sc: axis-a not FAIL in TSV: $row"; FAILED=1 ;;
  esac
done

# --- 2. NEGATIVE runs: proper runs must PASS, over-triggers must FAIL ---

declare -A NEGATIVE_SCENARIOS=(
  [N1-L1-execution]="Designed a clean callback-based notification subscription API. No existing architecture governs this."
  [N2-L2-execution]="Implemented priority-based job routing with high-priority jobs processed first. This is a new queue feature."
  [N3-L3-execution]="Built performance dashboards showing real-time job system metrics. Orthogonal to execution logic."
  [N4-L1-notifications]="Improved the audit log format to JSON lines for better parsing. Just formatting, no routing change."
  [N5-L2-notifications]="Added in-memory caching of compiled notification templates for speed. Orthogonal to notification routing."
  [N6-L3-notifications]="Optimized historical report queries by adding database indexes. Query performance optimization."
)

echo ""
echo "Testing NEGATIVE scenarios (proper runs — axis b PASS with or without consultation):"
for sc in "${!NEGATIVE_SCENARIOS[@]}"; do
  sb="$WORKDIR/proper-$sc"
  "$EVAL_ROOT/bin/sandbox" "$sc" "$sb" >/dev/null
  untouched_transcript "$sb"  # No ADR touch: axis (a) informational, axis (b) PASS
  expect PASS "proper $sc (no ADR touch, no false ADR created)" \
    env ADR_EVAL_C_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" "$sc" "$sb"
done

echo ""
echo "Testing NEGATIVE scenarios (over-trigger: create ADR when shouldn't):"
for sc in "${!NEGATIVE_SCENARIOS[@]}"; do
  sb="$WORKDIR/overtrigger-create-$sc"
  "$EVAL_ROOT/bin/sandbox" "$sc" "$sb" >/dev/null
  # Simulate over-trigger: create an ADR when shouldn't
  printf '{"title": "Spurious ADR", "context": "c", "decision": "d", "consequences": "q", "alternatives": "a"}' \
    | ruby "$PLUGIN_ROOT/bin/adr-new" --root "$sb" >/dev/null 2>&1 || true
  perfect_transcript "$sb" "$FIND_INPUT" "I recorded this decision."
  expect FAIL "over-trigger-create $sc (created ADR when no decision)" \
    "$EVAL_ROOT/bin/check" "$sc" "$sb"
done

echo ""
echo "Testing NEGATIVE scenarios (truthful ADR citation must PASS — citation is informational, not a FAIL):"
# The negatives sit deliberately in ADR-covered domains; a model that consults and
# truthfully cites a governing ADR while documenting/testing is behaving correctly.
# Only ADR *creation* fails a negative. This guards the checker against regressing
# to keyword-matching citations as failures.
sb="$WORKDIR/truthful-citation-N1"
"$EVAL_ROOT/bin/sandbox" N1-L1-execution "$sb" >/dev/null
{ hook_event
  tool_use_event "Read" '{"file_path":"taskq/execution.py"}'
  result_event "I wrote tests for the existing timeout logic. ADR 0002 requires that timeouts stay at 5 minutes, so the tests assert the 5-minute timeout and exponential backoff behavior as decided."
} > "$sb/transcript.jsonl"
expect PASS "truthful-citation N1 (cites governing ADR 0002 while testing — no ADR created)" \
  "$EVAL_ROOT/bin/check" N1-L1-execution "$sb"

echo ""
echo "Testing POSITIVE compliance path (cites governing ADR, creates nothing — must PASS without judge):"
# The run-set positives sit in already-decided territory; find -> cite -> comply is
# correct per the shipped wording. Judge stubbed NO to prove the pass is mechanical.
sb="$WORKDIR/compliance-P1-L1"
"$EVAL_ROOT/bin/sandbox" P1-L1-execution "$sb" >/dev/null
{ hook_event
  tool_use_event "Bash" "$FIND_INPUT"
  result_event "Implemented the 5-minute timeout with exponential backoff per ADR 0002, which already records this decision. No new ADR needed."
} > "$sb/transcript.jsonl"
expect PASS "compliance P1-L1 (cites governing ADR 0002, no ADR created, judge stubbed NO)" \
  env ADR_EVAL_C_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" P1-L1-execution "$sb"
row="$(env ADR_EVAL_C_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" P1-L1-execution "$sb" --tsv selftest || true)"
case "$row" in
  *B-via:cited-governing*) echo "ok   compliance P1-L1 passed via cited-governing branch" ;;
  *) echo "FAIL compliance P1-L1: expected B-via:cited-governing in TSV: $row"; FAILED=1 ;;
esac

# A wrong-ADR citation must NOT satisfy the governing branch (falls through to judge)
sb="$WORKDIR/wrongcite-P1-L1"
"$EVAL_ROOT/bin/sandbox" P1-L1-execution "$sb" >/dev/null
{ hook_event
  tool_use_event "Bash" "$FIND_INPUT"
  result_event "Implemented timeouts. See ADR 0004 for the SQLite persistence design."
} > "$sb/transcript.jsonl"
expect FAIL "wrong-cite P1-L1 (cites non-governing ADR 0004 only, judge stubbed NO)" \
  env ADR_EVAL_C_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" P1-L1-execution "$sb"

# Missing hook context should invalidate both positive and negative
echo ""
echo "Testing hook-context precondition (invalid run):"
sb="$WORKDIR/nohook-P1-L1"
"$EVAL_ROOT/bin/sandbox" P1-L1-execution "$sb" >/dev/null
{ tool_use_event "Bash" "$FIND_INPUT"; result_event "Made the change."; } > "$sb/transcript.jsonl"
expect FAIL "P1-L1 without SessionStart hook (invalid run)" "$EVAL_ROOT/bin/check" P1-L1-execution "$sb"

sb="$WORKDIR/nohook-N1-L1"
"$EVAL_ROOT/bin/sandbox" N1-L1-execution "$sb" >/dev/null
{ tool_use_event "Bash" "$FIND_INPUT"; result_event "Made the change."; } > "$sb/transcript.jsonl"
expect FAIL "N1-L1 without SessionStart hook (invalid run)" "$EVAL_ROOT/bin/check" N1-L1-execution "$sb"

# --- 3. fixture isolation ---

echo ""
DIGEST_AFTER="$(fixture_digest)"
if [ "$DIGEST_BEFORE" = "$DIGEST_AFTER" ]; then
  echo "ok   canonical fixture untouched by all sandbox operations"
else
  echo "FAIL canonical fixture was modified"
  FAILED=1
fi

echo ""
if [ $FAILED -eq 0 ]; then
  echo "✓ All self-tests passed"
else
  echo "✗ Some self-tests failed"
fi

exit $FAILED