225 lines
10 KiB
Bash
Executable File
225 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Self-test the Eval C harness in both directions, model-free:
|
|
#
|
|
# 1. For each POSITIVE scenario, fabricate a "perfect" transcript/sandbox
|
|
# state (model did right thing) -> bin/check must PASS.
|
|
# 2. For each POSITIVE scenario, fabricate an "untouched" run (never
|
|
# consulted ADR system) -> bin/check must FAIL, axis (a) FAIL.
|
|
# 3. For each NEGATIVE scenario, fabricate a "proper" transcript (no ADR
|
|
# touch, no false ADR creation) -> bin/check must PASS.
|
|
# 4. For each NEGATIVE scenario, fabricate an "over-trigger" run (consulted
|
|
# ADR system or created ADR when shouldn't) -> bin/check must FAIL.
|
|
# 5. Sandbox isolation: the canonical fixture must be byte-identical after
|
|
# all sandbox operations.
|
|
#
|
|
# The judge is stubbed via ADR_EVAL_C_JUDGE_CMD so this never spends tokens.
|
|
# It never touches the held-out scenario Task blocks.
|
|
#
|
|
# Usage: self-test [workdir] (default: a fresh mktemp dir)
|
|
set -euo pipefail
|
|
|
|
EVAL_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
PLUGIN_ROOT="$(cd "$EVAL_ROOT/.." && pwd)"
|
|
WORKDIR="${1:-$(mktemp -d /tmp/adr-eval-c-selftest.XXXXXX)}"
|
|
mkdir -p "$WORKDIR"
|
|
FAILED=0
|
|
|
|
fixture_digest() {
|
|
(cd "$EVAL_ROOT/fixture/project" && find . -type f ! -path "./.os-adr/*" -print0 \
|
|
| sort -z | xargs -0 md5sum | md5sum)
|
|
}
|
|
DIGEST_BEFORE="$(fixture_digest)"
|
|
|
|
# --- transcript fabrication -------------------------------------------------
|
|
|
|
hook_event() {
|
|
printf '%s\n' '{"type":"system","subtype":"hook_response","hook_name":"SessionStart:startup","output":"[os-adr] This project records architecture decisions in docs/adr/"}'
|
|
}
|
|
|
|
tool_use_event() { # $1 = tool name, $2 = input JSON
|
|
printf '{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","name":"%s","input":%s}]}}\n' "$1" "$2"
|
|
}
|
|
|
|
result_event() { # $1 = final text (single line)
|
|
python3 -c 'import json,sys; print(json.dumps({"type":"result","subtype":"success","result":sys.argv[1]}))' "$1"
|
|
}
|
|
|
|
perfect_transcript() { # $1 = sandbox, $2 = adr-touching tool input, $3 = final text
|
|
{ hook_event
|
|
tool_use_event "Bash" "$2"
|
|
result_event "$3"
|
|
} > "$1/transcript.jsonl"
|
|
}
|
|
|
|
untouched_transcript() { # $1 = sandbox
|
|
{ hook_event
|
|
tool_use_event "Read" '{"file_path":"taskq/execution.py"}'
|
|
result_event "I made the change you asked for."
|
|
} > "$1/transcript.jsonl"
|
|
}
|
|
|
|
expect() { # $1 = PASS|FAIL, $2 = label, then the check command
|
|
local want="$1" label="$2"; shift 2
|
|
local got=PASS
|
|
"$@" >/dev/null 2>&1 || got=FAIL
|
|
if [ "$got" = "$want" ]; then
|
|
echo "ok $label ($want)"
|
|
else
|
|
echo "FAIL $label: wanted $want, got $got"
|
|
"$@" || true
|
|
FAILED=1
|
|
fi
|
|
}
|
|
|
|
FIND_INPUT='{"command":"ruby '"$PLUGIN_ROOT"'/bin/adr-find --root . --paths taskq/execution.py"}'
|
|
|
|
# --- 1. POSITIVE runs: perfect runs must PASS, untouched must FAIL axis-a ---
|
|
|
|
declare -A POSITIVE_SCENARIOS=(
|
|
[P1-L1-execution]="Added 5-minute timeout with exponential backoff retry logic. Should we record this timeout strategy as an ADR?"
|
|
[P2-L2-execution]="Terminal failures now escalate to a review queue. This is a significant policy decision — should we record it?"
|
|
[P3-L3-execution]="Implemented guarantee that no work vanishes silently. All jobs either complete, fail with logging, or are explicitly abandoned. This is an architectural commitment."
|
|
[P4-L1-notifications]="Refactored all notifications to route through central NotificationService. This is a major coupling decision — should we record it as an ADR?"
|
|
[P5-L2-notifications]="Webhook delivery now retries with exponential backoff on failure. This is a reliability policy we should document as a decision."
|
|
[P6-L3-notifications]="Ensured all user-facing notifications go through one coherent pathway. This is a core architectural guarantee."
|
|
)
|
|
|
|
echo "Testing POSITIVE scenarios (perfect runs):"
|
|
for sc in "${!POSITIVE_SCENARIOS[@]}"; do
|
|
sb="$WORKDIR/perfect-$sc"
|
|
"$EVAL_ROOT/bin/sandbox" "$sc" "$sb" >/dev/null
|
|
|
|
# Create a new ADR to simulate the model recording the decision
|
|
title=$(echo "${POSITIVE_SCENARIOS[$sc]}" | head -c 60)
|
|
printf '{"title": "%s", "context": "c", "decision": "d", "consequences": "q", "alternatives": "a"}' \
|
|
"$title" | ruby "$PLUGIN_ROOT/bin/adr-new" --root "$sb" >/dev/null 2>&1 || true
|
|
|
|
perfect_transcript "$sb" "$FIND_INPUT" "${POSITIVE_SCENARIOS[$sc]}"
|
|
expect PASS "perfect $sc" "$EVAL_ROOT/bin/check" "$sc" "$sb"
|
|
done
|
|
|
|
echo ""
|
|
echo "Testing POSITIVE scenarios (untouched runs must FAIL axis-a):"
|
|
for sc in "${!POSITIVE_SCENARIOS[@]}"; do
|
|
sb="$WORKDIR/untouched-$sc"
|
|
"$EVAL_ROOT/bin/sandbox" "$sc" "$sb" >/dev/null
|
|
untouched_transcript "$sb"
|
|
expect FAIL "untouched $sc" \
|
|
env ADR_EVAL_C_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" "$sc" "$sb"
|
|
row="$(env ADR_EVAL_C_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" "$sc" "$sb" --tsv selftest || true)"
|
|
case "$row" in
|
|
*$'\t'A:FAIL$'\t'*) echo "ok untouched $sc axis-a FAIL confirmed" ;;
|
|
*) echo "FAIL untouched $sc: axis-a not FAIL in TSV: $row"; FAILED=1 ;;
|
|
esac
|
|
done
|
|
|
|
# --- 2. NEGATIVE runs: proper runs must PASS, over-triggers must FAIL ---
|
|
|
|
declare -A NEGATIVE_SCENARIOS=(
|
|
[N1-L1-execution]="Designed a clean callback-based notification subscription API. No existing architecture governs this."
|
|
[N2-L2-execution]="Implemented priority-based job routing with high-priority jobs processed first. This is a new queue feature."
|
|
[N3-L3-execution]="Built performance dashboards showing real-time job system metrics. Orthogonal to execution logic."
|
|
[N4-L1-notifications]="Improved the audit log format to JSON lines for better parsing. Just formatting, no routing change."
|
|
[N5-L2-notifications]="Added in-memory caching of compiled notification templates for speed. Orthogonal to notification routing."
|
|
[N6-L3-notifications]="Optimized historical report queries by adding database indexes. Query performance optimization."
|
|
)
|
|
|
|
echo ""
|
|
echo "Testing NEGATIVE scenarios (proper runs — axis b PASS with or without consultation):"
|
|
for sc in "${!NEGATIVE_SCENARIOS[@]}"; do
|
|
sb="$WORKDIR/proper-$sc"
|
|
"$EVAL_ROOT/bin/sandbox" "$sc" "$sb" >/dev/null
|
|
untouched_transcript "$sb" # No ADR touch: axis (a) informational, axis (b) PASS
|
|
expect PASS "proper $sc (no ADR touch, no false ADR created)" \
|
|
env ADR_EVAL_C_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" "$sc" "$sb"
|
|
done
|
|
|
|
echo ""
|
|
echo "Testing NEGATIVE scenarios (over-trigger: create ADR when shouldn't):"
|
|
for sc in "${!NEGATIVE_SCENARIOS[@]}"; do
|
|
sb="$WORKDIR/overtrigger-create-$sc"
|
|
"$EVAL_ROOT/bin/sandbox" "$sc" "$sb" >/dev/null
|
|
# Simulate over-trigger: create an ADR when shouldn't
|
|
printf '{"title": "Spurious ADR", "context": "c", "decision": "d", "consequences": "q", "alternatives": "a"}' \
|
|
| ruby "$PLUGIN_ROOT/bin/adr-new" --root "$sb" >/dev/null 2>&1 || true
|
|
perfect_transcript "$sb" "$FIND_INPUT" "I recorded this decision."
|
|
expect FAIL "over-trigger-create $sc (created ADR when no decision)" \
|
|
"$EVAL_ROOT/bin/check" "$sc" "$sb"
|
|
done
|
|
|
|
echo ""
|
|
echo "Testing NEGATIVE scenarios (truthful ADR citation must PASS — citation is informational, not a FAIL):"
|
|
# The negatives sit deliberately in ADR-covered domains; a model that consults and
|
|
# truthfully cites a governing ADR while documenting/testing is behaving correctly.
|
|
# Only ADR *creation* fails a negative. This guards the checker against regressing
|
|
# to keyword-matching citations as failures.
|
|
sb="$WORKDIR/truthful-citation-N1"
|
|
"$EVAL_ROOT/bin/sandbox" N1-L1-execution "$sb" >/dev/null
|
|
{ hook_event
|
|
tool_use_event "Read" '{"file_path":"taskq/execution.py"}'
|
|
result_event "I wrote tests for the existing timeout logic. ADR 0002 requires that timeouts stay at 5 minutes, so the tests assert the 5-minute timeout and exponential backoff behavior as decided."
|
|
} > "$sb/transcript.jsonl"
|
|
expect PASS "truthful-citation N1 (cites governing ADR 0002 while testing — no ADR created)" \
|
|
"$EVAL_ROOT/bin/check" N1-L1-execution "$sb"
|
|
|
|
echo ""
|
|
echo "Testing POSITIVE compliance path (cites governing ADR, creates nothing — must PASS without judge):"
|
|
# The run-set positives sit in already-decided territory; find -> cite -> comply is
|
|
# correct per the shipped wording. Judge stubbed NO to prove the pass is mechanical.
|
|
sb="$WORKDIR/compliance-P1-L1"
|
|
"$EVAL_ROOT/bin/sandbox" P1-L1-execution "$sb" >/dev/null
|
|
{ hook_event
|
|
tool_use_event "Bash" "$FIND_INPUT"
|
|
result_event "Implemented the 5-minute timeout with exponential backoff per ADR 0002, which already records this decision. No new ADR needed."
|
|
} > "$sb/transcript.jsonl"
|
|
expect PASS "compliance P1-L1 (cites governing ADR 0002, no ADR created, judge stubbed NO)" \
|
|
env ADR_EVAL_C_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" P1-L1-execution "$sb"
|
|
row="$(env ADR_EVAL_C_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" P1-L1-execution "$sb" --tsv selftest || true)"
|
|
case "$row" in
|
|
*B-via:cited-governing*) echo "ok compliance P1-L1 passed via cited-governing branch" ;;
|
|
*) echo "FAIL compliance P1-L1: expected B-via:cited-governing in TSV: $row"; FAILED=1 ;;
|
|
esac
|
|
|
|
# A wrong-ADR citation must NOT satisfy the governing branch (falls through to judge)
|
|
sb="$WORKDIR/wrongcite-P1-L1"
|
|
"$EVAL_ROOT/bin/sandbox" P1-L1-execution "$sb" >/dev/null
|
|
{ hook_event
|
|
tool_use_event "Bash" "$FIND_INPUT"
|
|
result_event "Implemented timeouts. See ADR 0004 for the SQLite persistence design."
|
|
} > "$sb/transcript.jsonl"
|
|
expect FAIL "wrong-cite P1-L1 (cites non-governing ADR 0004 only, judge stubbed NO)" \
|
|
env ADR_EVAL_C_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" P1-L1-execution "$sb"
|
|
|
|
# Missing hook context should invalidate both positive and negative
|
|
echo ""
|
|
echo "Testing hook-context precondition (invalid run):"
|
|
sb="$WORKDIR/nohook-P1-L1"
|
|
"$EVAL_ROOT/bin/sandbox" P1-L1-execution "$sb" >/dev/null
|
|
{ tool_use_event "Bash" "$FIND_INPUT"; result_event "Made the change."; } > "$sb/transcript.jsonl"
|
|
expect FAIL "P1-L1 without SessionStart hook (invalid run)" "$EVAL_ROOT/bin/check" P1-L1-execution "$sb"
|
|
|
|
sb="$WORKDIR/nohook-N1-L1"
|
|
"$EVAL_ROOT/bin/sandbox" N1-L1-execution "$sb" >/dev/null
|
|
{ tool_use_event "Bash" "$FIND_INPUT"; result_event "Made the change."; } > "$sb/transcript.jsonl"
|
|
expect FAIL "N1-L1 without SessionStart hook (invalid run)" "$EVAL_ROOT/bin/check" N1-L1-execution "$sb"
|
|
|
|
# --- 3. fixture isolation ---
|
|
|
|
echo ""
|
|
DIGEST_AFTER="$(fixture_digest)"
|
|
if [ "$DIGEST_BEFORE" = "$DIGEST_AFTER" ]; then
|
|
echo "ok canonical fixture untouched by all sandbox operations"
|
|
else
|
|
echo "FAIL canonical fixture was modified"
|
|
FAILED=1
|
|
fi
|
|
|
|
echo ""
|
|
if [ $FAILED -eq 0 ]; then
|
|
echo "✓ All self-tests passed"
|
|
else
|
|
echo "✗ Some self-tests failed"
|
|
fi
|
|
|
|
exit $FAILED
|