cc-os/plugins/os-adr/eval-b/bin/self-test

#!/usr/bin/env bash
# Self-test the Eval B harness in both directions, model-free:
#
#   1. For each scenario, fabricate a "perfect" transcript/sandbox state
#      (an agent that did the right thing) -> bin/check must PASS.
#   2. For each scenario, fabricate an "untouched" run (agent never consulted
#      the ADR system) -> bin/check must FAIL, axis (a) FAIL specifically.
#   3. Sandbox isolation: the canonical fixture must be byte-identical after
#      all sandbox operations.
#
# The W-scenario LLM judge is stubbed via ADR_EVAL_B_JUDGE_CMD so this never
# spends model tokens. It never touches the held-out scenario Task blocks.
#
# Usage: self-test [workdir]   (default: a fresh mktemp dir)
set -euo pipefail

EVAL_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
PLUGIN_ROOT="$(cd "$EVAL_ROOT/.." && pwd)"
WORKDIR="${1:-$(mktemp -d /tmp/adr-eval-b-selftest.XXXXXX)}"
mkdir -p "$WORKDIR"
FAILED=0

fixture_digest() {
  (cd "$EVAL_ROOT/fixture/project" && find . -type f ! -path "./graphify-out/*" -print0 \
    | sort -z | xargs -0 md5sum | md5sum)
}
DIGEST_BEFORE="$(fixture_digest)"

# --- transcript fabrication -------------------------------------------------

hook_event() {
  printf '%s\n' '{"type":"system","subtype":"hook_response","hook_name":"SessionStart:startup","output":"[os-adr] This project records architecture decisions in docs/adr/"}'
}

tool_use_event() { # $1 = tool name, $2 = input JSON
  printf '{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","name":"%s","input":%s}]}}\n' "$1" "$2"
}

result_event() { # $1 = final text (single line, no quotes needed beyond basic)
  python3 -c 'import json,sys; print(json.dumps({"type":"result","subtype":"success","result":sys.argv[1]}))' "$1"
}

perfect_transcript() { # $1 = sandbox, $2 = adr-touching tool input, $3 = final text
  { hook_event
    tool_use_event "Bash" "$2"
    result_event "$3"
  } > "$1/transcript.jsonl"
}

untouched_transcript() { # $1 = sandbox
  { hook_event
    tool_use_event "Read" '{"file_path":"lib/relay/delivery.rb"}'
    result_event "Done. I made the change you asked for."
  } > "$1/transcript.jsonl"
}

# --- expectations -----------------------------------------------------------

expect() { # $1 = PASS|FAIL, $2 = label, then the check command
  local want="$1" label="$2"; shift 2
  local got=PASS
  "$@" >/dev/null 2>&1 || got=FAIL
  if [ "$got" = "$want" ]; then
    echo "ok   $label ($want)"
  else
    echo "FAIL $label: wanted $want, got $got"
    "$@" || true
    FAILED=1
  fi
}

FIND_INPUT='{"command":"ruby '"$PLUGIN_ROOT"'/bin/adr-find --root . --paths lib/relay/delivery.rb"}'

# --- 1. perfect runs must PASS ----------------------------------------------

declare -A FINAL_TEXT=(
  [R1]="Stopped before implementing: ADR 0002 requires all outbound HTTP to go through the shared HttpClient, so a direct Net::HTTP call would conflict with that decision. I added ping via the wrapper instead."
  [R2]="Flat 2-second retries would violate ADR 0003 (exponential backoff, capped attempts). The older retry decision 0001 is superseded by 0003, so it does not apply. I did not change the backoff."
  [R3]="Per ADR 0005, recorded event times are UTC ISO-8601 strings, so the first-seen value is written with Time.now.utc.iso8601 — not local server time."
  [R4]="A retry-until-success loop in reports.rb would conflict with ADR 0003, which governs delivery retries one hop away: re-sends must follow the exponential backoff schedule. I routed redelivery through the backoff instead."
)

for sc in R1 R2 R3 R4; do
  sb="$WORKDIR/perfect-$sc"
  "$EVAL_ROOT/bin/sandbox" "$sc" "$sb" >/dev/null
  perfect_transcript "$sb" "$FIND_INPUT" "${FINAL_TEXT[$sc]}"
  expect PASS "perfect $sc" "$EVAL_ROOT/bin/check" "$sc" "$sb"
done

declare -A W_TITLE=(
  [W1]="Use SQLite for delivery history persistence"
  [W2]="Raise exceptions instead of returning Result objects"
  [W3]="Allow concurrent delivery dispatch, dropping strict ordering"
)

for sc in W1 W2 W3; do
  sb="$WORKDIR/perfect-$sc"
  "$EVAL_ROOT/bin/sandbox" "$sc" "$sb" >/dev/null
  printf '{"title": "%s", "context": "c", "decision": "d", "consequences": "q", "alternatives": "a"}' \
    "${W_TITLE[$sc]}" | ruby "$PLUGIN_ROOT/bin/adr-new" --root "$sb" >/dev/null
  perfect_transcript "$sb" '{"command":"ruby '"$PLUGIN_ROOT"'/bin/adr-new --root ."}' \
    "Recorded the decision as a new ADR before implementing."
  expect PASS "perfect $sc (mechanical new-ADR path)" "$EVAL_ROOT/bin/check" "$sc" "$sb"
done

# W judge fallback path: no new ADR file, final message proposes recording.
sb="$WORKDIR/perfect-W1-judge"
"$EVAL_ROOT/bin/sandbox" W1 "$sb" >/dev/null
perfect_transcript "$sb" "$FIND_INPUT" \
  "I implemented SQLite-backed history. This engine choice is hard to reverse — want me to record it as an ADR?"
expect PASS "perfect W1 (judge fallback, stub YES)" \
  env ADR_EVAL_B_JUDGE_CMD="printf YES" "$EVAL_ROOT/bin/check" W1 "$sb"

# --- 2. untouched runs must FAIL (axis a) -----------------------------------

for sc in W1 W2 W3 R1 R2 R3 R4 R4-nograph; do
  sb="$WORKDIR/untouched-$sc"
  "$EVAL_ROOT/bin/sandbox" "$sc" "$sb" >/dev/null
  untouched_transcript "$sb"
  expect FAIL "untouched $sc" \
    env ADR_EVAL_B_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" "$sc" "$sb"
  row="$(env ADR_EVAL_B_JUDGE_CMD="printf NO" "$EVAL_ROOT/bin/check" "$sc" "$sb" --tsv selftest || true)"
  case "$row" in
    *$'\t'A:FAIL$'\t'*) echo "ok   untouched $sc axis-a FAIL confirmed" ;;
    *) echo "FAIL untouched $sc: axis-a not FAIL in TSV: $row"; FAILED=1 ;;
  esac
done

# R2 distractor trap: cites superseded 0001 as if live -> axis (b) FAIL.
sb="$WORKDIR/distractor-R2"
"$EVAL_ROOT/bin/sandbox" R2 "$sb" >/dev/null
perfect_transcript "$sb" "$FIND_INPUT" \
  "ADR 0001 says deliveries retry three times, so I set the retries to match that decision."
expect FAIL "R2 citing superseded distractor 0001" "$EVAL_ROOT/bin/check" R2 "$sb"

# Missing hook context invalidates the run.
sb="$WORKDIR/nohook-R1"
"$EVAL_ROOT/bin/sandbox" R1 "$sb" >/dev/null
{ tool_use_event "Bash" "$FIND_INPUT"; result_event "${FINAL_TEXT[R1]}"; } > "$sb/transcript.jsonl"
expect FAIL "R1 without SessionStart hook context (invalid run)" "$EVAL_ROOT/bin/check" R1 "$sb"

# --- 3. fixture isolation ---------------------------------------------------

DIGEST_AFTER="$(fixture_digest)"
if [ "$DIGEST_BEFORE" = "$DIGEST_AFTER" ]; then
  echo "ok   canonical fixture untouched by all sandbox operations"
else
  echo "FAIL canonical fixture was modified"
  FAILED=1
fi

echo
if [ "$FAILED" = 0 ]; then
  echo "self-test PASS (workdir: $WORKDIR)"
else
  echo "self-test FAIL (workdir kept for inspection: $WORKDIR)"
fi
exit "$FAILED"