58 lines
2.2 KiB
Bash
Executable File
58 lines
2.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Headless runner for Eval C — the ONLY valid execution mode (fresh `claude -p`
|
|
# with cwd = sandbox so the real SessionStart hook fires).
|
|
#
|
|
# Usage: run <scenario> <model> <workdir> [--reps N] [--results FILE]
|
|
#
|
|
# scenario P*-L*|N*-L* (e.g., P1-L1-execution, N2-L2-execution)
|
|
# model haiku|sonnet|opus|...
|
|
# workdir sandboxes created under here as <scenario>-<model>-rN
|
|
# --reps repeated executions (default 1; design.md Decision 5)
|
|
#
|
|
# Each rep: fresh sandbox -> claude -p with only the scenario task prompt
|
|
# (no system-level hints) -> full stream-json transcript saved ->
|
|
# bin/check appends one TSV row.
|
|
set -euo pipefail
|
|
|
|
SCENARIO="${1:?usage: run <scenario> <model> <workdir> [--reps N] [--results FILE]}"
|
|
MODEL="${2:?model required (haiku|sonnet|...)}"
|
|
WORKDIR="${3:?workdir required}"
|
|
shift 3
|
|
|
|
REPS=1
|
|
RESULTS=""
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--reps) REPS="${2:?--reps needs a number}"; shift 2 ;;
|
|
--results) RESULTS="${2:?--results needs a path}"; shift 2 ;;
|
|
*) echo "unknown option: $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
EVAL_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
SCENARIO_FILE="$EVAL_ROOT/scenarios/${SCENARIO}.md"
|
|
if [ ! -f "$SCENARIO_FILE" ]; then
|
|
SCENARIO_FILE="$EVAL_ROOT/scenarios-reserve/${SCENARIO}.md"
|
|
fi
|
|
[ -f "$SCENARIO_FILE" ] || { echo "unknown scenario: $SCENARIO (checked scenarios/ and scenarios-reserve/)" >&2; exit 2; }
|
|
mkdir -p "$WORKDIR"
|
|
RESULTS="${RESULTS:-$WORKDIR/results.tsv}"
|
|
|
|
TASK="$(awk '/^## Task/{found=1; next} found' "$SCENARIO_FILE")"
|
|
|
|
for rep in $(seq 1 "$REPS"); do
|
|
SANDBOX="$WORKDIR/$SCENARIO-$MODEL-r$rep"
|
|
"$EVAL_ROOT/bin/sandbox" "$SCENARIO" "$SANDBOX" >/dev/null
|
|
|
|
# cwd = sandbox: the SessionStart hook resolves the project root from here.
|
|
(cd "$SANDBOX" && claude -p \
|
|
--model "$MODEL" \
|
|
--output-format stream-json --verbose \
|
|
--dangerously-skip-permissions \
|
|
"$TASK" > transcript.jsonl) || echo "claude exited non-zero for $SANDBOX" >&2
|
|
|
|
# bin/check exits 1 on FAIL; without the || true, set -euo pipefail would
|
|
# abort the rep loop on the first failing rep (found on the first grid run).
|
|
"$EVAL_ROOT/bin/check" "$SCENARIO" "$SANDBOX" --tsv "$MODEL" | tee -a "$RESULTS" || true
|
|
done
|