35 lines
1.2 KiB
Plaintext
35 lines
1.2 KiB
Plaintext
|
#!/usr/bin/env bash
|
||
|
|
# Headless fallback runner (burns more subscription credit than in-session subagents).
|
||
|
|
# Usage: run-headless <S1..S6> <model> <workdir> [results.tsv]
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
SCENARIO="${1:?usage: run-headless <scenario> <model> <workdir> [results.tsv]}"
|
||
|
|
MODEL="${2:?model required (haiku|sonnet|opus)}"
|
||
|
|
WORKDIR="${3:?workdir required}"
|
||
|
|
RESULTS="${4:-$WORKDIR/results.tsv}"
|
||
|
|
|
||
|
|
EVAL_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||
|
|
PLUGIN_ROOT="$(cd "$EVAL_ROOT/.." && pwd)"
|
||
|
|
SANDBOX="$WORKDIR/$SCENARIO-$MODEL-$(date +%s)"
|
||
|
|
|
||
|
|
case "$SCENARIO" in
|
||
|
|
S1|S2) SKILL=create ;;
|
||
|
|
S3|S4) SKILL=find ;;
|
||
|
|
S5) SKILL=init ;;
|
||
|
|
S6) SKILL=migrate ;;
|
||
|
|
*) echo "unknown scenario $SCENARIO" >&2; exit 2 ;;
|
||
|
|
esac
|
||
|
|
|
||
|
|
"$EVAL_ROOT/bin/sandbox" "$SCENARIO" "$SANDBOX" >/dev/null
|
||
|
|
TASK="$(awk '/^## Task/{found=1; next} found' "$EVAL_ROOT/scenarios/$SCENARIO.md")"
|
||
|
|
|
||
|
|
PROMPT="$(awk '/^---$/{found=1; next} found' "$EVAL_ROOT/runner-prompt.md" \
|
||
|
|
| sed -e "s|{{SANDBOX}}|$SANDBOX|g" \
|
||
|
|
-e "s|{{PLUGIN_ROOT}}|$PLUGIN_ROOT|g" \
|
||
|
|
-e "s|{{SKILL}}|$SKILL|g")"
|
||
|
|
PROMPT="${PROMPT//\{\{TASK\}\}/$TASK}"
|
||
|
|
|
||
|
|
(cd "$SANDBOX" && claude -p --model "$MODEL" --dangerously-skip-permissions "$PROMPT")
|
||
|
|
|
||
|
|
"$EVAL_ROOT/bin/check" "$SCENARIO" "$SANDBOX" --tsv "$MODEL" | tee -a "$RESULTS"
|