cc-os/plugins/os-context/eval/bin/run

#!/usr/bin/env bash
# Headless runner for the os-context eval — the ONLY valid execution mode
# (mid-session orchestration behavior under the real global SessionStart-injected
# prompts/session-start/10-orchestration.md; fresh `claude -p` per rep with cwd = sandbox).
#
# Usage: run <scenario> <model> <workdir> [--reps N] [--results FILE]
#
# E1P* scenarios get CLAUDE_CODE_SUBAGENT_MODEL=haiku injected into the child
# environment (the cluster-1 downgrade stub). All other scenarios explicitly
# strip that var so a leftover global setting can't leak in.
#
# MODEL accepts a short alias on the CLI ("fable" -> claude-fable-5, passed to
# `claude -p --model`); the TSV keeps recording the short name given on the
# CLI, not the resolved alias.
set -euo pipefail

SCENARIO="${1:?usage: run <scenario> <model> <workdir> [--reps N] [--results FILE]}"
MODEL="${2:?model required (haiku|sonnet|...)}"
WORKDIR="${3:?workdir required}"
shift 3

REPS=1
RESULTS=""
while [ $# -gt 0 ]; do
  case "$1" in
    --reps)    REPS="${2:?--reps needs a number}"; shift 2 ;;
    --results) RESULTS="${2:?--results needs a path}"; shift 2 ;;
    *) echo "unknown option: $1" >&2; exit 2 ;;
  esac
done

EVAL_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
SCENARIO_FILE="$EVAL_ROOT/scenarios/${SCENARIO}.md"
[ -f "$SCENARIO_FILE" ] || SCENARIO_FILE="$EVAL_ROOT/scenarios-reserve/${SCENARIO}.md"
[ -f "$SCENARIO_FILE" ] || { echo "unknown scenario: $SCENARIO" >&2; exit 2; }

mkdir -p "$WORKDIR"
RESULTS="${RESULTS:-$WORKDIR/results.tsv}"

TASK="$(awk '/^## Task/{found=1; next} found' "$SCENARIO_FILE")"

ENV_PREFIX=(env -u CLAUDE_CODE_SUBAGENT_MODEL)
case "$SCENARIO" in
  E1P*) ENV_PREFIX=(env CLAUDE_CODE_SUBAGENT_MODEL=haiku) ;;
esac
# E5* scenarios take the default env arm above (no downgrade stub) — same
# case statement, no new arm needed.

case "$MODEL" in
  fable) CLI_MODEL="claude-fable-5" ;;
  *)     CLI_MODEL="$MODEL" ;;
esac

for rep in $(seq 1 "$REPS"); do
  SANDBOX="$WORKDIR/$SCENARIO-$MODEL-r$rep"
  "$EVAL_ROOT/bin/sandbox" "$SCENARIO" "$SANDBOX" >/dev/null

  # cwd = sandbox: global plugins' SessionStart hooks fire for real.
  (cd "$SANDBOX" && timeout 1500 "${ENV_PREFIX[@]}" claude -p \
      --model "$CLI_MODEL" \
      --output-format stream-json --verbose \
      --dangerously-skip-permissions \
      "$TASK" > cli-output.jsonl) || echo "claude exited non-zero for $SANDBOX" >&2

  # check exits 1 on FAIL; guard so pipefail can't abort the remaining reps.
  "$EVAL_ROOT/bin/check" "$SCENARIO" "$SANDBOX" --tsv "$MODEL" "$rep" | tee -a "$RESULTS" || true
done