#!/usr/bin/env bash
# Headless runner for the os-vault write-behavior eval — the ONLY valid execution mode
# (fresh `claude -p` with cwd = sandbox/project so the real SessionStart hook fires).
#
# Usage: run <scenario> <model> <workdir> [--reps N] [--results FILE]
#
#   scenario  P*-L*|N*-L* (e.g., P1-L1-reporting, N4-L1-delivery)
#   model     haiku|sonnet|opus|...
#   workdir   sandboxes created under here as <scenario>-<model>-rN
#   --reps    repeated executions (default 1)
#
# Each rep: fresh sandbox -> claude -p (cwd = sandbox/project, OS_VAULT_PATH +
# OS_VAULT_SKIP_REBUILD set for that invocation only) -> full stream-json transcript
# saved to <sandbox>/transcript.jsonl -> bin/check appends one TSV row.
set -euo pipefail

SCENARIO="${1:?usage: run <scenario> <model> <workdir> [--reps N] [--results FILE]}"
MODEL="${2:?model required (haiku|sonnet|...)}"
WORKDIR="${3:?workdir required}"
shift 3

REPS=1
RESULTS=""
while [ $# -gt 0 ]; do
  case "$1" in
    --reps)    REPS="${2:?--reps needs a number}"; shift 2 ;;
    --results) RESULTS="${2:?--results needs a path}"; shift 2 ;;
    *) echo "unknown option: $1" >&2; exit 2 ;;
  esac
done

EVAL_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
SCENARIO_FILE="$EVAL_ROOT/scenarios/${SCENARIO}.md"
if [ ! -f "$SCENARIO_FILE" ]; then
  SCENARIO_FILE="$EVAL_ROOT/scenarios-reserve/${SCENARIO}.md"
fi
[ -f "$SCENARIO_FILE" ] || { echo "unknown scenario: $SCENARIO (checked scenarios/ and scenarios-reserve/)" >&2; exit 2; }
mkdir -p "$WORKDIR"
RESULTS="${RESULTS:-$WORKDIR/results.tsv}"

TASK="$(awk '/^## Task/{found=1; next} found' "$SCENARIO_FILE")"

for rep in $(seq 1 "$REPS"); do
  SANDBOX="$WORKDIR/$SCENARIO-$MODEL-r$rep"
  "$EVAL_ROOT/bin/sandbox" "$SCENARIO" "$SANDBOX" >/dev/null

  # cwd = sandbox/project: the SessionStart hook resolves the project root from here.
  # OS_VAULT_PATH/OS_VAULT_SKIP_REBUILD are exported only for this one command.
  (cd "$SANDBOX/project" && OS_VAULT_PATH="$SANDBOX/vault" OS_VAULT_SKIP_REBUILD=1 claude -p \
      --model "$MODEL" \
      --output-format stream-json --verbose \
      --dangerously-skip-permissions \
      "$TASK" > "$SANDBOX/transcript.jsonl") || echo "claude exited non-zero for $SANDBOX" >&2

  # bin/check exits 1 on FAIL; without the || true, set -euo pipefail would
  # abort the rep loop on the first failing rep (the eval-c pipefail lesson).
  "$EVAL_ROOT/bin/check" "$SCENARIO" "$SANDBOX" --tsv "$MODEL" | tee -a "$RESULTS" || true
done