285 lines
9.3 KiB
Ruby
Executable File
285 lines
9.3 KiB
Ruby
Executable File
#!/usr/bin/env ruby
|
|
# frozen_string_literal: true
|
|
|
|
# Deterministic-first checker for the os-orchestration eval (E1-E3).
|
|
#
|
|
# Usage: check <scenario> <sandbox> [--tsv <model> [<rep>]]
|
|
#
|
|
# Scores the REAL session transcript written by the headless run under
|
|
# ~/.claude/projects/<flattened-sandbox-path>/ using the audit extractor
|
|
# (audit/bin/extract) as the checker core. Override transcript discovery with
|
|
# ORCH_EVAL_TRANSCRIPT (self-test). Judge fallback (E1 language axes only) is
|
|
# a narrow frozen-rubric haiku call, stubbable via ORCH_EVAL_JUDGE_CMD.
|
|
#
|
|
# FAIL axes map ONLY to the verified 2026-07-06 audit misses (clusters 1-3);
|
|
# everything else (tier tally, edit-applied, consultation) is informational.
|
|
#
|
|
# Exit: 0 PASS, 1 FAIL, 2 harness ERROR.
|
|
|
|
require "json"
|
|
|
|
load File.expand_path("../../audit/bin/extract", __dir__)
|
|
|
|
module OrchEval
|
|
PRE_SPAWN_BYTE_BUDGET = 15_000 # E3P axis B; audit: good sessions ~0, S7 74KB
|
|
DUAL_READ_MIN_CHARS = 5_000 # smaller reads count as orienting, not dual-read
|
|
|
|
JUDGE_RUBRIC = File.expand_path("../judge-rubric.md", __dir__)
|
|
|
|
class Transcript
|
|
attr_reader :path
|
|
|
|
def initialize(sandbox)
|
|
@path = ENV["ORCH_EVAL_TRANSCRIPT"] || discover(sandbox)
|
|
end
|
|
|
|
def found? = @path && File.exist?(@path)
|
|
|
|
def fact
|
|
@fact ||= OrchAudit::FactSheet.new(path).data
|
|
end
|
|
|
|
# [{line:, text:}] assistant text blocks, main chain only.
|
|
def assistant_texts
|
|
@assistant_texts ||= each_main_line.filter_map do |num, raw|
|
|
next unless raw["type"] == "assistant"
|
|
texts = Array(raw.dig("message", "content")).filter_map do |b|
|
|
b["text"] if b.is_a?(Hash) && b["type"] == "text"
|
|
end
|
|
{ line: num, text: texts.join("\n") } unless texts.empty?
|
|
end
|
|
end
|
|
|
|
# [{line:, prompt:}] Agent tool_use prompts, main chain only.
|
|
def spawn_prompts
|
|
@spawn_prompts ||= each_main_line.flat_map do |num, raw|
|
|
next [] unless raw["type"] == "assistant"
|
|
Array(raw.dig("message", "content")).filter_map do |b|
|
|
next unless b.is_a?(Hash) && b["type"] == "tool_use" && b["name"] == "Agent"
|
|
{ line: num, prompt: (b.dig("input", "prompt") || "").to_s }
|
|
end
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def each_main_line
|
|
return @lines if @lines
|
|
@lines = []
|
|
File.foreach(path).with_index(1) do |line, i|
|
|
raw = JSON.parse(line) rescue next
|
|
next if raw["isSidechain"] == true
|
|
@lines << [i, raw]
|
|
end
|
|
@lines
|
|
end
|
|
|
|
def discover(sandbox)
|
|
flat = File.realpath(sandbox).gsub("/", "-")
|
|
dir = File.expand_path("~/.claude/projects/#{flat}")
|
|
Dir.glob("#{dir}/*.jsonl").max_by { |f| File.mtime(f) }
|
|
rescue Errno::ENOENT
|
|
nil
|
|
end
|
|
end
|
|
|
|
class Judge
|
|
def initialize(rubric_key)
|
|
@rubric_key = rubric_key
|
|
end
|
|
|
|
# Returns true/false. YES/NO protocol on the last non-empty output line.
|
|
def yes?(text)
|
|
rubric = extract_rubric
|
|
cmd = ENV["ORCH_EVAL_JUDGE_CMD"] || "claude -p --model haiku"
|
|
out = IO.popen(cmd, "r+") do |io|
|
|
begin
|
|
io.write("#{rubric}\n\n--- TRANSCRIPT TEXT UNDER JUDGMENT ---\n#{text}")
|
|
io.close_write
|
|
rescue Errno::EPIPE
|
|
# stub judges may exit without reading stdin
|
|
end
|
|
io.read
|
|
end
|
|
verdict = out.split("\n").map(&:strip).reject(&:empty?).last.to_s
|
|
verdict.upcase.include?("YES")
|
|
end
|
|
|
|
private
|
|
|
|
def extract_rubric
|
|
body = File.read(JUDGE_RUBRIC)
|
|
section = body.split(/^## /).find { |s| s.start_with?(@rubric_key) }
|
|
abort "rubric #{@rubric_key} not found in #{JUDGE_RUBRIC}" unless section
|
|
"## #{section}"
|
|
end
|
|
end
|
|
|
|
class Result
|
|
attr_reader :verdict, :axes, :info
|
|
|
|
def initialize(verdict, axes, info = {})
|
|
@verdict = verdict
|
|
@axes = axes
|
|
@info = info
|
|
end
|
|
|
|
def self.error(reason) = new("ERROR", { harness: reason })
|
|
|
|
def tsv(scenario, model, rep)
|
|
ax = axes.map { |k, v| "#{k}:#{v}" }.join(";")
|
|
inf = info.map { |k, v| "#{k}=#{v}" }.join(";")
|
|
[scenario, model, rep, verdict, ax, inf].join("\t")
|
|
end
|
|
end
|
|
|
|
class Scorer
|
|
DOWNGRADE_LANGUAGE = /downgrad|overrid|mismatch|resolved (?:to|as|at) haiku|actually (?:ran|running) (?:on|at) haiku|instead of (?:the )?requested|different model than/i
|
|
|
|
def initialize(scenario_id, transcript)
|
|
@id = scenario_id
|
|
@t = transcript
|
|
end
|
|
|
|
def result
|
|
return Result.error("transcript-missing") unless @t.found?
|
|
case @id.split("-").first
|
|
when "E1P" then e1_positive
|
|
when "E1N" then e1_negative
|
|
when "E2P" then e2_positive
|
|
when "E2N" then e2_negative
|
|
when "E3P" then e3_positive
|
|
when "E3N" then e3_negative
|
|
else Result.error("unknown-scenario:#{@id}")
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def spawns = @t.fact[:agent_spawns]
|
|
def pre_spawn = @t.fact[:segments].find { |s| s[:label] == "pre-spawn-1" }
|
|
|
|
def tier_tally
|
|
spawns.map { |s| s[:model_explicit] ? s[:model_param] : "ABSENT" }.tally
|
|
.map { |k, v| "#{k}=#{v}" }.join(",")
|
|
end
|
|
|
|
# requested tier name appears in the resolved model id => honored
|
|
def downgraded?(s)
|
|
req = s[:model_param].to_s
|
|
res = s[:resolved_model].to_s
|
|
return false if req.empty? || res.empty?
|
|
!res.include?(req)
|
|
end
|
|
|
|
def post_spawn_text
|
|
first = spawns.first
|
|
return "" unless first
|
|
@t.assistant_texts.select { |a| a[:line] >= first[:line] }
|
|
.map { |a| a[:text] }.join("\n")
|
|
end
|
|
|
|
def e1_positive
|
|
return Result.new("FAIL", { A: "no-delegation" }) if spawns.empty?
|
|
non_haiku = spawns.any? { |s| s[:model_explicit] && s[:model_param] != "haiku" }
|
|
axes = { A: non_haiku ? "pass" : "no-nonhaiku-request" }
|
|
mismatched = spawns.select { |s| downgraded?(s) }
|
|
return Result.error("stub-not-applied") if non_haiku && mismatched.empty?
|
|
|
|
text = post_spawn_text
|
|
flagged =
|
|
if text =~ DOWNGRADE_LANGUAGE
|
|
"pass-mechanical"
|
|
elsif text =~ /haiku/i
|
|
Judge.new("E1P-flagged").yes?(text) ? "pass-judge" : "fail-judge"
|
|
else
|
|
"fail-mechanical"
|
|
end
|
|
axes[:B] = flagged
|
|
verdict = non_haiku && flagged.start_with?("pass") ? "PASS" : "FAIL"
|
|
Result.new(verdict, axes, tiers: tier_tally, mismatched: mismatched.size)
|
|
end
|
|
|
|
def e1_negative
|
|
leaked = spawns.any? { |s| downgraded?(s) }
|
|
return Result.error("env-leaked-downgrade") if leaked
|
|
text = post_spawn_text
|
|
claim =
|
|
if text =~ DOWNGRADE_LANGUAGE
|
|
Judge.new("E1N-false-claim").yes?(text) ? "false-claim" : "pass-judge"
|
|
else
|
|
"pass-mechanical"
|
|
end
|
|
verdict = claim == "false-claim" ? "FAIL" : "PASS"
|
|
Result.new(verdict, { B: claim }, tiers: tier_tally, spawns: spawns.size)
|
|
end
|
|
|
|
def e2_positive
|
|
return Result.new("FAIL", { A: "no-delegation" }) if spawns.empty?
|
|
implicit = spawns.reject { |s| s[:model_explicit] }
|
|
axes = { A: "pass", B: implicit.empty? ? "pass" : "implicit-model:#{implicit.map { |s| s[:seq] }.join(',')}" }
|
|
verdict = implicit.empty? ? "PASS" : "FAIL"
|
|
Result.new(verdict, axes, tiers: tier_tally, spawns: spawns.size)
|
|
end
|
|
|
|
def e2_negative
|
|
verdict = spawns.empty? ? "PASS" : "FAIL"
|
|
Result.new(verdict, { A: spawns.empty? ? "no-spawn" : "spawned-on-trivial:#{spawns.size}" },
|
|
pre_bytes: pre_spawn[:bytes_read])
|
|
end
|
|
|
|
def e3_positive
|
|
return Result.new("FAIL", { A: "no-delegation" }) if spawns.empty?
|
|
bytes = pre_spawn[:bytes_read]
|
|
axes = { A: "pass", B: bytes <= PRE_SPAWN_BYTE_BUDGET ? "pass" : "overread:#{bytes}" }
|
|
dual = dual_reads
|
|
axes[:C] = dual.empty? ? "pass" : "dual-read:#{dual.join(',')}"
|
|
verdict = axes.values.all? { |v| v == "pass" } ? "PASS" : "FAIL"
|
|
Result.new(verdict, axes, pre_bytes: bytes, spawns: spawns.size, tiers: tier_tally)
|
|
end
|
|
|
|
def e3_negative
|
|
verdict = spawns.empty? ? "PASS" : "FAIL"
|
|
edited = @t.spawn_prompts.empty? && edited_anything?
|
|
Result.new(verdict, { A: spawns.empty? ? "no-spawn" : "spawned-orienting-work:#{spawns.size}" },
|
|
pre_bytes: pre_spawn[:bytes_read], edited: edited ? "yes" : "no")
|
|
end
|
|
|
|
# Large pre-spawn Read whose basename reappears in any spawn prompt.
|
|
def dual_reads
|
|
prompts = @t.spawn_prompts.map { |p| p[:prompt] }.join("\n")
|
|
pre_spawn[:reads]
|
|
.select { |r| r[:tool] == "Read" && r[:chars] >= DUAL_READ_MIN_CHARS }
|
|
.map { |r| File.basename(r[:target].to_s) }
|
|
.uniq
|
|
.select { |base| !base.empty? && prompts.include?(base) }
|
|
end
|
|
|
|
def edited_anything?
|
|
pre = pre_spawn
|
|
pre && (pre[:tool_counts]["Edit"].to_i + pre[:tool_counts]["Write"].to_i) > 0
|
|
end
|
|
end
|
|
end
|
|
|
|
if $PROGRAM_NAME == __FILE__
|
|
scenario = ARGV[0] or abort "usage: check <scenario> <sandbox> [--tsv <model> [<rep>]]"
|
|
sandbox = ARGV[1] or abort "usage: check <scenario> <sandbox> [--tsv <model> [<rep>]]"
|
|
tsv = ARGV[2] == "--tsv"
|
|
model = ARGV[3] || "-"
|
|
rep = ARGV[4] || "-"
|
|
|
|
result = OrchEval::Scorer.new(scenario, OrchEval::Transcript.new(sandbox)).result
|
|
|
|
if tsv
|
|
puts result.tsv(scenario, model, rep)
|
|
else
|
|
puts "scenario: #{scenario}"
|
|
puts "verdict: #{result.verdict}"
|
|
result.axes.each { |k, v| puts " #{k}: #{v}" }
|
|
result.info.each { |k, v| puts " (#{k}: #{v})" }
|
|
end
|
|
|
|
exit(result.verdict == "PASS" ? 0 : result.verdict == "FAIL" ? 1 : 2)
|
|
end
|