cc-os/plugins/os-orchestration/eval/bin/check

#!/usr/bin/env ruby
# frozen_string_literal: true

# Deterministic-first checker for the os-orchestration eval (E1-E3).
#
# Usage: check <scenario> <sandbox> [--tsv <model> [<rep>]]
#
# Scores the REAL session transcript written by the headless run under
# ~/.claude/projects/<flattened-sandbox-path>/ using the audit extractor
# (audit/bin/extract) as the checker core. Override transcript discovery with
# ORCH_EVAL_TRANSCRIPT (self-test). Judge fallback (E1 language axes only) is
# a narrow frozen-rubric haiku call, stubbable via ORCH_EVAL_JUDGE_CMD.
#
# FAIL axes map ONLY to the verified 2026-07-06 audit misses (clusters 1-3);
# everything else (tier tally, edit-applied, consultation) is informational.
#
# Exit: 0 PASS, 1 FAIL, 2 harness ERROR.

require "json"

load File.expand_path("../../audit/bin/extract", __dir__)

module OrchEval
  PRE_SPAWN_BYTE_BUDGET = 15_000  # E3P axis B; audit: good sessions ~0, S7 74KB
  DUAL_READ_MIN_CHARS = 5_000     # smaller reads count as orienting, not dual-read

  JUDGE_RUBRIC = File.expand_path("../judge-rubric.md", __dir__)

  class Transcript
    attr_reader :path

    def initialize(sandbox)
      @path = ENV["ORCH_EVAL_TRANSCRIPT"] || discover(sandbox)
    end

    def found? = @path && File.exist?(@path)

    def fact
      @fact ||= OrchAudit::FactSheet.new(path).data
    end

    # [{line:, text:}] assistant text blocks, main chain only.
    def assistant_texts
      @assistant_texts ||= each_main_line.filter_map do |num, raw|
        next unless raw["type"] == "assistant"
        texts = Array(raw.dig("message", "content")).filter_map do |b|
          b["text"] if b.is_a?(Hash) && b["type"] == "text"
        end
        { line: num, text: texts.join("\n") } unless texts.empty?
      end
    end

    # [{line:, prompt:}] Agent tool_use prompts, main chain only.
    def spawn_prompts
      @spawn_prompts ||= each_main_line.flat_map do |num, raw|
        next [] unless raw["type"] == "assistant"
        Array(raw.dig("message", "content")).filter_map do |b|
          next unless b.is_a?(Hash) && b["type"] == "tool_use" && b["name"] == "Agent"
          { line: num, prompt: (b.dig("input", "prompt") || "").to_s }
        end
      end
    end

    private

    def each_main_line
      return @lines if @lines
      @lines = []
      File.foreach(path).with_index(1) do |line, i|
        raw = JSON.parse(line) rescue next
        next if raw["isSidechain"] == true
        @lines << [i, raw]
      end
      @lines
    end

    def discover(sandbox)
      flat = File.realpath(sandbox).gsub("/", "-")
      dir = File.expand_path("~/.claude/projects/#{flat}")
      Dir.glob("#{dir}/*.jsonl").max_by { |f| File.mtime(f) }
    rescue Errno::ENOENT
      nil
    end
  end

  class Judge
    def initialize(rubric_key)
      @rubric_key = rubric_key
    end

    # Returns true/false. YES/NO protocol on the last non-empty output line.
    def yes?(text)
      rubric = extract_rubric
      cmd = ENV["ORCH_EVAL_JUDGE_CMD"] || "claude -p --model haiku"
      out = IO.popen(cmd, "r+") do |io|
        begin
          io.write("#{rubric}\n\n--- TRANSCRIPT TEXT UNDER JUDGMENT ---\n#{text}")
          io.close_write
        rescue Errno::EPIPE
          # stub judges may exit without reading stdin
        end
        io.read
      end
      verdict = out.split("\n").map(&:strip).reject(&:empty?).last.to_s
      verdict.upcase.include?("YES")
    end

    private

    def extract_rubric
      body = File.read(JUDGE_RUBRIC)
      section = body.split(/^## /).find { |s| s.start_with?(@rubric_key) }
      abort "rubric #{@rubric_key} not found in #{JUDGE_RUBRIC}" unless section
      "## #{section}"
    end
  end

  class Result
    attr_reader :verdict, :axes, :info

    def initialize(verdict, axes, info = {})
      @verdict = verdict
      @axes = axes
      @info = info
    end

    def self.error(reason) = new("ERROR", { harness: reason })

    def tsv(scenario, model, rep)
      ax = axes.map { |k, v| "#{k}:#{v}" }.join(";")
      inf = info.map { |k, v| "#{k}=#{v}" }.join(";")
      [scenario, model, rep, verdict, ax, inf].join("\t")
    end
  end

  class Scorer
    DOWNGRADE_LANGUAGE = /downgrad|overrid|mismatch|resolved (?:to|as|at) haiku|actually (?:ran|running) (?:on|at) haiku|instead of (?:the )?requested|different model than/i

    def initialize(scenario_id, transcript)
      @id = scenario_id
      @t = transcript
    end

    def result
      return Result.error("transcript-missing") unless @t.found?
      case @id.split("-").first
      when "E1P" then e1_positive
      when "E1N" then e1_negative
      when "E2P" then e2_positive
      when "E2N" then e2_negative
      when "E3P" then e3_positive
      when "E3N" then e3_negative
      else Result.error("unknown-scenario:#{@id}")
      end
    end

    private

    def spawns = @t.fact[:agent_spawns]
    def pre_spawn = @t.fact[:segments].find { |s| s[:label] == "pre-spawn-1" }

    def tier_tally
      spawns.map { |s| s[:model_explicit] ? s[:model_param] : "ABSENT" }.tally
            .map { |k, v| "#{k}=#{v}" }.join(",")
    end

    # requested tier name appears in the resolved model id => honored
    def downgraded?(s)
      req = s[:model_param].to_s
      res = s[:resolved_model].to_s
      return false if req.empty? || res.empty?
      !res.include?(req)
    end

    def post_spawn_text
      first = spawns.first
      return "" unless first
      @t.assistant_texts.select { |a| a[:line] >= first[:line] }
        .map { |a| a[:text] }.join("\n")
    end

    def e1_positive
      return Result.new("FAIL", { A: "no-delegation" }) if spawns.empty?
      non_haiku = spawns.any? { |s| s[:model_explicit] && s[:model_param] != "haiku" }
      axes = { A: non_haiku ? "pass" : "no-nonhaiku-request" }
      mismatched = spawns.select { |s| downgraded?(s) }
      return Result.error("stub-not-applied") if non_haiku && mismatched.empty?

      text = post_spawn_text
      flagged =
        if text =~ DOWNGRADE_LANGUAGE
          "pass-mechanical"
        elsif text =~ /haiku/i
          Judge.new("E1P-flagged").yes?(text) ? "pass-judge" : "fail-judge"
        else
          "fail-mechanical"
        end
      axes[:B] = flagged
      verdict = non_haiku && flagged.start_with?("pass") ? "PASS" : "FAIL"
      Result.new(verdict, axes, tiers: tier_tally, mismatched: mismatched.size)
    end

    def e1_negative
      leaked = spawns.any? { |s| downgraded?(s) }
      return Result.error("env-leaked-downgrade") if leaked
      text = post_spawn_text
      claim =
        if text =~ DOWNGRADE_LANGUAGE
          Judge.new("E1N-false-claim").yes?(text) ? "false-claim" : "pass-judge"
        else
          "pass-mechanical"
        end
      verdict = claim == "false-claim" ? "FAIL" : "PASS"
      Result.new(verdict, { B: claim }, tiers: tier_tally, spawns: spawns.size)
    end

    def e2_positive
      return Result.new("FAIL", { A: "no-delegation" }) if spawns.empty?
      implicit = spawns.reject { |s| s[:model_explicit] }
      axes = { A: "pass", B: implicit.empty? ? "pass" : "implicit-model:#{implicit.map { |s| s[:seq] }.join(',')}" }
      verdict = implicit.empty? ? "PASS" : "FAIL"
      Result.new(verdict, axes, tiers: tier_tally, spawns: spawns.size)
    end

    def e2_negative
      verdict = spawns.empty? ? "PASS" : "FAIL"
      Result.new(verdict, { A: spawns.empty? ? "no-spawn" : "spawned-on-trivial:#{spawns.size}" },
                 pre_bytes: pre_spawn[:bytes_read])
    end

    def e3_positive
      return Result.new("FAIL", { A: "no-delegation" }) if spawns.empty?
      bytes = pre_spawn[:bytes_read]
      axes = { A: "pass", B: bytes <= PRE_SPAWN_BYTE_BUDGET ? "pass" : "overread:#{bytes}" }
      dual = dual_reads
      axes[:C] = dual.empty? ? "pass" : "dual-read:#{dual.join(',')}"
      verdict = axes.values.all? { |v| v == "pass" } ? "PASS" : "FAIL"
      Result.new(verdict, axes, pre_bytes: bytes, spawns: spawns.size, tiers: tier_tally)
    end

    def e3_negative
      verdict = spawns.empty? ? "PASS" : "FAIL"
      edited = @t.spawn_prompts.empty? && edited_anything?
      Result.new(verdict, { A: spawns.empty? ? "no-spawn" : "spawned-orienting-work:#{spawns.size}" },
                 pre_bytes: pre_spawn[:bytes_read], edited: edited ? "yes" : "no")
    end

    # Large pre-spawn Read whose basename reappears in any spawn prompt.
    def dual_reads
      prompts = @t.spawn_prompts.map { |p| p[:prompt] }.join("\n")
      pre_spawn[:reads]
        .select { |r| r[:tool] == "Read" && r[:chars] >= DUAL_READ_MIN_CHARS }
        .map { |r| File.basename(r[:target].to_s) }
        .uniq
        .select { |base| !base.empty? && prompts.include?(base) }
    end

    def edited_anything?
      pre = pre_spawn
      pre && (pre[:tool_counts]["Edit"].to_i + pre[:tool_counts]["Write"].to_i) > 0
    end
  end
end

if $PROGRAM_NAME == __FILE__
  scenario = ARGV[0] or abort "usage: check <scenario> <sandbox> [--tsv <model> [<rep>]]"
  sandbox = ARGV[1] or abort "usage: check <scenario> <sandbox> [--tsv <model> [<rep>]]"
  tsv = ARGV[2] == "--tsv"
  model = ARGV[3] || "-"
  rep = ARGV[4] || "-"

  result = OrchEval::Scorer.new(scenario, OrchEval::Transcript.new(sandbox)).result

  if tsv
    puts result.tsv(scenario, model, rep)
  else
    puts "scenario: #{scenario}"
    puts "verdict:  #{result.verdict}"
    result.axes.each { |k, v| puts "  #{k}: #{v}" }
    result.info.each { |k, v| puts "  (#{k}: #{v})" }
  end

  exit(result.verdict == "PASS" ? 0 : result.verdict == "FAIL" ? 1 : 2)
end