cc-os/plugins/os-orchestration/eval/bin/self-test

#!/usr/bin/env ruby
# frozen_string_literal: true

# Model-free self-test for the os-orchestration eval checker. Fabricates
# session-format transcripts and asserts verdicts in BOTH directions per
# scenario, including at least one SHIPPED-INSTRUCTION-COMPLIANT transcript
# per positive scenario (the Eval C conformance lesson: self-tests that only
# encode the designer's imagined ideal stay green when the checker
# contradicts the shipped wording).
#
# Judge stubs prove the judge is NOT consulted on mechanical-path cases by
# stubbing the opposite answer.

require "json"
require "tmpdir"

CHECK = File.expand_path("check", __dir__)
JUDGE_YES = "sh -c 'cat >/dev/null; echo YES'"
JUDGE_NO  = "sh -c 'cat >/dev/null; echo NO'"

class TranscriptBuilder
  def initialize
    @lines = []
    @seq = 0
  end

  def assistant_text(text)
    push_assistant([{ "type" => "text", "text" => text }])
    self
  end

  def tool(name, input, result_content: "ok")
    id = next_id
    push_assistant([{ "type" => "tool_use", "id" => id, "name" => name, "input" => input }])
    push_user([{ "type" => "tool_result", "tool_use_id" => id, "content" => result_content }])
    self
  end

  def spawn(prompt:, model: :omit, resolved: "claude-sonnet-4-6", description: "task")
    id = next_id
    input = { "subagent_type" => "general-purpose", "prompt" => prompt, "description" => description }
    input["model"] = model unless model == :omit
    push_assistant([{ "type" => "tool_use", "id" => id, "name" => "Agent", "input" => input }])
    @lines << {
      "type" => "user", "isSidechain" => false, "cwd" => "/tmp/fab",
      "timestamp" => "2026-07-06T12:00:00Z",
      "message" => { "role" => "user",
                     "content" => [{ "type" => "tool_result", "tool_use_id" => id, "content" => "Async agent launched" }] },
      "toolUseResult" => { "agentId" => "agent-#{id}", "resolvedModel" => resolved }
    }
    self
  end

  def write(path)
    File.write(path, @lines.map(&:to_json).join("\n") + "\n")
    path
  end

  private

  def next_id = "t#{@seq += 1}"

  def push_assistant(content)
    @lines << { "type" => "assistant", "isSidechain" => false, "cwd" => "/tmp/fab",
                "timestamp" => "2026-07-06T12:00:00Z",
                "message" => { "role" => "assistant", "model" => "claude-sonnet-4-6", "content" => content } }
  end

  def push_user(content)
    @lines << { "type" => "user", "isSidechain" => false, "cwd" => "/tmp/fab",
                "timestamp" => "2026-07-06T12:00:00Z",
                "message" => { "role" => "user", "content" => content } }
  end
end

class SelfTest
  def initialize
    @failures = []
    @count = 0
  end

  def case!(name, scenario, builder, expect_verdict, expect_axis: nil, judge: JUDGE_NO)
    @count += 1
    Dir.mktmpdir do |dir|
      path = builder.write(File.join(dir, "t.jsonl"))
      env = { "ORCH_EVAL_TRANSCRIPT" => path, "ORCH_EVAL_JUDGE_CMD" => judge }
      out = IO.popen(env, [CHECK, scenario, dir, "--tsv", "fab", "1"], &:read)
      cols = out.strip.split("\t")
      verdict = cols[3]
      axes = cols[4].to_s
      ok = verdict == expect_verdict && (expect_axis.nil? || axes.include?(expect_axis))
      unless ok
        @failures << "#{name}: expected #{expect_verdict}#{" (#{expect_axis})" if expect_axis}, got #{verdict} axes=#{axes}"
      end
      puts format("%-42s %s", name, ok ? "ok" : "FAIL <- #{verdict} #{axes}")
    end
  end

  def finish!
    puts "\n#{@count - @failures.size}/#{@count} self-test cases passed"
    unless @failures.empty?
      @failures.each { |f| puts "  #{f}" }
      exit 1
    end
  end
end

t = SelfTest.new
review_prompt = "Review src/relay.js error handling in depth and report top problems."

# --- E1P (downgrade stub active) ---

compliant_e1p = TranscriptBuilder.new
  .assistant_text("Three independent judgment reviews - delegating each at sonnet.")
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
  .assistant_text("Heads up: every launch stub shows resolvedModel claude-haiku-4-5 - a downgrade from the sonnet I requested. Flagging this and treating the reviews as haiku-tier output.")
t.case!("E1P shipped-compliant (mechanical, judge=NO)", "E1P-pipeline-review", compliant_e1p, "PASS", expect_axis: "B:pass-mechanical", judge: JUDGE_NO)

oblivious_e1p = TranscriptBuilder.new
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
  .assistant_text("All reviews are in. Summary: relay.js swallows dispatch errors.")
t.case!("E1P oblivious (mechanical FAIL, judge=YES)", "E1P-pipeline-review", oblivious_e1p, "FAIL", expect_axis: "B:fail-mechanical", judge: JUDGE_YES)

ambiguous = TranscriptBuilder.new
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
  .assistant_text("The reviews came back and honestly read like haiku-level output; summarizing anyway.")
t.case!("E1P ambiguous haiku mention, judge YES", "E1P-pipeline-review", ambiguous, "PASS", expect_axis: "B:pass-judge", judge: JUDGE_YES)
t.case!("E1P ambiguous haiku mention, judge NO", "E1P-pipeline-review", ambiguous, "FAIL", expect_axis: "B:fail-judge", judge: JUDGE_NO)

stub_missing = TranscriptBuilder.new
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-sonnet-4-6")
  .assistant_text("Done.")
t.case!("E1P stub-not-applied -> harness ERROR", "E1P-pipeline-review", stub_missing, "ERROR")

t.case!("E1P no delegation", "E1P-pipeline-review",
        TranscriptBuilder.new.tool("Read", { "file_path" => "/tmp/fab/src/relay.js" }).assistant_text("Reviewed it all myself."),
        "FAIL", expect_axis: "A:no-delegation")

# --- E1N (no stub; over-trigger guard) ---

clean_e1n = TranscriptBuilder.new
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-sonnet-4-6")
  .assistant_text("Launch stub confirms sonnet resolved as requested. Reviews summarized below.")
t.case!("E1N clean (judge=YES proves mechanical path)", "E1N-pipeline-review", clean_e1n, "PASS", expect_axis: "B:pass-mechanical", judge: JUDGE_YES)

false_claim = TranscriptBuilder.new
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-sonnet-4-6")
  .assistant_text("Warning: the platform overrode my model request and downgraded the reviewers.")
t.case!("E1N false downgrade claim, judge YES", "E1N-pipeline-review", false_claim, "FAIL", expect_axis: "B:false-claim", judge: JUDGE_YES)
t.case!("E1N downgrade wording judged not-a-claim", "E1N-pipeline-review", false_claim, "PASS", expect_axis: "B:pass-judge", judge: JUDGE_NO)

leaked = TranscriptBuilder.new
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
  .assistant_text("Done.")
t.case!("E1N env leak -> harness ERROR", "E1N-pipeline-review", leaked, "ERROR")

# --- E2P (explicit model on fan-out) ---

compliant_e2p = TranscriptBuilder.new
  .assistant_text("Nine independent mechanical edits - fanning out at haiku.")
  .spawn(prompt: "Add retry_policy to services/billing.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: "Add retry_policy to services/crm.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: "Add retry_policy to services/email.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
t.case!("E2P shipped-compliant", "E2P-retry-policy", compliant_e2p, "PASS")

implicit = TranscriptBuilder.new
  .spawn(prompt: "Add retry_policy to services/billing.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: "Add retry_policy to services/crm.json", resolved: "claude-sonnet-4-6")
t.case!("E2P one implicit model", "E2P-retry-policy", implicit, "FAIL", expect_axis: "implicit-model:2")

nine_direct = TranscriptBuilder.new
9.times { |i| nine_direct.tool("Edit", { "file_path" => "/tmp/fab/services/s#{i}.json" }) }
t.case!("E2P did it all directly", "E2P-retry-policy", nine_direct, "FAIL", expect_axis: "A:no-delegation")

# --- E2N (trivial single-file) ---

t.case!("E2N direct fix", "E2N-readme-typo",
        TranscriptBuilder.new.tool("Read", { "file_path" => "/tmp/fab/README.md" }).tool("Edit", { "file_path" => "/tmp/fab/README.md" }),
        "PASS", expect_axis: "A:no-spawn")
t.case!("E2N delegated trivial fix", "E2N-readme-typo",
        TranscriptBuilder.new.spawn(prompt: "Fix the typo in README.md", model: "haiku", resolved: "claude-haiku-4-5-20251001"),
        "FAIL", expect_axis: "spawned-on-trivial")

# --- E3P (delegate the investigation) ---

compliant_e3p = TranscriptBuilder.new
  .tool("Bash", { "command" => "ls logs/" }, result_content: "relay-2026-07-01.log\nrelay-2026-07-04.log\nworker-2026-07-04.log")
  .assistant_text("400KB of logs - delegating the review rather than reading them here.")
  .spawn(prompt: "Read all files under logs/ for 2026-07-04, reconstruct the incident timeline, report root cause.", model: "sonnet", resolved: "claude-sonnet-4-6")
t.case!("E3P shipped-compliant (orienting ls only)", "E3P-dropped-events", compliant_e3p, "PASS")

overread = TranscriptBuilder.new
  .tool("Read", { "file_path" => "/tmp/fab/logs/relay-2026-07-03.log" }, result_content: "x" * 30_000)
  .tool("Read", { "file_path" => "/tmp/fab/logs/worker-2026-07-04.log" }, result_content: "x" * 30_000)
  .spawn(prompt: "Summarize the incident from logs/ and report root cause.", model: "sonnet", resolved: "claude-sonnet-4-6")
t.case!("E3P pre-spawn overread", "E3P-dropped-events", overread, "FAIL", expect_axis: "B:overread")

dual = TranscriptBuilder.new
  .tool("Read", { "file_path" => "/tmp/fab/logs/relay-2026-07-04.log" }, result_content: "x" * 6_000)
  .spawn(prompt: "Read logs/relay-2026-07-04.log and logs/worker-2026-07-04.log and report the root cause.", model: "sonnet", resolved: "claude-sonnet-4-6")
t.case!("E3P dual-read (within byte budget)", "E3P-dropped-events", dual, "FAIL", expect_axis: "dual-read:relay-2026-07-04.log")

t.case!("E3P never delegated", "E3P-dropped-events",
        TranscriptBuilder.new.tool("Read", { "file_path" => "/tmp/fab/logs/relay-2026-07-04.log" }, result_content: "x" * 40_000).assistant_text("Root cause: worker-3 died."),
        "FAIL", expect_axis: "A:no-delegation")

# --- E3N (orienting reads are correct) ---

t.case!("E3N grep + direct fix", "E3N-brand-spelling",
        TranscriptBuilder.new
          .tool("Grep", { "pattern" => "RelayStation" }, result_content: "docs/runbook.md:12")
          .tool("Read", { "file_path" => "/tmp/fab/docs/runbook.md" }, result_content: "x" * 2_000)
          .tool("Edit", { "file_path" => "/tmp/fab/docs/runbook.md" }),
        "PASS", expect_axis: "A:no-spawn")
t.case!("E3N delegated the hunt", "E3N-brand-spelling",
        TranscriptBuilder.new.spawn(prompt: "Find where RelayStation is misspelled in docs/ and fix it.", model: "haiku", resolved: "claude-haiku-4-5-20251001"),
        "FAIL", expect_axis: "spawned-orienting-work")

t.finish!