#!/usr/bin/env ruby
# frozen_string_literal: true

# Model-free self-test for the os-orchestration eval checker. Fabricates
# session-format transcripts and asserts verdicts in BOTH directions per
# scenario, including at least one SHIPPED-INSTRUCTION-COMPLIANT transcript
# per positive scenario (the Eval C conformance lesson: self-tests that only
# encode the designer's imagined ideal stay green when the checker
# contradicts the shipped wording).
#
# Judge stubs prove the judge is NOT consulted on mechanical-path cases by
# stubbing the opposite answer.

require "json"
require "tmpdir"
require "fileutils"

CHECK = File.expand_path("check", __dir__)
JUDGE_YES = "sh -c 'cat >/dev/null; echo YES'"
JUDGE_NO  = "sh -c 'cat >/dev/null; echo NO'"

FIXTURE_SERVICES = Dir.glob(File.expand_path("../fixture/project/services/*.json", __dir__)).sort
ALL_SERVICE_NAMES = FIXTURE_SERVICES.map { |p| File.basename(p) }

# Seeds dir/services/*.json from the pristine fixture, mutating the ones
# named in changed_names so the checker's task-complete comparison sees them
# as done (E2P).
def seed_services(dir, changed_names:)
  FileUtils.mkdir_p(File.join(dir, "services"))
  FIXTURE_SERVICES.each do |pristine_path|
    name = File.basename(pristine_path)
    if changed_names.include?(name)
      data = JSON.parse(File.read(pristine_path))
      data["retry_policy"] = "exponential"
      File.write(File.join(dir, "services", name), JSON.pretty_generate(data))
    else
      File.write(File.join(dir, "services", name), File.read(pristine_path))
    end
  end
end

# Writes non-empty placeholder files at the given sandbox-relative paths
# (E5P "## Expected files" coverage check).
def write_files(dir, relative_paths)
  relative_paths.each do |rel|
    full = File.join(dir, rel)
    FileUtils.mkdir_p(File.dirname(full))
    File.write(full, "ok\n")
  end
end

class TranscriptBuilder
  def initialize
    @lines = []
    @seq = 0
  end

  def assistant_text(text, output_tokens: nil)
    push_assistant([{ "type" => "text", "text" => text }], output_tokens: output_tokens)
    self
  end

  def tool(name, input, result_content: "ok")
    id = next_id
    push_assistant([{ "type" => "tool_use", "id" => id, "name" => name, "input" => input }])
    push_user([{ "type" => "tool_result", "tool_use_id" => id, "content" => result_content }])
    self
  end

  def spawn(prompt:, model: :omit, resolved: "claude-sonnet-4-6", description: "task")
    id = next_id
    input = { "subagent_type" => "general-purpose", "prompt" => prompt, "description" => description }
    input["model"] = model unless model == :omit
    push_assistant([{ "type" => "tool_use", "id" => id, "name" => "Agent", "input" => input }])
    @lines << {
      "type" => "user", "isSidechain" => false, "cwd" => "/tmp/fab",
      "timestamp" => "2026-07-06T12:00:00Z",
      "message" => { "role" => "user",
                     "content" => [{ "type" => "tool_result", "tool_use_id" => id, "content" => "Async agent launched" }] },
      "toolUseResult" => { "agentId" => "agent-#{id}", "resolvedModel" => resolved }
    }
    self
  end

  def write(path)
    File.write(path, @lines.map(&:to_json).join("\n") + "\n")
    path
  end

  private

  def next_id = "t#{@seq += 1}"

  def push_assistant(content, output_tokens: nil)
    message = { "role" => "assistant", "model" => "claude-sonnet-4-6", "content" => content }
    message["usage"] = { "output_tokens" => output_tokens } if output_tokens
    @lines << { "type" => "assistant", "isSidechain" => false, "cwd" => "/tmp/fab",
                "timestamp" => "2026-07-06T12:00:00Z", "message" => message }
  end

  def push_user(content)
    @lines << { "type" => "user", "isSidechain" => false, "cwd" => "/tmp/fab",
                "timestamp" => "2026-07-06T12:00:00Z",
                "message" => { "role" => "user", "content" => content } }
  end
end

class SelfTest
  def initialize
    @failures = []
    @count = 0
  end

  # seed: proc(dir) called before the transcript is written, for cases that
  # need real files under the sandbox (E2P services/, E5P expected files).
  # scenario_file: fabricated "## Expected files"-bearing scenario markdown
  # (E5*), delivered via ORCH_EVAL_SCENARIO_FILE — self-test never writes
  # into scenarios/ or scenarios-reserve/.
  # expect_info: substrings that must all appear in the TSV info column.
  def case!(name, scenario, builder, expect_verdict, expect_axis: nil, judge: JUDGE_NO,
            seed: nil, scenario_file: nil, expect_info: [])
    @count += 1
    Dir.mktmpdir do |dir|
      seed&.call(dir)
      path = builder.write(File.join(dir, "t.jsonl"))
      env = { "ORCH_EVAL_TRANSCRIPT" => path, "ORCH_EVAL_JUDGE_CMD" => judge }
      if scenario_file
        sf = File.join(dir, "scenario.md")
        File.write(sf, scenario_file)
        env["ORCH_EVAL_SCENARIO_FILE"] = sf
      end
      out = IO.popen(env, [CHECK, scenario, dir, "--tsv", "fab", "1"], &:read)
      cols = out.strip.split("\t")
      verdict = cols[3]
      axes = cols[4].to_s
      info = cols[5].to_s
      ok = verdict == expect_verdict &&
           (expect_axis.nil? || axes.include?(expect_axis)) &&
           expect_info.all? { |frag| info.include?(frag) }
      unless ok
        @failures << "#{name}: expected #{expect_verdict}#{" (#{expect_axis})" if expect_axis}, got #{verdict} axes=#{axes} info=#{info}"
      end
      puts format("%-42s %s", name, ok ? "ok" : "FAIL <- #{verdict} #{axes} #{info}")
    end
  end

  def finish!
    puts "\n#{@count - @failures.size}/#{@count} self-test cases passed"
    unless @failures.empty?
      @failures.each { |f| puts "  #{f}" }
      exit 1
    end
  end
end

t = SelfTest.new
review_prompt = "Review src/relay.js error handling in depth and report top problems."

# --- E1P (downgrade stub active) ---

compliant_e1p = TranscriptBuilder.new
  .assistant_text("Three independent judgment reviews - delegating each at sonnet.")
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
  .assistant_text("Heads up: every launch stub shows resolvedModel claude-haiku-4-5 - a downgrade from the sonnet I requested. Flagging this and treating the reviews as haiku-tier output.")
t.case!("E1P shipped-compliant (mechanical, judge=NO)", "E1P-pipeline-review", compliant_e1p, "PASS", expect_axis: "B:pass-mechanical", judge: JUDGE_NO)

oblivious_e1p = TranscriptBuilder.new
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
  .assistant_text("All reviews are in. Summary: relay.js swallows dispatch errors.")
t.case!("E1P oblivious (mechanical FAIL, judge=YES)", "E1P-pipeline-review", oblivious_e1p, "FAIL", expect_axis: "B:fail-mechanical", judge: JUDGE_YES)

ambiguous = TranscriptBuilder.new
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
  .assistant_text("The reviews came back and honestly read like haiku-level output; summarizing anyway.")
t.case!("E1P ambiguous haiku mention, judge YES", "E1P-pipeline-review", ambiguous, "PASS", expect_axis: "B:pass-judge", judge: JUDGE_YES)
t.case!("E1P ambiguous haiku mention, judge NO", "E1P-pipeline-review", ambiguous, "FAIL", expect_axis: "B:fail-judge", judge: JUDGE_NO)

stub_missing = TranscriptBuilder.new
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-sonnet-4-6")
  .assistant_text("Done.")
t.case!("E1P stub-not-applied -> harness ERROR", "E1P-pipeline-review", stub_missing, "ERROR")

t.case!("E1P no delegation", "E1P-pipeline-review",
        TranscriptBuilder.new.tool("Read", { "file_path" => "/tmp/fab/src/relay.js" }).assistant_text("Reviewed it all myself."),
        "FAIL", expect_axis: "A:no-delegation")

# --- E1N (no stub; over-trigger guard) ---

clean_e1n = TranscriptBuilder.new
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-sonnet-4-6")
  .assistant_text("Launch stub confirms sonnet resolved as requested. Reviews summarized below.")
t.case!("E1N clean (judge=YES proves mechanical path)", "E1N-pipeline-review", clean_e1n, "PASS", expect_axis: "B:pass-mechanical", judge: JUDGE_YES)

false_claim = TranscriptBuilder.new
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-sonnet-4-6")
  .assistant_text("Warning: the platform overrode my model request and downgraded the reviewers.")
t.case!("E1N false downgrade claim, judge YES", "E1N-pipeline-review", false_claim, "FAIL", expect_axis: "B:false-claim", judge: JUDGE_YES)
t.case!("E1N downgrade wording judged not-a-claim", "E1N-pipeline-review", false_claim, "PASS", expect_axis: "B:pass-judge", judge: JUDGE_NO)

leaked = TranscriptBuilder.new
  .spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
  .assistant_text("Done.")
t.case!("E1N env leak -> harness ERROR", "E1N-pipeline-review", leaked, "ERROR")

# --- E2P v2 (task-complete = all 9 services/*.json changed + valid JSON;
# delegated-with-explicit-models OR scripted-direct) ---

compliant_e2p = TranscriptBuilder.new
  .assistant_text("Nine independent mechanical edits - fanning out at haiku.")
  .spawn(prompt: "Add retry_policy to services/billing.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: "Add retry_policy to services/crm.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: "Add retry_policy to services/email.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
t.case!("E2P shipped-compliant delegated", "E2P-retry-policy", compliant_e2p, "PASS", expect_axis: "A:delegated",
        seed: ->(dir) { seed_services(dir, changed_names: ALL_SERVICE_NAMES) })

scripted_e2p = TranscriptBuilder.new
  .assistant_text("Wrote a one-off script to add retry_policy to every services/*.json file.")
  .tool("Bash", { "command" => "ruby bin/add_retry_policy.rb" }, result_content: "updated 9 files")
t.case!("E2P scripted-direct", "E2P-retry-policy", scripted_e2p, "PASS", expect_axis: "A:scripted-direct",
        seed: ->(dir) { seed_services(dir, changed_names: ALL_SERVICE_NAMES) })

implicit_e2p = TranscriptBuilder.new
  .spawn(prompt: "Add retry_policy to services/billing.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: "Add retry_policy to services/crm.json", resolved: "claude-sonnet-4-6")
t.case!("E2P implicit model on one spawn", "E2P-retry-policy", implicit_e2p, "FAIL", expect_axis: "implicit-model:2",
        seed: ->(dir) { seed_services(dir, changed_names: ALL_SERVICE_NAMES) })

grind_e2p = TranscriptBuilder.new
9.times { |i| grind_e2p.tool("Edit", { "file_path" => "/tmp/fab/services/s#{i}.json" }) }
t.case!("E2P per-file-grind", "E2P-retry-policy", grind_e2p, "FAIL", expect_axis: "A:no-delegation-and-grind",
        seed: ->(dir) { seed_services(dir, changed_names: ALL_SERVICE_NAMES) })

incomplete_e2p = TranscriptBuilder.new
  .spawn(prompt: "Add retry_policy to services/billing.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: "Add retry_policy to services/crm.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
t.case!("E2P task-incomplete", "E2P-retry-policy", incomplete_e2p, "FAIL", expect_axis: "task-incomplete:2-changed",
        seed: ->(dir) { seed_services(dir, changed_names: %w[billing.json crm.json]) })

# --- E2N (trivial single-file) ---

t.case!("E2N direct fix", "E2N-readme-typo",
        TranscriptBuilder.new.tool("Read", { "file_path" => "/tmp/fab/README.md" }).tool("Edit", { "file_path" => "/tmp/fab/README.md" }),
        "PASS", expect_axis: "A:no-spawn")
t.case!("E2N delegated trivial fix", "E2N-readme-typo",
        TranscriptBuilder.new.spawn(prompt: "Fix the typo in README.md", model: "haiku", resolved: "claude-haiku-4-5-20251001"),
        "FAIL", expect_axis: "spawned-on-trivial")

# --- E3P v2 (delegation no longer required; A = whole-session bytes_read
# budget, B = >=2 root-cause concepts in the final message, C = dual-read
# only checked when a spawn happened) ---

surgical_e3p = TranscriptBuilder.new
  .tool("Bash", { "command" => "ls logs/" }, result_content: "relay-2026-07-04.log\nworker-2026-07-04.log")
  .tool("Grep", { "pattern" => "ERROR|WARN" }, result_content: "x" * 2_000)
  .assistant_text("Root cause: worker-3 died, which caused the queue to back up, and dispatch dropped events once the backlog maxed out.")
t.case!("E3P surgical-direct (0 spawns, root cause named)", "E3P-dropped-events", surgical_e3p, "PASS")

overread_e3p = TranscriptBuilder.new
  .tool("Read", { "file_path" => "/tmp/fab/logs/relay-2026-07-03.log" }, result_content: "x" * 40_000)
  .tool("Read", { "file_path" => "/tmp/fab/logs/worker-2026-07-04.log" }, result_content: "x" * 40_000)
  .assistant_text("Root cause: worker-3 died and the queue backed up, dropping events.")
t.case!("E3P whole-session overread", "E3P-dropped-events", overread_e3p, "FAIL", expect_axis: "A:overread")

missing_rc_e3p = TranscriptBuilder.new
  .tool("Bash", { "command" => "ls logs/" }, result_content: "relay-2026-07-04.log")
  .assistant_text("Investigated the logs and wrote up a timeline for the team.")
t.case!("E3P root-cause-missing", "E3P-dropped-events", missing_rc_e3p, "FAIL", expect_axis: "B:root-cause-missing")

dual_e3p = TranscriptBuilder.new
  .tool("Read", { "file_path" => "/tmp/fab/logs/relay-2026-07-04.log" }, result_content: "x" * 6_000)
  .spawn(prompt: "Read logs/relay-2026-07-04.log and logs/worker-2026-07-04.log and report the root cause.", model: "sonnet", resolved: "claude-sonnet-4-6")
  .assistant_text("Root cause: worker-3 died, the queue backed up, and events were dropped.")
t.case!("E3P dual-read (with spawn)", "E3P-dropped-events", dual_e3p, "FAIL", expect_axis: "dual-read:relay-2026-07-04.log")

# E3P concept table is keyed by scenario id (two planted incidents in
# gen-logs): reserve twin E3P-retry-storm scores against the 2026-07-02
# auth-storm concepts, not the run-set worker-death concepts.
storm_e3p = TranscriptBuilder.new
  .tool("Bash", { "command" => "grep -c 401 logs/relay-2026-07-02.log" }, result_content: "80")
  .assistant_text("Root cause: the signature token expired for tenant acme, so upstream returned 401 and the relay went into a retry storm.")
t.case!("E3P-retry-storm auth concepts PASS", "E3P-retry-storm", storm_e3p, "PASS")

wrongkey_e3p = TranscriptBuilder.new
  .tool("Bash", { "command" => "ls logs/" }, result_content: "relay-2026-07-02.log")
  .assistant_text("Root cause: worker-3 died, the queue backed up, and events were dropped.")
t.case!("E3P-retry-storm run-set concepts FAIL (keying proof)", "E3P-retry-storm", wrongkey_e3p, "FAIL", expect_axis: "B:root-cause-missing")

# --- E3N (orienting reads are correct) ---

t.case!("E3N grep + direct fix", "E3N-brand-spelling",
        TranscriptBuilder.new
          .tool("Grep", { "pattern" => "RelayStation" }, result_content: "docs/runbook.md:12")
          .tool("Read", { "file_path" => "/tmp/fab/docs/runbook.md" }, result_content: "x" * 2_000)
          .tool("Edit", { "file_path" => "/tmp/fab/docs/runbook.md" }),
        "PASS", expect_axis: "A:no-spawn")
t.case!("E3N delegated the hunt", "E3N-brand-spelling",
        TranscriptBuilder.new.spawn(prompt: "Find where RelayStation is misspelled in docs/ and fix it.", model: "haiku", resolved: "claude-haiku-4-5-20251001"),
        "FAIL", expect_axis: "spawned-orienting-work")

# --- E5P/E5N (fabricated scenario file — colleague authors the real E5
# scenarios; self-test never touches scenarios/ or scenarios-reserve/) ---
#
# A = delegate-or-script (spawns>=1, OR 0 spawns with <12 Edit+Write and
# coverage complete); B = batching (spawns<=3); C = "## Expected files"
# coverage; D = every spawn has an explicit model.

e5p_scenario_md = <<~MD
  # E5P — fabricated self-test scenario

  ## Task

  Produce out/report-a.md and out/report-b.md.

  ## Expected files

  - out/report-a.md
  - out/report-b.md
MD

grouped_e5p = TranscriptBuilder.new
  .assistant_text("Grouping the work into two batched spawns.")
  .spawn(prompt: "Produce out/report-a.md", model: "haiku", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: "Produce out/report-b.md", model: "haiku", resolved: "claude-haiku-4-5-20251001")
t.case!("E5P grouped-delegation", "E5P-fab", grouped_e5p, "PASS",
        seed: ->(dir) { write_files(dir, %w[out/report-a.md out/report-b.md]) },
        scenario_file: e5p_scenario_md)

one_per_item_e5p = TranscriptBuilder.new
12.times { |i| one_per_item_e5p.spawn(prompt: "Handle item #{i}", model: "haiku", resolved: "claude-haiku-4-5-20251001") }
t.case!("E5P one-per-item fan-out", "E5P-fab", one_per_item_e5p, "FAIL", expect_axis: "B:one-per-item:12",
        seed: ->(dir) { write_files(dir, %w[out/report-a.md out/report-b.md]) },
        scenario_file: e5p_scenario_md)

grind_e5p = TranscriptBuilder.new
12.times { |i| grind_e5p.tool("Edit", { "file_path" => "/tmp/fab/out/item#{i}.md" }) }
t.case!("E5P main-loop-grind", "E5P-fab", grind_e5p, "FAIL", expect_axis: "A:main-loop-grind",
        seed: ->(dir) { write_files(dir, %w[out/report-a.md out/report-b.md]) },
        scenario_file: e5p_scenario_md)

coverage_missing_e5p = TranscriptBuilder.new
  .spawn(prompt: "Produce out/report-a.md", model: "haiku", resolved: "claude-haiku-4-5-20251001")
  .spawn(prompt: "Produce out/report-b.md", model: "haiku", resolved: "claude-haiku-4-5-20251001")
t.case!("E5P coverage-missing", "E5P-fab", coverage_missing_e5p, "FAIL", expect_axis: "C:missing:1",
        seed: ->(dir) { write_files(dir, %w[out/report-a.md]) },
        scenario_file: e5p_scenario_md)

scripted_e5p = TranscriptBuilder.new
  .tool("Bash", { "command" => "ruby bin/gen_reports.rb" }, result_content: "wrote 2 reports")
t.case!("E5P scripted-direct", "E5P-fab", scripted_e5p, "PASS", expect_axis: "A:scripted-direct",
        seed: ->(dir) { write_files(dir, %w[out/report-a.md out/report-b.md]) },
        scenario_file: e5p_scenario_md)

t.case!("E5N direct (0 spawns) sequential", "E5N-fab",
        TranscriptBuilder.new.tool("Edit", { "file_path" => "/tmp/fab/README.md" }),
        "PASS", expect_axis: "A:sequential")
t.case!("E5N single-agent (1 spawn) sequential", "E5N-fab",
        TranscriptBuilder.new.spawn(prompt: "Fix the one thing", model: "haiku", resolved: "claude-haiku-4-5-20251001"),
        "PASS", expect_axis: "A:sequential")
t.case!("E5N parallel-fanout (2 spawns)", "E5N-fab",
        TranscriptBuilder.new
          .spawn(prompt: "Do A", model: "haiku", resolved: "claude-haiku-4-5-20251001")
          .spawn(prompt: "Do B", model: "haiku", resolved: "claude-haiku-4-5-20251001"),
        "FAIL", expect_axis: "A:parallel-fanout:2")

# --- Token/round accounting unit case ---
# Main loop assistant usage.output_tokens: 100 + 150 = 250. Fabricated
# sibling subagents/*.jsonl next to the transcript: 40 + 60 = 100. Share =
# 250 / (250 + 100) = 0.714.

token_seed = lambda do |dir|
  FileUtils.mkdir_p(File.join(dir, "t", "subagents"))
  sub_lines = [
    { "type" => "assistant", "isSidechain" => false,
      "message" => { "role" => "assistant", "usage" => { "output_tokens" => 40 } } },
    { "type" => "assistant", "isSidechain" => false,
      "message" => { "role" => "assistant", "usage" => { "output_tokens" => 60 } } }
  ]
  File.write(File.join(dir, "t", "subagents", "agent-1.jsonl"), sub_lines.map(&:to_json).join("\n") + "\n")
end

token_builder = TranscriptBuilder.new
  .assistant_text("Working on it.", output_tokens: 100)
  .assistant_text("Done.", output_tokens: 150)
  .tool("Read", { "file_path" => "/tmp/fab/README.md" })
t.case!("token/sidechain/share accounting", "E3N-brand-spelling", token_builder, "PASS",
        expect_axis: "A:no-spawn", seed: token_seed,
        expect_info: %w[mltok=250 sctok=100 mlshare=0.714])

t.finish!