#!/usr/bin/env ruby
# frozen_string_literal: true

# Two-axis, deterministic-first checker for Eval B (unprompted behavior).
#
#   check <W1|W2|W3|R1|R2|R3|R4|R4-nograph> <sandbox-root> [--tsv <model-label>]
#
# Reads <sandbox>/transcript.jsonl (stream-json, written by bin/run) plus the
# sandbox file state.
#
#   axis (a) — unprompted consultation: mechanical. True iff any tool_use
#     block touches the ADR system (an os-adr:* skill, a bin/adr-* CLI, or a
#     read/glob/write under docs/adr/). Prose never counts; the SessionStart
#     hook note cannot false-positive because it appears only outside
#     tool_use blocks.
#   axis (b) — specific correctness:
#     R1..R4      mechanical: assistant text must cite the scenario's correct
#                 ADR ID (and flag the conflict); R2 additionally fails when
#                 the Superseded distractor is cited as if live.
#     W1..W3      mechanical PASS when a new docs/adr/NNNN-*.md matching the
#                 scenario's topic exists; otherwise a rubric-bound LLM judge
#                 (judge-rubric.md) reads ONLY the final message. Override the
#                 judge with ADR_EVAL_B_JUDGE_CMD (reads prompt on stdin,
#                 prints YES/NO) — the self-test uses this to stay model-free.
#
# Overall PASS requires both axes. R4-nograph runs R4's checks under its own
# label; on the degradation grid an axis-(b) FAIL there is the expected,
# correct outcome (see README).
#
# TSV mode: scenario, model, axis-a, axis-b, PASS|FAIL, reasons.
# Exit 0 on PASS, 1 on FAIL, 2 on usage error.

require "json"
require "open3"

module AdrEvalB
  BASELINE_ADR_IDS = %w[0001 0002 0003 0004 0005 0006].freeze
  CONFLICT = /conflict|violat|contradic|goes against|supersed|locked|decided|decision|record/i

  # The stream-json transcript bin/run captured. Deliberately structural:
  # axis (a) looks only inside tool_use blocks, never at raw text.
  class Transcript
    def self.load(path)
      return nil unless File.exist?(path)
      events = File.readlines(path).filter_map do |line|
        JSON.parse(line)
      rescue JSON::ParserError
        nil
      end
      new(events)
    end

    def initialize(events) = @events = events

    def hook_context_present?
      @events.any? do |e|
        e["type"] == "system" && e["subtype"] == "hook_response" &&
          e["hook_name"].to_s.start_with?("SessionStart") &&
          e.to_json.include?("[os-adr]")
      end
    end

    def tool_uses
      assistant_blocks.select { |b| b["type"] == "tool_use" }
    end

    # Everything the model said (assistant text blocks + the final result).
    def assistant_text
      texts = assistant_blocks.select { |b| b["type"] == "text" }.map { |b| b["text"] }
      texts << final_message
      texts.compact.join("\n")
    end

    def final_message
      result = @events.reverse.find { |e| e["type"] == "result" }
      result && result["result"].is_a?(String) ? result["result"] : nil
    end

    private

    def assistant_blocks
      @events.select { |e| e["type"] == "assistant" }
             .flat_map { |e| e.dig("message", "content") || [] }
    end
  end

  class Sandbox
    def initialize(root) = @root = root
    attr_reader :root

    def transcript = Transcript.load(File.join(@root, "transcript.jsonl"))

    def new_adr_files
      Dir.glob(File.join(@root, "docs/adr/[0-9]*.md")).sort.reject do |path|
        BASELINE_ADR_IDS.include?(File.basename(path)[0, 4])
      end
    end
  end

  class Result
    def initialize = @failures = []
    attr_reader :failures
    def pass? = @failures.empty?

    def expect(condition, reason)
      @failures << reason unless condition
      !!condition
    end
  end

  # Narrow LLM fallback for W-scenario axis (b): rubric + final message only.
  class Judge
    RUBRIC = File.read(File.expand_path("../judge-rubric.md", __dir__))
                 .split("---", 2).last.strip

    def self.command
      ENV.fetch("ADR_EVAL_B_JUDGE_CMD",
                "claude -p --model haiku --dangerously-skip-permissions")
    end

    def proposed_adr?(final_message)
      prompt = "#{RUBRIC}\n\n#{final_message}"
      output, status = Open3.capture2(self.class.command, stdin_data: prompt)
      raise "judge command failed: #{self.class.command}" unless status.success?
      verdict = output[/\b(YES|NO)\b/, 1]
      raise "judge gave no YES/NO verdict: #{output.inspect}" if verdict.nil?
      verdict == "YES"
    end
  end

  class Scenario
    def initialize(sandbox, judge: Judge.new)
      @sb = sandbox
      @judge = judge
    end

    def run
      axis_a = Result.new
      axis_b = Result.new
      transcript = @sb.transcript
      if transcript.nil?
        axis_a.expect(false, "no transcript.jsonl in sandbox")
        axis_b.expect(false, "no transcript.jsonl in sandbox")
      elsif !transcript.hook_context_present?
        axis_a.expect(false, "invalid run: SessionStart [os-adr] hook context missing")
        axis_b.expect(false, "invalid run: SessionStart [os-adr] hook context missing")
      else
        check_axis_a(axis_a, transcript)
        check_axis_b(axis_b, transcript)
      end
      [axis_a, axis_b]
    end

    private

    attr_reader :sb, :judge

    def check_axis_a(result, transcript)
      touched = transcript.tool_uses.any? { |tu| adr_touch?(tu) }
      result.expect(touched, "no tool_use touched the ADR system (os-adr:*, bin/adr-*, docs/adr/)")
    end

    def adr_touch?(tool_use)
      probe = "#{tool_use['name']} #{tool_use['input'].to_json}"
      probe.match?(%r{os-adr|bin/adr-|adr-(find|new|init|detect|migrate)\b|docs/adr})
    end
  end

  # --- Retrieval scenarios: mechanical ID + conflict-flag checks ------------

  class Retrieval < Scenario
    def check_axis_b(result, transcript)
      text = transcript.assistant_text
      result.expect(text.match?(/\b#{correct_id}\b/),
                    "assistant text never cites the correct ADR #{correct_id}")
      result.expect(text.match?(AdrEvalB::CONFLICT),
                    "assistant text never flags the conflict/decision")
      check_distractors(result, text)
    end

    def check_distractors(_result, _text) = nil
  end

  class R1 < Retrieval
    def correct_id = "0002"
  end

  class R2 < Retrieval
    def correct_id = "0003"

    # Citing the Superseded 0001 as if live is exactly the failure R2 exists
    # to catch; citing it while noting supersession is correct behavior.
    def check_distractors(result, text)
      if text.match?(/\b0001\b/)
        result.expect(text.match?(/supersed/i),
                      "cites Superseded distractor 0001 without noting it is superseded")
      end
    end
  end

  class R3 < Retrieval
    def correct_id = "0005"

    # Correctness for R3 is answering the question from the ADR: UTC ISO-8601.
    def check_axis_b(result, text_or_transcript)
      text = text_or_transcript.assistant_text
      result.expect(text.match?(/UTC/i), "answer never says UTC")
      result.expect(text.match?(/ISO.?8601/i), "answer never says ISO-8601")
    end
  end

  class R4 < Retrieval
    def correct_id = "0003"
  end

  # --- Write-trigger scenarios: new-ADR-file check, judge fallback ----------

  class WriteTrigger < Scenario
    def check_axis_b(result, transcript)
      new_files = sb.new_adr_files
      if new_files.any?
        matching = new_files.select { |p| File.read(p).match?(topic) }
        result.expect(matching.any?,
                      "new ADR exists but matches no topic term #{topic.inspect}")
        return
      end
      final = transcript.final_message
      unless final
        result.expect(false, "no new ADR file and no final message to judge")
        return
      end
      result.expect(judge.proposed_adr?(final),
                    "no new ADR file; judge: final message does not propose/ask to record the decision")
    end
  end

  class W1 < WriteTrigger
    def topic = /persist|stor(e|age|ing)|database|sqlite|history/i
  end

  class W2 < WriteTrigger
    def topic = /error|exception|result|raise|failure/i
  end

  class W3 < WriteTrigger
    def topic = /concurren|thread|parallel|order|sequential|queue/i
  end

  SCENARIOS = {
    "W1" => W1, "W2" => W2, "W3" => W3,
    "R1" => R1, "R2" => R2, "R3" => R3, "R4" => R4,
    "R4-nograph" => R4 # same checks; expected to FAIL axis (b) — see README
  }.freeze
end

scenario_id, sandbox_root = ARGV[0], ARGV[1]
tsv_model = ARGV[2] == "--tsv" ? (ARGV[3] || "unknown") : nil
klass = AdrEvalB::SCENARIOS[scenario_id]
abort "usage: check <#{AdrEvalB::SCENARIOS.keys.join('|')}> <sandbox-root> [--tsv <model>]" if klass.nil? || sandbox_root.nil?
abort "no such sandbox: #{sandbox_root}" unless File.directory?(sandbox_root)

axis_a, axis_b = klass.new(AdrEvalB::Sandbox.new(File.expand_path(sandbox_root))).run
overall = axis_a.pass? && axis_b.pass?
reasons = (axis_a.failures + axis_b.failures).join("; ")

if tsv_model
  puts [scenario_id, tsv_model,
        axis_a.pass? ? "A:PASS" : "A:FAIL",
        axis_b.pass? ? "B:PASS" : "B:FAIL",
        overall ? "PASS" : "FAIL", reasons].join("\t")
else
  puts "#{overall ? 'PASS' : 'FAIL'} #{scenario_id}  " \
       "(axis-a #{axis_a.pass? ? 'PASS' : 'FAIL'}, axis-b #{axis_b.pass? ? 'PASS' : 'FAIL'})"
  (axis_a.failures + axis_b.failures).each { |f| puts "  - #{f}" }
end
exit(overall ? 0 : 1)