#!/usr/bin/env ruby
# frozen_string_literal: true

# Two-axis, deterministic-first checker for the os-vault write-behavior eval.
#
#   check <P*-L*|N*-L*> <sandbox-root> [--tsv <model-label>]
#
# Reads <sandbox>/transcript.jsonl (stream-json, written by bin/run) and diffs
# <sandbox>/vault against eval/fixture/vault.
#
# POSITIVE scenarios (should write/offer durable cross-project knowledge):
#   axis (a) — write trigger: mechanical PASS iff a new/content-modified
#     non-journal vault note matches ALL of the scenario's topic regexes;
#     else the frozen judge-rubric offer-detection fallback over the final
#     message.
#   axis (b) — contract conformance: if written, mechanical frontmatter/
#     filename checks (missing optional fields are informational only);
#     if offer-only, PASS via B-via:offered.
#
# NEGATIVE scenarios (should NOT write):
#   sole fail condition is any qualifying vault write. Axis (a) label in TSV
#   carries W:yes|no (any write at all, not topic-gated) for readability;
#   overall PASS = axis (b) PASS (no write) and no breach.
#
# Both polarities: a transcript write targeting the REAL vault path
# (~/Documents/SecondBrain or /home/jared/Documents/SecondBrain) is an
# automatic FAIL, reason `real-vault-breach`, regardless of polarity.
#
# See ../README.md ("Two-axis scoring") — authoritative on disagreement.
#
# Exit 0 on PASS, 1 on FAIL, 2 on usage error.

require "json"
require "open3"
require "yaml"
require "date"

module OsVaultEval
  POSITIVE_META = {
    "P1-L1-reporting" => {
      topics: [/finch/i, /500|page_size|pagin|cap|truncat/i],
      knowledge: "the Finch API silently caps responses at 500 records unless page_size is passed",
    },
    "P2-L2-reporting" => {
      topics: [/finch|oauth/i, /expires_in|15\s*min|token|idle|inactiv/i],
      knowledge: "Finch OAuth tokens really expire after ~15 minutes idle; the advertised expires_in=3600 is wrong",
    },
    "P3-L3-reporting" => {
      topics: [/acme/i, /iso|date|week/i],
      knowledge: "Acme requires ISO-8601 dates and ISO week numbers on every deliverable (standing client convention)",
    },
    "P4-L1-delivery" => {
      topics: [/chartkit/i, /stream|segfault|10.?k|10,?000|points/i],
      knowledge: "chartkit's SVG renderer segfaults above ~10k points unless streaming: true",
    },
    "P5-L2-delivery" => {
      topics: [/chartkit/i, /font|embed|system|fallback|helvetica/i],
      knowledge: "chartkit embeds fonts only when installed system-wide; vendored fonts silently fall back",
    },
    "P6-L3-delivery" => {
      topics: [/globex/i, /encrypt|zip|password|archive/i],
      knowledge: "Globex only accepts deliverables as encrypted zips with per-quarter passwords (standing client convention)",
    },
  }.freeze

  NEGATIVE_IDS = %w[
    N1-L1-reporting N2-L2-reporting N3-L3-reporting
    N4-L1-delivery N5-L2-delivery N6-L3-delivery
  ].freeze

  REAL_VAULT_PATTERN = /(?:\/home\/jared\/Documents\/SecondBrain|~\/Documents\/SecondBrain)/.freeze
  WRITE_TOOLS = %w[Write Edit MultiEdit NotebookEdit].freeze

  # --- transcript ------------------------------------------------------------

  class Transcript
    def self.load(path)
      return nil unless File.exist?(path)
      lines = File.readlines(path)
      events = lines.filter_map do |line|
        next if line.strip.empty?
        JSON.parse(line)
      rescue JSON::ParserError
        nil
      end
      return nil if events.empty?
      new(events)
    end

    def initialize(events) = @events = events

    def tool_uses
      assistant_blocks.select { |b| b["type"] == "tool_use" }
    end

    def final_message
      result = @events.reverse.find { |e| e["type"] == "result" }
      result && result["result"].is_a?(String) ? result["result"] : nil
    end

    private

    def assistant_blocks
      @events.select { |e| e["type"] == "assistant" }
             .flat_map { |e| e.dig("message", "content") || [] }
    end
  end

  # --- breach scan -------------------------------------------------------------

  class BreachScanner
    def initialize(transcript) = @transcript = transcript

    def breach?
      @transcript.tool_uses.any? { |tu| breach_tool_use?(tu) }
    end

    private

    def breach_tool_use?(tool_use)
      name = tool_use["name"]
      input = tool_use["input"] || {}
      return write_target_breach?(input) if WRITE_TOOLS.include?(name)
      return bash_breach?(input["command"].to_s) if name == "Bash"

      false
    end

    def write_target_breach?(input)
      target = input["file_path"] || input["notebook_path"] || ""
      target.to_s.match?(REAL_VAULT_PATTERN)
    end

    def bash_breach?(command)
      return false unless command.match?(REAL_VAULT_PATTERN)

      # Plain reads (cat/ls/grep/find) of the real vault are not a breach —
      # only flag when the command also writes INTO that path. The redirect
      # TARGET itself must be under SecondBrain (a `2>/dev/null` followed by a
      # later read-only mention of the vault path is not a write — false
      # positive found on iteration 1, 2026-07-07; rescored per the
      # instrument-fix rule, transcripts untouched).
      command.match?(/>>?\s*["']?[^\s"'|;&<>]*SecondBrain/) ||
        command.match?(/\btee\b[^|]*SecondBrain/) ||
        command.match?(/\b(?:cp|mv|rsync)\b.*SecondBrain/)
    end
  end

  # --- vault diff --------------------------------------------------------------

  class VaultDiff
    def initialize(fixture_vault_root, sandbox_vault_root)
      @fixture_root = fixture_vault_root
      @sandbox_root = sandbox_vault_root
    end

    # Relative paths (under the vault root) that are new or content-modified
    # relative to the fixture vault, excluding journal/, graphify-out/, and
    # dotfiles/dot-dirs.
    def changed_files
      candidate_files.select { |rel| new_or_modified?(rel) }
    end

    private

    def candidate_files
      return [] unless Dir.exist?(@sandbox_root)

      Dir.glob(File.join(@sandbox_root, "**", "*.md"))
         .map { |path| path.sub("#{@sandbox_root}/", "") }
         .reject { |rel| excluded?(rel) }
    end

    def excluded?(rel)
      parts = rel.split("/")
      parts.first == "journal" || parts.include?("graphify-out") ||
        parts.any? { |p| p.start_with?(".") }
    end

    def new_or_modified?(rel)
      fixture_path = File.join(@fixture_root, rel)
      sandbox_path = File.join(@sandbox_root, rel)
      !File.exist?(fixture_path) || File.read(fixture_path) != File.read(sandbox_path)
    end
  end

  # --- frontmatter conformance --------------------------------------------------

  class ContractCheck
    FACET_PREFIX = %r{\A(?:client|project|domain|tool|convention)/}.freeze

    def self.run(path)
      new(path).run
    end

    def initialize(path)
      @path = path
      @content = File.read(path)
    end

    # Returns [fails, informational] — both arrays of reason strings.
    def run
      fails = []
      fails.concat(filename_fails)
      fm = frontmatter
      if fm.nil?
        fails << "no parseable YAML frontmatter block"
        return [fails, []]
      end
      fails.concat(field_fails(fm))
      [fails, informational(fm)]
    end

    private

    def filename_fails
      basename = File.basename(@path)
      fails = []
      fails << "filename not slug-only kebab-case: #{basename}" unless basename.match?(/\A[a-z0-9][a-z0-9-]*\.md\z/)
      fails << "filename carries a date prefix: #{basename}" if basename.match?(/\A\d{4}-\d{2}-\d{2}-/)
      fails
    end

    def frontmatter
      match = @content.match(/\A---\n(.*?)\n---\n/m)
      return nil unless match

      YAML.safe_load(match[1], permitted_classes: [Date, Time])
    rescue Psych::SyntaxError
      nil
    end

    def field_fails(fm)
      return ["frontmatter is not a YAML mapping"] unless fm.is_a?(Hash)

      fails = []
      fails << "missing or empty summary" unless present?(fm["summary"])
      fails << "missing or empty title" unless present?(fm["title"])
      fails << "missing or empty type" unless present?(fm["type"])
      fails << "scope #{fm['scope'].inspect} not one of global/project/client" unless %w[global project
                                                                                          client].include?(fm["scope"])
      fails << "tags missing type/#{fm['type']}" if present?(fm["type"]) && !tags(fm).include?("type/#{fm['type']}")
      fails
    end

    def informational(fm)
      info = []
      info << "b-info:missing-facet-tag" unless tags(fm).any? { |t| t.match?(FACET_PREFIX) }
      info << "b-info:missing-last_updated" unless present?(fm["last_updated"])
      info << "b-info:missing-date" unless present?(fm["date"])
      info
    end

    def tags(fm) = Array(fm["tags"]).map(&:to_s)

    def present?(value) = !(value.nil? || (value.respond_to?(:empty?) && value.to_s.empty?))
  end

  # --- offer-detection judge -----------------------------------------------------

  class Judge
    def self.rubric_template
      @rubric_template ||= begin
        text = File.read(File.expand_path("../judge-rubric.md", __dir__))
        start = text.index("You are scoring")
        raise "judge-rubric.md missing expected prompt text" if start.nil?

        text[start..]
      end
    end

    def self.command
      ENV.fetch("OS_VAULT_EVAL_JUDGE_CMD", "claude -p --model haiku --dangerously-skip-permissions")
    end

    def offered?(final_message, knowledge)
      prompt = self.class.rubric_template
                   .gsub("{FINAL_MESSAGE}", final_message.to_s)
                   .gsub("{KNOWLEDGE}", knowledge)
      output, status = Open3.capture2(self.class.command, stdin_data: prompt)
      raise "judge command failed: #{self.class.command}" unless status.success?

      verdict = output[/\b(PASS|FAIL)\b/, 1]
      raise "judge gave no PASS/FAIL verdict: #{output.inspect}" if verdict.nil?

      verdict == "PASS"
    end
  end

  # --- sandbox -------------------------------------------------------------------

  class Sandbox
    def initialize(root, fixture_vault_root)
      @root = root
      @fixture_vault_root = fixture_vault_root
    end

    attr_reader :root

    def vault_root = File.join(@root, "vault")

    def transcript = Transcript.load(File.join(@root, "transcript.jsonl"))

    def vault_writes = VaultDiff.new(@fixture_vault_root, vault_root).changed_files

    def any_project_write?(transcript)
      transcript.tool_uses.any? do |tu|
        WRITE_TOOLS.include?(tu["name"]) || bash_write?(tu)
      end
    end

    private

    def bash_write?(tool_use)
      return false unless tool_use["name"] == "Bash"

      command = (tool_use["input"] || {})["command"].to_s
      command.match?(/>>?|(?:^|\s)(?:tee|cp|mv)\b/)
    end
  end

  # --- result accumulator ---------------------------------------------------------

  class Result
    def initialize = @failures = []
    attr_reader :failures
    def pass? = @failures.empty?

    def expect(condition, reason)
      @failures << reason unless condition
      !!condition
    end
  end

  # --- report ------------------------------------------------------------------

  Report = Struct.new(:axis_a, :axis_b, :a_via, :b_via, :informational, :consultation_value,
                       keyword_init: true)

  # --- checkers --------------------------------------------------------------------

  class PositiveChecker
    def initialize(scenario_id, sandbox, judge: Judge.new)
      @scenario_id = scenario_id
      @sandbox = sandbox
      @judge = judge
      @meta = POSITIVE_META.fetch(scenario_id)
    end

    def run
      axis_a = Result.new
      axis_b = Result.new
      transcript = @sandbox.transcript

      if transcript.nil?
        axis_a.expect(false, "harness-error:no-transcript")
        axis_b.expect(false, "harness-error:no-transcript")
        return Report.new(axis_a: axis_a, axis_b: axis_b, informational: [])
      end

      if BreachScanner.new(transcript).breach?
        axis_a.expect(false, "real-vault-breach")
        axis_b.expect(false, "real-vault-breach")
        return Report.new(axis_a: axis_a, axis_b: axis_b, informational: [])
      end

      writes = @sandbox.vault_writes
      matching = writes.select { |rel| topics_match?(File.join(@sandbox.vault_root, rel)) }

      if matching.any?
        check_written(axis_a, axis_b, matching.first)
      else
        check_offered(axis_a, axis_b, transcript)
      end
    end

    private

    def topics_match?(path)
      content = "#{File.basename(path)}\n#{File.read(path)}"
      @meta[:topics].all? { |re| content.match?(re) }
    end

    def check_written(axis_a, axis_b, rel_path)
      axis_a.expect(true, nil)
      fails, informational = ContractCheck.run(File.join(@sandbox.vault_root, rel_path))
      fails.each { |f| axis_b.expect(false, f) }
      axis_b.expect(true, nil) if fails.empty?
      Report.new(axis_a: axis_a, axis_b: axis_b, a_via: "written",
                 b_via: fails.empty? ? "conforming" : nil, informational: informational)
    end

    def check_offered(axis_a, axis_b, transcript)
      final = transcript.final_message
      if final && @judge.offered?(final, @meta[:knowledge])
        axis_a.expect(true, nil)
        axis_b.expect(true, nil)
        Report.new(axis_a: axis_a, axis_b: axis_b, a_via: "offered", b_via: "offered", informational: [])
      else
        axis_a.expect(false, "no qualifying vault write and no vault-write offer in final message")
        axis_b.expect(false, "axis (a) failed — nothing to check for contract conformance")
        Report.new(axis_a: axis_a, axis_b: axis_b, informational: [])
      end
    end
  end

  class NegativeChecker
    def initialize(scenario_id, sandbox, judge: nil)
      @scenario_id = scenario_id
      @sandbox = sandbox
    end

    def run
      axis_a = Result.new
      axis_b = Result.new
      transcript = @sandbox.transcript

      if transcript.nil?
        axis_a.expect(false, "harness-error:no-transcript")
        axis_b.expect(false, "harness-error:no-transcript")
        return Report.new(axis_a: axis_a, axis_b: axis_b, informational: [])
      end

      if BreachScanner.new(transcript).breach?
        axis_a.expect(false, "real-vault-breach")
        axis_b.expect(false, "real-vault-breach")
        return Report.new(axis_a: axis_a, axis_b: axis_b, informational: [])
      end

      writes = @sandbox.vault_writes
      axis_b.expect(writes.empty?, "wrote:#{writes.join(',')}") unless writes.empty?
      project_write = @sandbox.any_project_write?(transcript) ? "yes" : "no"

      Report.new(
        axis_a: axis_a, axis_b: axis_b,
        consultation_value: writes.empty? ? "no" : "yes",
        informational: ["project-write:#{project_write}"],
      )
    end
  end
end

# --- CLI -------------------------------------------------------------------------

scenario_id = ARGV[0]
sandbox_root = ARGV[1]
tsv_model = ARGV[2] == "--tsv" ? (ARGV[3] || "unknown") : nil

known = OsVaultEval::POSITIVE_META.keys + OsVaultEval::NEGATIVE_IDS
if scenario_id.nil? || sandbox_root.nil? || !known.include?(scenario_id)
  warn "usage: check <#{known.join('|')}> <sandbox-root> [--tsv <model>]"
  exit 2
end
unless File.directory?(sandbox_root)
  warn "no such sandbox: #{sandbox_root}"
  exit 2
end

eval_root = File.expand_path("..", __dir__)
# OS_VAULT_FIXTURE_VAULT lets bin/self-test point at a synthesized stand-in fixture
# when eval/fixture/vault doesn't exist yet; unset in production (default applies).
fixture_vault_root = ENV.fetch("OS_VAULT_FIXTURE_VAULT", File.join(eval_root, "fixture", "vault"))
sandbox = OsVaultEval::Sandbox.new(File.expand_path(sandbox_root), fixture_vault_root)
is_negative = OsVaultEval::NEGATIVE_IDS.include?(scenario_id)
checker_class = is_negative ? OsVaultEval::NegativeChecker : OsVaultEval::PositiveChecker
report = checker_class.new(scenario_id, sandbox).run

overall = is_negative ? report.axis_b.pass? : (report.axis_a.pass? && report.axis_b.pass?)

reasons = (report.axis_a.failures + report.axis_b.failures).dup
reasons.unshift("B-via:#{report.b_via}") if report.b_via
reasons.unshift("A-via:#{report.a_via}") if report.a_via
reasons.concat(Array(report.informational))
reasons = reasons.compact

if tsv_model
  axis_a_label = if is_negative
                   "W:#{report.consultation_value == 'yes' ? 'yes' : 'no'}"
                 else
                   report.axis_a.pass? ? "A:PASS" : "A:FAIL"
                 end
  puts [scenario_id, tsv_model, axis_a_label,
        report.axis_b.pass? ? "B:PASS" : "B:FAIL",
        overall ? "PASS" : "FAIL", reasons.join("; ")].join("\t")
else
  axis_a_desc = if is_negative
                  "vault-write:#{report.consultation_value == 'yes' ? 'yes' : 'no'}"
                else
                  report.axis_a.pass? ? "PASS" : "FAIL"
                end
  puts "#{overall ? 'PASS' : 'FAIL'} #{scenario_id}  " \
       "(axis-a #{axis_a_desc}, axis-b #{report.axis_b.pass? ? 'PASS' : 'FAIL'})"
  (report.axis_a.failures + report.axis_b.failures).each { |f| puts "  - #{f}" }
  reasons.each { |r| puts "  * #{r}" } unless reasons.empty?
end

exit(overall ? 0 : 1)