277 lines
9.1 KiB
Plaintext
277 lines
9.1 KiB
Plaintext
|
#!/usr/bin/env ruby
|
||
|
|
# frozen_string_literal: true
|
||
|
|
|
||
|
|
# Two-axis, deterministic-first checker for Eval B (unprompted behavior).
|
||
|
|
#
|
||
|
|
# check <W1|W2|W3|R1|R2|R3|R4|R4-nograph> <sandbox-root> [--tsv <model-label>]
|
||
|
|
#
|
||
|
|
# Reads <sandbox>/transcript.jsonl (stream-json, written by bin/run) plus the
|
||
|
|
# sandbox file state.
|
||
|
|
#
|
||
|
|
# axis (a) — unprompted consultation: mechanical. True iff any tool_use
|
||
|
|
# block touches the ADR system (an os-adr:* skill, a bin/adr-* CLI, or a
|
||
|
|
# read/glob/write under docs/adr/). Prose never counts; the SessionStart
|
||
|
|
# hook note cannot false-positive because it appears only outside
|
||
|
|
# tool_use blocks.
|
||
|
|
# axis (b) — specific correctness:
|
||
|
|
# R1..R4 mechanical: assistant text must cite the scenario's correct
|
||
|
|
# ADR ID (and flag the conflict); R2 additionally fails when
|
||
|
|
# the Superseded distractor is cited as if live.
|
||
|
|
# W1..W3 mechanical PASS when a new docs/adr/NNNN-*.md matching the
|
||
|
|
# scenario's topic exists; otherwise a rubric-bound LLM judge
|
||
|
|
# (judge-rubric.md) reads ONLY the final message. Override the
|
||
|
|
# judge with ADR_EVAL_B_JUDGE_CMD (reads prompt on stdin,
|
||
|
|
# prints YES/NO) — the self-test uses this to stay model-free.
|
||
|
|
#
|
||
|
|
# Overall PASS requires both axes. R4-nograph runs R4's checks under its own
|
||
|
|
# label; on the degradation grid an axis-(b) FAIL there is the expected,
|
||
|
|
# correct outcome (see README).
|
||
|
|
#
|
||
|
|
# TSV mode: scenario, model, axis-a, axis-b, PASS|FAIL, reasons.
|
||
|
|
# Exit 0 on PASS, 1 on FAIL, 2 on usage error.
|
||
|
|
|
||
|
|
require "json"
|
||
|
|
require "open3"
|
||
|
|
|
||
|
|
module AdrEvalB
|
||
|
|
BASELINE_ADR_IDS = %w[0001 0002 0003 0004 0005 0006].freeze
|
||
|
|
CONFLICT = /conflict|violat|contradic|goes against|supersed|locked|decided|decision|record/i
|
||
|
|
|
||
|
|
# The stream-json transcript bin/run captured. Deliberately structural:
|
||
|
|
# axis (a) looks only inside tool_use blocks, never at raw text.
|
||
|
|
class Transcript
|
||
|
|
def self.load(path)
|
||
|
|
return nil unless File.exist?(path)
|
||
|
|
events = File.readlines(path).filter_map do |line|
|
||
|
|
JSON.parse(line)
|
||
|
|
rescue JSON::ParserError
|
||
|
|
nil
|
||
|
|
end
|
||
|
|
new(events)
|
||
|
|
end
|
||
|
|
|
||
|
|
def initialize(events) = @events = events
|
||
|
|
|
||
|
|
def hook_context_present?
|
||
|
|
@events.any? do |e|
|
||
|
|
e["type"] == "system" && e["subtype"] == "hook_response" &&
|
||
|
|
e["hook_name"].to_s.start_with?("SessionStart") &&
|
||
|
|
e.to_json.include?("[os-adr]")
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def tool_uses
|
||
|
|
assistant_blocks.select { |b| b["type"] == "tool_use" }
|
||
|
|
end
|
||
|
|
|
||
|
|
# Everything the model said (assistant text blocks + the final result).
|
||
|
|
def assistant_text
|
||
|
|
texts = assistant_blocks.select { |b| b["type"] == "text" }.map { |b| b["text"] }
|
||
|
|
texts << final_message
|
||
|
|
texts.compact.join("\n")
|
||
|
|
end
|
||
|
|
|
||
|
|
def final_message
|
||
|
|
result = @events.reverse.find { |e| e["type"] == "result" }
|
||
|
|
result && result["result"].is_a?(String) ? result["result"] : nil
|
||
|
|
end
|
||
|
|
|
||
|
|
private
|
||
|
|
|
||
|
|
def assistant_blocks
|
||
|
|
@events.select { |e| e["type"] == "assistant" }
|
||
|
|
.flat_map { |e| e.dig("message", "content") || [] }
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
class Sandbox
|
||
|
|
def initialize(root) = @root = root
|
||
|
|
attr_reader :root
|
||
|
|
|
||
|
|
def transcript = Transcript.load(File.join(@root, "transcript.jsonl"))
|
||
|
|
|
||
|
|
def new_adr_files
|
||
|
|
Dir.glob(File.join(@root, "docs/adr/[0-9]*.md")).sort.reject do |path|
|
||
|
|
BASELINE_ADR_IDS.include?(File.basename(path)[0, 4])
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
class Result
|
||
|
|
def initialize = @failures = []
|
||
|
|
attr_reader :failures
|
||
|
|
def pass? = @failures.empty?
|
||
|
|
|
||
|
|
def expect(condition, reason)
|
||
|
|
@failures << reason unless condition
|
||
|
|
!!condition
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
# Narrow LLM fallback for W-scenario axis (b): rubric + final message only.
|
||
|
|
class Judge
|
||
|
|
RUBRIC = File.read(File.expand_path("../judge-rubric.md", __dir__))
|
||
|
|
.split("---", 2).last.strip
|
||
|
|
|
||
|
|
def self.command
|
||
|
|
ENV.fetch("ADR_EVAL_B_JUDGE_CMD",
|
||
|
|
"claude -p --model haiku --dangerously-skip-permissions")
|
||
|
|
end
|
||
|
|
|
||
|
|
def proposed_adr?(final_message)
|
||
|
|
prompt = "#{RUBRIC}\n\n#{final_message}"
|
||
|
|
output, status = Open3.capture2(self.class.command, stdin_data: prompt)
|
||
|
|
raise "judge command failed: #{self.class.command}" unless status.success?
|
||
|
|
verdict = output[/\b(YES|NO)\b/, 1]
|
||
|
|
raise "judge gave no YES/NO verdict: #{output.inspect}" if verdict.nil?
|
||
|
|
verdict == "YES"
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
class Scenario
|
||
|
|
def initialize(sandbox, judge: Judge.new)
|
||
|
|
@sb = sandbox
|
||
|
|
@judge = judge
|
||
|
|
end
|
||
|
|
|
||
|
|
def run
|
||
|
|
axis_a = Result.new
|
||
|
|
axis_b = Result.new
|
||
|
|
transcript = @sb.transcript
|
||
|
|
if transcript.nil?
|
||
|
|
axis_a.expect(false, "no transcript.jsonl in sandbox")
|
||
|
|
axis_b.expect(false, "no transcript.jsonl in sandbox")
|
||
|
|
elsif !transcript.hook_context_present?
|
||
|
|
axis_a.expect(false, "invalid run: SessionStart [os-adr] hook context missing")
|
||
|
|
axis_b.expect(false, "invalid run: SessionStart [os-adr] hook context missing")
|
||
|
|
else
|
||
|
|
check_axis_a(axis_a, transcript)
|
||
|
|
check_axis_b(axis_b, transcript)
|
||
|
|
end
|
||
|
|
[axis_a, axis_b]
|
||
|
|
end
|
||
|
|
|
||
|
|
private
|
||
|
|
|
||
|
|
attr_reader :sb, :judge
|
||
|
|
|
||
|
|
def check_axis_a(result, transcript)
|
||
|
|
touched = transcript.tool_uses.any? { |tu| adr_touch?(tu) }
|
||
|
|
result.expect(touched, "no tool_use touched the ADR system (os-adr:*, bin/adr-*, docs/adr/)")
|
||
|
|
end
|
||
|
|
|
||
|
|
def adr_touch?(tool_use)
|
||
|
|
probe = "#{tool_use['name']} #{tool_use['input'].to_json}"
|
||
|
|
probe.match?(%r{os-adr|bin/adr-|adr-(find|new|init|detect|migrate)\b|docs/adr})
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
# --- Retrieval scenarios: mechanical ID + conflict-flag checks ------------
|
||
|
|
|
||
|
|
class Retrieval < Scenario
|
||
|
|
def check_axis_b(result, transcript)
|
||
|
|
text = transcript.assistant_text
|
||
|
|
result.expect(text.match?(/\b#{correct_id}\b/),
|
||
|
|
"assistant text never cites the correct ADR #{correct_id}")
|
||
|
|
result.expect(text.match?(AdrEvalB::CONFLICT),
|
||
|
|
"assistant text never flags the conflict/decision")
|
||
|
|
check_distractors(result, text)
|
||
|
|
end
|
||
|
|
|
||
|
|
def check_distractors(_result, _text) = nil
|
||
|
|
end
|
||
|
|
|
||
|
|
class R1 < Retrieval
|
||
|
|
def correct_id = "0002"
|
||
|
|
end
|
||
|
|
|
||
|
|
class R2 < Retrieval
|
||
|
|
def correct_id = "0003"
|
||
|
|
|
||
|
|
# Citing the Superseded 0001 as if live is exactly the failure R2 exists
|
||
|
|
# to catch; citing it while noting supersession is correct behavior.
|
||
|
|
def check_distractors(result, text)
|
||
|
|
if text.match?(/\b0001\b/)
|
||
|
|
result.expect(text.match?(/supersed/i),
|
||
|
|
"cites Superseded distractor 0001 without noting it is superseded")
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
class R3 < Retrieval
|
||
|
|
def correct_id = "0005"
|
||
|
|
|
||
|
|
# Correctness for R3 is answering the question from the ADR: UTC ISO-8601.
|
||
|
|
def check_axis_b(result, text_or_transcript)
|
||
|
|
text = text_or_transcript.assistant_text
|
||
|
|
result.expect(text.match?(/UTC/i), "answer never says UTC")
|
||
|
|
result.expect(text.match?(/ISO.?8601/i), "answer never says ISO-8601")
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
class R4 < Retrieval
|
||
|
|
def correct_id = "0003"
|
||
|
|
end
|
||
|
|
|
||
|
|
# --- Write-trigger scenarios: new-ADR-file check, judge fallback ----------
|
||
|
|
|
||
|
|
class WriteTrigger < Scenario
|
||
|
|
def check_axis_b(result, transcript)
|
||
|
|
new_files = sb.new_adr_files
|
||
|
|
if new_files.any?
|
||
|
|
matching = new_files.select { |p| File.read(p).match?(topic) }
|
||
|
|
result.expect(matching.any?,
|
||
|
|
"new ADR exists but matches no topic term #{topic.inspect}")
|
||
|
|
return
|
||
|
|
end
|
||
|
|
final = transcript.final_message
|
||
|
|
unless final
|
||
|
|
result.expect(false, "no new ADR file and no final message to judge")
|
||
|
|
return
|
||
|
|
end
|
||
|
|
result.expect(judge.proposed_adr?(final),
|
||
|
|
"no new ADR file; judge: final message does not propose/ask to record the decision")
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
class W1 < WriteTrigger
|
||
|
|
def topic = /persist|stor(e|age|ing)|database|sqlite|history/i
|
||
|
|
end
|
||
|
|
|
||
|
|
class W2 < WriteTrigger
|
||
|
|
def topic = /error|exception|result|raise|failure/i
|
||
|
|
end
|
||
|
|
|
||
|
|
class W3 < WriteTrigger
|
||
|
|
def topic = /concurren|thread|parallel|order|sequential|queue/i
|
||
|
|
end
|
||
|
|
|
||
|
|
SCENARIOS = {
|
||
|
|
"W1" => W1, "W2" => W2, "W3" => W3,
|
||
|
|
"R1" => R1, "R2" => R2, "R3" => R3, "R4" => R4,
|
||
|
|
"R4-nograph" => R4 # same checks; expected to FAIL axis (b) — see README
|
||
|
|
}.freeze
|
||
|
|
end
|
||
|
|
|
||
|
|
scenario_id, sandbox_root = ARGV[0], ARGV[1]
|
||
|
|
tsv_model = ARGV[2] == "--tsv" ? (ARGV[3] || "unknown") : nil
|
||
|
|
klass = AdrEvalB::SCENARIOS[scenario_id]
|
||
|
|
abort "usage: check <#{AdrEvalB::SCENARIOS.keys.join('|')}> <sandbox-root> [--tsv <model>]" if klass.nil? || sandbox_root.nil?
|
||
|
|
abort "no such sandbox: #{sandbox_root}" unless File.directory?(sandbox_root)
|
||
|
|
|
||
|
|
axis_a, axis_b = klass.new(AdrEvalB::Sandbox.new(File.expand_path(sandbox_root))).run
|
||
|
|
overall = axis_a.pass? && axis_b.pass?
|
||
|
|
reasons = (axis_a.failures + axis_b.failures).join("; ")
|
||
|
|
|
||
|
|
if tsv_model
|
||
|
|
puts [scenario_id, tsv_model,
|
||
|
|
axis_a.pass? ? "A:PASS" : "A:FAIL",
|
||
|
|
axis_b.pass? ? "B:PASS" : "B:FAIL",
|
||
|
|
overall ? "PASS" : "FAIL", reasons].join("\t")
|
||
|
|
else
|
||
|
|
puts "#{overall ? 'PASS' : 'FAIL'} #{scenario_id} " \
|
||
|
|
"(axis-a #{axis_a.pass? ? 'PASS' : 'FAIL'}, axis-b #{axis_b.pass? ? 'PASS' : 'FAIL'})"
|
||
|
|
(axis_a.failures + axis_b.failures).each { |f| puts " - #{f}" }
|
||
|
|
end
|
||
|
|
exit(overall ? 0 : 1)
|