446 lines
15 KiB
Ruby
Executable File
446 lines
15 KiB
Ruby
Executable File
#!/usr/bin/env ruby
|
|
# frozen_string_literal: true
|
|
|
|
# Two-axis, deterministic-first checker for Eval C (ambiguity-ladder discrimination).
|
|
#
|
|
# check <P*-L*|N*-L*> <sandbox-root> [--tsv <model-label>]
|
|
#
|
|
# Reads <sandbox>/transcript.jsonl (stream-json, written by bin/run).
|
|
#
|
|
# Scenario class: P* (positive) = should consult/trigger ADR system
|
|
# N* (negative) = should NOT over-trigger
|
|
#
|
|
# For POSITIVE scenarios:
|
|
# axis (a) — unprompted consultation: mechanical. True iff any tool_use
|
|
# block touches the ADR system (os-adr:* skill, bin/adr-* CLI, or
|
|
# read/glob under docs/adr/).
|
|
# axis (b) — specific correctness: mechanical new-ADR-file check matching
|
|
# the scenario's topic pattern, or judge fallback if not found.
|
|
#
|
|
# For NEGATIVE scenarios:
|
|
# axis (a) — over-trigger detection: mechanical. True iff NO tool_use
|
|
# touched the ADR system (inverse of positive).
|
|
# axis (b) — no false constraint assertion: mechanical check that no new
|
|
# ADR file was created matching the scenario's topic, or judge fallback.
|
|
#
|
|
# Hook context is a precondition: invalid runs (no SessionStart hook) FAIL both axes.
|
|
#
|
|
# Overall PASS requires both axes. TSV mode for autoresearch integration.
|
|
# Exit 0 on PASS, 1 on FAIL, 2 on usage error.
|
|
|
|
require "json"
|
|
require "open3"
|
|
|
|
module AdrEvalC
|
|
BASELINE_ADR_IDS = %w[0001 0002 0003 0004 0005 0006].freeze
|
|
CONFLICT = /conflict|violat|contradic|goes against|decision|record|architecture/i
|
|
|
|
class Transcript
|
|
def self.load(path)
|
|
return nil unless File.exist?(path)
|
|
events = File.readlines(path).filter_map do |line|
|
|
JSON.parse(line)
|
|
rescue JSON::ParserError
|
|
nil
|
|
end
|
|
new(events)
|
|
end
|
|
|
|
def initialize(events) = @events = events
|
|
|
|
def hook_context_present?
|
|
@events.any? do |e|
|
|
e["type"] == "system" && e["subtype"] == "hook_response" &&
|
|
e["hook_name"].to_s.start_with?("SessionStart") &&
|
|
e.to_json.include?("[os-adr]")
|
|
end
|
|
end
|
|
|
|
def tool_uses
|
|
assistant_blocks.select { |b| b["type"] == "tool_use" }
|
|
end
|
|
|
|
def assistant_text
|
|
texts = assistant_blocks.select { |b| b["type"] == "text" }.map { |b| b["text"] }
|
|
texts << final_message
|
|
texts.compact.join("\n")
|
|
end
|
|
|
|
def final_message
|
|
result = @events.reverse.find { |e| e["type"] == "result" }
|
|
result && result["result"].is_a?(String) ? result["result"] : nil
|
|
end
|
|
|
|
private
|
|
|
|
def assistant_blocks
|
|
@events.select { |e| e["type"] == "assistant" }
|
|
.flat_map { |e| e.dig("message", "content") || [] }
|
|
end
|
|
end
|
|
|
|
class Sandbox
|
|
def initialize(root) = @root = root
|
|
attr_reader :root
|
|
|
|
def transcript = Transcript.load(File.join(@root, "transcript.jsonl"))
|
|
|
|
def new_adr_files
|
|
Dir.glob(File.join(@root, "docs/adr/[0-9]*.md")).sort.reject do |path|
|
|
BASELINE_ADR_IDS.include?(File.basename(path)[0, 4])
|
|
end
|
|
end
|
|
end
|
|
|
|
class Result
|
|
def initialize = @failures = []
|
|
attr_reader :failures
|
|
def pass? = @failures.empty?
|
|
|
|
def expect(condition, reason)
|
|
@failures << reason unless condition
|
|
!!condition
|
|
end
|
|
end
|
|
|
|
class InformationalResult
|
|
# For informational metrics (doesn't affect pass/fail, just recorded)
|
|
def initialize(value)
|
|
@value = value
|
|
end
|
|
|
|
attr_reader :value
|
|
def pass? = true # Informational results never fail
|
|
end
|
|
|
|
class Judge
|
|
RUBRIC = File.read(File.expand_path("../judge-rubric.md", __dir__))
|
|
.split("---", 2).last.strip
|
|
|
|
def self.command
|
|
ENV.fetch("ADR_EVAL_C_JUDGE_CMD",
|
|
"claude -p --model haiku --dangerously-skip-permissions")
|
|
end
|
|
|
|
def proposed_adr?(final_message)
|
|
prompt = "#{RUBRIC}\n\n#{final_message}"
|
|
output, status = Open3.capture2(self.class.command, stdin_data: prompt)
|
|
raise "judge command failed: #{self.class.command}" unless status.success?
|
|
verdict = output[/\b(YES|NO)\b/, 1]
|
|
raise "judge gave no YES/NO verdict: #{output.inspect}" if verdict.nil?
|
|
verdict == "YES"
|
|
end
|
|
end
|
|
|
|
class Scenario
|
|
def initialize(sandbox, judge: Judge.new)
|
|
@sb = sandbox
|
|
@judge = judge
|
|
end
|
|
|
|
def run
|
|
axis_a = Result.new
|
|
axis_b = Result.new
|
|
transcript = @sb.transcript
|
|
if transcript.nil?
|
|
axis_a.expect(false, "no transcript.jsonl in sandbox")
|
|
axis_b.expect(false, "no transcript.jsonl in sandbox")
|
|
elsif !transcript.hook_context_present?
|
|
axis_a.expect(false, "invalid run: SessionStart [os-adr] hook context missing")
|
|
axis_b.expect(false, "invalid run: SessionStart [os-adr] hook context missing")
|
|
else
|
|
check_axis_a(axis_a, transcript)
|
|
check_axis_b(axis_b, transcript)
|
|
end
|
|
[axis_a, axis_b]
|
|
end
|
|
|
|
private
|
|
|
|
attr_reader :sb, :judge
|
|
|
|
def check_axis_a(result, transcript)
|
|
# Override in subclasses
|
|
raise NotImplementedError, "subclasses must override check_axis_a"
|
|
end
|
|
|
|
def adr_touch?(tool_use)
|
|
probe = "#{tool_use['name']} #{tool_use['input'].to_json}"
|
|
probe.match?(%r{os-adr|bin/adr-|adr-(find|new|init|detect|migrate)\b|docs/adr})
|
|
end
|
|
end
|
|
|
|
# --- Positive scenarios: should trigger ADR system ---
|
|
|
|
class PositiveScenario < Scenario
|
|
private
|
|
|
|
def check_axis_a(result, transcript)
|
|
touched = transcript.tool_uses.any? { |tu| adr_touch?(tu) }
|
|
result.expect(touched, "no tool_use touched the ADR system (os-adr:*, bin/adr-*, docs/adr/)")
|
|
end
|
|
end
|
|
|
|
class P1L1Execution < PositiveScenario
|
|
def check_axis_b(result, transcript)
|
|
new_files = sb.new_adr_files
|
|
if new_files.any?
|
|
matching = new_files.select { |p| File.read(p).match?(topic) }
|
|
result.expect(matching.any?,
|
|
"new ADR exists but matches no topic term #{topic.inspect}")
|
|
return
|
|
end
|
|
final = transcript.final_message
|
|
unless final
|
|
result.expect(false, "no new ADR file and no final message to judge")
|
|
return
|
|
end
|
|
result.expect(judge.proposed_adr?(final),
|
|
"no new ADR file; judge: final message does not propose/ask to record the decision")
|
|
end
|
|
|
|
def topic = /timeout|retry|backoff|exponential|attempt/i
|
|
end
|
|
|
|
class P2L2Execution < PositiveScenario
|
|
def check_axis_b(result, transcript)
|
|
new_files = sb.new_adr_files
|
|
if new_files.any?
|
|
matching = new_files.select { |p| File.read(p).match?(topic) }
|
|
result.expect(matching.any?,
|
|
"new ADR exists but matches no topic term #{topic.inspect}")
|
|
return
|
|
end
|
|
final = transcript.final_message
|
|
unless final
|
|
result.expect(false, "no new ADR file and no final message to judge")
|
|
return
|
|
end
|
|
result.expect(judge.proposed_adr?(final),
|
|
"no new ADR file; judge: final message does not propose/ask to record the decision")
|
|
end
|
|
|
|
def topic = /escalat|termin|failure|review|abandon/i
|
|
end
|
|
|
|
class P3L3Execution < PositiveScenario
|
|
def check_axis_b(result, transcript)
|
|
new_files = sb.new_adr_files
|
|
if new_files.any?
|
|
matching = new_files.select { |p| File.read(p).match?(topic) }
|
|
result.expect(matching.any?,
|
|
"new ADR exists but matches no topic term #{topic.inspect}")
|
|
return
|
|
end
|
|
final = transcript.final_message
|
|
unless final
|
|
result.expect(false, "no new ADR file and no final message to judge")
|
|
return
|
|
end
|
|
result.expect(judge.proposed_adr?(final),
|
|
"no new ADR file; judge: final message does not propose/ask to record the decision")
|
|
end
|
|
|
|
def topic = /abandon|finish|complet|reliability|guarant/i
|
|
end
|
|
|
|
class P4L1Notifications < PositiveScenario
|
|
def check_axis_b(result, transcript)
|
|
new_files = sb.new_adr_files
|
|
if new_files.any?
|
|
matching = new_files.select { |p| File.read(p).match?(topic) }
|
|
result.expect(matching.any?,
|
|
"new ADR exists but matches no topic term #{topic.inspect}")
|
|
return
|
|
end
|
|
final = transcript.final_message
|
|
unless final
|
|
result.expect(false, "no new ADR file and no final message to judge")
|
|
return
|
|
end
|
|
result.expect(judge.proposed_adr?(final),
|
|
"no new ADR file; judge: final message does not propose/ask to record the decision")
|
|
end
|
|
|
|
def topic = /notif|route|service|webhook|event/i
|
|
end
|
|
|
|
class P5L2Notifications < PositiveScenario
|
|
def check_axis_b(result, transcript)
|
|
new_files = sb.new_adr_files
|
|
if new_files.any?
|
|
matching = new_files.select { |p| File.read(p).match?(topic) }
|
|
result.expect(matching.any?,
|
|
"new ADR exists but matches no topic term #{topic.inspect}")
|
|
return
|
|
end
|
|
final = transcript.final_message
|
|
unless final
|
|
result.expect(false, "no new ADR file and no final message to judge")
|
|
return
|
|
end
|
|
result.expect(judge.proposed_adr?(final),
|
|
"no new ADR file; judge: final message does not propose/ask to record the decision")
|
|
end
|
|
|
|
def topic = /webhook|deliv|retry|notification/i
|
|
end
|
|
|
|
class P6L3Notifications < PositiveScenario
|
|
def check_axis_b(result, transcript)
|
|
new_files = sb.new_adr_files
|
|
if new_files.any?
|
|
matching = new_files.select { |p| File.read(p).match?(topic) }
|
|
result.expect(matching.any?,
|
|
"new ADR exists but matches no topic term #{topic.inspect}")
|
|
return
|
|
end
|
|
final = transcript.final_message
|
|
unless final
|
|
result.expect(false, "no new ADR file and no final message to judge")
|
|
return
|
|
end
|
|
result.expect(judge.proposed_adr?(final),
|
|
"no new ADR file; judge: final message does not propose/ask to record the decision")
|
|
end
|
|
|
|
def topic = /notif|channel|routing|delivery|unified/i
|
|
end
|
|
|
|
# --- Negative scenarios: should NOT create false ADRs or assert phantom constraints ---
|
|
|
|
class NegativeScenario < Scenario
|
|
def check_axis_a(result, transcript)
|
|
# Axis (a) for negatives is INFORMATIONAL ONLY (doesn't affect pass/fail)
|
|
# Record whether ADR system was consulted (yes/no)
|
|
touched = transcript.tool_uses.any? { |tu| adr_touch?(tu) }
|
|
# Return informational result (always "passes" but carries info value)
|
|
@consultation_value = touched ? "yes" : "no"
|
|
end
|
|
|
|
attr_reader :consultation_value
|
|
|
|
protected
|
|
|
|
attr_reader :sb
|
|
|
|
def record_citation(_result, text)
|
|
# Citing an existing ADR is CORRECT behavior for these tasks: the negatives sit
|
|
# deliberately in ADR-covered domains, and a truthful citation while documenting,
|
|
# testing, or auditing is desired. Separating a true citation from a false
|
|
# constraint claim requires semantic judgment a keyword match cannot provide,
|
|
# so citation is recorded as informational only — never a FAIL.
|
|
@citation_value = text.match?(/\b000[1-6]\b|ADR \d{4}/i) ? "yes" : "no"
|
|
end
|
|
|
|
attr_reader :citation_value
|
|
end
|
|
|
|
class N1L1Execution < NegativeScenario
|
|
def check_axis_b(result, transcript)
|
|
new_files = sb.new_adr_files
|
|
result.expect(new_files.empty?,
|
|
"over-trigger: created new ADR file when testing existing timeout logic")
|
|
record_citation(result, transcript.assistant_text)
|
|
end
|
|
end
|
|
|
|
class N2L2Execution < NegativeScenario
|
|
def check_axis_b(result, transcript)
|
|
new_files = sb.new_adr_files
|
|
result.expect(new_files.empty?,
|
|
"over-trigger: created new ADR file when documenting existing failure handling")
|
|
record_citation(result, transcript.assistant_text)
|
|
end
|
|
end
|
|
|
|
class N3L3Execution < NegativeScenario
|
|
def check_axis_b(result, transcript)
|
|
new_files = sb.new_adr_files
|
|
result.expect(new_files.empty?,
|
|
"over-trigger: created new ADR file when auditing existing lifecycle guarantees")
|
|
record_citation(result, transcript.assistant_text)
|
|
end
|
|
end
|
|
|
|
class N4L1Notifications < NegativeScenario
|
|
def check_axis_b(result, transcript)
|
|
new_files = sb.new_adr_files
|
|
result.expect(new_files.empty?,
|
|
"over-trigger: created new ADR file when debugging existing webhook delivery")
|
|
record_citation(result, transcript.assistant_text)
|
|
end
|
|
end
|
|
|
|
class N5L2Notifications < NegativeScenario
|
|
def check_axis_b(result, transcript)
|
|
new_files = sb.new_adr_files
|
|
result.expect(new_files.empty?,
|
|
"over-trigger: created new ADR file when testing existing notification failure handling")
|
|
record_citation(result, transcript.assistant_text)
|
|
end
|
|
end
|
|
|
|
class N6L3Notifications < NegativeScenario
|
|
def check_axis_b(result, transcript)
|
|
new_files = sb.new_adr_files
|
|
result.expect(new_files.empty?,
|
|
"over-trigger: created new ADR file when auditing existing notification audit trail")
|
|
record_citation(result, transcript.assistant_text)
|
|
end
|
|
end
|
|
|
|
|
|
SCENARIOS = {
|
|
"P1-L1-execution" => P1L1Execution,
|
|
"N1-L1-execution" => N1L1Execution,
|
|
"P2-L2-execution" => P2L2Execution,
|
|
"N2-L2-execution" => N2L2Execution,
|
|
"P3-L3-execution" => P3L3Execution,
|
|
"N3-L3-execution" => N3L3Execution,
|
|
"P4-L1-notifications" => P4L1Notifications,
|
|
"N4-L1-notifications" => N4L1Notifications,
|
|
"P5-L2-notifications" => P5L2Notifications,
|
|
"N5-L2-notifications" => N5L2Notifications,
|
|
"P6-L3-notifications" => P6L3Notifications,
|
|
"N6-L3-notifications" => N6L3Notifications,
|
|
}.freeze
|
|
end
|
|
|
|
scenario_id, sandbox_root = ARGV[0], ARGV[1]
|
|
tsv_model = ARGV[2] == "--tsv" ? (ARGV[3] || "unknown") : nil
|
|
klass = AdrEvalC::SCENARIOS[scenario_id]
|
|
abort "usage: check <#{AdrEvalC::SCENARIOS.keys.join('|')}> <sandbox-root> [--tsv <model>]" if klass.nil? || sandbox_root.nil?
|
|
abort "no such sandbox: #{sandbox_root}" unless File.directory?(sandbox_root)
|
|
|
|
checker = klass.new(AdrEvalC::Sandbox.new(File.expand_path(sandbox_root)))
|
|
axis_a, axis_b = checker.run
|
|
is_negative = scenario_id.start_with?("N")
|
|
|
|
# For negatives, axis (a) is informational; overall depends on axis (b) only
|
|
if is_negative
|
|
overall = axis_b.pass?
|
|
axis_a_label = "A:#{checker.consultation_value || 'unknown'}"
|
|
else
|
|
overall = axis_a.pass? && axis_b.pass?
|
|
axis_a_label = axis_a.pass? ? "A:PASS" : "A:FAIL"
|
|
end
|
|
|
|
reasons = (axis_a.failures + axis_b.failures).join("; ")
|
|
if is_negative && checker.respond_to?(:citation_value) && checker.citation_value
|
|
reasons = ["cited-adr:#{checker.citation_value}", reasons].reject(&:empty?).join("; ")
|
|
end
|
|
|
|
if tsv_model
|
|
puts [scenario_id, tsv_model,
|
|
axis_a_label,
|
|
axis_b.pass? ? "B:PASS" : "B:FAIL",
|
|
overall ? "PASS" : "FAIL", reasons].join("\t")
|
|
else
|
|
axis_a_desc = is_negative ? "consultation:#{checker.consultation_value || 'unknown'}" : (axis_a.pass? ? "PASS" : "FAIL")
|
|
puts "#{overall ? 'PASS' : 'FAIL'} #{scenario_id} " \
|
|
"(axis-a #{axis_a_desc}, axis-b #{axis_b.pass? ? 'PASS' : 'FAIL'})"
|
|
(axis_a.failures + axis_b.failures).each { |f| puts " - #{f}" }
|
|
end
|
|
exit(overall ? 0 : 1)
|