234 lines
8.8 KiB
Ruby
Executable File
234 lines
8.8 KiB
Ruby
Executable File
#!/usr/bin/env ruby
|
|
# frozen_string_literal: true
|
|
|
|
# Deterministic per-scenario checker for the os-adr model-tier eval (Eval A).
|
|
#
|
|
# check <S1..S6> <sandbox-root> [--tsv <model-label>]
|
|
#
|
|
# Reads the sandbox produced by bin/sandbox after a model run. Scenarios S3/S4
|
|
# additionally require the model's final answer saved to <sandbox>/ANSWER.md.
|
|
# Human mode prints PASS/FAIL + reasons; --tsv prints one tab-separated line
|
|
# (scenario, model, PASS|FAIL, reasons) for autoresearch's *-results.tsv.
|
|
# Exit 0 on PASS, 1 on FAIL, 2 on usage error.
|
|
|
|
require "date"
|
|
|
|
module AdrEval
|
|
PENDING = /_pending LLM fill/i
|
|
|
|
# Minimal frontmatter reader — deliberately independent of lib/adr so the
|
|
# checker cannot inherit a bug from the code it is scoring around.
|
|
class Frontmatter
|
|
def self.parse(text)
|
|
block = text[/\A---\n(.*?)\n---\n/m, 1] or return nil
|
|
fields = {}
|
|
block.scan(/^([\w-]+):[ \t]*(.*)$/) { |k, v| fields[k] = v.strip.gsub(/\A"|"\z/, "") }
|
|
new(fields)
|
|
end
|
|
|
|
def initialize(fields) = @fields = fields
|
|
def [](key) = @fields[key]
|
|
def list(key) = (@fields[key] || "").scan(/"([^"]*)"|([^\[\],\s]+)/).flatten.compact
|
|
end
|
|
|
|
class Sandbox
|
|
def initialize(root, eval_root)
|
|
@root = root
|
|
@eval_root = eval_root
|
|
end
|
|
|
|
attr_reader :root
|
|
|
|
def read(rel) = File.read(File.join(@root, rel))
|
|
def exist?(rel) = File.exist?(File.join(@root, rel))
|
|
def adr_files = Dir.glob(File.join(@root, "docs/adr/[0-9]*.md")).sort
|
|
def answer = exist?("ANSWER.md") ? read("ANSWER.md") : nil
|
|
|
|
def unchanged_from_fixture?(rel, fixture)
|
|
original = File.join(@eval_root, "fixture", fixture, rel)
|
|
File.exist?(original) && exist?(rel) && File.read(original) == read(rel)
|
|
end
|
|
|
|
def index = exist?("docs/adr/README.md") ? read("docs/adr/README.md") : ""
|
|
end
|
|
|
|
class Result
|
|
def initialize = @failures = []
|
|
def pass? = @failures.empty?
|
|
attr_reader :failures
|
|
|
|
# Returns the condition so callers can bail early on failure.
|
|
def expect(condition, reason)
|
|
@failures << reason unless condition
|
|
!!condition
|
|
end
|
|
end
|
|
|
|
class Scenario
|
|
def initialize(sandbox) = @sb = sandbox
|
|
|
|
def run
|
|
result = Result.new
|
|
check(result)
|
|
result
|
|
end
|
|
|
|
private
|
|
|
|
attr_reader :sb
|
|
|
|
# A well-formed, fully-authored ADR file at the given path.
|
|
def expect_complete_adr(result, path, id:)
|
|
unless sb.exist?(path)
|
|
result.expect(false, "missing #{path}")
|
|
return nil
|
|
end
|
|
text = sb.read(path)
|
|
fm = Frontmatter.parse(text)
|
|
return nil unless result.expect(!fm.nil?, "#{path}: no frontmatter")
|
|
result.expect(fm["id"] == id, "#{path}: id #{fm['id'].inspect} != #{id.inspect}")
|
|
result.expect(text !~ AdrEval::PENDING, "#{path}: pending-fill markers remain")
|
|
%w[Context Decision Consequences].each do |section|
|
|
body = text[/^## #{section}\n(.*?)(?=^## |\z)/m, 1].to_s.strip
|
|
result.expect(!body.empty?, "#{path}: empty #{section} section")
|
|
end
|
|
fm
|
|
end
|
|
|
|
def expect_indexed(result, id)
|
|
result.expect(sb.index.include?(id), "index README.md has no row for #{id}")
|
|
end
|
|
|
|
def expect_fixture_adrs_untouched(result, except: [])
|
|
%w[0001 0002 0003 0004].each do |id|
|
|
next if except.include?(id)
|
|
rel = Dir.glob(File.join(sb.root, "docs/adr/#{id}-*.md")).first
|
|
next result.expect(false, "fixture ADR #{id} missing") unless rel
|
|
rel = rel.sub("#{sb.root}/", "")
|
|
result.expect(sb.unchanged_from_fixture?(rel, "project"), "fixture ADR #{id} was modified")
|
|
end
|
|
end
|
|
end
|
|
|
|
class S1 < Scenario # create: new decision recorded via CLI
|
|
def check(result)
|
|
fm = expect_complete_adr(result, new_adr_path, id: "0005")
|
|
return unless fm
|
|
result.expect(fm["status"] == "Accepted", "0005 status #{fm['status'].inspect} != Accepted")
|
|
result.expect(!fm.list("affected-paths").empty?, "0005 affected-paths empty")
|
|
expect_indexed(result, "0005")
|
|
expect_fixture_adrs_untouched(result)
|
|
end
|
|
|
|
def new_adr_path
|
|
path = Dir.glob(File.join(sb.root, "docs/adr/0005-*.md")).first
|
|
path ? path.sub("#{sb.root}/", "") : "docs/adr/0005-*.md"
|
|
end
|
|
end
|
|
|
|
class S2 < S1 # create with supersession of 0001
|
|
def check(result)
|
|
fm = expect_complete_adr(result, new_adr_path, id: "0005")
|
|
return unless fm
|
|
result.expect(fm["supersedes"] == "0001",
|
|
"0005 supersedes #{fm['supersedes'].inspect} != 0001")
|
|
old = Frontmatter.parse(sb.read(old_adr_rel))
|
|
result.expect(old["status"] == "Superseded", "0001 status #{old['status'].inspect} != Superseded")
|
|
result.expect(old["superseded-by"] == "0005", "0001 superseded-by #{old['superseded-by'].inspect} != 0005")
|
|
expect_indexed(result, "0005")
|
|
expect_fixture_adrs_untouched(result, except: %w[0001])
|
|
end
|
|
|
|
def old_adr_rel
|
|
Dir.glob(File.join(sb.root, "docs/adr/0001-*.md")).first.sub("#{sb.root}/", "")
|
|
end
|
|
end
|
|
|
|
class FindScenario < Scenario
|
|
def check(result)
|
|
answer = sb.answer
|
|
return unless result.expect(!answer.nil?, "no ANSWER.md saved in sandbox")
|
|
result.expect(answer.include?("adr-find"), "answer never mentions running adr-find")
|
|
check_answer(result, answer)
|
|
expect_fixture_adrs_untouched(result)
|
|
end
|
|
end
|
|
|
|
class S3 < FindScenario # find: direct path match + conflict recognition
|
|
def check_answer(result, answer)
|
|
result.expect(answer.include?("0001"), "answer does not surface ADR 0001")
|
|
result.expect(answer.match?(/conflict|violat|supersed|contradic|locked/i),
|
|
"answer does not flag the conflict with 0001")
|
|
end
|
|
end
|
|
|
|
class S4 < FindScenario # find: distractor discrimination (0003 not superseded 0002)
|
|
def check_answer(result, answer)
|
|
result.expect(answer.include?("0003"), "answer does not surface ADR 0003")
|
|
if answer.include?("0002")
|
|
result.expect(answer.match?(/supersed/i),
|
|
"answer cites superseded 0002 without noting it is superseded")
|
|
end
|
|
end
|
|
end
|
|
|
|
class S5 < Scenario # init on the legacy project
|
|
def check(result)
|
|
result.expect(sb.exist?("docs/adr/README.md"), "docs/adr/README.md not created")
|
|
result.expect(sb.index.include?("<!--"), "index missing generated markers")
|
|
gitignore = sb.exist?(".gitignore") ? sb.read(".gitignore") : ""
|
|
result.expect(gitignore.include?(".os-adr"), ".os-adr/ not gitignored")
|
|
result.expect(sb.unchanged_from_fixture?("DECISIONS.md", "legacy-project"),
|
|
"DECISIONS.md was modified")
|
|
end
|
|
end
|
|
|
|
class S6 < Scenario # migrate the legacy DECISIONS.md, sources untouched, fills applied
|
|
STATUS_EXPECTATIONS = { "0001" => "Accepted", "0002" => "Accepted", "0003" => "Proposed" }.freeze
|
|
|
|
def check(result)
|
|
result.expect(sb.unchanged_from_fixture?("DECISIONS.md", "legacy-project"),
|
|
"DECISIONS.md was modified (sources must stay byte-identical)")
|
|
result.expect(sb.exist?("docs/adr/migration-report.md"), "migration-report.md missing")
|
|
result.expect(sb.adr_files.size >= 3, "expected 3 migrated ADRs, found #{sb.adr_files.size}")
|
|
STATUS_EXPECTATIONS.each do |id, expected|
|
|
path = Dir.glob(File.join(sb.root, "docs/adr/#{id}-*.md")).first
|
|
next result.expect(false, "migrated ADR #{id} missing") unless path
|
|
fm = Frontmatter.parse(File.read(path))
|
|
result.expect(fm && fm["status"] == expected,
|
|
"#{id} status #{fm && fm['status'].inspect} != #{expected} " \
|
|
"(SETTLED→Accepted / OPEN→Proposed mapping)")
|
|
end
|
|
first = Dir.glob(File.join(sb.root, "docs/adr/0001-*.md")).first
|
|
if first
|
|
text = File.read(first)
|
|
%w[Consequences Alternatives].each do |section|
|
|
body = text[/^## #{section}.*?\n(.*?)(?=^## |\z)/m, 1].to_s
|
|
result.expect(body !~ AdrEval::PENDING && !body.strip.empty?,
|
|
"0001 #{section} not filled from source material")
|
|
end
|
|
end
|
|
expect_indexed(result, "0001")
|
|
end
|
|
end
|
|
|
|
SCENARIOS = { "S1" => S1, "S2" => S2, "S3" => S3, "S4" => S4, "S5" => S5, "S6" => S6 }.freeze
|
|
end
|
|
|
|
scenario_id, sandbox_root = ARGV[0], ARGV[1]
|
|
tsv_model = ARGV[2] == "--tsv" ? (ARGV[3] || "unknown") : nil
|
|
klass = AdrEval::SCENARIOS[scenario_id]
|
|
abort "usage: check <#{AdrEval::SCENARIOS.keys.join('|')}> <sandbox-root> [--tsv <model>]" if klass.nil? || sandbox_root.nil?
|
|
abort "no such sandbox: #{sandbox_root}" unless File.directory?(sandbox_root)
|
|
|
|
eval_root = File.expand_path("..", __dir__)
|
|
result = klass.new(AdrEval::Sandbox.new(File.expand_path(sandbox_root), eval_root)).run
|
|
|
|
if tsv_model
|
|
puts [scenario_id, tsv_model, result.pass? ? "PASS" : "FAIL", result.failures.join("; ")].join("\t")
|
|
else
|
|
puts result.pass? ? "PASS #{scenario_id}" : "FAIL #{scenario_id}"
|
|
result.failures.each { |f| puts " - #{f}" }
|
|
end
|
|
exit(result.pass? ? 0 : 1)
|