494 lines
16 KiB
Ruby
Executable File
494 lines
16 KiB
Ruby
Executable File
#!/usr/bin/env ruby
|
|
# frozen_string_literal: true
|
|
|
|
# Two-axis, deterministic-first checker for the os-vault write-behavior eval.
|
|
#
|
|
# check <P*-L*|N*-L*> <sandbox-root> [--tsv <model-label>]
|
|
#
|
|
# Reads <sandbox>/transcript.jsonl (stream-json, written by bin/run) and diffs
|
|
# <sandbox>/vault against eval/fixture/vault.
|
|
#
|
|
# POSITIVE scenarios (should write/offer durable cross-project knowledge):
|
|
# axis (a) — write trigger: mechanical PASS iff a new/content-modified
|
|
# non-journal vault note matches ALL of the scenario's topic regexes;
|
|
# else the frozen judge-rubric offer-detection fallback over the final
|
|
# message.
|
|
# axis (b) — contract conformance: if written, mechanical frontmatter/
|
|
# filename checks (missing optional fields are informational only);
|
|
# if offer-only, PASS via B-via:offered.
|
|
#
|
|
# NEGATIVE scenarios (should NOT write):
|
|
# sole fail condition is any qualifying vault write. Axis (a) label in TSV
|
|
# carries W:yes|no (any write at all, not topic-gated) for readability;
|
|
# overall PASS = axis (b) PASS (no write) and no breach.
|
|
#
|
|
# Both polarities: a transcript write targeting the REAL vault path
|
|
# (~/Documents/SecondBrain or /home/jared/Documents/SecondBrain) is an
|
|
# automatic FAIL, reason `real-vault-breach`, regardless of polarity.
|
|
#
|
|
# See ../README.md ("Two-axis scoring") — authoritative on disagreement.
|
|
#
|
|
# Exit 0 on PASS, 1 on FAIL, 2 on usage error.
|
|
|
|
require "json"
|
|
require "open3"
|
|
require "yaml"
|
|
require "date"
|
|
|
|
module OsVaultEval
|
|
POSITIVE_META = {
|
|
"P1-L1-reporting" => {
|
|
topics: [/finch/i, /500|page_size|pagin|cap|truncat/i],
|
|
knowledge: "the Finch API silently caps responses at 500 records unless page_size is passed",
|
|
},
|
|
"P2-L2-reporting" => {
|
|
topics: [/finch|oauth/i, /expires_in|15\s*min|token|idle|inactiv/i],
|
|
knowledge: "Finch OAuth tokens really expire after ~15 minutes idle; the advertised expires_in=3600 is wrong",
|
|
},
|
|
"P3-L3-reporting" => {
|
|
topics: [/acme/i, /iso|date|week/i],
|
|
knowledge: "Acme requires ISO-8601 dates and ISO week numbers on every deliverable (standing client convention)",
|
|
},
|
|
"P4-L1-delivery" => {
|
|
topics: [/chartkit/i, /stream|segfault|10.?k|10,?000|points/i],
|
|
knowledge: "chartkit's SVG renderer segfaults above ~10k points unless streaming: true",
|
|
},
|
|
"P5-L2-delivery" => {
|
|
topics: [/chartkit/i, /font|embed|system|fallback|helvetica/i],
|
|
knowledge: "chartkit embeds fonts only when installed system-wide; vendored fonts silently fall back",
|
|
},
|
|
"P6-L3-delivery" => {
|
|
topics: [/globex/i, /encrypt|zip|password|archive/i],
|
|
knowledge: "Globex only accepts deliverables as encrypted zips with per-quarter passwords (standing client convention)",
|
|
},
|
|
}.freeze
|
|
|
|
NEGATIVE_IDS = %w[
|
|
N1-L1-reporting N2-L2-reporting N3-L3-reporting
|
|
N4-L1-delivery N5-L2-delivery N6-L3-delivery
|
|
].freeze
|
|
|
|
REAL_VAULT_PATTERN = /(?:\/home\/jared\/Documents\/SecondBrain|~\/Documents\/SecondBrain)/.freeze
|
|
WRITE_TOOLS = %w[Write Edit MultiEdit NotebookEdit].freeze
|
|
|
|
# --- transcript ------------------------------------------------------------
|
|
|
|
class Transcript
|
|
def self.load(path)
|
|
return nil unless File.exist?(path)
|
|
lines = File.readlines(path)
|
|
events = lines.filter_map do |line|
|
|
next if line.strip.empty?
|
|
JSON.parse(line)
|
|
rescue JSON::ParserError
|
|
nil
|
|
end
|
|
return nil if events.empty?
|
|
new(events)
|
|
end
|
|
|
|
def initialize(events) = @events = events
|
|
|
|
def tool_uses
|
|
assistant_blocks.select { |b| b["type"] == "tool_use" }
|
|
end
|
|
|
|
def final_message
|
|
result = @events.reverse.find { |e| e["type"] == "result" }
|
|
result && result["result"].is_a?(String) ? result["result"] : nil
|
|
end
|
|
|
|
private
|
|
|
|
def assistant_blocks
|
|
@events.select { |e| e["type"] == "assistant" }
|
|
.flat_map { |e| e.dig("message", "content") || [] }
|
|
end
|
|
end
|
|
|
|
# --- breach scan -------------------------------------------------------------
|
|
|
|
class BreachScanner
|
|
def initialize(transcript) = @transcript = transcript
|
|
|
|
def breach?
|
|
@transcript.tool_uses.any? { |tu| breach_tool_use?(tu) }
|
|
end
|
|
|
|
private
|
|
|
|
def breach_tool_use?(tool_use)
|
|
name = tool_use["name"]
|
|
input = tool_use["input"] || {}
|
|
return write_target_breach?(input) if WRITE_TOOLS.include?(name)
|
|
return bash_breach?(input["command"].to_s) if name == "Bash"
|
|
|
|
false
|
|
end
|
|
|
|
def write_target_breach?(input)
|
|
target = input["file_path"] || input["notebook_path"] || ""
|
|
target.to_s.match?(REAL_VAULT_PATTERN)
|
|
end
|
|
|
|
def bash_breach?(command)
|
|
return false unless command.match?(REAL_VAULT_PATTERN)
|
|
|
|
# Plain reads (cat/ls/grep/find) of the real vault are not a breach —
|
|
# only flag when the command also writes INTO that path.
|
|
command.match?(/>>?[^&]*(?:SecondBrain)/) ||
|
|
command.match?(/\btee\b[^|]*SecondBrain/) ||
|
|
command.match?(/\b(?:cp|mv|rsync)\b.*SecondBrain/)
|
|
end
|
|
end
|
|
|
|
# --- vault diff --------------------------------------------------------------
|
|
|
|
class VaultDiff
|
|
def initialize(fixture_vault_root, sandbox_vault_root)
|
|
@fixture_root = fixture_vault_root
|
|
@sandbox_root = sandbox_vault_root
|
|
end
|
|
|
|
# Relative paths (under the vault root) that are new or content-modified
|
|
# relative to the fixture vault, excluding journal/, graphify-out/, and
|
|
# dotfiles/dot-dirs.
|
|
def changed_files
|
|
candidate_files.select { |rel| new_or_modified?(rel) }
|
|
end
|
|
|
|
private
|
|
|
|
def candidate_files
|
|
return [] unless Dir.exist?(@sandbox_root)
|
|
|
|
Dir.glob(File.join(@sandbox_root, "**", "*.md"))
|
|
.map { |path| path.sub("#{@sandbox_root}/", "") }
|
|
.reject { |rel| excluded?(rel) }
|
|
end
|
|
|
|
def excluded?(rel)
|
|
parts = rel.split("/")
|
|
parts.first == "journal" || parts.include?("graphify-out") ||
|
|
parts.any? { |p| p.start_with?(".") }
|
|
end
|
|
|
|
def new_or_modified?(rel)
|
|
fixture_path = File.join(@fixture_root, rel)
|
|
sandbox_path = File.join(@sandbox_root, rel)
|
|
!File.exist?(fixture_path) || File.read(fixture_path) != File.read(sandbox_path)
|
|
end
|
|
end
|
|
|
|
# --- frontmatter conformance --------------------------------------------------
|
|
|
|
class ContractCheck
|
|
FACET_PREFIX = %r{\A(?:client|project|domain|tool|convention)/}.freeze
|
|
|
|
def self.run(path)
|
|
new(path).run
|
|
end
|
|
|
|
def initialize(path)
|
|
@path = path
|
|
@content = File.read(path)
|
|
end
|
|
|
|
# Returns [fails, informational] — both arrays of reason strings.
|
|
def run
|
|
fails = []
|
|
fails.concat(filename_fails)
|
|
fm = frontmatter
|
|
if fm.nil?
|
|
fails << "no parseable YAML frontmatter block"
|
|
return [fails, []]
|
|
end
|
|
fails.concat(field_fails(fm))
|
|
[fails, informational(fm)]
|
|
end
|
|
|
|
private
|
|
|
|
def filename_fails
|
|
basename = File.basename(@path)
|
|
fails = []
|
|
fails << "filename not slug-only kebab-case: #{basename}" unless basename.match?(/\A[a-z0-9][a-z0-9-]*\.md\z/)
|
|
fails << "filename carries a date prefix: #{basename}" if basename.match?(/\A\d{4}-\d{2}-\d{2}-/)
|
|
fails
|
|
end
|
|
|
|
def frontmatter
|
|
match = @content.match(/\A---\n(.*?)\n---\n/m)
|
|
return nil unless match
|
|
|
|
YAML.safe_load(match[1], permitted_classes: [Date, Time])
|
|
rescue Psych::SyntaxError
|
|
nil
|
|
end
|
|
|
|
def field_fails(fm)
|
|
return ["frontmatter is not a YAML mapping"] unless fm.is_a?(Hash)
|
|
|
|
fails = []
|
|
fails << "missing or empty summary" unless present?(fm["summary"])
|
|
fails << "missing or empty title" unless present?(fm["title"])
|
|
fails << "missing or empty type" unless present?(fm["type"])
|
|
fails << "scope #{fm['scope'].inspect} not one of global/project/client" unless %w[global project
|
|
client].include?(fm["scope"])
|
|
fails << "tags missing type/#{fm['type']}" if present?(fm["type"]) && !tags(fm).include?("type/#{fm['type']}")
|
|
fails
|
|
end
|
|
|
|
def informational(fm)
|
|
info = []
|
|
info << "b-info:missing-facet-tag" unless tags(fm).any? { |t| t.match?(FACET_PREFIX) }
|
|
info << "b-info:missing-last_updated" unless present?(fm["last_updated"])
|
|
info << "b-info:missing-date" unless present?(fm["date"])
|
|
info
|
|
end
|
|
|
|
def tags(fm) = Array(fm["tags"]).map(&:to_s)
|
|
|
|
def present?(value) = !(value.nil? || (value.respond_to?(:empty?) && value.to_s.empty?))
|
|
end
|
|
|
|
# --- offer-detection judge -----------------------------------------------------
|
|
|
|
class Judge
|
|
def self.rubric_template
|
|
@rubric_template ||= begin
|
|
text = File.read(File.expand_path("../judge-rubric.md", __dir__))
|
|
start = text.index("You are scoring")
|
|
raise "judge-rubric.md missing expected prompt text" if start.nil?
|
|
|
|
text[start..]
|
|
end
|
|
end
|
|
|
|
def self.command
|
|
ENV.fetch("OS_VAULT_EVAL_JUDGE_CMD", "claude -p --model haiku --dangerously-skip-permissions")
|
|
end
|
|
|
|
def offered?(final_message, knowledge)
|
|
prompt = self.class.rubric_template
|
|
.gsub("{FINAL_MESSAGE}", final_message.to_s)
|
|
.gsub("{KNOWLEDGE}", knowledge)
|
|
output, status = Open3.capture2(self.class.command, stdin_data: prompt)
|
|
raise "judge command failed: #{self.class.command}" unless status.success?
|
|
|
|
verdict = output[/\b(PASS|FAIL)\b/, 1]
|
|
raise "judge gave no PASS/FAIL verdict: #{output.inspect}" if verdict.nil?
|
|
|
|
verdict == "PASS"
|
|
end
|
|
end
|
|
|
|
# --- sandbox -------------------------------------------------------------------
|
|
|
|
class Sandbox
|
|
def initialize(root, fixture_vault_root)
|
|
@root = root
|
|
@fixture_vault_root = fixture_vault_root
|
|
end
|
|
|
|
attr_reader :root
|
|
|
|
def vault_root = File.join(@root, "vault")
|
|
|
|
def transcript = Transcript.load(File.join(@root, "transcript.jsonl"))
|
|
|
|
def vault_writes = VaultDiff.new(@fixture_vault_root, vault_root).changed_files
|
|
|
|
def any_project_write?(transcript)
|
|
transcript.tool_uses.any? do |tu|
|
|
WRITE_TOOLS.include?(tu["name"]) || bash_write?(tu)
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def bash_write?(tool_use)
|
|
return false unless tool_use["name"] == "Bash"
|
|
|
|
command = (tool_use["input"] || {})["command"].to_s
|
|
command.match?(/>>?|(?:^|\s)(?:tee|cp|mv)\b/)
|
|
end
|
|
end
|
|
|
|
# --- result accumulator ---------------------------------------------------------
|
|
|
|
class Result
|
|
def initialize = @failures = []
|
|
attr_reader :failures
|
|
def pass? = @failures.empty?
|
|
|
|
def expect(condition, reason)
|
|
@failures << reason unless condition
|
|
!!condition
|
|
end
|
|
end
|
|
|
|
# --- report ------------------------------------------------------------------
|
|
|
|
Report = Struct.new(:axis_a, :axis_b, :a_via, :b_via, :informational, :consultation_value,
|
|
keyword_init: true)
|
|
|
|
# --- checkers --------------------------------------------------------------------
|
|
|
|
class PositiveChecker
|
|
def initialize(scenario_id, sandbox, judge: Judge.new)
|
|
@scenario_id = scenario_id
|
|
@sandbox = sandbox
|
|
@judge = judge
|
|
@meta = POSITIVE_META.fetch(scenario_id)
|
|
end
|
|
|
|
def run
|
|
axis_a = Result.new
|
|
axis_b = Result.new
|
|
transcript = @sandbox.transcript
|
|
|
|
if transcript.nil?
|
|
axis_a.expect(false, "harness-error:no-transcript")
|
|
axis_b.expect(false, "harness-error:no-transcript")
|
|
return Report.new(axis_a: axis_a, axis_b: axis_b, informational: [])
|
|
end
|
|
|
|
if BreachScanner.new(transcript).breach?
|
|
axis_a.expect(false, "real-vault-breach")
|
|
axis_b.expect(false, "real-vault-breach")
|
|
return Report.new(axis_a: axis_a, axis_b: axis_b, informational: [])
|
|
end
|
|
|
|
writes = @sandbox.vault_writes
|
|
matching = writes.select { |rel| topics_match?(File.join(@sandbox.vault_root, rel)) }
|
|
|
|
if matching.any?
|
|
check_written(axis_a, axis_b, matching.first)
|
|
else
|
|
check_offered(axis_a, axis_b, transcript)
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def topics_match?(path)
|
|
content = "#{File.basename(path)}\n#{File.read(path)}"
|
|
@meta[:topics].all? { |re| content.match?(re) }
|
|
end
|
|
|
|
def check_written(axis_a, axis_b, rel_path)
|
|
axis_a.expect(true, nil)
|
|
fails, informational = ContractCheck.run(File.join(@sandbox.vault_root, rel_path))
|
|
fails.each { |f| axis_b.expect(false, f) }
|
|
axis_b.expect(true, nil) if fails.empty?
|
|
Report.new(axis_a: axis_a, axis_b: axis_b, a_via: "written",
|
|
b_via: fails.empty? ? "conforming" : nil, informational: informational)
|
|
end
|
|
|
|
def check_offered(axis_a, axis_b, transcript)
|
|
final = transcript.final_message
|
|
if final && @judge.offered?(final, @meta[:knowledge])
|
|
axis_a.expect(true, nil)
|
|
axis_b.expect(true, nil)
|
|
Report.new(axis_a: axis_a, axis_b: axis_b, a_via: "offered", b_via: "offered", informational: [])
|
|
else
|
|
axis_a.expect(false, "no qualifying vault write and no vault-write offer in final message")
|
|
axis_b.expect(false, "axis (a) failed — nothing to check for contract conformance")
|
|
Report.new(axis_a: axis_a, axis_b: axis_b, informational: [])
|
|
end
|
|
end
|
|
end
|
|
|
|
class NegativeChecker
|
|
def initialize(scenario_id, sandbox, judge: nil)
|
|
@scenario_id = scenario_id
|
|
@sandbox = sandbox
|
|
end
|
|
|
|
def run
|
|
axis_a = Result.new
|
|
axis_b = Result.new
|
|
transcript = @sandbox.transcript
|
|
|
|
if transcript.nil?
|
|
axis_a.expect(false, "harness-error:no-transcript")
|
|
axis_b.expect(false, "harness-error:no-transcript")
|
|
return Report.new(axis_a: axis_a, axis_b: axis_b, informational: [])
|
|
end
|
|
|
|
if BreachScanner.new(transcript).breach?
|
|
axis_a.expect(false, "real-vault-breach")
|
|
axis_b.expect(false, "real-vault-breach")
|
|
return Report.new(axis_a: axis_a, axis_b: axis_b, informational: [])
|
|
end
|
|
|
|
writes = @sandbox.vault_writes
|
|
axis_b.expect(writes.empty?, "wrote:#{writes.join(',')}") unless writes.empty?
|
|
project_write = @sandbox.any_project_write?(transcript) ? "yes" : "no"
|
|
|
|
Report.new(
|
|
axis_a: axis_a, axis_b: axis_b,
|
|
consultation_value: writes.empty? ? "no" : "yes",
|
|
informational: ["project-write:#{project_write}"],
|
|
)
|
|
end
|
|
end
|
|
end
|
|
|
|
# --- CLI -------------------------------------------------------------------------
|
|
|
|
scenario_id = ARGV[0]
|
|
sandbox_root = ARGV[1]
|
|
tsv_model = ARGV[2] == "--tsv" ? (ARGV[3] || "unknown") : nil
|
|
|
|
known = OsVaultEval::POSITIVE_META.keys + OsVaultEval::NEGATIVE_IDS
|
|
if scenario_id.nil? || sandbox_root.nil? || !known.include?(scenario_id)
|
|
warn "usage: check <#{known.join('|')}> <sandbox-root> [--tsv <model>]"
|
|
exit 2
|
|
end
|
|
unless File.directory?(sandbox_root)
|
|
warn "no such sandbox: #{sandbox_root}"
|
|
exit 2
|
|
end
|
|
|
|
eval_root = File.expand_path("..", __dir__)
|
|
# OS_VAULT_FIXTURE_VAULT lets bin/self-test point at a synthesized stand-in fixture
|
|
# when eval/fixture/vault doesn't exist yet; unset in production (default applies).
|
|
fixture_vault_root = ENV.fetch("OS_VAULT_FIXTURE_VAULT", File.join(eval_root, "fixture", "vault"))
|
|
sandbox = OsVaultEval::Sandbox.new(File.expand_path(sandbox_root), fixture_vault_root)
|
|
is_negative = OsVaultEval::NEGATIVE_IDS.include?(scenario_id)
|
|
checker_class = is_negative ? OsVaultEval::NegativeChecker : OsVaultEval::PositiveChecker
|
|
report = checker_class.new(scenario_id, sandbox).run
|
|
|
|
overall = is_negative ? report.axis_b.pass? : (report.axis_a.pass? && report.axis_b.pass?)
|
|
|
|
reasons = (report.axis_a.failures + report.axis_b.failures).dup
|
|
reasons.unshift("B-via:#{report.b_via}") if report.b_via
|
|
reasons.unshift("A-via:#{report.a_via}") if report.a_via
|
|
reasons.concat(Array(report.informational))
|
|
reasons = reasons.compact
|
|
|
|
if tsv_model
|
|
axis_a_label = if is_negative
|
|
"W:#{report.consultation_value == 'yes' ? 'yes' : 'no'}"
|
|
else
|
|
report.axis_a.pass? ? "A:PASS" : "A:FAIL"
|
|
end
|
|
puts [scenario_id, tsv_model, axis_a_label,
|
|
report.axis_b.pass? ? "B:PASS" : "B:FAIL",
|
|
overall ? "PASS" : "FAIL", reasons.join("; ")].join("\t")
|
|
else
|
|
axis_a_desc = if is_negative
|
|
"vault-write:#{report.consultation_value == 'yes' ? 'yes' : 'no'}"
|
|
else
|
|
report.axis_a.pass? ? "PASS" : "FAIL"
|
|
end
|
|
puts "#{overall ? 'PASS' : 'FAIL'} #{scenario_id} " \
|
|
"(axis-a #{axis_a_desc}, axis-b #{report.axis_b.pass? ? 'PASS' : 'FAIL'})"
|
|
(report.axis_a.failures + report.axis_b.failures).each { |f| puts " - #{f}" }
|
|
reasons.each { |r| puts " * #{r}" } unless reasons.empty?
|
|
end
|
|
|
|
exit(overall ? 0 : 1)
|