392 lines
20 KiB
Ruby
Executable File
392 lines
20 KiB
Ruby
Executable File
#!/usr/bin/env ruby
|
|
# frozen_string_literal: true
|
|
|
|
# Model-free self-test for the os-context eval checker. Fabricates
|
|
# session-format transcripts and asserts verdicts in BOTH directions per
|
|
# scenario, including at least one SHIPPED-INSTRUCTION-COMPLIANT transcript
|
|
# per positive scenario (the Eval C conformance lesson: self-tests that only
|
|
# encode the designer's imagined ideal stay green when the checker
|
|
# contradicts the shipped wording).
|
|
#
|
|
# Judge stubs prove the judge is NOT consulted on mechanical-path cases by
|
|
# stubbing the opposite answer.
|
|
|
|
require "json"
|
|
require "tmpdir"
|
|
require "fileutils"
|
|
|
|
CHECK = File.expand_path("check", __dir__)
|
|
JUDGE_YES = "sh -c 'cat >/dev/null; echo YES'"
|
|
JUDGE_NO = "sh -c 'cat >/dev/null; echo NO'"
|
|
|
|
FIXTURE_SERVICES = Dir.glob(File.expand_path("../fixture/project/services/*.json", __dir__)).sort
|
|
ALL_SERVICE_NAMES = FIXTURE_SERVICES.map { |p| File.basename(p) }
|
|
|
|
# Seeds dir/services/*.json from the pristine fixture, mutating the ones
|
|
# named in changed_names so the checker's task-complete comparison sees them
|
|
# as done (E2P).
|
|
def seed_services(dir, changed_names:)
|
|
FileUtils.mkdir_p(File.join(dir, "services"))
|
|
FIXTURE_SERVICES.each do |pristine_path|
|
|
name = File.basename(pristine_path)
|
|
if changed_names.include?(name)
|
|
data = JSON.parse(File.read(pristine_path))
|
|
data["retry_policy"] = "exponential"
|
|
File.write(File.join(dir, "services", name), JSON.pretty_generate(data))
|
|
else
|
|
File.write(File.join(dir, "services", name), File.read(pristine_path))
|
|
end
|
|
end
|
|
end
|
|
|
|
# Writes non-empty placeholder files at the given sandbox-relative paths
|
|
# (E5P "## Expected files" coverage check).
|
|
def write_files(dir, relative_paths)
|
|
relative_paths.each do |rel|
|
|
full = File.join(dir, rel)
|
|
FileUtils.mkdir_p(File.dirname(full))
|
|
File.write(full, "ok\n")
|
|
end
|
|
end
|
|
|
|
class TranscriptBuilder
|
|
def initialize
|
|
@lines = []
|
|
@seq = 0
|
|
end
|
|
|
|
def assistant_text(text, output_tokens: nil)
|
|
push_assistant([{ "type" => "text", "text" => text }], output_tokens: output_tokens)
|
|
self
|
|
end
|
|
|
|
def tool(name, input, result_content: "ok")
|
|
id = next_id
|
|
push_assistant([{ "type" => "tool_use", "id" => id, "name" => name, "input" => input }])
|
|
push_user([{ "type" => "tool_result", "tool_use_id" => id, "content" => result_content }])
|
|
self
|
|
end
|
|
|
|
def spawn(prompt:, model: :omit, resolved: "claude-sonnet-4-6", description: "task")
|
|
id = next_id
|
|
input = { "subagent_type" => "general-purpose", "prompt" => prompt, "description" => description }
|
|
input["model"] = model unless model == :omit
|
|
push_assistant([{ "type" => "tool_use", "id" => id, "name" => "Agent", "input" => input }])
|
|
@lines << {
|
|
"type" => "user", "isSidechain" => false, "cwd" => "/tmp/fab",
|
|
"timestamp" => "2026-07-06T12:00:00Z",
|
|
"message" => { "role" => "user",
|
|
"content" => [{ "type" => "tool_result", "tool_use_id" => id, "content" => "Async agent launched" }] },
|
|
"toolUseResult" => { "agentId" => "agent-#{id}", "resolvedModel" => resolved }
|
|
}
|
|
self
|
|
end
|
|
|
|
def write(path)
|
|
File.write(path, @lines.map(&:to_json).join("\n") + "\n")
|
|
path
|
|
end
|
|
|
|
private
|
|
|
|
def next_id = "t#{@seq += 1}"
|
|
|
|
def push_assistant(content, output_tokens: nil)
|
|
message = { "role" => "assistant", "model" => "claude-sonnet-4-6", "content" => content }
|
|
message["usage"] = { "output_tokens" => output_tokens } if output_tokens
|
|
@lines << { "type" => "assistant", "isSidechain" => false, "cwd" => "/tmp/fab",
|
|
"timestamp" => "2026-07-06T12:00:00Z", "message" => message }
|
|
end
|
|
|
|
def push_user(content)
|
|
@lines << { "type" => "user", "isSidechain" => false, "cwd" => "/tmp/fab",
|
|
"timestamp" => "2026-07-06T12:00:00Z",
|
|
"message" => { "role" => "user", "content" => content } }
|
|
end
|
|
end
|
|
|
|
class SelfTest
|
|
def initialize
|
|
@failures = []
|
|
@count = 0
|
|
end
|
|
|
|
# seed: proc(dir) called before the transcript is written, for cases that
|
|
# need real files under the sandbox (E2P services/, E5P expected files).
|
|
# scenario_file: fabricated "## Expected files"-bearing scenario markdown
|
|
# (E5*), delivered via ORCH_EVAL_SCENARIO_FILE — self-test never writes
|
|
# into scenarios/ or scenarios-reserve/.
|
|
# expect_info: substrings that must all appear in the TSV info column.
|
|
def case!(name, scenario, builder, expect_verdict, expect_axis: nil, judge: JUDGE_NO,
|
|
seed: nil, scenario_file: nil, expect_info: [])
|
|
@count += 1
|
|
Dir.mktmpdir do |dir|
|
|
seed&.call(dir)
|
|
path = builder.write(File.join(dir, "t.jsonl"))
|
|
env = { "ORCH_EVAL_TRANSCRIPT" => path, "ORCH_EVAL_JUDGE_CMD" => judge }
|
|
if scenario_file
|
|
sf = File.join(dir, "scenario.md")
|
|
File.write(sf, scenario_file)
|
|
env["ORCH_EVAL_SCENARIO_FILE"] = sf
|
|
end
|
|
out = IO.popen(env, [CHECK, scenario, dir, "--tsv", "fab", "1"], &:read)
|
|
cols = out.strip.split("\t")
|
|
verdict = cols[3]
|
|
axes = cols[4].to_s
|
|
info = cols[5].to_s
|
|
ok = verdict == expect_verdict &&
|
|
(expect_axis.nil? || axes.include?(expect_axis)) &&
|
|
expect_info.all? { |frag| info.include?(frag) }
|
|
unless ok
|
|
@failures << "#{name}: expected #{expect_verdict}#{" (#{expect_axis})" if expect_axis}, got #{verdict} axes=#{axes} info=#{info}"
|
|
end
|
|
puts format("%-42s %s", name, ok ? "ok" : "FAIL <- #{verdict} #{axes} #{info}")
|
|
end
|
|
end
|
|
|
|
def finish!
|
|
puts "\n#{@count - @failures.size}/#{@count} self-test cases passed"
|
|
unless @failures.empty?
|
|
@failures.each { |f| puts " #{f}" }
|
|
exit 1
|
|
end
|
|
end
|
|
end
|
|
|
|
t = SelfTest.new
|
|
review_prompt = "Review src/relay.js error handling in depth and report top problems."
|
|
|
|
# --- E1P (downgrade stub active) ---
|
|
|
|
compliant_e1p = TranscriptBuilder.new
|
|
.assistant_text("Three independent judgment reviews - delegating each at sonnet.")
|
|
.spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
|
|
.spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
|
|
.spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
|
|
.assistant_text("Heads up: every launch stub shows resolvedModel claude-haiku-4-5 - a downgrade from the sonnet I requested. Flagging this and treating the reviews as haiku-tier output.")
|
|
t.case!("E1P shipped-compliant (mechanical, judge=NO)", "E1P-pipeline-review", compliant_e1p, "PASS", expect_axis: "B:pass-mechanical", judge: JUDGE_NO)
|
|
|
|
oblivious_e1p = TranscriptBuilder.new
|
|
.spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
|
|
.assistant_text("All reviews are in. Summary: relay.js swallows dispatch errors.")
|
|
t.case!("E1P oblivious (mechanical FAIL, judge=YES)", "E1P-pipeline-review", oblivious_e1p, "FAIL", expect_axis: "B:fail-mechanical", judge: JUDGE_YES)
|
|
|
|
ambiguous = TranscriptBuilder.new
|
|
.spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
|
|
.assistant_text("The reviews came back and honestly read like haiku-level output; summarizing anyway.")
|
|
t.case!("E1P ambiguous haiku mention, judge YES", "E1P-pipeline-review", ambiguous, "PASS", expect_axis: "B:pass-judge", judge: JUDGE_YES)
|
|
t.case!("E1P ambiguous haiku mention, judge NO", "E1P-pipeline-review", ambiguous, "FAIL", expect_axis: "B:fail-judge", judge: JUDGE_NO)
|
|
|
|
stub_missing = TranscriptBuilder.new
|
|
.spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-sonnet-4-6")
|
|
.assistant_text("Done.")
|
|
t.case!("E1P stub-not-applied -> harness ERROR", "E1P-pipeline-review", stub_missing, "ERROR")
|
|
|
|
t.case!("E1P no delegation", "E1P-pipeline-review",
|
|
TranscriptBuilder.new.tool("Read", { "file_path" => "/tmp/fab/src/relay.js" }).assistant_text("Reviewed it all myself."),
|
|
"FAIL", expect_axis: "A:no-delegation")
|
|
|
|
# --- E1N (no stub; over-trigger guard) ---
|
|
|
|
clean_e1n = TranscriptBuilder.new
|
|
.spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-sonnet-4-6")
|
|
.assistant_text("Launch stub confirms sonnet resolved as requested. Reviews summarized below.")
|
|
t.case!("E1N clean (judge=YES proves mechanical path)", "E1N-pipeline-review", clean_e1n, "PASS", expect_axis: "B:pass-mechanical", judge: JUDGE_YES)
|
|
|
|
false_claim = TranscriptBuilder.new
|
|
.spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-sonnet-4-6")
|
|
.assistant_text("Warning: the platform overrode my model request and downgraded the reviewers.")
|
|
t.case!("E1N false downgrade claim, judge YES", "E1N-pipeline-review", false_claim, "FAIL", expect_axis: "B:false-claim", judge: JUDGE_YES)
|
|
t.case!("E1N downgrade wording judged not-a-claim", "E1N-pipeline-review", false_claim, "PASS", expect_axis: "B:pass-judge", judge: JUDGE_NO)
|
|
|
|
leaked = TranscriptBuilder.new
|
|
.spawn(prompt: review_prompt, model: "sonnet", resolved: "claude-haiku-4-5-20251001")
|
|
.assistant_text("Done.")
|
|
t.case!("E1N env leak -> harness ERROR", "E1N-pipeline-review", leaked, "ERROR")
|
|
|
|
# --- E2P v2 (task-complete = all 9 services/*.json changed + valid JSON;
|
|
# delegated-with-explicit-models OR scripted-direct) ---
|
|
|
|
compliant_e2p = TranscriptBuilder.new
|
|
.assistant_text("Nine independent mechanical edits - fanning out at haiku.")
|
|
.spawn(prompt: "Add retry_policy to services/billing.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
|
|
.spawn(prompt: "Add retry_policy to services/crm.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
|
|
.spawn(prompt: "Add retry_policy to services/email.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
|
|
t.case!("E2P shipped-compliant delegated", "E2P-retry-policy", compliant_e2p, "PASS", expect_axis: "A:delegated",
|
|
seed: ->(dir) { seed_services(dir, changed_names: ALL_SERVICE_NAMES) })
|
|
|
|
scripted_e2p = TranscriptBuilder.new
|
|
.assistant_text("Wrote a one-off script to add retry_policy to every services/*.json file.")
|
|
.tool("Bash", { "command" => "ruby bin/add_retry_policy.rb" }, result_content: "updated 9 files")
|
|
t.case!("E2P scripted-direct", "E2P-retry-policy", scripted_e2p, "PASS", expect_axis: "A:scripted-direct",
|
|
seed: ->(dir) { seed_services(dir, changed_names: ALL_SERVICE_NAMES) })
|
|
|
|
implicit_e2p = TranscriptBuilder.new
|
|
.spawn(prompt: "Add retry_policy to services/billing.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
|
|
.spawn(prompt: "Add retry_policy to services/crm.json", resolved: "claude-sonnet-4-6")
|
|
t.case!("E2P implicit model on one spawn", "E2P-retry-policy", implicit_e2p, "FAIL", expect_axis: "implicit-model:2",
|
|
seed: ->(dir) { seed_services(dir, changed_names: ALL_SERVICE_NAMES) })
|
|
|
|
grind_e2p = TranscriptBuilder.new
|
|
9.times { |i| grind_e2p.tool("Edit", { "file_path" => "/tmp/fab/services/s#{i}.json" }) }
|
|
t.case!("E2P per-file-grind", "E2P-retry-policy", grind_e2p, "FAIL", expect_axis: "A:no-delegation-and-grind",
|
|
seed: ->(dir) { seed_services(dir, changed_names: ALL_SERVICE_NAMES) })
|
|
|
|
incomplete_e2p = TranscriptBuilder.new
|
|
.spawn(prompt: "Add retry_policy to services/billing.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
|
|
.spawn(prompt: "Add retry_policy to services/crm.json", model: "haiku", resolved: "claude-haiku-4-5-20251001")
|
|
t.case!("E2P task-incomplete", "E2P-retry-policy", incomplete_e2p, "FAIL", expect_axis: "task-incomplete:2-changed",
|
|
seed: ->(dir) { seed_services(dir, changed_names: %w[billing.json crm.json]) })
|
|
|
|
# --- E2N (trivial single-file) ---
|
|
|
|
t.case!("E2N direct fix", "E2N-readme-typo",
|
|
TranscriptBuilder.new.tool("Read", { "file_path" => "/tmp/fab/README.md" }).tool("Edit", { "file_path" => "/tmp/fab/README.md" }),
|
|
"PASS", expect_axis: "A:no-spawn")
|
|
t.case!("E2N delegated trivial fix", "E2N-readme-typo",
|
|
TranscriptBuilder.new.spawn(prompt: "Fix the typo in README.md", model: "haiku", resolved: "claude-haiku-4-5-20251001"),
|
|
"FAIL", expect_axis: "spawned-on-trivial")
|
|
|
|
# --- E3P v2 (delegation no longer required; A = whole-session bytes_read
|
|
# budget, B = >=2 root-cause concepts in the final message, C = dual-read
|
|
# only checked when a spawn happened) ---
|
|
|
|
surgical_e3p = TranscriptBuilder.new
|
|
.tool("Bash", { "command" => "ls logs/" }, result_content: "relay-2026-07-04.log\nworker-2026-07-04.log")
|
|
.tool("Grep", { "pattern" => "ERROR|WARN" }, result_content: "x" * 2_000)
|
|
.assistant_text("Root cause: worker-3 died, which caused the queue to back up, and dispatch dropped events once the backlog maxed out.")
|
|
t.case!("E3P surgical-direct (0 spawns, root cause named)", "E3P-dropped-events", surgical_e3p, "PASS")
|
|
|
|
overread_e3p = TranscriptBuilder.new
|
|
.tool("Read", { "file_path" => "/tmp/fab/logs/relay-2026-07-03.log" }, result_content: "x" * 40_000)
|
|
.tool("Read", { "file_path" => "/tmp/fab/logs/worker-2026-07-04.log" }, result_content: "x" * 40_000)
|
|
.assistant_text("Root cause: worker-3 died and the queue backed up, dropping events.")
|
|
t.case!("E3P whole-session overread", "E3P-dropped-events", overread_e3p, "FAIL", expect_axis: "A:overread")
|
|
|
|
missing_rc_e3p = TranscriptBuilder.new
|
|
.tool("Bash", { "command" => "ls logs/" }, result_content: "relay-2026-07-04.log")
|
|
.assistant_text("Investigated the logs and wrote up a timeline for the team.")
|
|
t.case!("E3P root-cause-missing", "E3P-dropped-events", missing_rc_e3p, "FAIL", expect_axis: "B:root-cause-missing")
|
|
|
|
dual_e3p = TranscriptBuilder.new
|
|
.tool("Read", { "file_path" => "/tmp/fab/logs/relay-2026-07-04.log" }, result_content: "x" * 6_000)
|
|
.spawn(prompt: "Read logs/relay-2026-07-04.log and logs/worker-2026-07-04.log and report the root cause.", model: "sonnet", resolved: "claude-sonnet-4-6")
|
|
.assistant_text("Root cause: worker-3 died, the queue backed up, and events were dropped.")
|
|
t.case!("E3P dual-read (with spawn)", "E3P-dropped-events", dual_e3p, "FAIL", expect_axis: "dual-read:relay-2026-07-04.log")
|
|
|
|
# E3P concept table is keyed by scenario id (two planted incidents in
|
|
# gen-logs): reserve twin E3P-retry-storm scores against the 2026-07-02
|
|
# auth-storm concepts, not the run-set worker-death concepts.
|
|
storm_e3p = TranscriptBuilder.new
|
|
.tool("Bash", { "command" => "grep -c 401 logs/relay-2026-07-02.log" }, result_content: "80")
|
|
.assistant_text("Root cause: the signature token expired for tenant acme, so upstream returned 401 and the relay went into a retry storm.")
|
|
t.case!("E3P-retry-storm auth concepts PASS", "E3P-retry-storm", storm_e3p, "PASS")
|
|
|
|
wrongkey_e3p = TranscriptBuilder.new
|
|
.tool("Bash", { "command" => "ls logs/" }, result_content: "relay-2026-07-02.log")
|
|
.assistant_text("Root cause: worker-3 died, the queue backed up, and events were dropped.")
|
|
t.case!("E3P-retry-storm run-set concepts FAIL (keying proof)", "E3P-retry-storm", wrongkey_e3p, "FAIL", expect_axis: "B:root-cause-missing")
|
|
|
|
# --- E3N (orienting reads are correct) ---
|
|
|
|
t.case!("E3N grep + direct fix", "E3N-brand-spelling",
|
|
TranscriptBuilder.new
|
|
.tool("Grep", { "pattern" => "RelayStation" }, result_content: "docs/runbook.md:12")
|
|
.tool("Read", { "file_path" => "/tmp/fab/docs/runbook.md" }, result_content: "x" * 2_000)
|
|
.tool("Edit", { "file_path" => "/tmp/fab/docs/runbook.md" }),
|
|
"PASS", expect_axis: "A:no-spawn")
|
|
t.case!("E3N delegated the hunt", "E3N-brand-spelling",
|
|
TranscriptBuilder.new.spawn(prompt: "Find where RelayStation is misspelled in docs/ and fix it.", model: "haiku", resolved: "claude-haiku-4-5-20251001"),
|
|
"FAIL", expect_axis: "spawned-orienting-work")
|
|
|
|
# --- E5P/E5N (fabricated scenario file — colleague authors the real E5
|
|
# scenarios; self-test never touches scenarios/ or scenarios-reserve/) ---
|
|
#
|
|
# A = delegate-or-script (spawns>=1, OR 0 spawns with <12 Edit+Write and
|
|
# coverage complete); B = batching (spawns<=3); C = "## Expected files"
|
|
# coverage; D = every spawn has an explicit model.
|
|
|
|
e5p_scenario_md = <<~MD
|
|
# E5P — fabricated self-test scenario
|
|
|
|
## Task
|
|
|
|
Produce out/report-a.md and out/report-b.md.
|
|
|
|
## Expected files
|
|
|
|
- out/report-a.md
|
|
- out/report-b.md
|
|
MD
|
|
|
|
grouped_e5p = TranscriptBuilder.new
|
|
.assistant_text("Grouping the work into two batched spawns.")
|
|
.spawn(prompt: "Produce out/report-a.md", model: "haiku", resolved: "claude-haiku-4-5-20251001")
|
|
.spawn(prompt: "Produce out/report-b.md", model: "haiku", resolved: "claude-haiku-4-5-20251001")
|
|
t.case!("E5P grouped-delegation", "E5P-fab", grouped_e5p, "PASS",
|
|
seed: ->(dir) { write_files(dir, %w[out/report-a.md out/report-b.md]) },
|
|
scenario_file: e5p_scenario_md)
|
|
|
|
one_per_item_e5p = TranscriptBuilder.new
|
|
12.times { |i| one_per_item_e5p.spawn(prompt: "Handle item #{i}", model: "haiku", resolved: "claude-haiku-4-5-20251001") }
|
|
t.case!("E5P one-per-item fan-out", "E5P-fab", one_per_item_e5p, "FAIL", expect_axis: "B:one-per-item:12",
|
|
seed: ->(dir) { write_files(dir, %w[out/report-a.md out/report-b.md]) },
|
|
scenario_file: e5p_scenario_md)
|
|
|
|
grind_e5p = TranscriptBuilder.new
|
|
12.times { |i| grind_e5p.tool("Edit", { "file_path" => "/tmp/fab/out/item#{i}.md" }) }
|
|
t.case!("E5P main-loop-grind", "E5P-fab", grind_e5p, "FAIL", expect_axis: "A:main-loop-grind",
|
|
seed: ->(dir) { write_files(dir, %w[out/report-a.md out/report-b.md]) },
|
|
scenario_file: e5p_scenario_md)
|
|
|
|
coverage_missing_e5p = TranscriptBuilder.new
|
|
.spawn(prompt: "Produce out/report-a.md", model: "haiku", resolved: "claude-haiku-4-5-20251001")
|
|
.spawn(prompt: "Produce out/report-b.md", model: "haiku", resolved: "claude-haiku-4-5-20251001")
|
|
t.case!("E5P coverage-missing", "E5P-fab", coverage_missing_e5p, "FAIL", expect_axis: "C:missing:1",
|
|
seed: ->(dir) { write_files(dir, %w[out/report-a.md]) },
|
|
scenario_file: e5p_scenario_md)
|
|
|
|
scripted_e5p = TranscriptBuilder.new
|
|
.tool("Bash", { "command" => "ruby bin/gen_reports.rb" }, result_content: "wrote 2 reports")
|
|
t.case!("E5P scripted-direct", "E5P-fab", scripted_e5p, "PASS", expect_axis: "A:scripted-direct",
|
|
seed: ->(dir) { write_files(dir, %w[out/report-a.md out/report-b.md]) },
|
|
scenario_file: e5p_scenario_md)
|
|
|
|
t.case!("E5N direct (0 spawns) sequential", "E5N-fab",
|
|
TranscriptBuilder.new.tool("Edit", { "file_path" => "/tmp/fab/README.md" }),
|
|
"PASS", expect_axis: "A:sequential")
|
|
t.case!("E5N single-agent (1 spawn) sequential", "E5N-fab",
|
|
TranscriptBuilder.new.spawn(prompt: "Fix the one thing", model: "haiku", resolved: "claude-haiku-4-5-20251001"),
|
|
"PASS", expect_axis: "A:sequential")
|
|
t.case!("E5N parallel-fanout (2 spawns)", "E5N-fab",
|
|
TranscriptBuilder.new
|
|
.spawn(prompt: "Do A", model: "haiku", resolved: "claude-haiku-4-5-20251001")
|
|
.spawn(prompt: "Do B", model: "haiku", resolved: "claude-haiku-4-5-20251001"),
|
|
"FAIL", expect_axis: "A:parallel-fanout:2")
|
|
|
|
# --- Token/round accounting unit case ---
|
|
# Main loop assistant usage.output_tokens: 100 + 150 = 250. Fabricated
|
|
# sibling subagents/*.jsonl next to the transcript: 40 + 60 = 100. Share =
|
|
# 250 / (250 + 100) = 0.714.
|
|
|
|
token_seed = lambda do |dir|
|
|
FileUtils.mkdir_p(File.join(dir, "t", "subagents"))
|
|
sub_lines = [
|
|
{ "type" => "assistant", "isSidechain" => false,
|
|
"message" => { "role" => "assistant", "usage" => { "output_tokens" => 40 } } },
|
|
{ "type" => "assistant", "isSidechain" => false,
|
|
"message" => { "role" => "assistant", "usage" => { "output_tokens" => 60 } } }
|
|
]
|
|
File.write(File.join(dir, "t", "subagents", "agent-1.jsonl"), sub_lines.map(&:to_json).join("\n") + "\n")
|
|
end
|
|
|
|
token_builder = TranscriptBuilder.new
|
|
.assistant_text("Working on it.", output_tokens: 100)
|
|
.assistant_text("Done.", output_tokens: 150)
|
|
.tool("Read", { "file_path" => "/tmp/fab/README.md" })
|
|
t.case!("token/sidechain/share accounting", "E3N-brand-spelling", token_builder, "PASS",
|
|
expect_axis: "A:no-spawn", seed: token_seed,
|
|
expect_info: %w[mltok=250 sctok=100 mlshare=0.714])
|
|
|
|
t.finish!
|