jared
/
cc-os
1
0
Fork 0
Code Issues 18 Pull Requests Packages Projects Releases Wiki Activity

os-orchestration eval: key E3P root-cause concepts by scenario id 

The reserve twin E3P-retry-storm targets gen-logs' other planted incident
(2026-07-02 auth-token-expiry 401 storm) and was authored before axis B
existed; the single concept table would have failed a compliant model.
Fixed pre-reserve-measurement without reading reserve files (incident chain
comes from gen-logs). Run-set scoring byte-identical; self-test 34/34 incl.
keying proof both directions.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jared 2026-07-08 11:05:03 -04:00
parent a771163663
commit ab258d5ba1
2 changed files with 34 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
plugins/os-orchestration/eval/bin/check
View File

@ -38,15 +38,30 @@ module OrchEval
FIXTURE_SERVICES_DIR = File.expand_path("../fixture/project/services", __dir__)
# E3P axis B: at least two of the three planted-incident concepts (worker
# death, queue backup, dropped events — see fixture/project/bin/gen-logs)
# must be named in the final assistant message.
ROOT_CAUSE_CONCEPTS = {
# E3P axis B: at least two of the three planted-incident concepts must be
# named in the final assistant message. Concepts are keyed by scenario id
# because the fixture plants two distinct incidents (see
# fixture/project/bin/gen-logs): the 2026-07-04 worker-death chain (run-set
# E3P-dropped-events) and the 2026-07-02 auth-token-expiry 401 retry storm
# (reserve E3P-retry-storm). The reserve twin was authored before this axis
# existed; keying by id fixes the instrument WITHOUT reading reserve files —
# the incident chain comes from gen-logs, not the scenario text.
WORKER_DEATH_CONCEPTS = {
worker: /\bworkers?(?:-3)?\b.{0,60}?\b(died|dead|crash(?:ed)?|down|failed|failure)\b|\b(died|dead|crash(?:ed)?|down|failed|failure)\b.{0,60}?\bworkers?(?:-3)?\b/i,
queue: /\bqueue\b.{0,60}?\b(back ?up|backlog|full|filled|overflow(?:ed)?|saturat\w*|depth)\b|\b(back ?up|backlog|overflow(?:ed)?|saturat\w*)\b.{0,60}?\bqueue\b/i,
dropped: /\b(dropped|drop(?:ping|s)?|lost|loss)\b.{0,60}?\bevents?\b|\bevents?\b.{0,60}?\b(dropped|drop(?:ping|s)?|lost|loss)\b/i
}.freeze
AUTH_STORM_CONCEPTS = {
token: /\b(signature\s+)?(auth\s+)?tokens?\b.{0,60}?\bexpir\w*\b|\bexpir\w*\b.{0,60}?\btokens?\b/i,
unauthorized: /\b401\b|\bunauthori[sz]ed\b|\bauth(entication)?\b.{0,40}?\b(fail\w*|error)\b/i,
retry_storm: /\bretr(y|ies|ying)\b.{0,60}?\b(storm|spike|flood|loop|repeated\w*|surge|cascade)\b|\b(storm|spike|flood|surge|cascade)\b.{0,60}?\bretr(y|ies|ying)\b|\bscheduling retry\b/i
}.freeze
ROOT_CAUSE_CONCEPTS_BY_SCENARIO = Hash.new(WORKER_DEATH_CONCEPTS).merge(
"E3P-retry-storm" => AUTH_STORM_CONCEPTS
).freeze
JUDGE_RUBRIC = File.expand_path("../judge-rubric.md", __dir__)
class Transcript
@ -298,7 +313,8 @@ module OrchEval
bytes = whole_session_bytes_read
axes = { A: bytes <= MAIN_LOOP_BYTE_BUDGET ? "pass" : "overread:#{bytes}" }
matched = ROOT_CAUSE_CONCEPTS.select { |_, re| @t.final_assistant_text =~ re }.keys
concepts = ROOT_CAUSE_CONCEPTS_BY_SCENARIO[@id]
matched = concepts.select { |_, re| @t.final_assistant_text =~ re }.keys
axes[:B] = matched.size >= 2 ? "pass" : "root-cause-missing:#{matched.join(',')}"
if spawns.empty?

13
plugins/os-orchestration/eval/bin/self-test
View File

@ -274,6 +274,19 @@ dual_e3p = TranscriptBuilder.new
.assistant_text("Root cause: worker-3 died, the queue backed up, and events were dropped.")
t.case!("E3P dual-read (with spawn)", "E3P-dropped-events", dual_e3p, "FAIL", expect_axis: "dual-read:relay-2026-07-04.log")
# E3P concept table is keyed by scenario id (two planted incidents in
# gen-logs): reserve twin E3P-retry-storm scores against the 2026-07-02
# auth-storm concepts, not the run-set worker-death concepts.
storm_e3p = TranscriptBuilder.new
.tool("Bash", { "command" => "grep -c 401 logs/relay-2026-07-02.log" }, result_content: "80")
.assistant_text("Root cause: the signature token expired for tenant acme, so upstream returned 401 and the relay went into a retry storm.")
t.case!("E3P-retry-storm auth concepts PASS", "E3P-retry-storm", storm_e3p, "PASS")
wrongkey_e3p = TranscriptBuilder.new
.tool("Bash", { "command" => "ls logs/" }, result_content: "relay-2026-07-02.log")
.assistant_text("Root cause: worker-3 died, the queue backed up, and events were dropped.")
t.case!("E3P-retry-storm run-set concepts FAIL (keying proof)", "E3P-retry-storm", wrongkey_e3p, "FAIL", expect_axis: "B:root-cause-missing")
# --- E3N (orienting reads are correct) ---
t.case!("E3N grep + direct fix", "E3N-brand-spelling",