From ab258d5ba1266ed9e16963de736f70e6a3ba5634 Mon Sep 17 00:00:00 2001 From: jared Date: Wed, 8 Jul 2026 11:05:03 -0400 Subject: [PATCH] os-orchestration eval: key E3P root-cause concepts by scenario id The reserve twin E3P-retry-storm targets gen-logs' other planted incident (2026-07-02 auth-token-expiry 401 storm) and was authored before axis B existed; the single concept table would have failed a compliant model. Fixed pre-reserve-measurement without reading reserve files (incident chain comes from gen-logs). Run-set scoring byte-identical; self-test 34/34 incl. keying proof both directions. Co-Authored-By: Claude Fable 5 --- plugins/os-orchestration/eval/bin/check | 26 +++++++++++++++++---- plugins/os-orchestration/eval/bin/self-test | 13 +++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/plugins/os-orchestration/eval/bin/check b/plugins/os-orchestration/eval/bin/check index 2500d69..d50589b 100755 --- a/plugins/os-orchestration/eval/bin/check +++ b/plugins/os-orchestration/eval/bin/check @@ -38,15 +38,30 @@ module OrchEval FIXTURE_SERVICES_DIR = File.expand_path("../fixture/project/services", __dir__) - # E3P axis B: at least two of the three planted-incident concepts (worker - # death, queue backup, dropped events — see fixture/project/bin/gen-logs) - # must be named in the final assistant message. - ROOT_CAUSE_CONCEPTS = { + # E3P axis B: at least two of the three planted-incident concepts must be + # named in the final assistant message. Concepts are keyed by scenario id + # because the fixture plants two distinct incidents (see + # fixture/project/bin/gen-logs): the 2026-07-04 worker-death chain (run-set + # E3P-dropped-events) and the 2026-07-02 auth-token-expiry 401 retry storm + # (reserve E3P-retry-storm). The reserve twin was authored before this axis + # existed; keying by id fixes the instrument WITHOUT reading reserve files — + # the incident chain comes from gen-logs, not the scenario text. + WORKER_DEATH_CONCEPTS = { worker: /\bworkers?(?:-3)?\b.{0,60}?\b(died|dead|crash(?:ed)?|down|failed|failure)\b|\b(died|dead|crash(?:ed)?|down|failed|failure)\b.{0,60}?\bworkers?(?:-3)?\b/i, queue: /\bqueue\b.{0,60}?\b(back ?up|backlog|full|filled|overflow(?:ed)?|saturat\w*|depth)\b|\b(back ?up|backlog|overflow(?:ed)?|saturat\w*)\b.{0,60}?\bqueue\b/i, dropped: /\b(dropped|drop(?:ping|s)?|lost|loss)\b.{0,60}?\bevents?\b|\bevents?\b.{0,60}?\b(dropped|drop(?:ping|s)?|lost|loss)\b/i }.freeze + AUTH_STORM_CONCEPTS = { + token: /\b(signature\s+)?(auth\s+)?tokens?\b.{0,60}?\bexpir\w*\b|\bexpir\w*\b.{0,60}?\btokens?\b/i, + unauthorized: /\b401\b|\bunauthori[sz]ed\b|\bauth(entication)?\b.{0,40}?\b(fail\w*|error)\b/i, + retry_storm: /\bretr(y|ies|ying)\b.{0,60}?\b(storm|spike|flood|loop|repeated\w*|surge|cascade)\b|\b(storm|spike|flood|surge|cascade)\b.{0,60}?\bretr(y|ies|ying)\b|\bscheduling retry\b/i + }.freeze + + ROOT_CAUSE_CONCEPTS_BY_SCENARIO = Hash.new(WORKER_DEATH_CONCEPTS).merge( + "E3P-retry-storm" => AUTH_STORM_CONCEPTS + ).freeze + JUDGE_RUBRIC = File.expand_path("../judge-rubric.md", __dir__) class Transcript @@ -298,7 +313,8 @@ module OrchEval bytes = whole_session_bytes_read axes = { A: bytes <= MAIN_LOOP_BYTE_BUDGET ? "pass" : "overread:#{bytes}" } - matched = ROOT_CAUSE_CONCEPTS.select { |_, re| @t.final_assistant_text =~ re }.keys + concepts = ROOT_CAUSE_CONCEPTS_BY_SCENARIO[@id] + matched = concepts.select { |_, re| @t.final_assistant_text =~ re }.keys axes[:B] = matched.size >= 2 ? "pass" : "root-cause-missing:#{matched.join(',')}" if spawns.empty? diff --git a/plugins/os-orchestration/eval/bin/self-test b/plugins/os-orchestration/eval/bin/self-test index da56502..f0f8174 100755 --- a/plugins/os-orchestration/eval/bin/self-test +++ b/plugins/os-orchestration/eval/bin/self-test @@ -274,6 +274,19 @@ dual_e3p = TranscriptBuilder.new .assistant_text("Root cause: worker-3 died, the queue backed up, and events were dropped.") t.case!("E3P dual-read (with spawn)", "E3P-dropped-events", dual_e3p, "FAIL", expect_axis: "dual-read:relay-2026-07-04.log") +# E3P concept table is keyed by scenario id (two planted incidents in +# gen-logs): reserve twin E3P-retry-storm scores against the 2026-07-02 +# auth-storm concepts, not the run-set worker-death concepts. +storm_e3p = TranscriptBuilder.new + .tool("Bash", { "command" => "grep -c 401 logs/relay-2026-07-02.log" }, result_content: "80") + .assistant_text("Root cause: the signature token expired for tenant acme, so upstream returned 401 and the relay went into a retry storm.") +t.case!("E3P-retry-storm auth concepts PASS", "E3P-retry-storm", storm_e3p, "PASS") + +wrongkey_e3p = TranscriptBuilder.new + .tool("Bash", { "command" => "ls logs/" }, result_content: "relay-2026-07-02.log") + .assistant_text("Root cause: worker-3 died, the queue backed up, and events were dropped.") +t.case!("E3P-retry-storm run-set concepts FAIL (keying proof)", "E3P-retry-storm", wrongkey_e3p, "FAIL", expect_axis: "B:root-cause-missing") + # --- E3N (orienting reads are correct) --- t.case!("E3N grep + direct fix", "E3N-brand-spelling",