From 3e42aecc130b50f386d5f2dfaaf3624cb6a7d30a Mon Sep 17 00:00:00 2001 From: jared Date: Fri, 10 Jul 2026 12:00:10 -0400 Subject: [PATCH] os-orchestration: semi-automate the biweekly IRL audit (Option C) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deterministic layer: audit/bin/audit-stats (Ruby) composes the existing extract fact-sheets across all production transcripts since the last run, writes per-run stats/flags/sheets to ~/.local/state/os-orchestration/audit/, and appends the metrics.tsv trend ledger (seeded with the 2026-07-10 run: 23 sessions). Judgment layer: new /os-orchestration:audit-sessions skill — sonnet auditor fan-out over precomputed flagged regions only, tiered synthesis with run-over-run trend verdicts, stops before any wording edit. Nudge: os-status check orchestration-audit-due warns when the ledger is >=14 days stale (missing ledger = machine not opted in). Planka recurrence entry deferred: gem offsets are +1y/+6m/+3m/+1m by design; +2w blocked on parked gem versioning decisions. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XknQRvihHDpYE47RTmUR4N --- docs/implementation-status.md | 15 ++ .../os-orchestration/audit/bin/audit-stats | 252 ++++++++++++++++++ .../skills/audit-sessions/SKILL.md | 68 +++++ .../audit-sessions/references/rubric.md | 55 ++++ plugins/os-status/hooks/checks.py | 36 +++ plugins/os-status/tests/hook_test.py | 32 +++ 6 files changed, 458 insertions(+) create mode 100755 plugins/os-orchestration/audit/bin/audit-stats create mode 100644 plugins/os-orchestration/skills/audit-sessions/SKILL.md create mode 100644 plugins/os-orchestration/skills/audit-sessions/references/rubric.md diff --git a/docs/implementation-status.md b/docs/implementation-status.md index 7907fcc..356927a 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -269,6 +269,21 @@ into `~/.claude/plugins/os-vault` settled-design drift 8/19 sessions). Side finding needing separate repro: harness ignored explicit `model: "sonnet"` pins on general-purpose agents (env override unset — distinct from WS1 cause); the self-report line caught it. +- **IRL audit semi-automated (2026-07-10):** recurring biweekly audit formalized, working + from any cwd. Deterministic layer: `audit/bin/audit-stats` (Ruby, composes the existing + `audit/bin/extract` fact-sheets) enumerates production transcripts since the last run + (excludes `-tmp-orch-*`/`-tmp-claude-*` sandboxes, `*-eval` dirs, active/trivial + sessions), writes `~/.local/state/os-orchestration/audit/run-/` + (stats.tsv/flags.md/sheets) and appends the trend ledger `metrics.tsv` (seeded with the + 2026-07-10 run: 23 sessions, 52 spawns, 1 missing model). Judgment layer: new skill + `/os-orchestration:audit-sessions` (rubric in `references/`) — sonnet auditor fan-out + over flagged regions only, tiered synthesis, run-over-run trend verdicts, stops before + any wording edit. Nudge: os-status check `orchestration-audit-due` warns when the + ledger's last row is ≥14 days old (missing ledger = machine not opted in, silent; env + override `OS_ORCH_AUDIT_LEDGER` for tests). Planka recurrence-manifest entry skipped: + the gem's offsets are deliberately `+1y/+6m/+3m/+1m` only (issue 11) — a `+2w` offset is + a gem change blocked on the parked publish/versioning decisions; the os-status nudge is + the trigger for now. **Global os-status plugin** — `cc-os/plugins/os-status/` (git-tracked, 2026-07-06); symlinked into `~/.claude/plugins/os-status` diff --git a/plugins/os-orchestration/audit/bin/audit-stats b/plugins/os-orchestration/audit/bin/audit-stats new file mode 100755 index 0000000..9c7b4d2 --- /dev/null +++ b/plugins/os-orchestration/audit/bin/audit-stats @@ -0,0 +1,252 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# audit-stats — deterministic run-level driver for the orchestration IRL audit. +# +# Enumerates production session transcripts modified since a given date, +# reduces each to a delegation fact-sheet (audit/bin/extract), and writes a +# run directory under the audit state dir (~/.local/state/os-orchestration/audit): +# +# run-/stats.tsv one row per audited session (deterministic columns) +# run-/flags.md heuristic missed-delegation candidates, per session, +# for auditor judgment (mechanical vs judgment-dependent) +# run-/sheets/ full fact-sheet JSON per session +# +# and appends one row to metrics.tsv at the state-dir root — the trend ledger +# that the os-status due-check and run-over-run comparison read. Everything in +# this file is code-only: no model involvement, reproducible from transcripts. +# +# Usage: audit-stats [--since YYYY-MM-DD] [--exclude SESSION_ID] [--state-dir DIR] +# [--projects-dir DIR] [--dry-run] +# +# --since defaults to the date of the last metrics.tsv row; the first run must +# pass it explicitly. Transcripts written to within the last 10 minutes are +# treated as active sessions and deferred to the next run. + +require "date" +require "fileutils" +require "json" +require "optparse" +# `extract` is extensionless (a CLI); its __FILE__ guard makes load safe here. +load File.expand_path("extract", __dir__) + +module OrchAudit + DEFAULT_STATE_DIR = File.expand_path("~/.local/state/os-orchestration/audit") + DEFAULT_PROJECTS_DIR = File.expand_path("~/.claude/projects") + + class TranscriptSet + # Non-production sessions: eval sandboxes (-tmp-orch-*), headless runs + # spawned from session scratchpads (-tmp-claude-*), and sessions whose + # cwd was an eval harness directory (…-eval). + EXCLUDED_PROJECT_DIRS = [/-tmp-orch-/, /-tmp-claude-/, /-eval\z/].freeze + ACTIVE_WINDOW_SECONDS = 600 + + def initialize(projects_dir:, since:, excluded_sessions: [], now: Time.now) + @projects_dir = projects_dir + @since = since + @excluded_sessions = excluded_sessions + @now = now + end + + def paths + Dir.glob(File.join(@projects_dir, "*", "*.jsonl")).sort.select { |p| candidate?(p) } + end + + private + + def candidate?(path) + return false if EXCLUDED_PROJECT_DIRS.any? { |re| File.dirname(path).match?(re) } + return false if @excluded_sessions.include?(File.basename(path, ".jsonl")) + mtime = File.mtime(path) + mtime.to_date >= @since && (@now - mtime) > ACTIVE_WINDOW_SECONDS + end + end + + class SessionStats + MIN_ASSISTANT_TURNS = 8 + + def initialize(path) + @path = path + @sheet = FactSheet.new(path).data + end + + attr_reader :path, :sheet + + def session_id = File.basename(@path, ".jsonl") + def project = File.basename(File.dirname(@path)) + def trivial? = sheet[:metadata][:assistant_turns] < MIN_ASSISTANT_TURNS + def spawns = sheet[:agent_spawns] + def flagged_runs = sheet[:missed_delegation_candidates] + + def spawns_missing_model + spawns.count { |s| !s[:model_explicit] } + end + + # A pin mismatch is deterministic evidence of the harness ignoring an + # explicit model param (the 2026-07-10 finding): param present, resolved + # model known, and the resolved id does not contain the requested tier. + def model_pin_mismatches + spawns.count do |s| + s[:model_explicit] && s[:model_param] && s[:resolved_model] && + !s[:resolved_model].to_s.include?(s[:model_param].to_s) + end + end + + def sendmessage_followups + spawns.sum { |s| s[:rounds] - 1 } + end + + def row + m = sheet[:metadata] + [ + session_id, project, m[:assistant_turns], m[:main_loop_output_tokens], + m[:main_loop_share], spawns.size, spawns_missing_model, + model_pin_mismatches, sendmessage_followups, flagged_runs.size + ] + end + end + + class AuditRun + STATS_HEADER = %w[ + session project assistant_turns main_loop_output_tokens main_loop_share + spawns spawns_missing_model model_pin_mismatches sendmessage_followups + flagged_runs + ].freeze + LEDGER_HEADER = %w[ + date since sessions trivial_skipped spawns spawns_missing_model + model_pin_mismatches sendmessage_followups flagged_runs + median_main_loop_share + ].freeze + + def initialize(state_dir:, projects_dir:, since:, excluded_sessions:, today: Date.today) + @state_dir = state_dir + @since = since + @today = today + @transcripts = TranscriptSet.new( + projects_dir: projects_dir, since: since, excluded_sessions: excluded_sessions + ) + end + + def ledger_path = File.join(@state_dir, "metrics.tsv") + def run_dir = File.join(@state_dir, "run-#{@today.iso8601}") + + def plan + @transcripts.paths + end + + def execute + audited, trivial = partition_sessions + write_run_dir(audited) + append_ledger(audited, trivial) + { run_dir: run_dir, audited: audited.size, trivial: trivial.size } + end + + private + + def partition_sessions + @transcripts.paths.map { |p| SessionStats.new(p) }.partition { |s| !s.trivial? } + end + + def write_run_dir(audited) + sheets_dir = File.join(run_dir, "sheets") + FileUtils.mkdir_p(sheets_dir) + File.write(File.join(run_dir, "stats.tsv"), stats_tsv(audited)) + File.write(File.join(run_dir, "flags.md"), flags_md(audited)) + audited.each do |s| + File.write(File.join(sheets_dir, "#{s.session_id}.json"), JSON.pretty_generate(s.sheet)) + end + end + + def stats_tsv(audited) + lines = [STATS_HEADER.join("\t")] + audited.each { |s| lines << s.row.join("\t") } + lines.join("\n") + "\n" + end + + def flags_md(audited) + md = +"# Flagged regions for auditor judgment — run #{@today.iso8601}\n\n" + md << "Heuristic candidates only; sequential-dependent work is a valid reason not\n" + md << "to delegate. Auditors judge mechanical vs judgment-dependent per region.\n" + audited.each do |s| + md << "\n## #{s.session_id} (#{s.project})\n\n" + if s.flagged_runs.empty? + md << "No flagged runs.\n" + next + end + s.flagged_runs.each do |m| + md << "- **#{m[:tool]} x#{m[:length]}** (jsonl lines #{m[:lines].join('-')}, " + md << "#{m[:distinct_targets]} distinct targets)\n" + m[:targets_sample].each { |t| md << " - `#{t}`\n" } + end + end + md + end + + def append_ledger(audited, trivial) + FileUtils.mkdir_p(@state_dir) + File.write(ledger_path, LEDGER_HEADER.join("\t") + "\n") unless File.exist?(ledger_path) + File.open(ledger_path, "a") { |f| f.puts(ledger_row(audited, trivial).join("\t")) } + end + + def ledger_row(audited, trivial) + [ + @today.iso8601, @since.iso8601, audited.size, trivial.size, + audited.sum { |s| s.spawns.size }, + audited.sum(&:spawns_missing_model), + audited.sum(&:model_pin_mismatches), + audited.sum(&:sendmessage_followups), + audited.sum { |s| s.flagged_runs.size }, + median(audited.map { |s| s.sheet[:metadata][:main_loop_share] }) + ] + end + + def median(values) + return 0 if values.empty? + sorted = values.sort + mid = sorted.size / 2 + (sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0).round(3) + end + end + + class Ledger + def self.last_run_date(path) + return nil unless File.exist?(path) + last = File.readlines(path).map(&:strip).reject(&:empty?)[1..]&.last + last && (Date.iso8601(last.split("\t").first) rescue nil) + end + end +end + +if $PROGRAM_NAME == __FILE__ + options = { + state_dir: OrchAudit::DEFAULT_STATE_DIR, + projects_dir: OrchAudit::DEFAULT_PROJECTS_DIR, + exclude: [], + dry_run: false + } + OptionParser.new do |o| + o.banner = "Usage: audit-stats [--since YYYY-MM-DD] [--exclude SESSION_ID] [options]" + o.on("--since DATE") { |d| options[:since] = Date.iso8601(d) } + o.on("--exclude ID", "Session id to skip (repeatable)") { |id| options[:exclude] << id } + o.on("--state-dir DIR") { |d| options[:state_dir] = File.expand_path(d) } + o.on("--projects-dir DIR") { |d| options[:projects_dir] = File.expand_path(d) } + o.on("--dry-run", "List transcripts that would be audited, then exit") { options[:dry_run] = true } + end.parse! + + since = options[:since] || + OrchAudit::Ledger.last_run_date(File.join(options[:state_dir], "metrics.tsv")) + abort "No --since given and no prior ledger row to infer it from" unless since + + run = OrchAudit::AuditRun.new( + state_dir: options[:state_dir], projects_dir: options[:projects_dir], + since: since, excluded_sessions: options[:exclude] + ) + if options[:dry_run] + puts run.plan + exit 0 + end + result = run.execute + puts "Audited #{result[:audited]} sessions (#{result[:trivial]} trivial skipped)" + puts "Run dir: #{result[:run_dir]}" + puts "Ledger: #{run.ledger_path}" +end diff --git a/plugins/os-orchestration/skills/audit-sessions/SKILL.md b/plugins/os-orchestration/skills/audit-sessions/SKILL.md new file mode 100644 index 0000000..57ed428 --- /dev/null +++ b/plugins/os-orchestration/skills/audit-sessions/SKILL.md @@ -0,0 +1,68 @@ +--- +description: Run the biweekly orchestration IRL audit - deterministic transcript stats precompute, then auditor fan-out judging flagged regions against ORCHESTRATION.md rules, then a tiered tune-up report. Works from any directory. Trigger via the os-status due nudge or the Planka recurrence card. +--- + +# Audit production sessions against the ORCHESTRATION.md rules + +You are running the recurring (biweekly) IRL audit of the shipped delegation-economics +rules. The signal source is real production session transcripts — NEVER eval grid +scenarios or reserve sets (contaminated for wording tuning since 2026-07-08). This skill +works from any cwd; nothing here requires being inside the cc-os repo. + +## Step 1 — Deterministic stats precompute (code, no judgment) + +Run the driver at `/../../audit/bin/audit-stats` (the skill base +directory is `/skills/audit-sessions`, the driver lives at +`/audit/bin/audit-stats`): + +```bash +/audit/bin/audit-stats --exclude +``` + +(If the current session id is unknown, omit `--exclude` — the 10-minute active-window +guard already defers the live session to the next run.) + +- `--since` defaults to the last ledger row's date. It refuses to run without a prior row; + a first-ever run needs an explicit `--since YYYY-MM-DD`. +- Output: `~/.local/state/os-orchestration/audit/run-/` containing `stats.tsv` + (per-session deterministic columns), `flags.md` (heuristic missed-delegation regions), + and `sheets/*.json` (full per-session fact-sheets). One row is appended to + `~/.local/state/os-orchestration/audit/metrics.tsv` — the trend ledger. Never edit the + ledger by hand; never re-derive these numbers with a model. + +## Step 2 — Auditor fan-out (judgment only where code can't judge) + +Read `stats.tsv` and `flags.md` from the run dir. Sessions with zero spawns AND zero +flagged runs AND main_loop_share 1.0 at low turn counts usually need no auditor — +include them only in the aggregate. Batch the remaining sessions into ~3 parallel +auditor agents (5–8 sessions each), `model: "sonnet"`, background, one round each. + +Each auditor prompt must include: +- The rubric: read `references/rubric.md` (sibling to this SKILL.md) and paste it in. +- The session's fact-sheet JSON path and flagged regions — auditors judge + **mechanical vs judgment-dependent** for each flagged region by reading the transcript + selectively around the cited jsonl lines (jq on line ranges, never the raw file). +- The self-report line: "State the exact model ID you are running as in the first line + of your report." Cross-check it; flag any pin mismatch in the report. +- The exact per-session report format from the rubric. + +## Step 3 — Synthesize and compare (main loop) + +- Aggregate auditor findings into the tiered format: Tier 1 (HIGH, recurring), + Tier 2 (MEDIUM), Tier 3 (solved surfaces — explicitly name what NOT to spend + wording budget on). +- Compare this run's ledger row against prior rows in `metrics.tsv` — the previous + run's Tier 1 items are hypotheses this run tests (e.g. did settled-design drift + drop after the 2026-07-10 wording patch?). State each verdict explicitly. +- Write `report.md` into the run dir. + +## Step 4 — Stop for the human + +Report the tiers and trend verdicts, then STOP. Do not edit ORCHESTRATION.md without +explicit approval — wording changes are a human decision (injected-token budget; grid +validation is unavailable). If approved later: edit +`~/dev/cc-os/plugins/os-orchestration/ORCHESTRATION.md`, run `~/dev/cc-os/bin/refresh-plugins`, +and record the change in `~/dev/cc-os/docs/implementation-status.md`. + +Finally, if a Planka card for this audit exists on the backlog board, move it to +Review (never to Done). diff --git a/plugins/os-orchestration/skills/audit-sessions/references/rubric.md b/plugins/os-orchestration/skills/audit-sessions/references/rubric.md new file mode 100644 index 0000000..2b1ba13 --- /dev/null +++ b/plugins/os-orchestration/skills/audit-sessions/references/rubric.md @@ -0,0 +1,55 @@ +# IRL audit rubric — shipped ORCHESTRATION.md delegation-economics rules + +Audit each assigned session against these shipped rules. A "finding" is a concrete +violation, near-miss, or friction point, with evidence (jsonl line number / short quote). +The deterministic fact-sheet (stats + flagged regions) is precomputed — do NOT recount +tool calls or spawns; your job is the judgment layer: mechanical vs judgment-dependent, +severity, and category. + +## Rule surfaces (categories) + +1. **missed-delegation** — main loop ran a mechanical sequence beyond ~3 tool calls with no + judgment between steps (multi-file edits, lookups, conversions, long log review, wide + grep-and-synthesize) instead of delegating down-tier. Includes: write-N-files fan-out + from a settled design done directly; the operator's own eval/benchmark/extraction loops. +2. **model-param** — an Agent spawn with no explicit `model:` param, or mechanical work + sent to sonnet/opus when haiku would do, or a sonnet/opus spawn whose prompt lacks the + self-report line, or a downgraded/mismatched resolved model treated silently as + judgment-tier. +3. **spawn-batching** — many small spawns where ~5–8 similar items should have been grouped + into one agent prompt; or a follow-up on an agent's result sent as a fresh spawn instead + of SendMessage to the live agent. +4. **async-usage** — main loop idle-waiting on a synchronous agent when background + + continue was possible; sleep-polling a background job; or conversely + babysitting/polling that the rules say to do directly. +5. **redundant-context** — delegated investigation re-covering ground the main loop already + read, or the main loop reading many files itself before delegating the investigation + anyway. +6. **over-delegation** — delegation where direct work was correct: ≤2 tool-call ops, + judgment-dependent steps, interactive troubleshooting, or a uniform change coverable by + one scripted Bash loop. +7. **drift** — session starts disciplined then drifts into long direct runs mid-session, + especially after a design/decision settles. + +## Per-session report format (return exactly this) + +``` +SESSION: +PROJECT/TOPIC: +FINDINGS: +- [] — | evidence: +...or "none" +FLAGGED-REGION VERDICTS: +POSITIVE: +``` + +## Method notes + +- The fact-sheet JSON gives spawns (with model params and resolved models), per-segment + tool profiles, and flagged same-tool runs with jsonl line ranges. Read the transcript + ONLY selectively around those lines, e.g. + `sed -n '120,180p' file.jsonl | jq -r 'select(.type=="assistant") | .message.content[]? | select(.type=="text") | .text'` +- Judge "mechanical vs judgment" from tool names + inputs + the surrounding + user/assistant text: did each result change what happened next? +- Sequential-dependent work is a valid reason not to delegate; interactive + troubleshooting (user replying every few turns) is a valid reason too. diff --git a/plugins/os-status/hooks/checks.py b/plugins/os-status/hooks/checks.py index b03de3d..5259e2b 100644 --- a/plugins/os-status/hooks/checks.py +++ b/plugins/os-status/hooks/checks.py @@ -11,8 +11,10 @@ invariants.md). Do not paraphrase them. from __future__ import annotations import json +import os import re from dataclasses import dataclass +from datetime import date from pathlib import Path from typing import Callable, Optional @@ -160,6 +162,39 @@ def vault_hub_note_present(ctx: Ctx) -> CheckResult: ) +AUDIT_LEDGER_ENV = "OS_ORCH_AUDIT_LEDGER" +DEFAULT_AUDIT_LEDGER = "~/.local/state/os-orchestration/audit/metrics.tsv" +AUDIT_INTERVAL_DAYS = 14 + + +def _last_audit_date(ledger: Path): + try: + rows = [l for l in ledger.read_text().splitlines() if l.strip()][1:] + return date.fromisoformat(rows[-1].split("\t")[0]) + except Exception: + return None + + +def orchestration_audit_due(ctx: Ctx) -> CheckResult: + """Nudge when the biweekly orchestration IRL audit is overdue. The ledger + (metrics.tsv) is appended deterministically by os-orchestration's + audit/bin/audit-stats; this check only reads its last row's date.""" + ledger = Path( + os.path.expanduser(ctx.environ.get(AUDIT_LEDGER_ENV, DEFAULT_AUDIT_LEDGER)) + ) + last = _last_audit_date(ledger) + if last is None: + return OK # no ledger — this machine hasn't opted into audits yet + days = (date.today() - last).days + if days < AUDIT_INTERVAL_DAYS: + return OK + return warn( + f"orchestration IRL audit due (last run {last.isoformat()}, {days} days" + " ago) — run /os-orchestration:audit-sessions (works from any project;" + " deterministic stats precompute, then auditor fan-out)." + ) + + @dataclass(frozen=True) class Check: name: str @@ -171,4 +206,5 @@ REGISTRY = [ Check("subagent-model-env-override", subagent_model_env_override, project_scoped=False), Check("adr-system-present", adr_system_present, project_scoped=True), Check("vault-hub-note-present", vault_hub_note_present, project_scoped=True), + Check("orchestration-audit-due", orchestration_audit_due, project_scoped=False), ] diff --git a/plugins/os-status/tests/hook_test.py b/plugins/os-status/tests/hook_test.py index efda2a8..57c4b13 100644 --- a/plugins/os-status/tests/hook_test.py +++ b/plugins/os-status/tests/hook_test.py @@ -198,6 +198,38 @@ class HubNoteCheckTest(unittest.TestCase): self.assertEqual("warn", result.status) +class AuditDueCheckTest(unittest.TestCase): + def ledger_env(self, tmp, rows): + path = Path(tmp) / "metrics.tsv" + header = "date\tsince\tsessions\n" + path.write_text(header + "".join(f"{r}\t2026-01-01\t5\n" for r in rows)) + return {checks.AUDIT_LEDGER_ENV: str(path)} + + def test_missing_ledger_is_silent_machine_not_opted_in(self): + ctx = make_ctx(environ={checks.AUDIT_LEDGER_ENV: "/nonexistent/metrics.tsv"}) + self.assertEqual("ok", checks.orchestration_audit_due(ctx).status) + + def test_recent_run_is_ok(self): + with TemporaryDirectory() as tmp: + env = self.ledger_env(tmp, [date.today().isoformat()]) + self.assertEqual("ok", checks.orchestration_audit_due(make_ctx(environ=env)).status) + + def test_stale_run_warns_with_last_date(self): + with TemporaryDirectory() as tmp: + env = self.ledger_env(tmp, ["2026-01-01", "2026-02-01"]) + result = checks.orchestration_audit_due(make_ctx(environ=env)) + self.assertEqual("warn", result.status) + self.assertIn("2026-02-01", result.message) + self.assertIn("/os-orchestration:audit-sessions", result.message) + + def test_corrupt_ledger_reads_as_missing(self): + with TemporaryDirectory() as tmp: + path = Path(tmp) / "metrics.tsv" + path.write_text("header only\nnot-a-date\tx\n") + env = {checks.AUDIT_LEDGER_ENV: str(path)} + self.assertEqual("ok", checks.orchestration_audit_due(make_ctx(environ=env)).status) + + class RunnerTest(unittest.TestCase): """run() contract: routing, aggregation, snooze/suppress, isolation."""