cc-os/plugins/os-orchestration/audit/bin/audit-stats

#!/usr/bin/env ruby
# frozen_string_literal: true

# audit-stats — deterministic run-level driver for the orchestration IRL audit.
#
# Enumerates production session transcripts modified since a given date,
# reduces each to a delegation fact-sheet (audit/bin/extract), and writes a
# run directory under the audit state dir (~/.local/state/os-orchestration/audit):
#
#   run-<date>/stats.tsv    one row per audited session (deterministic columns)
#   run-<date>/flags.md     heuristic missed-delegation candidates, per session,
#                           for auditor judgment (mechanical vs judgment-dependent)
#   run-<date>/sheets/      full fact-sheet JSON per session
#
# and appends one row to metrics.tsv at the state-dir root — the trend ledger
# that the os-status due-check and run-over-run comparison read. Everything in
# this file is code-only: no model involvement, reproducible from transcripts.
#
# Usage: audit-stats [--since YYYY-MM-DD] [--exclude SESSION_ID] [--state-dir DIR]
#                    [--projects-dir DIR] [--dry-run]
#
# --since defaults to the date of the last metrics.tsv row; the first run must
# pass it explicitly. Transcripts written to within the last 10 minutes are
# treated as active sessions and deferred to the next run.

require "date"
require "fileutils"
require "json"
require "optparse"
# `extract` is extensionless (a CLI); its __FILE__ guard makes load safe here.
load File.expand_path("extract", __dir__)

module OrchAudit
  DEFAULT_STATE_DIR = File.expand_path("~/.local/state/os-orchestration/audit")
  DEFAULT_PROJECTS_DIR = File.expand_path("~/.claude/projects")

  class TranscriptSet
    # Non-production sessions: eval sandboxes (-tmp-orch-*), headless runs
    # spawned from session scratchpads (-tmp-claude-*), and sessions whose
    # cwd was an eval harness directory (…-eval).
    EXCLUDED_PROJECT_DIRS = [/-tmp-orch-/, /-tmp-claude-/, /-eval\z/].freeze
    ACTIVE_WINDOW_SECONDS = 600

    def initialize(projects_dir:, since:, excluded_sessions: [], now: Time.now)
      @projects_dir = projects_dir
      @since = since
      @excluded_sessions = excluded_sessions
      @now = now
    end

    def paths
      Dir.glob(File.join(@projects_dir, "*", "*.jsonl")).sort.select { |p| candidate?(p) }
    end

    private

    def candidate?(path)
      return false if EXCLUDED_PROJECT_DIRS.any? { |re| File.dirname(path).match?(re) }
      return false if @excluded_sessions.include?(File.basename(path, ".jsonl"))
      mtime = File.mtime(path)
      mtime.to_date >= @since && (@now - mtime) > ACTIVE_WINDOW_SECONDS
    end
  end

  class SessionStats
    MIN_ASSISTANT_TURNS = 8

    def initialize(path)
      @path = path
      @sheet = FactSheet.new(path).data
    end

    attr_reader :path, :sheet

    def session_id = File.basename(@path, ".jsonl")
    def project = File.basename(File.dirname(@path))
    def trivial? = sheet[:metadata][:assistant_turns] < MIN_ASSISTANT_TURNS
    def spawns = sheet[:agent_spawns]
    def flagged_runs = sheet[:missed_delegation_candidates]

    def spawns_missing_model
      spawns.count { |s| !s[:model_explicit] }
    end

    # A pin mismatch is deterministic evidence of the harness ignoring an
    # explicit model param (the 2026-07-10 finding): param present, resolved
    # model known, and the resolved id does not contain the requested tier.
    def model_pin_mismatches
      spawns.count do |s|
        s[:model_explicit] && s[:model_param] && s[:resolved_model] &&
          !s[:resolved_model].to_s.include?(s[:model_param].to_s)
      end
    end

    def sendmessage_followups
      spawns.sum { |s| s[:rounds] - 1 }
    end

    def row
      m = sheet[:metadata]
      [
        session_id, project, m[:assistant_turns], m[:main_loop_output_tokens],
        m[:main_loop_share], spawns.size, spawns_missing_model,
        model_pin_mismatches, sendmessage_followups, flagged_runs.size
      ]
    end
  end

  class AuditRun
    STATS_HEADER = %w[
      session project assistant_turns main_loop_output_tokens main_loop_share
      spawns spawns_missing_model model_pin_mismatches sendmessage_followups
      flagged_runs
    ].freeze
    LEDGER_HEADER = %w[
      date since sessions trivial_skipped spawns spawns_missing_model
      model_pin_mismatches sendmessage_followups flagged_runs
      median_main_loop_share
    ].freeze

    def initialize(state_dir:, projects_dir:, since:, excluded_sessions:, today: Date.today)
      @state_dir = state_dir
      @since = since
      @today = today
      @transcripts = TranscriptSet.new(
        projects_dir: projects_dir, since: since, excluded_sessions: excluded_sessions
      )
    end

    def ledger_path = File.join(@state_dir, "metrics.tsv")
    def run_dir = File.join(@state_dir, "run-#{@today.iso8601}")

    def plan
      @transcripts.paths
    end

    def execute
      audited, trivial = partition_sessions
      write_run_dir(audited)
      append_ledger(audited, trivial)
      { run_dir: run_dir, audited: audited.size, trivial: trivial.size }
    end

    private

    def partition_sessions
      @transcripts.paths.map { |p| SessionStats.new(p) }.partition { |s| !s.trivial? }
    end

    def write_run_dir(audited)
      sheets_dir = File.join(run_dir, "sheets")
      FileUtils.mkdir_p(sheets_dir)
      File.write(File.join(run_dir, "stats.tsv"), stats_tsv(audited))
      File.write(File.join(run_dir, "flags.md"), flags_md(audited))
      audited.each do |s|
        File.write(File.join(sheets_dir, "#{s.session_id}.json"), JSON.pretty_generate(s.sheet))
      end
    end

    def stats_tsv(audited)
      lines = [STATS_HEADER.join("\t")]
      audited.each { |s| lines << s.row.join("\t") }
      lines.join("\n") + "\n"
    end

    def flags_md(audited)
      md = +"# Flagged regions for auditor judgment — run #{@today.iso8601}\n\n"
      md << "Heuristic candidates only; sequential-dependent work is a valid reason not\n"
      md << "to delegate. Auditors judge mechanical vs judgment-dependent per region.\n"
      audited.each do |s|
        md << "\n## #{s.session_id} (#{s.project})\n\n"
        if s.flagged_runs.empty?
          md << "No flagged runs.\n"
          next
        end
        s.flagged_runs.each do |m|
          md << "- **#{m[:tool]} x#{m[:length]}** (jsonl lines #{m[:lines].join('-')}, "
          md << "#{m[:distinct_targets]} distinct targets)\n"
          m[:targets_sample].each { |t| md << "  - `#{t}`\n" }
        end
      end
      md
    end

    def append_ledger(audited, trivial)
      FileUtils.mkdir_p(@state_dir)
      File.write(ledger_path, LEDGER_HEADER.join("\t") + "\n") unless File.exist?(ledger_path)
      File.open(ledger_path, "a") { |f| f.puts(ledger_row(audited, trivial).join("\t")) }
    end

    def ledger_row(audited, trivial)
      [
        @today.iso8601, @since.iso8601, audited.size, trivial.size,
        audited.sum { |s| s.spawns.size },
        audited.sum(&:spawns_missing_model),
        audited.sum(&:model_pin_mismatches),
        audited.sum(&:sendmessage_followups),
        audited.sum { |s| s.flagged_runs.size },
        median(audited.map { |s| s.sheet[:metadata][:main_loop_share] })
      ]
    end

    def median(values)
      return 0 if values.empty?
      sorted = values.sort
      mid = sorted.size / 2
      (sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0).round(3)
    end
  end

  class Ledger
    def self.last_run_date(path)
      return nil unless File.exist?(path)
      last = File.readlines(path).map(&:strip).reject(&:empty?)[1..]&.last
      last && (Date.iso8601(last.split("\t").first) rescue nil)
    end
  end
end

if $PROGRAM_NAME == __FILE__
  options = {
    state_dir: OrchAudit::DEFAULT_STATE_DIR,
    projects_dir: OrchAudit::DEFAULT_PROJECTS_DIR,
    exclude: [],
    dry_run: false
  }
  OptionParser.new do |o|
    o.banner = "Usage: audit-stats [--since YYYY-MM-DD] [--exclude SESSION_ID] [options]"
    o.on("--since DATE") { |d| options[:since] = Date.iso8601(d) }
    o.on("--exclude ID", "Session id to skip (repeatable)") { |id| options[:exclude] << id }
    o.on("--state-dir DIR") { |d| options[:state_dir] = File.expand_path(d) }
    o.on("--projects-dir DIR") { |d| options[:projects_dir] = File.expand_path(d) }
    o.on("--dry-run", "List transcripts that would be audited, then exit") { options[:dry_run] = true }
  end.parse!

  since = options[:since] ||
          OrchAudit::Ledger.last_run_date(File.join(options[:state_dir], "metrics.tsv"))
  abort "No --since given and no prior ledger row to infer it from" unless since

  run = OrchAudit::AuditRun.new(
    state_dir: options[:state_dir], projects_dir: options[:projects_dir],
    since: since, excluded_sessions: options[:exclude]
  )
  if options[:dry_run]
    puts run.plan
    exit 0
  end
  result = run.execute
  puts "Audited #{result[:audited]} sessions (#{result[:trivial]} trivial skipped)"
  puts "Run dir: #{result[:run_dir]}"
  puts "Ledger:  #{run.ledger_path}"
end