src/model-learning.ts

Functions

Imports

It imports src/model-routing.ts, src/time.ts. Imported by src/tools/jobs.ts.

Source

1import { MIN_SAMPLE, RATE_FLOOR, RATE_GAIN, ROUTING_RULES, ROUTING_RULES_VERSION, TRIAL_ARMS, cellsOf, modelAlias, proposeRuleChanges, trialStatus, type Cell, type OutcomeFact, type Proposal, type TrialStatus } from "./model-routing";
2import { DAY_MS } from "./time";
3
4// WHAT THE ROUTING LEARNS (capsid/research/design-model-routing.md): per kind and model,
5// the share of jobs merged with green CI and not undone since, the corrections spent,
6// and the cost where telemetry recorded one, read from job_outcomes. Merge and CI are
7// GitHub's, read at complete, not the driver's own account (docs/work-queue.md). A merge
8// is a weak label on its own (Peralta et al., MSR 2026: reviewers fix and merge, or
9// merge and revert), so a job whose pull request was later reverted (src/merge-safety.ts,
10// `job-pr-reverted`) or whose deploy was rolled back does not count as work that held.
11// The result proposes rule changes and applies none: the rules are code and the seat
12// approves each edit.
13
14const LEARNING_WINDOW_DAYS = 90;
15// One read, bounded. A window with more outcomes than this says so, never a partial table
16// presented as the whole.
17const LEARNING_ROWS_MAX = 5000;
18
19export interface ModelLearning {
20  generated: string;
21  window_days: number;
22  outcomes_read: number;
23  truncated: boolean;
24  min_sample: number;
25  rate_floor: number;
26  rate_gain: number;
27  rules_version: number;
28  rules: ReadonlyArray<{ when: Record<string, string>; model: string; effort: string }>;
29  trial_arms: ReadonlyArray<{ kind: string; model: string; percent: number }>;
30  cells: Cell[];
31  proposals: Proposal[];
32  // Each trial arm against its kind's rule (src/model-routing.ts TRIAL_ARMS).
33  trials: TrialStatus[];
34  note: string;
35}
36
37interface Row {
38  kind: string | null;
39  model: string | null;
40  model_actual: string | null;
41  prs_merged: number | null;
42  ci_green: number | null;
43  corrections: number | null;
44  cost_usd: number | null;
45  tokens: number | null;
46  undone: number;
47}
48
49// An outcome is learned under the model Capsid chose, which the Worker recorded.
50// model_actual is the session's own report, so it never moves an outcome into another
51// model's cell; an outcome whose session reports a different model is left out, since
52// its work was not shown to be done on the chosen one. opusplan runs as Opus and Sonnet.
53function sameModel(r: Row): boolean {
54  const actual = modelAlias(r.model_actual);
55  if (actual === null || r.model === null) return true;
56  return r.model === "opusplan" ? actual === "opus" || actual === "sonnet" || actual === "opusplan" : actual === r.model;
57}
58
59export async function readModelLearning(db: D1Database, now: Date): Promise<ModelLearning> {
60  const since = new Date(now.getTime() - LEARNING_WINDOW_DAYS * DAY_MS).toISOString();
61  const { results } = await db
62    .prepare(
63      `SELECT o.job_kind AS kind, o.model_chosen AS model, o.model_actual AS model_actual, o.prs_merged AS prs_merged, o.ci_green AS ci_green,
64              j.corrections_count AS corrections, o.cost_usd AS cost_usd,
65              CASE WHEN o.tokens_input IS NULL AND o.tokens_output IS NULL THEN NULL ELSE COALESCE(o.tokens_input, 0) + COALESCE(o.tokens_output, 0) END AS tokens,
66              (EXISTS (SELECT 1 FROM audit_log a WHERE a.action = 'job-pr-reverted' AND json_extract(a.params, '$.job_id') = o.job_id)
67               OR EXISTS (SELECT 1 FROM job_outcome_prs p JOIN deploy_outcomes d
68                            ON lower(p.pr_url) = lower('https://github.com/' || d.repo || '/pull/' || d.pr)
69                          WHERE p.job_id = o.job_id AND d.outcome IN ('rolled_back', 'rollback_failed'))) AS undone
70       FROM job_outcomes o LEFT JOIN jobs j ON j.id = o.job_id
71       WHERE o.job_kind IS NOT NULL AND o.recorded_at >= ?1
72       ORDER BY o.recorded_at DESC LIMIT ?2`
73    )
74    .bind(since, LEARNING_ROWS_MAX + 1)
75    .all<Row>();
76  const rows = results ?? [];
77  const truncated = rows.length > LEARNING_ROWS_MAX;
78  const read = truncated ? rows.slice(0, LEARNING_ROWS_MAX) : rows;
79  const facts: OutcomeFact[] = read.filter(sameModel).map((r) => ({
80    kind: r.kind, model: r.model, prs_merged: r.prs_merged, ci_green: r.ci_green, corrections: r.corrections, cost_usd: r.cost_usd, tokens: r.tokens, undone: r.undone === 1,
81  }));
82  const mismatched = read.length - facts.length;
83  const cells = cellsOf(facts);
84  const proposals = proposeRuleChanges(cells);
85  const enough = cells.filter((c) => c.enough).length;
86  return {
87    generated: now.toISOString(),
88    window_days: LEARNING_WINDOW_DAYS,
89    outcomes_read: read.length,
90    truncated,
91    min_sample: MIN_SAMPLE,
92    rate_floor: RATE_FLOOR,
93    rate_gain: RATE_GAIN,
94    rules_version: ROUTING_RULES_VERSION,
95    rules: ROUTING_RULES.map((r) => ({ when: Object.fromEntries(Object.entries(r.when)) as Record<string, string>, model: r.model, effort: r.effort })),
96    trial_arms: TRIAL_ARMS.map((a) => ({ kind: a.kind, model: a.model, percent: a.percent })),
97    cells,
98    proposals,
99    trials: trialStatus(cells),
100    note:
101      read.length === 0
102        ? "No routed outcome in the window yet: outcomes written before migration 0032 carry no kind or model. Nothing is proposed from no evidence."
103        : `${cells.length} kind and model cells, ${enough} with at least ${MIN_SAMPLE} outcomes; only those are acted on. ${proposals.length} proposed change${proposals.length === 1 ? "" : "s"}, none applied: the seat approves each rule edit.${mismatched ? ` ${mismatched} outcome${mismatched === 1 ? " was" : "s were"} left out because the session reported a model other than the one chosen.` : ""}${truncated ? ` The window holds more than ${LEARNING_ROWS_MAX} outcomes, so this is the newest ${LEARNING_ROWS_MAX}.` : ""}`,
104  };
105}