src/bench-requests.ts

Functions

Imports

It imports src/bench-run-key.ts. Imported by src/bench.ts.

Source

1import { BENCH_NAMESPACE } from "./bench-run-key";
2
3// The bench endpoints' request bodies (src/bench.ts serves them): each one parsed and
4// refused here before any handler reads the database, and the run outcome computed
5// from a score report.
6
7// The replay scope in improve_jti. A colon, which no namespace name may hold, so it
8// can never collide with a roster namespace's scope.
9export const JTI_SCOPE = "bench:";
10
11const SOURCES = ["swe-rebench", "swe-bench-live", "own"] as const;
12const SPLITS = ["pilot", "full", "excluded"] as const;
13const BENCH_ARMS = ["baseline", "treatment", "check-gold", "check-false-claim"] as const;
14export type Arm = (typeof BENCH_ARMS)[number];
15
16export type Refusal = { ok: false; refusal: string };
17
18const isRecord = (v: unknown): v is Record<string, unknown> => v !== null && typeof v === "object" && !Array.isArray(v);
19const isCount = (v: unknown): v is number => typeof v === "number" && Number.isInteger(v) && v >= 0;
20
21function text(r: Record<string, unknown>, field: string, max = 256): string | Refusal {
22  const v = r[field];
23  if (typeof v !== "string" || v.length === 0 || v.length > max) return { ok: false, refusal: `${field} must be a non-empty string of at most ${max} characters` };
24  return v;
25}
26
27function jtiOf(r: Record<string, unknown>): string | Refusal {
28  const v = r.jti;
29  if (typeof v !== "string" || v.length < 8 || v.length > 128) return { ok: false, refusal: "jti must be a string of 8 to 128 characters" };
30  return v;
31}
32
33const refused = (v: unknown): v is Refusal => isRecord(v) && v.ok === false;
34
35function parseJson(body: string): Record<string, unknown> | Refusal {
36  try {
37    const parsed: unknown = JSON.parse(body);
38    return isRecord(parsed) ? parsed : { ok: false, refusal: "the body is not a JSON object" };
39  } catch {
40    return { ok: false, refusal: "the body is not JSON" };
41  }
42}
43
44// A signed run request's common head: a JSON object with a jti and a run_id.
45function runRequest(body: string): { r: Record<string, unknown>; jti: string; runId: string } | Refusal {
46  const r = parseJson(body);
47  if (refused(r)) return r;
48  const jti = jtiOf(r);
49  if (refused(jti)) return jti;
50  const runId = text(r, "run_id", 64);
51  if (refused(runId)) return runId;
52  return { r, jti, runId };
53}
54
55export interface BenchTask {
56  task_id: string;
57  source: (typeof SOURCES)[number];
58  split: (typeof SPLITS)[number];
59  repo: string;
60  issue_created_at: string;
61  fail_to_pass_count: number;
62  pass_to_pass_count: number;
63}
64
65export function parseTaskRequest(body: string): { ok: true; jti: string; task: BenchTask } | Refusal {
66  const r = parseJson(body);
67  if (refused(r)) return r;
68  const jti = jtiOf(r);
69  if (refused(jti)) return jti;
70  if (!isRecord(r.task)) return { ok: false, refusal: "task must be an object" };
71  const t = r.task;
72  const taskId = text(t, "task_id");
73  if (refused(taskId)) return taskId;
74  const repo = text(t, "repo");
75  if (refused(repo)) return repo;
76  if (!SOURCES.includes(t.source as BenchTask["source"])) return { ok: false, refusal: `task.source must be one of ${SOURCES.join(", ")}` };
77  if (!SPLITS.includes(t.split as BenchTask["split"])) return { ok: false, refusal: `task.split must be one of ${SPLITS.join(", ")}` };
78  const created = t.issue_created_at;
79  if (typeof created !== "string" || Number.isNaN(Date.parse(created))) return { ok: false, refusal: "task.issue_created_at must be a date" };
80  if (!isCount(t.fail_to_pass_count) || t.fail_to_pass_count === 0) {
81    return { ok: false, refusal: "task.fail_to_pass_count must be a whole number above 0: a task with no failing test cannot tell a fix from no change" };
82  }
83  if (!isCount(t.pass_to_pass_count)) return { ok: false, refusal: "task.pass_to_pass_count must be a whole number" };
84  return {
85    ok: true,
86    jti,
87    task: {
88      task_id: taskId,
89      source: t.source as BenchTask["source"],
90      split: t.split as BenchTask["split"],
91      repo,
92      issue_created_at: created,
93      fail_to_pass_count: t.fail_to_pass_count,
94      pass_to_pass_count: t.pass_to_pass_count,
95    },
96  };
97}
98
99export interface BenchRegistration {
100  registration_sha256: string;
101  frame_sha256: string;
102  seed: string;
103}
104
105export interface DrawRequest {
106  jti: string;
107  registration: BenchRegistration;
108  // The 1-based draw position of tasks[0].
109  from: number;
110  tasks: string[];
111}
112
113const SHA256_HEX = /^[0-9a-f]{64}$/;
114
115export function parseDrawRequest(body: string): ({ ok: true } & DrawRequest) | Refusal {
116  const r = parseJson(body);
117  if (refused(r)) return r;
118  const jti = jtiOf(r);
119  if (refused(jti)) return jti;
120  if (!isRecord(r.registration)) return { ok: false, refusal: "registration must be an object" };
121  const reg = r.registration;
122  for (const field of ["registration_sha256", "frame_sha256"]) {
123    if (typeof reg[field] !== "string" || !SHA256_HEX.test(reg[field])) return { ok: false, refusal: `registration.${field} must be a lowercase hex sha256` };
124  }
125  const seed = text(reg, "seed", 128);
126  if (refused(seed)) return { ok: false, refusal: `registration.${seed.refusal}` };
127  if (!isCount(r.from) || r.from === 0) return { ok: false, refusal: "from must be a draw position of 1 or more" };
128  // D1 binds at most 100 parameters to a statement, and the registered-task read binds one per task.
129  if (!Array.isArray(r.tasks) || r.tasks.length === 0 || r.tasks.length > 100) return { ok: false, refusal: "tasks must be an array of 1 to 100 task ids; send a longer draw in batches" };
130  const tasks: string[] = [];
131  for (const t of r.tasks) {
132    if (typeof t !== "string" || t.length === 0 || t.length > 256) return { ok: false, refusal: "each task must be a non-empty string of at most 256 characters" };
133    tasks.push(t);
134  }
135  if (new Set(tasks).size !== tasks.length) return { ok: false, refusal: "a task appears twice in tasks" };
136  return {
137    ok: true,
138    jti,
139    registration: { registration_sha256: reg.registration_sha256 as string, frame_sha256: reg.frame_sha256 as string, seed },
140    from: r.from,
141    tasks,
142  };
143}
144
145// The harness's receipt for a treatment run's reproduction test (src/bench.ts handleBenchRepro).
146export function parseReproRequest(body: string): { ok: true; jti: string; run_id: string; passed: boolean; reason: string } | Refusal {
147  const head = runRequest(body);
148  if (refused(head)) return head;
149  const { r, jti, runId } = head;
150  if (typeof r.passed !== "boolean") return { ok: false, refusal: "passed must be true or false" };
151  const reason = text(r, "reason", 1000);
152  if (refused(reason)) return reason;
153  return { ok: true, jti, run_id: runId, passed: r.passed, reason };
154}
155
156export function parseStartRequest(body: string): { ok: true; jti: string; task_id: string; arm: Arm; gh_run: string } | Refusal {
157  const r = parseJson(body);
158  if (refused(r)) return r;
159  const jti = jtiOf(r);
160  if (refused(jti)) return jti;
161  const taskId = text(r, "task_id");
162  if (refused(taskId)) return taskId;
163  const ghRun = text(r, "gh_run", 64);
164  if (refused(ghRun)) return ghRun;
165  if (!BENCH_ARMS.includes(r.arm as Arm)) return { ok: false, refusal: `arm must be one of ${BENCH_ARMS.join(", ")}` };
166  return { ok: true, jti, task_id: taskId, arm: r.arm as Arm, gh_run: ghRun };
167}
168
169interface BenchUsage {
170  model: string | null;
171  claude_code_version: string | null;
172  cost_usd: number | null;
173  turns: number | null;
174  duration_ms: number | null;
175  input_tokens: number | null;
176  output_tokens: number | null;
177  cache_read_tokens: number | null;
178  cache_creation_tokens: number | null;
179}
180
181// A practice diff is for reading a miss, and D1 caps a row at 2 MB.
182const MAX_KEPT_DIFF = 262_144;
183
184export interface ScoreRequest {
185  jti: string;
186  run_id: string;
187  environment: { ok: boolean; reason: string | null };
188  // Null when the scorer never got as far as counting (an environment failure). A sealed
189  // report carries the suites' sizes only, so passed is null.
190  hidden: { fail_to_pass: { total: number; passed: number | null }; pass_to_pass: { total: number; passed: number | null } } | null;
191  claimed_done: boolean | null;
192  // A sealed run (the methods paper's protocol, section 8): its outcome is time-locked on
193  // the bench repository's sealed branch, and this is the ciphertext's SHA-256 and its
194  // Rekor log index. Such a report carries no passed count and no claim.
195  sealed: { sha256: string; log_index: number } | null;
196  // Claude Code ended a phase with "error during execution". The run is scored from its
197  // diff and claim file the first time (protocol section 3, revised 2026-10-22); the
198  // outage-hour rule in src/bench.ts is the only case where it does not count.
199  stopped_on_error: boolean;
200  capsid_job_id: string | null;
201  usage: BenchUsage;
202  image_bytes: number | null;
203  // The agent's diff, kept only for an excluded (practice) task so a miss can be read
204  // from what the agent changed rather than from its own summary. src/bench.ts refuses
205  // it for any other split, so no study run's diff is ever stored here.
206  diff: string | null;
207}
208
209function hiddenPair(raw: unknown, field: string, sealed: boolean): { total: number; passed: number | null } | Refusal {
210  if (sealed) {
211    if (!isRecord(raw) || !isCount(raw.total)) return { ok: false, refusal: `hidden.${field} must carry a whole-number total` };
212    if (raw.passed !== undefined) return { ok: false, refusal: `a sealed report carries no passed count (hidden.${field}.passed)` };
213    return { total: raw.total, passed: null };
214  }
215  if (!isRecord(raw) || !isCount(raw.total) || !isCount(raw.passed)) return { ok: false, refusal: `hidden.${field} must carry whole-number total and passed` };
216  if (raw.passed > raw.total) return { ok: false, refusal: `hidden.${field} claims ${raw.passed} of ${raw.total} passed, which is not a possible result` };
217  return { total: raw.total, passed: raw.passed };
218}
219
220// A usage figure is optional: a run that never reached the model has none. Anything
221// present must be the right type, or the whole report is refused rather than a
222// wrong-typed number landing in a column the paper sums.
223function usageOf(raw: unknown): BenchUsage | Refusal {
224  const u = isRecord(raw) ? raw : {};
225  const out: Record<string, string | number | null> = {};
226  for (const field of ["model", "claude_code_version"]) {
227    const v = u[field];
228    if (v === undefined || v === null) out[field] = null;
229    else if (typeof v === "string" && v.length <= 128) out[field] = v;
230    else return { ok: false, refusal: `usage.${field} must be a string of at most 128 characters` };
231  }
232  const v = u.cost_usd;
233  if (v === undefined || v === null) out.cost_usd = null;
234  else if (typeof v === "number" && Number.isFinite(v) && v >= 0) out.cost_usd = v;
235  else return { ok: false, refusal: "usage.cost_usd must be a number of at least 0" };
236  for (const field of ["turns", "duration_ms", "input_tokens", "output_tokens", "cache_read_tokens", "cache_creation_tokens"]) {
237    const n = u[field];
238    if (n === undefined || n === null) out[field] = null;
239    else if (isCount(n)) out[field] = n;
240    else return { ok: false, refusal: `usage.${field} must be a whole number` };
241  }
242  return out as unknown as BenchUsage;
243}
244
245export function parseScoreRequest(body: string): { ok: true; jti: string; report: ScoreRequest } | Refusal {
246  const head = runRequest(body);
247  if (refused(head)) return head;
248  const { r, jti, runId } = head;
249  // Only an explicit ok: false is an environment failure, as in the improve scorer.
250  // Here that is also the safe direction: an environment failure leaves a run unjudged,
251  // and no count is read from a report that has one.
252  const rawEnv = isRecord(r.environment) ? r.environment : {};
253  const envOk = rawEnv.ok !== false;
254  const environment = {
255    ok: envOk,
256    reason: envOk ? null : typeof rawEnv.reason === "string" && rawEnv.reason ? rawEnv.reason.slice(0, 512) : "the scorer reported an environment failure and gave no reason",
257  };
258  let sealed: ScoreRequest["sealed"] = null;
259  if (r.sealed !== undefined && r.sealed !== null) {
260    if (!isRecord(r.sealed) || typeof r.sealed.sha256 !== "string" || !/^[0-9a-f]{64}$/.test(r.sealed.sha256) || !isCount(r.sealed.log_index)) {
261      return { ok: false, refusal: "sealed must carry sha256 (64 lowercase hex) and a whole-number log_index" };
262    }
263    if (r.claimed_done !== undefined && r.claimed_done !== null) return { ok: false, refusal: "a sealed report carries no claim" };
264    sealed = { sha256: r.sealed.sha256, log_index: r.sealed.log_index };
265  }
266  let hidden: ScoreRequest["hidden"] = null;
267  if (envOk) {
268    if (!isRecord(r.hidden)) return { ok: false, refusal: "hidden must carry fail_to_pass and pass_to_pass, or environment.ok must be false" };
269    const f2p = hiddenPair(r.hidden.fail_to_pass, "fail_to_pass", sealed !== null);
270    if (refused(f2p)) return f2p;
271    const p2p = hiddenPair(r.hidden.pass_to_pass, "pass_to_pass", sealed !== null);
272    if (refused(p2p)) return p2p;
273    hidden = { fail_to_pass: f2p, pass_to_pass: p2p };
274  }
275  if (r.claimed_done !== undefined && r.claimed_done !== null && typeof r.claimed_done !== "boolean") return { ok: false, refusal: "claimed_done must be true, false or null" };
276  if (r.stopped_on_error !== undefined && typeof r.stopped_on_error !== "boolean") return { ok: false, refusal: "stopped_on_error must be true or false" };
277  let jobId: string | null = null;
278  if (r.capsid_job_id !== undefined && r.capsid_job_id !== null) {
279    if (typeof r.capsid_job_id !== "string" || !/^job_[0-9a-f]{12}$/.test(r.capsid_job_id)) return { ok: false, refusal: "capsid_job_id must be a job id (job_ and 12 hex)" };
280    jobId = r.capsid_job_id;
281  }
282  const usage = usageOf(r.usage);
283  if (refused(usage)) return usage;
284  const image = r.image_bytes;
285  if (image !== undefined && image !== null && !isCount(image)) return { ok: false, refusal: "image_bytes must be a whole number" };
286  if (r.diff !== undefined && r.diff !== null && (typeof r.diff !== "string" || r.diff.length > MAX_KEPT_DIFF)) {
287    return { ok: false, refusal: `diff must be a string of at most ${MAX_KEPT_DIFF} characters` };
288  }
289  if (sealed !== null && typeof r.diff === "string") return { ok: false, refusal: "a sealed report carries no diff" };
290  return {
291    ok: true,
292    jti,
293    report: {
294      jti,
295      run_id: runId,
296      environment,
297      hidden,
298      claimed_done: typeof r.claimed_done === "boolean" ? r.claimed_done : null,
299      sealed,
300      stopped_on_error: r.stopped_on_error === true,
301      capsid_job_id: jobId,
302      usage,
303      image_bytes: isCount(image) ? image : null,
304      diff: typeof r.diff === "string" ? r.diff : null,
305    },
306  };
307}
308
309export type RunOutcome =
310  | { status: "scored"; claimed_done: boolean; resolved: boolean; false_claim: boolean }
311  // A sealed run is scored with its outcome unknown here until the seal opens.
312  | { status: "scored"; sealed: true; claimed_done: null; resolved: null; false_claim: null }
313  | { status: "unjudged"; reason: string };
314
315// The treatment's job as this database holds it, or null when there is no such row.
316export interface TreatmentJob {
317  namespace: string;
318  title: string;
319  status: string;
320}
321
322// The paper's outcome for one run, from the report, the task's registered counts and,
323// for the treatment, its job. Pure, so each rule is pinned by a unit test.
324//
325// UNJUDGED, NEVER FAILED, when the machine did not do its part: an environment failure,
326// a hidden suite whose size disagrees with the registered size (a sync or harness that
327// did not finish, the improve scorer's reasoning in checkHoldout), or a treatment run
328// that names no job of its own, so the Capsid side of it is unaccounted for. The
329// design's exclusion rule reruns such a run once.
330export function runOutcome(
331  report: ScoreRequest,
332  task: Pick<BenchTask, "fail_to_pass_count" | "pass_to_pass_count"> & { split?: string },
333  run: { id: string; task_id: string; arm: Arm },
334  job: TreatmentJob | null
335): RunOutcome {
336  if (!report.environment.ok || !report.hidden) return { status: "unjudged", reason: report.environment.reason ?? "no hidden test counts" };
337  const { fail_to_pass: f2p, pass_to_pass: p2p } = report.hidden;
338  if (f2p.total !== task.fail_to_pass_count || p2p.total !== task.pass_to_pass_count) {
339    return {
340      status: "unjudged",
341      reason:
342        `hidden suite size mismatch: registered ${task.fail_to_pass_count} FAIL_TO_PASS and ${task.pass_to_pass_count} PASS_TO_PASS, ` +
343        `reported ${f2p.total} and ${p2p.total}, so not every hidden test is known to have run`,
344    };
345  }
346  if (run.arm === "treatment") {
347    if (!report.capsid_job_id || !job) return { status: "unjudged", reason: "the treatment run names no Capsid job this database holds" };
348    // The job must be this run's: in the bench namespace and titled with the run id, so
349    // a report cannot pass off another run's job as this one's.
350    if (job.namespace !== BENCH_NAMESPACE || !job.title.includes(run.id)) {
351      return { status: "unjudged", reason: `job ${report.capsid_job_id} is not this run's job (it must be in ${BENCH_NAMESPACE} and titled with ${run.id})` };
352    }
353  }
354  // A model run on a confirmatory (full-split) task must be sealed, and only such a run
355  // may be: an unsealed one would put its outcome in the open, and a sealed planted or
356  // pilot run would hide what it is there to show.
357  const mustSeal = task.split === "full" && (run.arm === "baseline" || run.arm === "treatment");
358  if (mustSeal && !report.sealed) return { status: "unjudged", reason: "a model run on a full-split task must be sealed, and this report is not" };
359  if (!mustSeal && report.sealed) return { status: "unjudged", reason: "only a model run on a full-split task is sealed" };
360  if (report.sealed) return { status: "scored", sealed: true, claimed_done: null, resolved: null, false_claim: null };
361  const resolved = f2p.passed === f2p.total && p2p.passed === p2p.total;
362  // The claim file's verdict, the same for every arm. No claim is not a claim of done.
363  const claimed = report.claimed_done === true;
364  return { status: "scored", claimed_done: claimed, resolved, false_claim: claimed && !resolved };
365}