src/bench-run-key.ts

Functions

Imports

It imports src/agents-schema.ts, src/agents.ts, src/encoding.ts, src/time.ts. Imported by src/agents.ts, src/bench-requests.ts, src/bench.ts, src/jobs-seat.ts.

Source

1import { safeEqual } from "@dustinedwards/site-runtime/compare";
2import { noFlags } from "./agents-schema";
3import type { Agent } from "./agents";
4import { bytesToHex } from "./encoding";
5import { HOUR_MS } from "./time";
6
7// THE RUN KEY (the methods paper's protocol, section 4, "The run key"; decided
8// 2026-10-19). Each treatment run gets its own Capsid key, minted by /bench/start for
9// that run id and held by no other run. The harness posts the run's one job with its
10// own driver key, which the agent never holds. The run key's only successful calls are
11// claim, and list by id, of that one job (src/scope.ts benchRunRefusal). It stops
12// working when the run closes, or once the run is past the stale window.
13//
14// Only the key's sha256 is stored, on the run's own row (bench_runs.run_key_hash,
15// migrations/0050), so the key's life is the row's: no agents row, nothing to revoke.
16
17// A running row older than this no longer holds a slot (src/bench.ts), and its key no
18// longer resolves.
19export const BENCH_STALE_MS = 3 * HOUR_MS;
20export const BENCH_NAMESPACE = "bench";
21
22export function newRunKey(): string {
23  return `capsid_run_${bytesToHex(crypto.getRandomValues(new Uint8Array(32)))}`;
24}
25
26// The job a run works: the one job in the bench namespace titled as the harness titles
27// it, `bench <task_id> <run_id>` (capsid-bench scripts/post-job.mjs). Null unless there
28// is exactly one, so a run whose job is missing or doubled can work nothing.
29export function runJobTitle(taskId: string, runId: string): string {
30  return `bench ${taskId} ${runId}`;
31}
32
33/** The caller a run key resolves to, or null when no running treatment run holds it. */
34export async function runKeyAgent(db: D1Database, hash: string, now: Date): Promise<Agent | null> {
35  const row = await db
36    .prepare(
37      `SELECT id, task_id, run_key_hash FROM bench_runs
38       WHERE run_key_hash = ?1 AND arm = 'treatment' AND status = 'running' AND started_at > ?2`
39    )
40    .bind(hash, new Date(now.getTime() - BENCH_STALE_MS).toISOString())
41    .first<{ id: string; task_id: string; run_key_hash: string }>();
42  if (!row || !await safeEqual(row.run_key_hash, hash)) return null;
43  const { results: jobs } = await db
44    .prepare("SELECT id FROM jobs WHERE namespace = ?1 AND title = ?2 LIMIT 2")
45    .bind(BENCH_NAMESPACE, runJobTitle(row.task_id, row.id))
46    .all<{ id: string }>();
47  const actor = `agent:bench-run-${row.id}`;
48  return {
49    id: actor,
50    name: `bench-run-${row.id}`,
51    kind: "driver",
52    actor,
53    scopes: { namespaces: [BENCH_NAMESPACE], repos: [], tools: ["capsid_work_jobs"], grants: ["read", "write"], flags: noFlags() },
54    admin: false,
55    row: null,
56    benchRun: { run: row.id, job: jobs?.length === 1 ? jobs[0].id : null },
57  };
58}
59
60// THE REPRODUCTION RECEIPT (the methods paper's protocol, section 2, "Prove it before
61// done"). At the phase break the harness runs the test the agent named on the base with
62// only that file applied, then on the agent's diff, and posts what it saw to
63// /bench/repro (src/bench.ts). It is kept as one audit row per run, written once, and a
64// run key's complete is accepted only when it says the test failed on the base and
65// passed on the diff (src/jobs-seat.ts completeAsCaller). The agent's own labels count
66// for nothing.
67export const BENCH_REPRO_ACTION = "bench-repro";
68
69export interface ReproReceipt {
70  passed: boolean;
71  reason: string;
72}
73
74/** The harness's receipt for a run, or null when none was posted. */
75export async function readReproReceipt(db: D1Database, runId: string): Promise<ReproReceipt | null> {
76  const row = await db
77    .prepare("SELECT params FROM audit_log WHERE action = ?1 AND namespace = ?2 AND path = ?3 ORDER BY id LIMIT 1")
78    .bind(BENCH_REPRO_ACTION, BENCH_NAMESPACE, runId)
79    .first<{ params: string }>();
80  if (!row) return null;
81  const parsed = JSON.parse(row.params) as Partial<ReproReceipt>;
82  return { passed: parsed.passed === true, reason: typeof parsed.reason === "string" ? parsed.reason : "" };
83}