src/job-claims.ts

Functions

Imports

It imports src/job-outcomes.ts, src/limits.ts, src/ops-types/claims.ts. Imported by src/jobs-holder.ts, src/jobs-seat.ts, src/tools/jobs.ts.

Source

1import { z } from "zod";
2import type { EvidenceVerdict, JobEvidence } from "./job-outcomes";
3import { bounded } from "./limits";
4import type { JobClaimRow as StoredClaimRow } from "./ops-types/claims";
5
6// WHAT THE AGENT SAID, kept apart from what the Worker verified (migrations/0023).
7//
8// job_outcomes stores GitHub's number in place of the driver's wherever GitHub could
9// answer, so the driver's own account was overwritten at the moment it was recorded.
10// This module records that account first, in job_claims, from the arguments exactly as
11// they arrived, and then one job_evaluations row per check with the claim and the
12// verified value side by side. Both are written in the transition's own batch
13// (holderTransition, src/jobs-holder.ts), so a claim exists exactly when the
14// transition it describes does.
15//
16// NULL IS NOT FALSE, on 0011's reasoning: a field the agent did not state is NULL,
17// never 0 and never false. "Nobody said" and "said none" are different facts.
18
19const CLAIM_TESTS_RESULTS = ["pass", "fail", "partial", "not_run"] as const;
20const CLAIM_DEPLOY_STATES = ["none", "pending", "deployed", "verified", "failed"] as const;
21
22// Bounds on what one claim may carry. A claim is stored whole, so each list and each
23// string is bounded where it arrives rather than truncated where it is written.
24export const MAX_CLAIM_PRS = 10;
25const MAX_CLAIM_URL = 512;
26const MAX_CLAIM_FILES = 500;
27const MAX_CLAIM_PATH = 512;
28const MAX_CLAIM_VERSION = 128;
29
30const count = z.number().int().nonnegative();
31const version = bounded(MAX_CLAIM_VERSION);
32// A usage figure the agent read off its own session. zod's number() already refuses NaN
33// and Infinity, which JSON cannot carry but a caller object can; the claim test pins that,
34// so neither can reach SUM().
35const usageAmount = z.number().nonnegative();
36
37// Strict at every level: an unknown key is refused, never dropped. A dropped key is a
38// claim the agent made and the record does not have, which is the loss this table
39// exists to prevent.
40export const claimSchema = z
41  .object({
42    prs_opened: z.array(bounded(MAX_CLAIM_URL)).max(MAX_CLAIM_PRS).optional().describe("Pull request URLs this job opened."),
43    prs_merged: z.array(bounded(MAX_CLAIM_URL)).max(MAX_CLAIM_PRS).optional().describe("Pull request URLs this job says are merged."),
44    tests: z
45      .object({
46        run: count.optional(),
47        passed: count.optional(),
48        failed: count.optional(),
49        result: z.enum(CLAIM_TESTS_RESULTS).optional(),
50      })
51      .strict()
52      .optional()
53      .describe("Tests the agent ran, and their result."),
54    deploy_state: z.enum(CLAIM_DEPLOY_STATES).optional(),
55    files_touched: z.array(bounded(MAX_CLAIM_PATH)).max(MAX_CLAIM_FILES).optional().describe("Repo-relative paths the agent touched."),
56    usage: z
57      .object({
58        cost_usd: usageAmount.optional(),
59        active_seconds: usageAmount.optional(),
60        tokens: z
61          .object({
62            input: count.optional(),
63            output: count.optional(),
64            cache_read: count.optional(),
65            cache_creation: count.optional(),
66          })
67          .strict()
68          .optional(),
69      })
70      .strict()
71      .optional()
72      .describe(
73        "What this session used, as the session reports it: cost_usd, active_seconds and tokens {input, output, cache_read, cache_creation}. This session's own total, not a running total across earlier claims on the job. Self-reported, recorded and never used to authorize anything. Telemetry (docs/telemetry.md) stays the measured source; the claims aggregate shows both side by side."
74      ),
75    versions: z
76      .object({
77        model_id: version.optional(),
78        client_name: version.optional(),
79        client_version: version.optional(),
80        permission_mode: version.optional(),
81      })
82      .strict()
83      .optional()
84      .describe("Self-reported, recorded and never used to authorize anything."),
85  })
86  .strict();
87
88export type JobClaim = z.infer<typeof claimSchema>;
89
90// A claim arrives as an object or as a JSON string, for the reason evidence does
91// (parseEvidence, src/job-outcomes.ts). Unlike evidence, a field that does not fit is
92// refused rather than dropped, because the claim is the record.
93export type ClaimInput = JobClaim | string | undefined;
94
95export function parseClaim(input: unknown): { claim: JobClaim | undefined } | { error: string } {
96  if (input === undefined || input === null) return { claim: undefined };
97  let value: unknown = input;
98  if (typeof input === "string") {
99    const text = input.trim();
100    // Blank is nothing sent, as for evidence.
101    if (text.length === 0) return { claim: undefined };
102    try {
103      value = JSON.parse(text);
104    } catch {
105      return { error: `claim was sent as a string that is not JSON: ${text.slice(0, 120)}. Nothing was written.` };
106    }
107  }
108  if (value === null || typeof value !== "object" || Array.isArray(value)) {
109    return { error: "claim must be an object (or a JSON string of one). Nothing was written." };
110  }
111  const parsed = claimSchema.safeParse(value);
112  if (!parsed.success) {
113    const problems = parsed.error.issues.map((issue) => `claim${issue.path.length ? "." + issue.path.map(String).join(".") : ""}: ${issue.message}`);
114    return { error: `claim was refused, not trimmed: ${problems.join("; ")}. Nothing was written.` };
115  }
116  return { claim: parsed.data };
117}
118
119// The claim-bearing arguments exactly as sent. Recorded whole in job_claims.raw so the
120// structured columns can always be checked against what was actually said.
121export interface ClaimRaw {
122  evidence?: unknown;
123  claim?: unknown;
124  result_summary?: string;
125  reason?: string;
126  result_ref?: string | null;
127  command?: string;
128}
129
130// evidence and claim may have arrived as JSON strings; raw keeps the parsed value, so
131// the column is one JSON document rather than JSON with JSON inside it. A string that
132// does not parse was refused before this runs, so it is kept as sent, not dropped.
133function parsedValue(value: unknown): unknown {
134  if (typeof value !== "string") return value;
135  const text = value.trim();
136  if (text.length === 0) return undefined;
137  try {
138    return JSON.parse(text);
139  } catch {
140    return value;
141  }
142}
143
144export function rawJson(raw: ClaimRaw): string {
145  // JSON.stringify omits a key whose value is undefined, which is the rule: a key the
146  // caller did not send is absent, not null.
147  return JSON.stringify({
148    evidence: parsedValue(raw.evidence),
149    claim: parsedValue(raw.claim),
150    result_summary: raw.result_summary,
151    reason: raw.reason,
152    result_ref: raw.result_ref ?? undefined,
153    command: raw.command,
154  });
155}
156
157// The row as inserted: the stored row (src/ops-types/claims.ts) before D1 gives it an id.
158export type JobClaimRow = Omit<StoredClaimRow, "id">;
159type ClaimAction = JobClaimRow["action"];
160
161const listColumn = (list: readonly string[] | undefined): string | null => (list === undefined ? null : JSON.stringify([...list]));
162
163/**
164 * The job_claims row for one complete, fail or block call. Pure, and built from the
165 * arguments before verifyEvidence runs, so nothing GitHub says can reach it: commits,
166 * files_changed and tests_added are the driver's numbers as sent. An empty list is
167 * stated ("said none") and stored as [] and 0; an absent one is NULL.
168 */
169export function claimRow(input: {
170  job_id: string;
171  action: ClaimAction;
172  agent: string;
173  namespace: string;
174  raw: ClaimRaw;
175  claim?: JobClaim;
176  evidence?: JobEvidence;
177  capsid_sha?: string | null;
178  now: Date;
179}): JobClaimRow {
180  const { claim, evidence } = input;
181  const opened = claim?.prs_opened ?? evidence?.prs;
182  const merged = claim?.prs_merged;
183  return {
184    job_id: input.job_id,
185    action: input.action,
186    agent: input.agent,
187    namespace: input.namespace,
188    raw: rawJson(input.raw),
189    prs_opened_urls: listColumn(opened),
190    prs_merged_urls: listColumn(merged),
191    prs_opened: opened === undefined ? null : opened.length,
192    prs_merged: merged === undefined ? null : merged.length,
193    commits: evidence?.commits ?? null,
194    files_changed: evidence?.files_changed ?? null,
195    tests_added: evidence?.tests_added ?? null,
196    tests_run: claim?.tests?.run ?? null,
197    tests_passed: claim?.tests?.passed ?? null,
198    tests_failed: claim?.tests?.failed ?? null,
199    tests_result: claim?.tests?.result ?? null,
200    deploy_state: claim?.deploy_state ?? null,
201    files_touched: listColumn(claim?.files_touched),
202    model_id: claim?.versions?.model_id ?? null,
203    client_name: claim?.versions?.client_name ?? null,
204    client_version: claim?.versions?.client_version ?? null,
205    permission_mode: claim?.versions?.permission_mode ?? null,
206    capsid_sha: input.capsid_sha ?? null,
207    recorded_at: input.now.toISOString(),
208  };
209}
210
211// Append-only: migrations/0023's triggers refuse UPDATE and DELETE, so this INSERT is
212// the only statement that ever touches the table from code.
213export function claimStatement(db: D1Database, row: JobClaimRow): D1PreparedStatement {
214  return db
215    .prepare(
216      `INSERT INTO job_claims (job_id, action, agent, namespace, raw, prs_opened_urls, prs_merged_urls, prs_opened,
217         prs_merged, commits, files_changed, tests_added, tests_run, tests_passed, tests_failed, tests_result,
218         deploy_state, files_touched, model_id, client_name, client_version, permission_mode, capsid_sha, recorded_at)
219       VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24)`
220    )
221    .bind(
222      row.job_id,
223      row.action,
224      row.agent,
225      row.namespace,
226      row.raw,
227      row.prs_opened_urls,
228      row.prs_merged_urls,
229      row.prs_opened,
230      row.prs_merged,
231      row.commits,
232      row.files_changed,
233      row.tests_added,
234      row.tests_run,
235      row.tests_passed,
236      row.tests_failed,
237      row.tests_result,
238      row.deploy_state,
239      row.files_touched,
240      row.model_id,
241      row.client_name,
242      row.client_version,
243      row.permission_mode,
244      row.capsid_sha,
245      row.recorded_at
246    );
247}
248
249// EVALUATIONS. The worker's own checks, named for OpenTelemetry's
250// gen_ai.evaluation.result event. hidden_tests and scope_respected are in the table's
251// vocabulary for a later evaluator (a model, a human); the worker writes none of them.
252export const WORKER_EVALUATIONS = ["pr_merged", "prs_opened", "commits", "files_changed", "ci_green"] as const;
253type WorkerEvaluation = (typeof WORKER_EVALUATIONS)[number];
254
255export type Agreement = "agree" | "disagree" | "unclaimed" | "unchecked";
256type ScoreLabel = "pass" | "fail" | "unknown";
257
258export interface JobEvaluationRow {
259  job_id: string;
260  name: WorkerEvaluation;
261  score_value: number | null;
262  score_label: ScoreLabel;
263  claimed: string | null;
264  verified: string | null;
265  agreement: Agreement;
266  evaluator: "worker";
267  evaluator_id: string;
268  explanation: string | null;
269  recorded_at: string;
270}
271
272/** Computed once, at write time, from the two values the row stores. */
273export function agreementOf(claimed: number | null, verified: number | null): Agreement {
274  if (claimed === null) return "unclaimed";
275  if (verified === null) return "unchecked";
276  return claimed === verified ? "agree" : "disagree";
277}
278
279export const workerEvaluatorId = (buildSha: string | null | undefined) => `capsid@${buildSha ?? "unknown"}`;
280
281// The column holds a JSON array the row builder wrote, or NULL.
282function urlsOf(column: string | null): string[] | null {
283  if (column === null) return null;
284  const parsed = JSON.parse(column) as unknown;
285  return Array.isArray(parsed) ? parsed.filter((u): u is string => typeof u === "string") : null;
286}
287
288const json = (value: number | null): string | null => (value === null ? null : JSON.stringify(value));
289
290/**
291 * One row per worker check, from the claim row and what verifyEvidence read off
292 * GitHub. Pure. A value the Worker could not read is NULL with label unknown, and an
293 * unstated claim is NULL with agreement unclaimed: neither is ever a 0 or a false.
294 *
295 * Only the pull requests in evidence.prs, and a result_ref that is one, are read (verifyEvidence), so a URL the claim
296 * names that evidence does not is unread, and a check over it is unchecked rather than
297 * guessed.
298 */
299export function evaluationRows(
300  claim: JobClaimRow,
301  verdict: Pick<EvidenceVerdict, "github">,
302  ctx: { evaluator_id: string; now: Date }
303): JobEvaluationRow[] {
304  const merged = verdict.github?.merged ?? {};
305  const read = (url: string) => Object.hasOwn(merged, url);
306  const row = (name: WorkerEvaluation, claimed: number | null, verified: number | null, label: ScoreLabel, explanation: string | null): JobEvaluationRow => ({
307    job_id: claim.job_id,
308    name,
309    score_value: verified,
310    score_label: verified === null ? "unknown" : label,
311    claimed: json(claimed),
312    verified: json(verified),
313    agreement: agreementOf(claimed, verified),
314    evaluator: "worker",
315    evaluator_id: ctx.evaluator_id,
316    explanation,
317    recorded_at: ctx.now.toISOString(),
318  });
319  const unread = (urls: string[]) => urls.filter((u) => !read(u)).length;
320  // A count passes when GitHub's number does not contradict the driver's.
321  const countLabel = (claimed: number | null, verified: number | null): ScoreLabel => (agreementOf(claimed, verified) === "disagree" ? "fail" : "pass");
322
323  // pr_merged: the pull requests the claim says are merged, else the ones evidence
324  // named. Verified is how many of them GitHub reads as merged, known only when every
325  // one was read.
326  const mergedClaim = urlsOf(claim.prs_merged_urls);
327  const mergedSet = mergedClaim ?? urlsOf(claim.prs_opened_urls) ?? [];
328  const mergedMissing = unread(mergedSet);
329  const mergedVerified = mergedSet.length > 0 && mergedMissing === 0 ? mergedSet.filter((u) => merged[u]).length : null;
330  const prMerged = row(
331    "pr_merged",
332    claim.prs_merged,
333    mergedVerified,
334    mergedVerified !== null && mergedVerified === mergedSet.length ? "pass" : "fail",
335    mergedSet.length === 0
336      ? "no pull request was named, so there was nothing to check"
337      : mergedMissing > 0
338        ? `${mergedMissing} of ${mergedSet.length} named pull requests could not be read (only evidence.prs and a pull request result_ref are read)`
339        : null
340  );
341
342  // prs_opened: every pull request the claim names exists on GitHub. Verified is the
343  // count, known only when every one was read.
344  const openedSet = urlsOf(claim.prs_opened_urls) ?? [];
345  const openedMissing = unread(openedSet);
346  const openedVerified = openedSet.length > 0 && openedMissing === 0 ? openedSet.length : null;
347  const prsOpened = row(
348    "prs_opened",
349    claim.prs_opened,
350    openedVerified,
351    countLabel(claim.prs_opened, openedVerified),
352    openedSet.length === 0
353      ? "no pull request was named, so there was nothing to check"
354      : openedMissing > 0
355        ? `${openedMissing} of ${openedSet.length} named pull requests could not be read (only evidence.prs and a pull request result_ref are read)`
356        : null
357  );
358
359  // commits and files_changed: GitHub's sum over every named pull request, known only
360  // when every one was read, against the driver's number as sent.
361  const sums = "GitHub's sum is known only when every pull request in evidence.prs and result_ref was read";
362  const commitsVerified = verdict.github?.commits ?? null;
363  const filesVerified = verdict.github?.files_changed ?? null;
364  const commits = row("commits", claim.commits, commitsVerified, countLabel(claim.commits, commitsVerified), commitsVerified === null ? sums : null);
365  const files = row("files_changed", claim.files_changed, filesVerified, countLabel(claim.files_changed, filesVerified), filesVerified === null ? sums : null);
366
367  // ci_green: no claim field says anything about CI, so the claim is NULL (unclaimed).
368  const ciVerified = verdict.github?.ci_green ?? null;
369  const ci = row(
370    "ci_green",
371    null,
372    ciVerified,
373    ciVerified === 1 ? "pass" : "fail",
374    ciVerified === null ? "CI on the last named pull request's head was not read, or had not finished" : null
375  );
376
377  return [prMerged, prsOpened, commits, files, ci];
378}
379
380// claim_id is the claim row this batch inserted. Its id is not known before the batch
381// runs, and the batch is one transaction, so the newest claim for the job is that row.
382export function evaluationStatements(db: D1Database, rows: readonly JobEvaluationRow[]): D1PreparedStatement[] {
383  return rows.map((row) =>
384    db
385      .prepare(
386        `INSERT INTO job_evaluations (job_id, claim_id, name, score_value, score_label, claimed, verified, agreement,
387           evaluator, evaluator_id, explanation, recorded_at)
388         VALUES (?1, (SELECT MAX(id) FROM job_claims WHERE job_id = ?1), ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)`
389      )
390      .bind(
391        row.job_id,
392        row.name,
393        row.score_value,
394        row.score_label,
395        row.claimed,
396        row.verified,
397        row.agreement,
398        row.evaluator,
399        row.evaluator_id,
400        row.explanation,
401        row.recorded_at
402      )
403  );
404}