src/deploy-outcomes.ts

Functions

Imports

It imports src/env.ts, src/github/client.ts, src/provenance.ts, src/roster.ts, src/watcher-detect.ts. Imported by src/watcher-gather.ts.

Source

1import type { Env } from "./env";
2import { getDefaultBranch, ghFetch, resolveRepo } from "./github/client";
3import { APP_ROSTER } from "./roster";
4import { externalFence } from "./provenance";
5import type { Finding } from "./watcher-detect";
6
7// THE MEMORY OF A ROLLBACK (docs/design/design-merge-pipeline.md section 6, PR 6). Each
8// watcher pass reads, for every DEPLOY_ROSTER repo, its newest completed default-branch push
9// runs, finds the job named "live gate", or "<caller job> / live gate" when the gate runs in
10// a called workflow (capsid's in .github/workflows/ci.yml; carrel's, "deploy / live gate",
11// in devkit's reusable worker-deploy.yml), and records what became of the deploy in
12// deploy_outcomes (migrations/0039). A rollback, or one that did not take effect,
13// is a watcher finding, so it reaches Needs you and the queue; the push is PR 7.
14//
15// GitHub serves no step summary through its API, so the outcome is read from what it does
16// serve: the job's conclusion, the conclusion of its rollback step, and the job's check-run
17// annotations, where the step's ::error:: lines land ("rolled back: was A ... now B", "not
18// rolling back", "the rollback did not take effect"). No CI secret is added: the App
19// already reads CI.
20
21const LIVE_JOB = "live gate";
22const LIVE_JOB_SUFFIX = ` / ${LIVE_JOB}`;
23// The namespaces the deploy watcher reads. carrel's live gate runs in devkit's reusable
24// worker-deploy.yml; carrel is not on APP_ROSTER (src/roster.ts).
25export const DEPLOY_ROSTER = [...APP_ROSTER, "carrel"] as const;
26
27/** A live gate job: "live gate", or a called workflow's "<caller job> / live gate". */
28export function isLiveGateJob(name: string): boolean {
29  return name === LIVE_JOB || name.endsWith(LIVE_JOB_SUFFIX);
30}
31
32const ROLLBACK_STEP = /^Roll back/i;
33const RUNS_READ = 3;
34
35type DeployOutcome = "verified" | "rolled_back" | "rollback_failed" | "rollback_skipped" | "unverified";
36
37interface Job {
38  id: number;
39  name: string;
40  conclusion: string | null;
41  steps?: Array<{ name: string; conclusion: string | null }>;
42}
43
44export interface OutcomeRead {
45  outcome: DeployOutcome;
46  rollback_from: string | null;
47  rollback_to: string | null;
48  detail: string;
49}
50
51/** What a live gate job's conclusion, its rollback step and its annotations say. Null for a
52 *  run whose live gate did not run (a run with no deploy). */
53export function outcomeFrom(job: Job | undefined, annotations: string[]): OutcomeRead | null {
54  if (!job || job.conclusion === "skipped" || job.conclusion === null || job.conclusion === "cancelled") return null;
55  const rollback = (job.steps ?? []).find((s) => ROLLBACK_STEP.test(s.name));
56  const said = (re: RegExp) => annotations.map((a) => re.exec(a)).find((m) => m !== null) ?? null;
57  if (job.conclusion === "success" && (!rollback || rollback.conclusion === "skipped" || rollback.conclusion === null)) {
58    return { outcome: "verified", rollback_from: null, rollback_to: null, detail: "the live gate passed" };
59  }
60  if (rollback?.conclusion === "failure") {
61    const m = said(/the rollback did not take effect: \/health still reports (\S+)/);
62    return { outcome: "rollback_failed", rollback_from: null, rollback_to: m?.[1] ?? null, detail: "the live gate refused the deploy and the rollback did not take effect" };
63  }
64  if (rollback?.conclusion === "success") {
65    const back = said(/rolled back: was (\S+) \(this run's commit \S+\), now ([^.\s]+)/);
66    if (back) return { outcome: "rolled_back", rollback_from: back[1]!, rollback_to: back[2]!, detail: "the live gate refused the deploy and it was rolled back" };
67    if (said(/not rolling back/)) return { outcome: "rollback_skipped", rollback_from: null, rollback_to: null, detail: "the live gate refused the deploy and the guard declined to roll back: master and production may disagree" };
68    return { outcome: "rollback_skipped", rollback_from: null, rollback_to: null, detail: "the rollback step succeeded but said neither that it rolled back nor that it declined" };
69  }
70  return { outcome: "unverified", rollback_from: null, rollback_to: null, detail: `the live gate concluded ${job.conclusion} with no rollback (it could not run, or the deploy was not this run's)` };
71}
72
73interface RunRow {
74  id: number;
75  head_sha: string;
76  created_at: string;
77  html_url: string;
78}
79
80export interface DeployRow extends OutcomeRead {
81  run_id: number;
82  head_sha: string;
83  pr: number | null;
84  run_url: string | null;
85  run_at: string;
86}
87
88const mk = (namespace: string, fingerprint: string, headline: string, evidence: string[]): Finding => ({
89  fingerprint,
90  namespace,
91  title: `Watcher: ${headline} [${fingerprint}]`,
92  body: [
93    "The live gate refused a deploy (docs/design/design-merge-pipeline.md section 6). Read",
94    "the run, then land a fix or revert the commit; master still points at it.",
95    "",
96    "## Evidence",
97    "",
98    externalFence("watcher-evidence", fingerprint, evidence.map((line) => `- ${line}`).join("\n")),
99  ].join("\n"),
100  evidence,
101});
102
103/** The finding a deploy row raises: a rollback, or one that did not take effect. A verified
104 *  deploy, a declined rollback and an unverified gate raise none here (a red CI run is the
105 *  ci check's finding). */
106export function deployFindings(namespace: string, row: DeployRow): Finding[] {
107  const sha = row.head_sha.slice(0, 7);
108  const evidence = [`run: ${row.run_url ?? row.run_id}`, `commit: ${row.head_sha}${row.pr ? ` (#${row.pr})` : ""}`, row.detail];
109  if (row.outcome === "rolled_back") {
110    return [mk(namespace, `deploy-rolled-back-${namespace}`, `${namespace} deploy ${sha} was rolled back`, [...evidence, `production went from ${row.rollback_from} back to ${row.rollback_to}`])];
111  }
112  if (row.outcome === "rollback_failed") {
113    return [mk(namespace, `deploy-rollback-failed-${namespace}`, `${namespace} deploy ${sha} was refused and its rollback did not take effect`, [...evidence, `production may still serve ${row.rollback_to ?? "the refused deploy"}`])];
114  }
115  return [];
116}
117
118/** Reads one repo's newest completed push runs on its default branch, records each new one
119 *  whose live gate ran, and returns the newest recorded deploy (or null when none ran). */
120export async function readDeployOutcomes(env: Env, namespace: string): Promise<DeployRow | null> {
121  const { owner, repo, full } = await resolveRepo(env, namespace);
122  const branch = await getDefaultBranch(env, owner, repo);
123  const listed = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/actions/runs?branch=${encodeURIComponent(branch)}&event=push&status=completed&per_page=${RUNS_READ}`);
124  if (!listed.ok) throw new Error(`the run list returned ${listed.status}`);
125  const runs = (((await listed.json()) as { workflow_runs?: RunRow[] }).workflow_runs ?? []).sort((a, b) => b.id - a.id);
126  if (!runs.length) return null;
127  const { results } = await env.DB.prepare(
128    `SELECT run_id, head_sha, pr, outcome, rollback_from, rollback_to, detail, run_url, run_at FROM deploy_outcomes
129     WHERE repo = ?1 AND run_id IN (SELECT value FROM json_each(?2))`
130  )
131    .bind(full, JSON.stringify(runs.map((r) => r.id)))
132    .all<DeployRow>();
133  const known = new Map((results ?? []).map((r) => [r.run_id, r]));
134  let newest: DeployRow | null = null;
135  for (const run of runs) {
136    let row = known.get(run.id) ?? null;
137    if (!row) {
138      const jobsResp = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/actions/runs/${run.id}/jobs?per_page=100`);
139      if (!jobsResp.ok) throw new Error(`the jobs of run ${run.id} returned ${jobsResp.status}`);
140      const job = (((await jobsResp.json()) as { jobs?: Job[] }).jobs ?? []).find((j) => isLiveGateJob(j.name));
141      let annotations: string[] = [];
142      if (job && job.conclusion !== "success" && job.conclusion !== "skipped") {
143        const notes = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/check-runs/${job.id}/annotations?per_page=100`);
144        if (!notes.ok) throw new Error(`the annotations of job ${job.id} returned ${notes.status}`);
145        annotations = ((await notes.json()) as Array<{ message?: string }>).map((a) => a.message ?? "");
146      }
147      const read = outcomeFrom(job, annotations);
148      if (!read) continue;
149      // The pull request the commit came from, when GitHub knows one.
150      const pulls = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/commits/${run.head_sha}/pulls`);
151      const pr = pulls.ok ? ((((await pulls.json()) as Array<{ number?: number }>)[0]?.number as number | undefined) ?? null) : null;
152      row = { ...read, run_id: run.id, head_sha: run.head_sha, pr, run_url: run.html_url ?? null, run_at: run.created_at };
153      await env.DB.prepare(
154        `INSERT OR IGNORE INTO deploy_outcomes (repo, namespace, run_id, head_sha, pr, outcome, rollback_from, rollback_to, detail, run_url, run_at)
155         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)`
156      )
157        .bind(full, namespace, row.run_id, row.head_sha, row.pr, row.outcome, row.rollback_from, row.rollback_to, row.detail, row.run_url, row.run_at)
158        .run();
159    }
160    newest ??= row;
161  }
162  return newest;
163}