src/runs.ts

Functions

Imports

It imports src/dump.ts, src/r2.ts, src/types.ts. Imported by src/backup.ts, src/drill.ts, src/index.ts.

Source

1import { COMPLETE_MARKER, DEFAULT_PREFIX, type CompleteMarker } from "./dump.js";
2import { deleteInChunks, listAllKeys } from "./r2.js";
3import type { R2Like } from "./types.js";
4
5// A daily dump plus a 2 hour grace, as Capsid's /health uses.
6export const DEFAULT_STALE_HOURS = 26;
7export const DEFAULT_RETENTION_DAYS = 90;
8export const DEFAULT_MIN_KEPT = 14;
9
10// Run id -> its keys, newest run first. A run id is the ISO time with ':' and '.'
11// replaced, so it sorts as the time does.
12async function runsOf(bucket: R2Like, prefix: string): Promise<Array<{ id: string; keys: string[]; complete: boolean }>> {
13  const byRun = new Map<string, string[]>();
14  for (const key of await listAllKeys(bucket, prefix)) {
15    const id = key.slice(prefix.length).split("/")[0];
16    if (!id || !key.slice(prefix.length).includes("/")) continue;
17    byRun.set(id, [...(byRun.get(id) ?? []), key]);
18  }
19  return [...byRun.entries()]
20    .sort(([a], [b]) => (a < b ? 1 : a > b ? -1 : 0))
21    .map(([id, keys]) => ({ id, keys, complete: keys.includes(`${prefix}${id}/${COMPLETE_MARKER}`) }));
22}
23
24export interface LatestDump {
25  run_id: string;
26  prefix: string;
27  exported_at: string;
28}
29
30// The newest complete dump, or null when there is none.
31export async function latestDump(bucket: R2Like, options: { prefix?: string } = {}): Promise<LatestDump | null> {
32  const prefix = options.prefix ?? DEFAULT_PREFIX;
33  for (const run of await runsOf(bucket, prefix)) {
34    const obj = await bucket.get(`${prefix}${run.id}/${COMPLETE_MARKER}`);
35    if (!obj) continue;
36    const marker = JSON.parse(await obj.text()) as CompleteMarker;
37    return { run_id: run.id, prefix: `${prefix}${run.id}/`, exported_at: marker.exported_at };
38  }
39  return null;
40}
41
42// The shape Capsid's /health reports under `backup`, which the console's operations
43// page reads as backup.age_hours. A failure to read is a warning, never an exception,
44// so a health route that calls this still answers.
45export interface BackupHealth {
46  last_ok: string | null;
47  age_hours: number | null;
48  warning?: string;
49}
50
51export async function backupHealth(
52  bucket: R2Like,
53  options: { prefix?: string; now?: Date; staleHours?: number } = {}
54): Promise<BackupHealth> {
55  const staleHours = options.staleHours ?? DEFAULT_STALE_HOURS;
56  let latest: LatestDump | null;
57  try {
58    latest = await latestDump(bucket, { prefix: options.prefix });
59  } catch (err) {
60    return { last_ok: null, age_hours: null, warning: `backup freshness unreadable: ${(err instanceof Error ? err.message : String(err)).slice(0, 80)}` };
61  }
62  if (!latest) return { last_ok: null, age_hours: null, warning: "no complete dump found" };
63  const stamped = Date.parse(latest.exported_at);
64  if (Number.isNaN(stamped)) return { last_ok: latest.exported_at, age_hours: null, warning: `the newest dump's time is not parseable: ${String(latest.exported_at).slice(0, 40)}` };
65  const ageMs = (options.now ?? new Date()).getTime() - stamped;
66  const ageHours = Math.round((ageMs / 3_600_000) * 10) / 10;
67  const result: BackupHealth = { last_ok: latest.exported_at, age_hours: ageHours };
68  if (ageMs > staleHours * 3_600_000) result.warning = `last complete dump was ${ageHours}h ago, over the ${staleHours}h threshold`;
69  return result;
70}
71
72// kept: runs retention keeps. pruned: stale runs deleted. locked: stale runs a bucket lock
73// rule refused to delete, left in place for a later prune; kept + pruned + locked is
74// every run under the prefix.
75export interface PruneResult {
76  kept: number;
77  pruned: number;
78  locked: number;
79}
80
81// R2 refuses a delete of an object a bucket lock rule still holds with error 10069,
82// ObjectLockedByBucketPolicy (developers.cloudflare.com/r2/api/error-codes/). The Workers
83// binding throws an R2Error carrying it as `code` and at the end of its message.
84const R2_OBJECT_LOCKED = 10069;
85function isLockRefusal(err: unknown): boolean {
86  if (typeof err !== "object" || err === null) return false;
87  if ((err as { code?: unknown }).code === R2_OBJECT_LOCKED) return true;
88  const message = (err as { message?: unknown }).message;
89  return typeof message === "string" && /\(10069\)|ObjectLockedByBucketPolicy/.test(message);
90}
91
92// Graduated retention, the default (capsid/rulings/shared-homes-2026-10-06.md): the
93// newest complete run of each of the last `daily` UTC days, of each of the last
94// `weekly` ISO weeks (Monday to Sunday) and of each of the last `monthly` calendar
95// months, counting the current one in each case. The three sets overlap, so the default
96// 14 + 8 + 6 keeps about 25 runs from a daily history and reaches back six months.
97export interface RetentionPolicy {
98  daily: number;
99  weekly: number;
100  monthly: number;
101}
102export const DEFAULT_RETENTION: RetentionPolicy = { daily: 14, weekly: 8, monthly: 6 };
103
104const DAY_MS = 86_400_000;
105const dayOf = (ms: number): string => new Date(ms).toISOString().slice(0, 10);
106// The Monday of the ISO week a UTC day falls in, as a day string. Keyed this way the
107// year boundary needs no week-number arithmetic.
108function mondayOf(day: string): string {
109  const ms = Date.parse(`${day}T00:00:00.000Z`);
110  return dayOf(ms - ((new Date(ms).getUTCDay() + 6) % 7) * DAY_MS);
111}
112
113function wantedKeys(now: Date, policy: RetentionPolicy): { days: Set<string>; weeks: Set<string>; months: Set<string> } {
114  const today = Date.parse(`${dayOf(now.getTime())}T00:00:00.000Z`);
115  const days = new Set<string>();
116  for (let k = 0; k < policy.daily; k++) days.add(dayOf(today - k * DAY_MS));
117  const thisMonday = Date.parse(`${mondayOf(dayOf(today))}T00:00:00.000Z`);
118  const weeks = new Set<string>();
119  for (let k = 0; k < policy.weekly; k++) weeks.add(dayOf(thisMonday - k * 7 * DAY_MS));
120  const months = new Set<string>();
121  const here = new Date(today);
122  for (let k = 0; k < policy.monthly; k++) {
123    months.add(new Date(Date.UTC(here.getUTCFullYear(), here.getUTCMonth() - k, 1)).toISOString().slice(0, 7));
124  }
125  return { days, weeks, months };
126}
127
128export interface RunRef {
129  id: string;
130  complete: boolean;
131}
132
133export interface SelectOptions {
134  now?: Date;
135  policy?: RetentionPolicy;
136  retentionDays?: number;
137  minKept?: number;
138}
139
140// Which runs retention deletes, from a list of run ids and the time. Pure: no bucket, no
141// disk, so a caller that holds its runs some other way (a directory of run folders) uses
142// the same rule pruneDumps does. With no retentionDays or minKept it applies the
143// graduated policy (options.policy, DEFAULT_RETENTION). Passing retentionDays or minKept
144// asks for the older flat rule instead: a run older than retentionDays is deleted whole,
145// except that the minKept newest COMPLETE runs are kept whatever their age.
146//
147// Either way: the newest complete run is never pruned; an incomplete run holds no slot
148// and is kept only while it is inside the daily window (flat: inside retentionDays), so
149// it stays as evidence for a while and then ages out; and nothing is selected while no
150// complete run exists, since that is what an emptied or misbound bucket looks like.
151// Returns the ids to delete, newest first; the input order does not matter.
152export function selectStaleRuns(runs: RunRef[], options: SelectOptions = {}): string[] {
153  const now = options.now ?? new Date();
154  const sorted = [...runs].sort((x, y) => (x.id < y.id ? 1 : x.id > y.id ? -1 : 0));
155  const complete = sorted.filter((r) => r.complete);
156  if (complete.length === 0) return [];
157
158  let stale: RunRef[];
159  if (options.retentionDays !== undefined || options.minKept !== undefined) {
160    const retentionDays = options.retentionDays ?? DEFAULT_RETENTION_DAYS;
161    const minKept = options.minKept ?? DEFAULT_MIN_KEPT;
162    const floor = new Set(complete.slice(0, minKept).map((r) => r.id));
163    const cutoffDay = new Date(now.getTime() - retentionDays * DAY_MS).toISOString().slice(0, 10);
164    stale = sorted.filter((r) => !floor.has(r.id) && r.id.slice(0, 10) < cutoffDay);
165  } else {
166    const policy = options.policy ?? DEFAULT_RETENTION;
167    const { days, weeks, months } = wantedKeys(now, policy);
168    const keep = new Set<string>([complete[0].id]);
169    // `complete` is newest first, so the first run seen for a key is the newest of it.
170    const seenDay = new Set<string>();
171    const seenWeek = new Set<string>();
172    const seenMonth = new Set<string>();
173    for (const r of complete) {
174      const day = r.id.slice(0, 10);
175      const week = mondayOf(day);
176      const month = day.slice(0, 7);
177      if (days.has(day) && !seenDay.has(day)) keep.add(r.id);
178      if (weeks.has(week) && !seenWeek.has(week)) keep.add(r.id);
179      if (months.has(month) && !seenMonth.has(month)) keep.add(r.id);
180      seenDay.add(day);
181      seenWeek.add(week);
182      seenMonth.add(month);
183    }
184    stale = sorted.filter((r) => !keep.has(r.id) && !(!r.complete && days.has(r.id.slice(0, 10))));
185  }
186  return stale.map((r) => r.id);
187}
188
189// Prunes dumps in a bucket: selectStaleRuns over the runs under the prefix, each stale run
190// deleted whole. A run is deleted marker first, in its own call, then the rest, so a run
191// that is only partly deleted is never left looking complete. A run whose delete a bucket
192// lock refuses is skipped and counted in `locked`, and the prune goes on to the next run;
193// it is selected again by a later prune and goes once the lock has aged off it. Any other
194// delete failure throws.
195export async function pruneDumps(
196  bucket: R2Like,
197  options: { prefix?: string; now?: Date; policy?: RetentionPolicy; retentionDays?: number; minKept?: number } = {}
198): Promise<PruneResult> {
199  const prefix = options.prefix ?? DEFAULT_PREFIX;
200  const runs = await runsOf(bucket, prefix);
201  const gone = new Set(selectStaleRuns(runs, options));
202  const stale = runs.filter((r) => gone.has(r.id));
203  let locked = 0;
204  for (const r of stale) {
205    const marker = `${prefix}${r.id}/${COMPLETE_MARKER}`;
206    try {
207      if (r.complete) await bucket.delete(marker);
208      await deleteInChunks(bucket, r.keys.filter((k) => k !== marker));
209    } catch (err) {
210      if (!isLockRefusal(err)) throw err;
211      locked++;
212    }
213  }
214  return { kept: runs.length - stale.length, pruned: stale.length - locked, locked };
215}