app/lib/media/backup.server.ts

Functions

Imports

It imports app/lib/operator/auth.server.ts. Imported by app/lib/health/checks.server.ts, app/lib/operator/sync-tools.server.ts, workers/media-events.ts.

Source

1// NO SITE CODE PATH EVER DELETES FROM THE BACKUP (check:destructive enforces it), and nothing copies
2// backup to media, which would let a stale mirror overwrite a live object.
3
4import type { OperatorEnv } from "~/lib/operator/auth.server";
5
6type BackupEnv = Pick<OperatorEnv, "MEDIA" | "MEDIA_BACKUP">;
7
8interface TwinComparison {
9  key: string;
10  present: boolean;
11  identical: boolean;
12  reason: string;
13}
14
15interface BackupStatus {
16  objects: number;
17  /**
18   * IDENTICAL twins only: a backup copy that differs is counted in `mismatched`, not here. Named
19   * `twins` because the operator API reports it under that key.
20   */
21  twins: number;
22  missing: string[];
23  mismatched: string[];
24}
25
26async function listAll(bucket: R2Bucket) {
27  const out: R2Object[] = [];
28  let cursor: string | undefined;
29  for (;;) {
30    const page = await bucket.list({ cursor, limit: 1000 });
31    out.push(...page.objects);
32    if (!page.truncated) return out;
33    cursor = page.cursor;
34  }
35}
36
37// ETag is the MD5 of a single-part object, corroborated by size (a same-length corruption is what a
38// mirror is for). A multipart etag (`-<parts>`) is not a digest, so that case degrades to size and says so.
39function compare(source: R2Object, twin: R2Object | null): TwinComparison {
40  if (!twin) {
41    return { key: source.key, present: false, identical: false, reason: "no twin" };
42  }
43  const multipart = source.etag.includes("-") || twin.etag.includes("-");
44  if (multipart) {
45    const identical = source.size === twin.size;
46    return {
47      key: source.key,
48      present: true,
49      identical,
50      reason: identical
51        ? "size only (a multipart etag is not a content digest)"
52        : `size differs: ${source.size} against ${twin.size}`,
53    };
54  }
55  if (source.etag !== twin.etag) {
56    return {
57      key: source.key,
58      present: true,
59      identical: false,
60      reason: `etag differs: ${source.etag} against ${twin.etag}`,
61    };
62  }
63  if (source.size !== twin.size) {
64    // Should be impossible; reported rather than trusted.
65    return {
66      key: source.key,
67      present: true,
68      identical: false,
69      reason: `etag matches but size differs: ${source.size} against ${twin.size}`,
70    };
71  }
72  return { key: source.key, present: true, identical: true, reason: "etag and size match" };
73}
74
75// The mismatched keys travel beside the display strings, so a repair never parses a key back out of
76// prose: a key containing " (" would be cut short and the wrong object copied.
77async function compareBuckets(env: BackupEnv) {
78  const [sources, twins] = await Promise.all([listAll(env.MEDIA), listAll(env.MEDIA_BACKUP)]);
79  const twinByKey = new Map(twins.map((t) => [t.key, t]));
80
81  const missing: string[] = [];
82  const mismatched: string[] = [];
83  const mismatchedKeys: string[] = [];
84  let identicalTwins = 0;
85
86  for (const source of sources) {
87    const verdict = compare(source, twinByKey.get(source.key) ?? null);
88    if (verdict.identical) identicalTwins += 1;
89    else if (verdict.present) {
90      mismatched.push(`${verdict.key} (${verdict.reason})`);
91      mismatchedKeys.push(verdict.key);
92    } else missing.push(verdict.key);
93  }
94
95  const status: BackupStatus = { objects: sources.length, twins: identicalTwins, missing, mismatched };
96  return { status, mismatchedKeys };
97}
98
99// `objects` is reported so zero missing cannot be mistaken for zero examined.
100export async function backupStatus(env: BackupEnv): Promise<BackupStatus> {
101  return (await compareBuckets(env)).status;
102}
103
104// R2 bindings have no server-side copy. Re-reads the source rather than trusting a queue message, so
105// replay converges; a source that is gone is not an error, since the twin staying is the point.
106export async function mirrorObject(env: BackupEnv, key: string): Promise<"copied" | "gone"> {
107  const object = await env.MEDIA.get(key);
108  if (!object) return "gone";
109  await env.MEDIA_BACKUP.put(key, object.body, {
110    httpMetadata: object.httpMetadata,
111    customMetadata: object.customMetadata,
112  });
113  return "copied";
114}
115
116// Mismatched twins are re-copied too: the source bucket is the truth.
117export async function copyMissingTwins(env: BackupEnv): Promise<{
118  copied: number;
119  gone: number;
120  status: BackupStatus;
121}> {
122  const before = await compareBuckets(env);
123  const work = [...before.status.missing, ...before.mismatchedKeys];
124
125  let copied = 0;
126  let gone = 0;
127  for (const key of work) {
128    const result = await mirrorObject(env, key);
129    if (result === "copied") copied += 1;
130    else gone += 1;
131  }
132
133  // Read back rather than trusting the loop's counters.
134  const status = await backupStatus(env);
135  return { copied, gone, status };
136}