app/lib/media/rebuild.server.ts

Functions

Imports

It imports app/db/index.ts, app/lib/error-message.mjs, app/lib/health/verdicts.mjs, app/lib/media/classify.mjs, app/lib/media/core.server.ts. Imported by app/lib/health/checks.server.ts, app/lib/media/actions.server.ts, app/lib/operator/sync-tools.server.ts.

Source

1import assetManifest from "../../../content/generated/assets.json";
2
3import { deleteMediaRecord, listMediaRecords, upsertDerivedMedia } from "~/db";
4import { bucketFor, classify, isRaster, roleOf, storageOf } from "./classify.mjs";
5import { measureDimensions, placeholderFor } from "./core.server";
6import { errorMessage } from "~/lib/error-message.mjs";
7import { setDrift } from "~/lib/health/verdicts.mjs";
8
9// `alt`, `caption`, `focal_x` and `focal_y` are AUTHORED and recoverable from nothing, so a rebuild
10// upserts derived columns only, never delete-then-insert. R2 wins: rows are removed, objects never.
11
12type RebuildReport = {
13  scannedObjects: number;
14  scannedFiles: number;
15  indexed: number;
16  removed: number;
17  failures: string[];
18};
19
20type Measured = { dimensions: { width: number; height: number } | null; failure: string | null };
21
22// A failed read still writes the row (the upsert leaves stored sizes alone) but is reported as a
23// failure, so the rebuild is never counted converged over sizes it could not measure.
24async function dimensionsFor(env: Env, body: ReadableStream | null): Promise<Measured> {
25  if (!body) return { dimensions: null, failure: "the object vanished before it could be measured" };
26  try {
27    return { dimensions: await measureDimensions(env, body), failure: null };
28  } catch (error) {
29    return { dimensions: null, failure: errorMessage(error) };
30  }
31}
32
33type MediaSource = { key: string; size: number; uploaded: string };
34
35// Shared with the reconciliation so the two cannot disagree about what exists. Exhaustive to the end of
36// the cursor: a partial listing makes every absent key look like a deletion. A Worker cannot list its
37// own static assets (the binding only has `fetch()`), so the static half is the committed manifest.
38async function enumerateMediaSources(
39  env: Env,
40): Promise<{ objects: MediaSource[]; files: string[] }> {
41  const objects: MediaSource[] = [];
42  for (const bucket of [env.MEDIA, env.OG]) {
43    let cursor: string | undefined;
44    for (;;) {
45      const page = await bucket.list({ limit: 1000, ...(cursor ? { cursor } : {}) });
46      for (const object of page.objects) {
47        objects.push({
48          key: object.key,
49          size: object.size,
50          uploaded:
51            object.uploaded instanceof Date
52              ? object.uploaded.toISOString()
53              : String(object.uploaded ?? ""),
54        });
55      }
56      if (!page.truncated) break;
57      cursor = (page as { cursor?: string }).cursor;
58      if (!cursor) throw new Error("R2 reported a truncated listing with no cursor");
59    }
60  }
61
62  const files: string[] = assetManifest.paths;
63
64  if (objects.length === 0 && files.length === 0) {
65    throw new Error(
66      "rebuild found neither R2 objects nor static files. Refusing to run: it would read as " +
67        "'everything was deleted' and remove every row.",
68    );
69  }
70
71  return { objects, files };
72}
73
74// Both sides read after any write. `missing` and `extra` are kept apart so they cannot cancel out.
75export async function mediaIndexStatus(env: Env): Promise<{
76  expected: number;
77  present: number;
78  missing: string[];
79  extra: string[];
80}> {
81  const [{ objects, files }, rows] = await Promise.all([
82    enumerateMediaSources(env),
83    listMediaRecords(env),
84  ]);
85
86  const expected = new Set<string>([...objects.map((o) => o.key), ...files]);
87  const present = new Set<string>(rows.map((r) => r.key));
88
89  return setDrift(expected, present);
90}
91
92// The failure to report for one R2 object, or null once it is indexed and measured.
93async function indexObject(env: Env, object: MediaSource): Promise<string | null> {
94  try {
95    const { kind, mime } = classify(object.key);
96    // An OG card lives in the OG bucket, so the read follows the key rather than assuming MEDIA.
97    const bucket = bucketFor(env, object.key);
98    // Two GETs: a body is a stream and can be read only once.
99    const measurable = isRaster(object.key);
100    const measured = measurable
101      ? await bucket.get(object.key).then((o) => dimensionsFor(env, o?.body ?? null))
102      : null;
103    const dimensions = measured?.dimensions ?? null;
104    const placeholder = measurable
105      ? await bucket.get(object.key).then((o) => (o ? placeholderFor(env, o.body) : null))
106      : null;
107
108    // The filename is re-derived from the object's custom metadata, so `original_name` is truly derived.
109    const named = await bucket.head(object.key);
110
111    await upsertDerivedMedia(env, {
112      key: object.key,
113      storage: storageOf(object.key),
114      kind,
115      role: roleOf(object.key),
116      originalName: named?.customMetadata?.originalName ?? null,
117      mime,
118      bytes: object.size,
119      width: dimensions?.width ?? null,
120      height: dimensions?.height ?? null,
121      placeholder,
122      uploadedAt: object.uploaded,
123    });
124    return measured?.failure ? `${object.key}: ${measured.failure}` : null;
125  } catch (error) {
126    return `${object.key}: ${errorMessage(error)}`;
127  }
128}
129
130// The failure to report for one static file, or null once it is indexed and measured.
131async function indexStaticFile(env: Env, path: string): Promise<string | null> {
132  try {
133    const { kind, mime } = classify(path);
134    // The ASSETS binding ignores the hostname and matches only the pathname.
135    const response = await env.ASSETS.fetch(new Request(`https://assets.local${path}`));
136    if (!response.ok) return `${path}: ASSETS returned ${response.status}`;
137    const buffer = await response.arrayBuffer();
138
139    const measurable = isRaster(path);
140    const measured = measurable
141      ? await dimensionsFor(env, new Response(buffer.slice(0)).body as ReadableStream)
142      : null;
143    const dimensions = measured?.dimensions ?? null;
144    const placeholder = measurable
145      ? await placeholderFor(env, new Response(buffer.slice(0)).body as ReadableStream)
146      : null;
147
148    await upsertDerivedMedia(env, {
149      key: path,
150      storage: storageOf(path),
151      kind,
152      role: roleOf(path),
153      mime,
154      bytes: buffer.byteLength,
155      width: dimensions?.width ?? null,
156      height: dimensions?.height ?? null,
157      placeholder,
158      // No upload event: a build-machine mtime would be an invented fact.
159      uploadedAt: null,
160    });
161    return measured?.failure ? `${path}: ${measured.failure}` : null;
162  } catch (error) {
163    return `${path}: ${errorMessage(error)}`;
164  }
165}
166
167export async function rebuildMediaIndex(env: Env): Promise<RebuildReport> {
168  const failures: string[] = [];
169
170  const { objects, files } = await enumerateMediaSources(env);
171
172  let indexed = 0;
173
174  for (const object of objects) {
175    const failure = await indexObject(env, object);
176    if (failure) failures.push(failure);
177    else indexed += 1;
178  }
179
180  for (const path of files) {
181    const failure = await indexStaticFile(env, path);
182    if (failure) failures.push(failure);
183    else indexed += 1;
184  }
185
186  // Last, and from the same enumerations just written from, so a transient read failure cannot delete.
187  const live = new Set<string>([...objects.map((o) => o.key), ...files]);
188  const rows = await listMediaRecords(env);
189  let removed = 0;
190  for (const row of rows) {
191    if (live.has(row.key)) continue;
192    await deleteMediaRecord(env, row.key);
193    removed += 1;
194  }
195
196  return {
197    scannedObjects: objects.length,
198    scannedFiles: files.length,
199    indexed,
200    removed,
201    failures,
202  };
203}