app/lib/editor/publish.server.ts

Functions

Imports

It imports app/db/index.ts, app/lib/cache-purge.server.ts, app/lib/content/load-pipeline.server.ts, app/lib/content/slug.mjs, app/lib/editor/converge.mjs, app/lib/editor/divergence.server.ts, app/lib/editor/github.server.ts, app/lib/editor/post-rows.server.ts, app/lib/editor/publish-policy.mjs, app/lib/editor/write-path.server.ts, app/lib/error-message.mjs, app/lib/media/classify.mjs, app/lib/media/manifest.mjs, app/lib/preview-links.server.ts, app/lib/search/ask.server.ts. Imported by app/kb/procedures/save.server.ts, app/lib/admin/posts-actions.server.ts, app/lib/carrel/errors.server.ts, app/lib/carrel/site-adapter.server.ts, app/lib/editor/action.server.ts, app/lib/editor/head.server.ts, app/lib/operator/api.server.ts, app/lib/operator/sync-tools.server.ts, app/routes/admin.posts.$slug.edit.tsx, app/routes/admin.preview.ts.

Source

1// Nothing reaches D1 that is not already committed. The gates run here because a commit made
2// through the GitHub API never passes the local hooks.
3
4import { imageSize } from "image-size";
5
6import { purgePosts, type PurgeOutcome } from "~/lib/cache-purge.server";
7import { askAvailable, removeAskPost, syncAskPost } from "~/lib/search/ask.server";
8
9import { postPath } from "~/lib/content/slug.mjs";
10import { loadPipeline } from "~/lib/content/load-pipeline.server";
11import { dimensionsFromKey } from "~/lib/media/classify.mjs";
12import { ASSET_MANIFEST_PATH } from "~/lib/media/manifest.mjs";
13import {
14  commitFiles,
15  getHead,
16  listPostFiles,
17  readFile,
18  readBinaryFile,
19  GitHubError,
20  blobGuard,
21} from "./github.server";
22import { convergeWithRetry } from "./converge.mjs";
23import { clearDivergence, recordDivergence } from "./divergence.server";
24import { decide, decideDelete, PolicyError, type Actor } from "./publish-policy.mjs";
25import { commitUnlessUnchanged } from "./write-path.server";
26import { listPostCorpusForRelated, listPostDerivedLists, listPostLinkCorpus } from "~/db";
27import { revokeAllPreviewLinks } from "~/lib/preview-links.server";
28import { errorMessage } from "~/lib/error-message.mjs";
29import { deletePostFromD1, syncPostToD1, type PostRecord, type PublishEnv } from "./post-rows.server";
30
31export { GitHubError, PolicyError };
32export type { Actor };
33
34export class EditorError extends Error {
35  field?: string;
36  line?: number;
37
38  constructor(message: string, options: { field?: string; line?: number } = {}) {
39    super(message);
40    this.name = "EditorError";
41    this.field = options.field;
42    this.line = options.line;
43  }
44}
45
46/** Not an error: thrown rather than returned so a caller that forgets a flag cannot skip the ceremony. */
47export class FirstPublishConfirmationRequired extends Error {
48  constructor() {
49    super("This post has never been public. The first publication needs confirming.");
50    this.name = "FirstPublishConfirmationRequired";
51  }
52}
53
54type AssetManifest = { placeholders?: Record<string, { sha: string; lqip: string }> };
55
56/**
57 * Must agree with scripts/lib/content.mjs, which builds from a clone: /media/* from the key, public/*
58 * and its placeholder from the repo at the pinned ref, never this Worker's manifest from the last deploy.
59 */
60export function makeResolveImage(env: PublishEnv) {
61  let manifest: Promise<AssetManifest> | null = null;
62  const placeholders = async () => {
63    // A missing or unparseable manifest refuses the save, as the build from a clone does: rendering
64    // without placeholders would commit HTML that disagrees with the next build.
65    manifest ??= readFile(env, ASSET_MANIFEST_PATH).then((file) => {
66      if (!file) {
67        throw new EditorError(
68          `${ASSET_MANIFEST_PATH} is not in the repository, so no placeholder can be read.`,
69        );
70      }
71      try {
72        return JSON.parse(file.content) as AssetManifest;
73      } catch (error) {
74        throw new EditorError(
75          `${ASSET_MANIFEST_PATH} did not parse: ` +
76            `${errorMessage(error)}`,
77        );
78      }
79    });
80    return (await manifest).placeholders ?? {};
81  };
82
83  return async (src: string) => {
84    if (!src.startsWith("/")) {
85      throw new EditorError(`Image src "${src}" must be a site-absolute path.`);
86    }
87
88    if (src.startsWith("/media/")) {
89      const dimensions = dimensionsFromKey(src);
90      if (!dimensions) {
91        throw new EditorError(
92          `Image "${src}" carries no dimensions in its key. Uploaded images ` +
93            `are keyed dustin-edwards-[<name>-]<hash>-<width>x<height>.<ext>; re-upload it to get a ` +
94            `key the build can measure.`,
95        );
96      }
97      return dimensions;
98    }
99
100    const bytes = await readBinaryFile(env, `public${src}`);
101    if (!bytes) {
102      throw new EditorError(
103        `Image "${src}" is not in the repository at public${src}.`,
104      );
105    }
106
107    const size = imageSize(bytes);
108    if (!size.width || !size.height) {
109      throw new EditorError(`Image "${src}" has no readable dimensions.`);
110    }
111    const placeholder = (await placeholders())[src]?.lqip;
112    return { width: size.width, height: size.height, ...(placeholder ? { placeholder } : {}) };
113  };
114}
115
116export async function validateAndRender(
117  env: PublishEnv,
118  slug: string,
119  raw: string,
120) {
121  const { ContentError, findWideDashes, renderPost } = await loadPipeline();
122  const dashes = findWideDashes(raw);
123  const first = dashes[0];
124  if (first) {
125    throw new EditorError(
126      `Wide dash ${first.char} on line ${first.line}, column ${first.column}. ` +
127        `House style uses commas, periods, parentheses or colons. Context: "${first.excerpt}"` +
128        (dashes.length > 1 ? ` (${dashes.length - 1} more)` : ""),
129      { line: first.line },
130    );
131  }
132
133  try {
134    return await renderPost({
135      file: postPath(slug),
136      raw,
137      expectedSlug: slug,
138      resolveImage: makeResolveImage(env),
139    });
140  } catch (error) {
141    if (error instanceof ContentError) {
142      throw new EditorError(error.message.replace(`${postPath(slug)}: `, ""));
143    }
144    throw error;
145  }
146}
147
148/**
149 * Runs a whole-corpus computation for the saved post alone and returns its entry. The other posts
150 * come from D1 with their status and date in the pipeline's shape; the saved post comes from its
151 * fresh render and goes LAST, so its entry is at a position that cannot miss.
152 */
153function computeForSaved<C extends { slug: string; status: string; publishAt: Date | null }, E>(
154  record: PostRecord,
155  corpus: C[],
156  fields: (post: C) => E,
157  saved: E,
158  compute: (posts: unknown[]) => Array<{ related?: unknown[]; backlinks?: unknown[] }>,
159) {
160  const others = corpus
161    .filter((p) => p.slug !== record.slug)
162    .map((p) => ({
163      slug: p.slug,
164      ...fields(p),
165      draft: p.status === "draft",
166      // ISO strings: string order over ISO timestamps is chronological order.
167      publishAt: p.publishAt ? p.publishAt.toISOString() : "",
168    }));
169  const computed = compute([
170    ...others,
171    { slug: record.slug, ...saved, draft: record.draft, publishAt: record.publishAt },
172  ]);
173  const entry = computed[computed.length - 1];
174  if (!entry) throw new Error(`the corpus computation returned nothing for ${record.slug}`);
175  return entry;
176}
177
178/** Only this post's list is computed here: relatedness is corpus-wide, and the other rows follow in refreshNeighbours. */
179async function relatedFor(env: PublishEnv, record: PostRecord) {
180  const { withRelated } = await loadPipeline();
181  return computeForSaved(
182    record,
183    await listPostCorpusForRelated(env),
184    (p) => ({ title: p.title, description: p.description, tags: p.tags }),
185    { title: record.title, description: record.description, tags: record.tags },
186    withRelated,
187  ).related;
188}
189
190/** Only this post's list is computed here, as in relatedFor. */
191async function backlinksFor(env: PublishEnv, record: PostRecord) {
192  const { withBacklinks } = await loadPipeline();
193  return computeForSaved(
194    record,
195    await listPostLinkCorpus(env),
196    (p) => ({ title: p.title, html: p.html ?? "" }),
197    { title: record.title, html: record.html },
198    withBacklinks,
199  ).backlinks;
200}
201
202/**
203 * The row renderAndWrite writes, without writing it: the render plus the corpus-wide related and
204 * backlinks lists. The Carrel preview renders this, so a preview is the page the save would publish.
205 */
206export async function renderRecord(env: PublishEnv, slug: string, raw: string) {
207  const record: PostRecord = await validateAndRender(env, slug, raw);
208  record.related = await relatedFor(env, record);
209  record.backlinks = await backlinksFor(env, record);
210  return record;
211}
212
213/**
214 * The only writer of a rendered row. blobSha proves the rendered bytes are the committed bytes, so a
215 * truncated fetch fails here by name.
216 */
217export async function renderAndWrite(
218  env: PublishEnv,
219  slug: string,
220  raw: string,
221  blobSha?: string | null,
222) {
223  const record = await renderRecord(env, slug, raw);
224
225  if (blobSha && record.sourceBlobSha !== blobSha) {
226    throw new EditorError(
227      `the markdown rendered for "${slug}" is not the markdown the repository holds: ` +
228        `the rendered bytes hash to ${record.sourceBlobSha}, the repository file is ` +
229        `${blobSha}. The read was truncated or the file changed mid-flight; re-read ` +
230        `the file and retry.`,
231    );
232  }
233
234  await syncPostToD1(env, record);
235
236  // Purges on a draft save too: an unpublish writes a draft row while changing every public listing.
237  // A condition, if ever needed, must read the PREVIOUS status, never the incoming record. A failed purge
238  // never fails the write, but it is returned: the public pages stay stale until the cache expires.
239  const purged = await purgePosts(`renderAndWrite ${slug}`);
240
241  return { record, purged };
242}
243
244/**
245 * After a save, the OTHER posts whose related or backlinks lists the save changed: a post the saved one
246 * now links to or shares a tag with, and one that had it in a list and no longer does. The lists are
247 * derived over the whole corpus by the pipeline's own withRelated and withBacklinks, the pass the bulk
248 * sync runs, since relatedness is a property of the set; that is in memory, over rows a save already
249 * reads. Only rows whose stored lists differ are written, in one batch, and only those two columns:
250 * another post's save must not move its `updated_at`, blob sha or render hash.
251 *
252 * Returns the slugs it rewrote. Throws on a failed read or write: the caller reports it.
253 */
254async function refreshNeighbours(env: PublishEnv, savedSlug: string) {
255  const { withRelated, withBacklinks } = await loadPipeline();
256  const [related, links, stored] = await Promise.all([
257    listPostCorpusForRelated(env),
258    listPostLinkCorpus(env),
259    listPostDerivedLists(env),
260  ]);
261  const htmlBySlug = new Map(links.map((p) => [p.slug, p.html ?? ""]));
262  const derived = withBacklinks(
263    withRelated(
264      related.map((p) => ({
265        slug: p.slug,
266        title: p.title,
267        description: p.description,
268        tags: p.tags,
269        html: htmlBySlug.get(p.slug) ?? "",
270        draft: p.status === "draft",
271        publishAt: p.publishAt ? p.publishAt.toISOString() : "",
272      })),
273    ),
274  ) as Array<{ slug: string; related: unknown[]; backlinks: unknown[] }>;
275
276  const storedBySlug = new Map(stored.map((p) => [p.slug, p]));
277  const changed = derived.flatMap((post) => {
278    // The saved row was written from the same derivation a moment ago.
279    const row = storedBySlug.get(post.slug);
280    if (post.slug === savedSlug || !row) return [];
281    const relatedJson = JSON.stringify(post.related);
282    const backlinksJson = JSON.stringify(post.backlinks);
283    // The bulk sync stores an empty backlinks list as NULL and a save as "[]": the same answer.
284    if (relatedJson === (row.related ?? "[]") && backlinksJson === (row.backlinks ?? "[]")) return [];
285    return [{ slug: post.slug, relatedJson, backlinksJson }];
286  });
287  if (changed.length === 0) return [];
288
289  await env.DB.batch(
290    changed.map((c) =>
291      env.DB.prepare(`UPDATE posts SET related = ?1, backlinks = ?2 WHERE slug = ?3`).bind(
292        c.relatedJson,
293        c.backlinksJson,
294        c.slug,
295      ),
296    ),
297  );
298  return changed.map((c) => c.slug);
299}
300
301/**
302 * The save's neighbour refresh, reported rather than raised: the commit and the saved row have landed, and
303 * undoing either would destroy the source to repair derived lists. A failure is returned for the caller
304 * to show and logged as an alert; the next save of any post re-derives and rewrites every stale list,
305 * as does the ship's sync.
306 */
307async function refreshNeighboursAfterSave(env: PublishEnv, slug: string) {
308  try {
309    const updated = await refreshNeighbours(env, slug);
310    // The save's own purge ran before these rows changed, so a page re-stored in between would keep the old lists.
311    const purged = updated.length > 0 ? await purgePosts(`refreshNeighbours ${slug}`) : null;
312    return { ok: true as const, updated, purged };
313  } catch (error) {
314    console.error(
315      JSON.stringify({ alert: "neighbour-refresh-failed", slug, detail: errorMessage(error) }),
316    );
317    return {
318      ok: false as const,
319      message:
320        `The post saved, but the related and backlinks lists of the other posts could not be ` +
321        `refreshed: ${errorMessage(error)}. Their lists stay as they were until the next save ` +
322        `of any post or the next content sync.`,
323    };
324  }
325}
326
327export async function currentHead(env: PublishEnv) {
328  return (await getHead(env)).commitSha;
329}
330
331/** True when D1's row for the post was rendered from exactly this file (the git blob sha matches). */
332async function postRowIsCurrent(env: PublishEnv, slug: string, blobSha: string | null | undefined) {
333  if (!blobSha) return false;
334  const row = await env.DB.prepare("SELECT source_blob_sha FROM posts WHERE slug = ?1")
335    .bind(slug)
336    .first<{ source_blob_sha: string | null }>();
337  return row?.source_blob_sha === blobSha;
338}
339
340/**
341 * Renders twice by design: as the gate before the commit, and inside renderAndWrite after it, since
342 * D1 is written only from a row that door produced.
343 */
344export async function savePost(
345  env: PublishEnv,
346  options: {
347    slug: string;
348    raw: string;
349    expectedHeadSha?: string | null;
350    /** The blob sha of this post's file as the caller loaded it; Carrel's version of a post. Undefined: no check. */
351    expectedBlobSha?: string | null;
352    isNew: boolean;
353    /** Required: there is no safe default for an identity. */
354    actor: Actor;
355    /** Absent means this caller has no ceremony, so the check is === false, not falsiness. */
356    firstPublishConfirmed?: boolean;
357  },
358) {
359  const { actor } = options;
360
361  // Read before rendering: it is the only authority on "ever published", and a refusal should not pay for a render.
362  const existing = await readFile(env, postPath(options.slug));
363
364  if (options.isNew && existing) {
365    throw new EditorError(
366      `A post with the slug "${options.slug}" already exists.`,
367      { field: "slug" },
368    );
369  }
370
371  // Throws PolicyError on an operator's first publish, and stamps first_published over what the caller sent.
372  const decision = decide({
373    actor,
374    incomingRaw: options.raw,
375    priorRaw: existing ? existing.content : null,
376  });
377  const raw = decision.raw;
378
379  // After decide(), so a credential that may not publish at all is refused rather than asked to confirm.
380  if (decision.outcome === "published-first" && options.firstPublishConfirmed === false) {
381    throw new FirstPublishConfirmationRequired();
382  }
383
384  // The gate: rendered from the STAMPED markdown before the commit, and its result is discarded.
385  const gated = await validateAndRender(env, options.slug, raw);
386
387  const written = await commitUnlessUnchanged(env, {
388    existing,
389    raw,
390    rowIsCurrent: () => postRowIsCurrent(env, options.slug, gated.sourceBlobSha),
391    commit: () =>
392      commitFiles(env, {
393        expectedHeadSha: options.expectedHeadSha,
394        expectedBlobs: blobGuard(postPath(options.slug), options.expectedBlobSha),
395        message: commitMessage(actor, options.isNew ? "Add" : "Update", gated.title),
396        changes: [{ path: postPath(options.slug), content: raw }],
397      }),
398  });
399  const unchanged = written.action !== "commit";
400  if (written.action === "noop") {
401    return {
402      commitSha: written.commitSha,
403      record: gated,
404      askSync: null,
405      neighbours: null,
406      purged: null as PurgeOutcome,
407      d1Retried: false,
408      previewLinksRevoked: null as number | null,
409      firstPublished: decision.firstPublished,
410      published: false,
411      outcome: decision.outcome,
412      unchanged: true,
413    };
414  }
415  const { commitSha, blobShas } = written;
416
417  // No compensating revert, ever: undoing the commit would destroy the source to repair a derived index.
418  let record = gated;
419  // False until a write lands: an unreached purge leaves the pages as stale as a failed one.
420  let purged: PurgeOutcome = false;
421  const converged = await convergeWithRetry({
422    write: async () => {
423      ({ record, purged } = await renderAndWrite(
424        env,
425        options.slug,
426        raw,
427        blobShas[postPath(options.slug)] ?? null,
428      ));
429    },
430    recordDivergence: (facts) => recordDivergence(env, facts),
431    slug: options.slug,
432    commitSha,
433  });
434
435  // Clear a stale divergence, or the status surface keeps reporting a repaired fault. Never fails the save.
436  try {
437    await clearDivergence(env, options.slug);
438  } catch (error) {
439    console.error(JSON.stringify({ alert: "divergence-clear-failed", detail: errorMessage(error) }));
440  }
441
442  const neighbours = await refreshNeighboursAfterSave(env, record.slug);
443
444  // Revoked AFTER the D1 sync: the read path checks status = 'draft', so revoking first opens a window.
445  // A surviving token is inert, so a failure does not throw.
446  let previewLinksRevoked: number | null = null;
447  if (decision.published) {
448    try {
449      previewLinksRevoked = await revokeAllPreviewLinks(env, options.slug);
450    } catch (error) {
451      // null already means "not attempted", so a failure needs its own value.
452      console.error(JSON.stringify({ alert: "preview-link-revocation-failed", detail: errorMessage(error) }));
453      previewLinksRevoked = -1;
454    }
455  }
456
457  // The Ask index must not fail a save: the commit has landed, and a failure here would split repo and D1.
458  const askSync = await syncAskForPost(env, record);
459
460  return {
461    commitSha,
462    record,
463    askSync,
464    /** Other posts' related and backlinks lists rewritten by this save, or why they could not be. */
465    neighbours,
466    /** False when the cache purge failed or was never reached, null when the runtime cannot purge. */
467    purged,
468    d1Retried: converged.retried,
469    /** null: not attempted; -1: the attempt threw. */
470    previewLinksRevoked,
471    firstPublished: decision.firstPublished,
472    published: decision.published,
473    outcome: decision.outcome,
474    unchanged,
475  };
476}
477
478function commitMessage(actor: Actor, verb: string, title: string) {
479  const base = `${verb} post: ${title}`;
480  if (actor.kind === "operator") return `${base} [operator:${actor.id}]`;
481  // The change id joins this commit to Carrel's authorship record, and records who published.
482  if (actor.kind === "carrel") return `${base} [carrel:${actor.changeId}]`;
483  return base;
484}
485
486async function syncAskForPost(env: PublishEnv, record: { slug: string }) {
487  if (!askAvailable(env)) return null;
488  try {
489    const result = await syncAskPost(env, record);
490    if (result.failed.length > 0) {
491      return {
492        ok: false as const,
493        message:
494          `${result.failed.length} of ${result.failed.length + result.uploaded} Ask ` +
495          `record(s) failed to upload after a retry: ${result.failed.join(", ")}. ` +
496          `The post saved. Re-run the Ask sync from /admin/posts to finish indexing it.`,
497      };
498    }
499    return { ok: true as const, ...result };
500  } catch (error) {
501    console.error(JSON.stringify({ alert: "ask-sync-failed-after-save", detail: errorMessage(error) }));
502    return { ok: false as const, message: errorMessage(error) };
503  }
504}
505
506export async function deletePost(
507  env: PublishEnv,
508  options: { slug: string; expectedHeadSha?: string | null; expectedBlobSha?: string; actor: Actor },
509) {
510  const { actor } = options;
511
512  // First statement: after the file read, two different errors would reveal which slugs exist.
513  decideDelete({ actor });
514
515  const existing = await readFile(env, postPath(options.slug));
516  if (!existing) {
517    throw new EditorError(`No post file exists for "${options.slug}".`);
518  }
519
520  const { commitSha } = await commitFiles(env, {
521    expectedHeadSha: options.expectedHeadSha,
522    // Carrel names a post by its blob sha: a file changed since it was loaded is refused at the head.
523    expectedBlobs: blobGuard(postPath(options.slug), options.expectedBlobSha),
524    message: commitMessage(actor, "Remove", options.slug),
525    changes: [{ path: postPath(options.slug), content: null }],
526  });
527
528  const { purged } = await deletePostFromD1(env, options.slug);
529
530  return { commitSha, purged, askRemoval: await removeAskForPost(env, options.slug) };
531}
532
533/**
534 * The Ask index cannot fail the delete, but a post still answerable through Ask is reported, not
535 * swallowed. Null means Ask is not configured; a failure is `ok: false`, never the same null.
536 */
537async function removeAskForPost(env: PublishEnv, slug: string) {
538  if (!askAvailable(env)) return null;
539  try {
540    return { ok: true as const, removed: await removeAskPost(env, slug) };
541  } catch (error) {
542    console.error(JSON.stringify({ alert: "ask-removal-failed-after-delete", detail: errorMessage(error) }));
543    return {
544      ok: false as const,
545      message:
546        `The post was deleted but its Ask records were not removed, so Ask can still quote it: ` +
547        `${errorMessage(error)}. Re-run the Ask sync to prune them.`,
548    };
549  }
550}
551
552/** Every row arrives through renderAndWrite, the same door a save uses, with the blob sha to prove its bytes. */
553export async function regenerateAllFromRepo(env: PublishEnv) {
554  const files = await listPostFiles(env);
555
556  // An empty listing is a broken read or a deleted directory; converging on it would delete every row.
557  if (files.length === 0) {
558    throw new EditorError(
559      "content/posts listed no markdown files. An empty corpus is not a state " +
560        "this site has ever had, so this refuses rather than deleting every row.",
561    );
562  }
563
564  // Through deletePostFromD1, the one delete door: a bare row delete would leave the post in /search
565  // and its media_refs blocking its images' deletion forever.
566  const keep = files.map((f) => f.slug);
567  const placeholders = keep.map((_, i) => `?${i + 1}`).join(", ");
568  const orphans = await env.DB.prepare(
569    `SELECT slug FROM posts WHERE source_path IS NOT NULL AND slug NOT IN (${placeholders})`,
570  )
571    .bind(...keep)
572    .all<{ slug: string }>();
573  let unpurged = 0;
574  for (const { slug } of orphans.results ?? []) {
575    if ((await deletePostFromD1(env, slug)).purged === false) unpurged += 1;
576  }
577
578  for (const entry of files) {
579    const slug = entry.slug;
580    const file = await readFile(env, entry.path);
581    if (!file) {
582      throw new EditorError(
583        `"${entry.path}" vanished between the directory listing and the read. ` +
584          `Main moved mid-rebuild; re-run the regenerate.`,
585      );
586    }
587    if ((await renderAndWrite(env, slug, file.content, entry.sha)).purged === false) unpurged += 1;
588  }
589
590  return { synced: files.length, removed: orphans.results?.length ?? 0, unpurged };
591}