app/lib/site-urls.server.ts

Functions

Imports

It imports app/db/index.ts, app/db/pages.ts, app/db/posts.ts, app/db/procedures.ts, app/db/publications.ts, app/db/registry.ts, app/kb/bases.mjs, app/kb/libraries.mjs, app/kb/registry/catalog.mjs, app/kb/registry/kinds.mjs, app/kb/registry/route.ts, app/lib/content-pages.mjs, app/lib/pages/route.ts, app/lib/publications/paths.mjs, app/lib/seo.ts, app/lib/series-path.mjs, app/lib/site-map.mjs, app/lib/source-dates.generated.mjs, app/lib/tag-path.mjs. Imported by app/routes/site-map.tsx, app/routes/sitemap.ts.

Source

1import { listBlogPosts, listBlogSeries, listBlogTags, nextScheduledPublishAt } from "~/db";
2import { listPublishedPageSources, listPublishedPageTitles } from "~/db/pages";
3import { listPostsWithImages } from "~/db/posts";
4import { listPublishedProcedures } from "~/db/procedures";
5import { listPublishedRegistrySources, listRegistry } from "~/db/registry";
6import { listPublishedPublicationSources, listPublishedPublications } from "~/db/publications";
7import { BASES } from "~/kb/bases.mjs";
8import { LIBRARY_BASES } from "~/kb/libraries.mjs";
9import { KINDS } from "~/kb/registry/kinds.mjs";
10import { LAB_PATH, hasItemPage, itemPath, kindPath } from "~/kb/registry/catalog.mjs";
11import { REGISTRY_CACHE_TAG } from "~/kb/registry/route";
12import { CONTENT_PAGE_PATHS, CONTENT_PAGES_FROM_DATA, CONTENT_PAGES_OWN_ROUTE } from "~/lib/content-pages.mjs";
13import { CONTENT_PAGES_CACHE_TAG } from "~/lib/pages/route";
14import { paperPath } from "~/lib/publications/paths.mjs";
15import { POSTS_AND_PROCEDURES_CACHE_TAGS } from "~/lib/seo";
16import { seriesPath } from "~/lib/series-path.mjs";
17import type { SiteUrl, Tally } from "~/lib/site-map.mjs";
18import { SOURCE_DATES } from "~/lib/source-dates.generated.mjs";
19import { tagPath } from "~/lib/tag-path.mjs";
20
21/**
22 * The public pages with a route of their own, each with the title the site map shows. Kept in step with
23 * `routes.ts` by hand: every public page route is listed here, drawn from data below, or exempted by name in
24 * test/site-map.test.mjs with a reason. check:links resolves each one. In the order the sitemap lists
25 * them; a page with a row of its own (About) is listed only while its row is published. `sources` are the
26 * files the page's own words live in, which date it (an index is also as new as the newest thing it lists).
27 * @public read by test/site-map.test.mjs and check:links
28 */
29export const STATIC_PAGES = [
30  { path: "/", title: "Home", sources: ["app/routes/home.tsx"] },
31  { path: "/about", title: "About", sources: ["app/routes/about.tsx"] },
32  { path: "/writing", title: "Articles", sources: ["app/routes/blog._index.tsx"] },
33  { path: "/research/publications", title: "Publications", sources: ["app/routes/publications.tsx"] },
34  { path: "/contact", title: "Contact", sources: ["app/routes/contact.tsx"] },
35  // SITE_MAP_PATH, written out so check:links can read it off the syntax; test/site-map.test.mjs holds the two equal.
36  { path: "/site-map", title: "Site map", sources: ["app/routes/site-map.tsx"] },
37  { path: "/colophon", title: "Colophon", sources: ["app/routes/colophon.tsx", "content/stack-notes.json"] },
38  { path: "/systems", title: "System map", sources: ["app/routes/systems.tsx", "data/system-model.json"] },
39] as const;
40
41/** The CV, page and PDF, is made from its files and the papers it lists. */
42const CV_SOURCE_TREES = ["content/cv/", "content/publications/"];
43
44/** The CV is generated from data and has no page row (CONTENT_PAGES_FROM_DATA), so its title is stated here. */
45const DATA_PAGE_TITLES: Record<string, string> = { "/cv": "CV" };
46
47/**
48 * Every document both lists, so a save that changes either is purged from both: posts, tags and series,
49 * procedures, the prose pages and the registry.
50 */
51export const SITE_URLS_CACHE_TAGS = `${POSTS_AND_PROCEDURES_CACHE_TAGS},${CONTENT_PAGES_CACHE_TAG},${REGISTRY_CACHE_TAG}`;
52
53const POSTS: Tally = { key: "posts", label: "Articles", to: "/writing", merge: true };
54// No tag index page and no stable anchor on /writing's tag list, so the count is plain text (site-map.mjs).
55const TAGS: Tally = { key: "tags", label: "Tags", to: "/writing", merge: false };
56const PAPERS: Tally = { key: "papers", label: "Publications", to: "/research/publications", merge: true };
57const REGISTRY_ITEMS: Tally = { key: "registry", label: "Lab registry", to: LAB_PATH, merge: true };
58
59/** A procedure is counted under its base's library: Protocols, Software how-tos or Recipes. */
60function procedureTally(path: string): Tally | undefined {
61  const base = BASES.find((b) => path.startsWith(b.entryRoot));
62  return base ? { key: `procedures:${base.id}`, label: base.name, to: base.library, merge: true } : undefined;
63}
64
65/** Each tracked file's last commit day (scripts/lib/source-dates.mjs). Empty when the build had no history. */
66const DATES: Readonly<Record<string, string>> = SOURCE_DATES;
67
68/** The newest of some dates, or null when there are none. */
69function latest(dates: Iterable<Date | null | undefined>): Date | null {
70  let newest: Date | null = null;
71  for (const date of dates) if (date && (!newest || date > newest)) newest = date;
72  return newest;
73}
74
75/** The day the newest of these files last changed, or null when none is dated. */
76function sourceDate(files: Iterable<string | null | undefined>): Date | null {
77  const days = [...files].map((file) => (file ? DATES[file] : undefined));
78  return latest(days.map((day) => (day ? new Date(`${day}T00:00:00.000Z`) : null)));
79}
80
81/** The day the newest file under any of these directories last changed. */
82const treeDate = (prefixes: readonly string[]) => sourceDate(Object.keys(DATES).filter((file) => prefixes.some((p) => file.startsWith(p))));
83
84/** The images a page's HTML shows, as site paths. A diagram's dark twin is the same picture, so only its light one is listed. */
85function imagesIn(html: string | null): string[] {
86  const found = new Set<string>();
87  for (const [tag] of (html ?? "").matchAll(/<img\b[^>]*>/g)) {
88    const src = /\ssrc="(\/[^"/][^"]*)"/.exec(tag)?.[1];
89    if (src && !/\bdiagram-dark\b/.test(tag)) found.add(src);
90  }
91  return [...found];
92}
93
94/** Files crawlers should find that are not pages, so the sitemap lists them and the site map page does not. */
95export type SiteFile = { path: string; lastmod: Date | null };
96
97const kindTitle = (kind: string) => {
98  const plural = KINDS[kind as keyof typeof KINDS]?.plural ?? kind;
99  return plural.charAt(0).toUpperCase() + plural.slice(1);
100};
101
102/**
103 * Every public URL, the one list sitemap.xml and the site map page are both drawn from. Drafts,
104 * unpublished rows and empty libraries are left out here, so neither can list one. Also returns the next
105 * scheduled publish, which both use for the edge cache's lifetime, and the files only sitemap.xml lists.
106 *
107 * `lastmod` is the day the page's content last changed, never a sync or deploy time, which would mark every
108 * page changed together and teach crawlers to ignore the field. A post or procedure uses the `updated` date
109 * its author wrote; any other page uses its source file's last commit (scripts/lib/source-dates.mjs); an index,
110 * tag or series is as new as the newest page it lists. With no dated source, `lastmod` is left out.
111 */
112export async function gatherSiteUrls(env: Env): Promise<{ urls: SiteUrl[]; files: SiteFile[]; nextPublishAt: Date | null }> {
113  /* No `kind = 'page'` read: both writers into `posts` write only 'post'. Recheck that before relying on it. */
114  const [
115    blog,
116    tagList,
117    seriesList,
118    nextPublishAt,
119    procedureRows,
120    papers,
121    pageTitles,
122    registryItems,
123    pageSources,
124    paperSources,
125    registrySources,
126    postsWithImages,
127  ] = await Promise.all([
128    listBlogPosts(env, { perPage: 1000 }),
129    listBlogTags(env),
130    listBlogSeries(env),
131    nextScheduledPublishAt(env),
132    // Procedures live in D1, so one saved through the operator API is listed without a deploy.
133    listPublishedProcedures(env),
134    // Papers live in D1 too, so one saved through Carrel is listed without a deploy.
135    listPublishedPublications(env),
136    // The prose pages live in D1 too (docs/PAGES.md): a draft is left out, and so is a path whose row is missing.
137    listPublishedPageTitles(env),
138    // The lab registry lives in D1 too (docs/REGISTRY.md): an item saved through Carrel is listed without a deploy.
139    listRegistry(env, { published: true }),
140    listPublishedPageSources(env),
141    listPublishedPublicationSources(env),
142    listPublishedRegistrySources(env),
143    listPostsWithImages(env),
144  ]);
145  const titleOf = (path: string) => pageTitles.get(path) ?? DATA_PAGE_TITLES[path];
146  const cvDate = treeDate(CV_SOURCE_TREES);
147  const pageDate = (path: string) => (CONTENT_PAGES_FROM_DATA.includes(path) ? cvDate : sourceDate([pageSources.get(path)]));
148  const postImages = new Map(postsWithImages.map((p) => [p.slug, imagesIn(p.html)]));
149  const itemSource = (item: { kind: string; id: string }) => registrySources.get(`${item.kind}/${item.id}`);
150
151  const paperUrls = papers.map((p) => ({ path: paperPath(p.slug), lastmod: sourceDate([paperSources.get(p.slug)]), title: p.title, tally: PAPERS }));
152  const procedureUrls = procedureRows.map((p) => ({
153    path: p.path,
154    lastmod: p.updated ? new Date(`${p.updated}T00:00:00.000Z`) : sourceDate([p.sourcePath]),
155    title: p.title,
156    tally: procedureTally(p.path),
157  }));
158  const itemDate = (kind?: string) => sourceDate(registryItems.filter((item) => !kind || item.kind === kind).map(itemSource));
159  const postUrls = blog.posts.map((p) => {
160    const images = postImages.get(p.slug) ?? [];
161    return { path: `/writing/${p.slug}`, lastmod: p.updatedAt, title: p.title, tally: POSTS, ...(images.length > 0 ? { images } : {}) };
162  });
163  const newestPost = latest(postUrls.map((u) => u.lastmod));
164  const newestPaper = latest(paperUrls.map((u) => u.lastmod));
165  /** What an index lists makes it new, as well as its own words. */
166  const listed: Record<string, Date | null> = {
167    "/": latest([newestPost, newestPaper, ...procedureUrls.map((u) => u.lastmod)]),
168    "/writing": newestPost,
169    "/research/publications": newestPaper,
170  };
171
172  const urls: SiteUrl[] = [
173    // A page with a route of its own (About) is listed in its place, and only while its row is published.
174    ...STATIC_PAGES.filter(({ path }) => !CONTENT_PAGES_OWN_ROUTE.includes(path) || pageTitles.has(path)).map(({ path, title, sources }) => ({
175      path,
176      lastmod: latest([sourceDate(sources), pageDate(path), listed[path]]),
177      title: pageTitles.get(path) ?? title,
178    })),
179    // In the registry's order, which is the order the sitemap has always listed them in.
180    ...CONTENT_PAGE_PATHS.filter((path) => CONTENT_PAGES_FROM_DATA.includes(path) || pageTitles.has(path)).map((path) => ({
181      path,
182      lastmod: pageDate(path),
183      title: titleOf(path) ?? path,
184    })),
185    // No showcase filter: a crawler has no double-counting problem.
186    ...paperUrls,
187    // A knowledge base's library is listed once it has a published entry; until then it is not a page (kb-library.tsx).
188    ...LIBRARY_BASES.filter((base) => procedureRows.some((p) => p.path.startsWith(base.entryRoot))).map((base) => ({
189      path: base.library,
190      lastmod: latest(procedureUrls.filter((u) => u.path.startsWith(base.entryRoot)).map((u) => u.lastmod)),
191      title: base.name,
192    })),
193    ...procedureUrls,
194    // The registry's inventory, each kind that has an item, and each item.
195    { path: LAB_PATH, lastmod: itemDate(), title: "Lab registry" },
196    ...[...new Set(registryItems.map((item) => item.kind))].map((kind) => ({ path: kindPath(kind), lastmod: itemDate(kind), title: kindTitle(kind) })),
197    ...registryItems
198      .filter((item) => hasItemPage(item.kind))
199      .map((item) => ({ path: itemPath(item.kind, item.id), lastmod: sourceDate([itemSource(item)]), title: item.name, tally: REGISTRY_ITEMS })),
200    ...postUrls,
201    // A tag or series is as new as its newest post.
202    ...tagList.map((t) => ({ path: tagPath(t.slug), lastmod: t.lastmod, title: t.name, tally: TAGS })),
203    ...seriesList.flatMap((row) => (row.name === null ? [] : [{ path: seriesPath(row.name), lastmod: row.lastmod, title: row.name }])),
204  ];
205  // The site map page lists every page, so it is as new as the newest of them.
206  const siteMap = urls.find((url) => url.path === "/site-map");
207  if (siteMap) siteMap.lastmod = latest(urls.map((url) => url.lastmod));
208
209  const files: SiteFile[] = [{ path: "/dustin-edwards-cv.pdf", lastmod: cvDate }];
210  return { urls, files, nextPublishAt };
211}