robots.mjs

Functions

Imports

It imports no other module of this repo. No module of this repo imports it.

Source

1/**
2 * The family's crawler policy (Dustin, 2026-10-25): search engines and the bots that put a page in an AI answer with a link
3 * are always allowed; training crawlers are allowed here so they reach Cloudflare's Pay Per Crawl price, where AI Crawl
4 * Control charges or blocks them; Bytespider is denied, since it ignores payment requests and has no answer surface.
5 */
6export const SEARCH_ENGINE_BOTS = ["Googlebot", "Bingbot", "DuckDuckBot", "Applebot"];
7export const AI_SEARCH_BOTS = [
8  "OAI-SearchBot",
9  "ChatGPT-User",
10  "Claude-SearchBot",
11  "Claude-User",
12  "PerplexityBot",
13  "Perplexity-User",
14];
15export const AI_TRAINING_BOTS = ["GPTBot", "ClaudeBot", "anthropic-ai", "CCBot", "Google-Extended", "Applebot-Extended"];
16export const DENIED_BOTS = ["Bytespider"];
17
18/**
19 * @typedef {object} RobotsInput
20 * @property {string} origin the production origin, for the Sitemap line
21 * @property {string[]} [disallow] paths no crawler is asked to fetch (admin, previews, billed endpoints)
22 * @property {string[]} [allowBots] named groups; defaults to the search engines, the AI search bots and the training bots
23 * @property {string[]} [denyBots] defaults to `DENIED_BOTS`
24 * @property {string[]} [sitemaps] defaults to `${origin}/sitemap.xml`
25 * @property {string[]} [notes] lines after the Sitemap lines, each written as a `#` comment (an llms.txt pointer, say)
26 * @property {boolean} [preview] a preview or workers.dev copy: every crawler is told `Disallow: /`
27 */
28
29/**
30 * robots.txt. Every named group repeats the full disallow list: a crawler that matches a named group uses only that group
31 * (RFC 9309 2.2.1, and Google's documentation: groups are not combined), so a named group with a bare `Allow: /` would open
32 * every path the wildcard excludes. Advisory only; the real controls stay in the site (noindex headers, POST-only routes).
33 *
34 * @param {RobotsInput} input
35 * @returns {string}
36 */
37export function robotsDocument(input) {
38  if (input.preview) return "User-agent: *\nDisallow: /\n";
39  const {
40    origin,
41    disallow = [],
42    allowBots = [...SEARCH_ENGINE_BOTS, ...AI_SEARCH_BOTS, ...AI_TRAINING_BOTS],
43    denyBots = DENIED_BOTS,
44    sitemaps = [`${origin}/sitemap.xml`],
45    notes = [],
46  } = input;
47  const group = (/** @type {string} */ agent) => [
48    `User-agent: ${agent}`,
49    "Allow: /",
50    ...disallow.map((path) => `Disallow: ${path}`),
51    "",
52  ];
53  return [
54    ...group("*"),
55    ...allowBots.flatMap(group),
56    ...denyBots.flatMap((agent) => [`User-agent: ${agent}`, "Disallow: /", ""]),
57    ...sitemaps.map((url) => `Sitemap: ${url}`),
58    ...notes.map((note) => `# ${note}`),
59    "",
60  ].join("\n");
61}
62
63/**
64 * The TDM Reservation Protocol header (W3C TDMRep): text and data mining rights are reserved, the EU DSM Article 4 opt-out,
65 * which is what lets a training crawler be charged rather than mine for free. Every public response carries it. There is no
66 * `tdm-policy`: TDMRep reads one only as ODRL JSON or an HTML page, and a plain-text ai.txt is neither.
67 */
68export const TDM_RESERVATION_HEADERS = { "tdm-reservation": "1" };
69
70/**
71 * @typedef {object} AiTxtInput
72 * @property {string} origin the production origin
73 * @property {string} name the site's name
74 * @property {string} contact where a licensing request goes, an email address
75 * @property {string[]} [disallow] the same list robots.txt carries
76 * @property {string[]} [policy] the site's own wording of the policy, in place of the family's; a line not already a `#`
77 *   comment is written as one, so edited text cannot turn into a directive
78 */
79
80/**
81 * ai.txt, the family's AI usage policy in the robots.txt form: answer engines may cite and link, training rights are
82 * reserved (as `TDM_RESERVATION_HEADERS` says) and licensed on request.
83 *
84 * @param {AiTxtInput} input
85 * @returns {string}
86 */
87export function aiTxt({ origin, name, contact, disallow = [], policy }) {
88  const wording = policy ?? [
89    "Citation and search (answer engines, retrieval): allowed. Please cite the page and link its canonical URL.",
90    "Training data: rights reserved (TDMRep tdm-reservation: 1). Ask for a license at the address below.",
91  ];
92  return [
93    `# ai.txt: AI usage policy for ${name} (${origin})`,
94    "# The same policy as /robots.txt and the TDMRep tdm-reservation header.",
95    "",
96    ...wording.map((line) => (line.startsWith("#") ? line : `# ${line}`.trimEnd())),
97    "",
98    "User-agent: *",
99    "Allow: /",
100    ...disallow.map((path) => `Disallow: ${path}`),
101    "",
102    "# Training data: rights reserved. Licensing:",
103    `# ${contact}`,
104    "Training: disallow",
105    "",
106    `Sitemap: ${origin}/sitemap.xml`,
107    `Content-index: ${origin}/llms.txt`,
108    "",
109  ].join("\n");
110}