app/routes/robots.ts

Functions

Imports

It imports app/lib/context.ts, app/lib/seo.ts, app/lib/worker-preview.ts. No module of this repo imports it.

Source

1import { getEnv } from "~/lib/context";
2import { HOURLY_CACHE_CONTROL, SITE_ORIGIN } from "~/lib/seo";
3import { isWorkerPreview } from "~/lib/worker-preview";
4import type { Route } from "./+types/robots";
5
6const AI_AGENTS = [
7  "GPTBot",
8  "OAI-SearchBot",
9  "ChatGPT-User",
10  "ClaudeBot",
11  "PerplexityBot",
12  "Google-Extended",
13];
14
15export function loader({ context }: Route.LoaderArgs) {
16  const origin = SITE_ORIGIN;
17
18  // A Worker Preview is a PR's draft of the site: nothing on it is for crawling. The gateway's
19  // X-Robots-Tag: noindex is the real control; this is the advisory half.
20  if (isWorkerPreview(getEnv(context))) {
21    return new Response("User-agent: *\nDisallow: /\n", {
22      headers: {
23        "content-type": "text/plain; charset=utf-8",
24        "cache-control": "no-store",
25      },
26    });
27  }
28
29  /*
30   * Advisory only. The real controls: `X-Robots-Tag` and `no-store` on /preview, and POST-only on
31   * /search/ask, which is disallowed because every answer is billed.
32   */
33  const block = (agent: string) =>
34    `User-agent: ${agent}\nAllow: /\nDisallow: /admin\nDisallow: /lab\nDisallow: /preview\nDisallow: /search/ask\n`;
35
36  const body = [
37    ...AI_AGENTS.map(block),
38    block("*"),
39    `Sitemap: ${origin}/sitemap.xml`,
40    `# LLM guide: ${origin}/llms.txt`,
41    "",
42  ].join("\n");
43
44  return new Response(body, {
45    headers: {
46      "content-type": "text/plain; charset=utf-8",
47      "cache-control": HOURLY_CACHE_CONTROL,
48    },
49  });
50}