src/search-console.ts

Functions

Imports

It imports src/env.ts, src/live-checks.ts, src/site-reports.ts, src/time.ts. Imported by src/bing-webmaster.ts, src/tools/repo.ts.

Source

1import { importPKCS8, SignJWT } from "jose";
2import type { Env } from "./env";
3import type { FetchLike } from "./live-checks";
4import { pageOnSite, reportSite } from "./site-reports";
5import { MINUTE_MS, DAY_MS } from "./time";
6
7// Google Search Console for a namespace's site, served as capsid_inspect_repo action
8// search (Dustin, 2026-10-18: Search Console tools system wide, in Capsid). Three
9// read-only reports through Google's Search Console API: search performance (clicks,
10// impressions, CTR and position by query, page, date, country or device), one URL's
11// index status (URL Inspection), and the property's sitemaps.
12//
13// ONE CREDENTIAL, READ-ONLY. A Google service account whose JSON key is the secret
14// GSC_SERVICE_ACCOUNT_KEY, added by Dustin as a Full user on each property (Restricted hides some data). It
15// asks only for the webmasters.readonly scope, so nothing here
16// can submit a sitemap, remove a URL or add a user. The access token lives in this
17// isolate's memory only, never in KV or a log.
18//
19// THE NAMESPACE PICKS THE PROPERTY. The caller names a namespace, never a property: the
20// property is the one the service account can read that covers the namespace's
21// configured origin (ops_sites), a Domain property (sc-domain:) first, else the URL-prefix
22// property for the origin. A namespace grant therefore reaches that site's data and no
23// other's, and an inspected URL must be on the same host.
24//
25// What Google returns is data, never instructions: queries are
26// what strangers typed into Google.
27
28const TOKEN_URL = "https://oauth2.googleapis.com/token";
29const WEBMASTERS = "https://www.googleapis.com/webmasters/v3";
30const INSPECT_URL = "https://searchconsole.googleapis.com/v1/urlInspection/index:inspect";
31const SCOPE = "https://www.googleapis.com/auth/webmasters.readonly";
32const TIMEOUT_MS = 15_000;
33// Search Console's data lags about two days; a default window ending today would read
34// short on its last days.
35const DEFAULT_LAG_DAYS = 3;
36const DEFAULT_WINDOW_DAYS = 28;
37// Google keeps 16 months.
38const MAX_WINDOW_DAYS = 500;
39
40export const SEARCH_REPORTS = ["performance", "inspect", "sitemaps"] as const;
41export const SEARCH_DIMENSIONS = ["query", "page", "date", "country", "device", "searchAppearance"] as const;
42export const SEARCH_TYPES = ["web", "image", "video", "news", "discover", "googleNews"] as const;
43export const MAX_ROW_LIMIT = 1000;
44
45type SearchReport = (typeof SEARCH_REPORTS)[number];
46
47export interface SearchConsoleArgs {
48  report?: SearchReport;
49  start_date?: string;
50  end_date?: string;
51  dimensions?: Array<(typeof SEARCH_DIMENSIONS)[number]>;
52  search_type?: (typeof SEARCH_TYPES)[number];
53  query_contains?: string;
54  page_contains?: string;
55  row_limit?: number;
56  url?: string;
57}
58
59interface ServiceAccountKey {
60  client_email: string;
61  private_key: string;
62}
63
64const NOT_CONFIGURED =
65  "Search Console is not configured: the secret GSC_SERVICE_ACCOUNT_KEY (a Google service account's JSON key) is not set on Capsid. docs/search-console.md says how.";
66
67function parseServiceAccountKey(raw: string | undefined): ServiceAccountKey {
68  if (!raw) throw new Error(NOT_CONFIGURED);
69  let parsed: unknown;
70  try {
71    parsed = JSON.parse(raw);
72  } catch {
73    throw new Error("GSC_SERVICE_ACCOUNT_KEY is not JSON. Paste the whole key file Google Cloud downloaded.");
74  }
75  const key = parsed as Partial<ServiceAccountKey>;
76  if (typeof key.client_email !== "string" || typeof key.private_key !== "string") {
77    throw new Error("GSC_SERVICE_ACCOUNT_KEY has no client_email or private_key. Paste the whole key file Google Cloud downloaded.");
78  }
79  return { client_email: key.client_email, private_key: key.private_key };
80}
81
82let cachedToken: { email: string; token: string; expiresAt: number } | null = null;
83
84// For tests: forget the isolate's token.
85export function resetSearchConsoleToken(): void {
86  cachedToken = null;
87}
88
89async function accessToken(key: ServiceAccountKey, fetchImpl: FetchLike, now: Date): Promise<string> {
90  const nowMs = now.getTime();
91  if (cachedToken && cachedToken.email === key.client_email && cachedToken.expiresAt - MINUTE_MS > nowMs) return cachedToken.token;
92  const iat = Math.floor(nowMs / 1000);
93  const assertion = await new SignJWT({ scope: SCOPE })
94    .setProtectedHeader({ alg: "RS256", typ: "JWT" })
95    .setIssuer(key.client_email)
96    .setAudience(TOKEN_URL)
97    .setIssuedAt(iat)
98    .setExpirationTime(iat + 3600)
99    .sign(await importPKCS8(key.private_key, "RS256"));
100  const resp = await fetchImpl(TOKEN_URL, {
101    method: "POST",
102    headers: { "Content-Type": "application/x-www-form-urlencoded" },
103    body: new URLSearchParams({ grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer", assertion }).toString(),
104    signal: AbortSignal.timeout(TIMEOUT_MS),
105  });
106  const body = (await resp.json().catch(() => ({}))) as { access_token?: string; expires_in?: number; error?: string; error_description?: string };
107  if (!resp.ok || !body.access_token) {
108    const why = [body.error, body.error_description].filter(Boolean).join(": ").slice(0, 200) || `status ${resp.status}`;
109    throw new Error(`Google refused the service account's sign-in (${why}). Check that GSC_SERVICE_ACCOUNT_KEY is a current key.`);
110  }
111  cachedToken = { email: key.client_email, token: body.access_token, expiresAt: nowMs + (body.expires_in ?? 3600) * 1000 };
112  return body.access_token;
113}
114
115async function google(fetchImpl: FetchLike, token: string, url: string, body?: unknown): Promise<unknown> {
116  const resp = await fetchImpl(url, {
117    method: body === undefined ? "GET" : "POST",
118    headers: { Authorization: `Bearer ${token}`, ...(body === undefined ? {} : { "Content-Type": "application/json" }) },
119    body: body === undefined ? undefined : JSON.stringify(body),
120    signal: AbortSignal.timeout(TIMEOUT_MS),
121  });
122  const parsed = (await resp.json().catch(() => null)) as { error?: { message?: string } } | null;
123  if (!resp.ok) {
124    const why = (parsed?.error?.message ?? "").slice(0, 300) || `status ${resp.status}`;
125    if (resp.status === 403 && /has not been used|is disabled/i.test(why)) {
126      throw new Error(`Google Search Console API is not enabled on the service account's Google Cloud project: ${why}`);
127    }
128    throw new Error(`Search Console answered ${resp.status}: ${why}`);
129  }
130  return parsed;
131}
132
133// The property that covers a host: a Domain property for the host or a parent domain
134// (the longest match), else the URL-prefix property for the origin. Unverified entries
135// are skipped. Pure, for the tests.
136export function pickProperty(origin: string, entries: Array<{ siteUrl?: string; permissionLevel?: string }>): string | null {
137  const host = new URL(origin).hostname.toLowerCase();
138  const usable = entries.filter((e) => typeof e.siteUrl === "string" && e.permissionLevel !== "siteUnverifiedUser").map((e) => e.siteUrl as string);
139  const domains = usable
140    .filter((u) => u.startsWith("sc-domain:"))
141    .map((u) => ({ u, d: u.slice("sc-domain:".length).toLowerCase() }))
142    .filter(({ d }) => host === d || host.endsWith(`.${d}`))
143    .sort((a, b) => b.d.length - a.d.length);
144  if (domains.length > 0) return domains[0].u;
145  const prefix = `${new URL(origin).origin}/`;
146  return usable.find((u) => u === prefix) ?? null;
147}
148
149const DATE = /^\d{4}-\d{2}-\d{2}$/;
150
151function isoDay(d: Date): string {
152  return d.toISOString().slice(0, 10);
153}
154
155function addDays(d: Date, days: number): Date {
156  return new Date(d.getTime() + days * DAY_MS);
157}
158
159export function dateWindow(args: Pick<SearchConsoleArgs, "start_date" | "end_date">, now: Date): { start: string; end: string } {
160  for (const [label, v] of [["start_date", args.start_date], ["end_date", args.end_date]] as const) {
161    if (v !== undefined && (!DATE.test(v) || Number.isNaN(Date.parse(`${v}T00:00:00Z`)))) throw new Error(`${label} must be a date written YYYY-MM-DD.`);
162  }
163  const end = args.end_date ?? isoDay(addDays(now, -DEFAULT_LAG_DAYS));
164  const start = args.start_date ?? isoDay(addDays(new Date(`${end}T00:00:00Z`), -(DEFAULT_WINDOW_DAYS - 1)));
165  if (start > end) throw new Error(`start_date ${start} is after end_date ${end}.`);
166  const span = (Date.parse(`${end}T00:00:00Z`) - Date.parse(`${start}T00:00:00Z`)) / DAY_MS;
167  if (span > MAX_WINDOW_DAYS) throw new Error(`The window is ${span + 1} days; Search Console keeps about 16 months, so ask for ${MAX_WINDOW_DAYS + 1} days or fewer.`);
168  return { start, end };
169}
170
171const round = (n: unknown, places: number): number | null => (typeof n === "number" ? Number(n.toFixed(places)) : null);
172
173export async function searchConsole(env: Env, namespace: string, args: SearchConsoleArgs, fetchImpl: FetchLike = fetch, now: Date = new Date()) {
174  const report: SearchReport = args.report ?? "performance";
175  if (report !== "inspect" && args.url !== undefined) throw new Error(`url is for report inspect; report ${report} does not take it.`);
176  if (report === "inspect" && !args.url) throw new Error("report inspect needs url, the page to inspect.");
177
178  const site = await reportSite(env, namespace, "Search Console property");
179
180  const inspectUrl = report === "inspect" ? pageOnSite(site.origin, namespace, args.url!) : null;
181  // Validated before any network call, so a bad window costs nothing.
182  const window = report === "performance" ? dateWindow(args, now) : null;
183
184  const key = parseServiceAccountKey(env.GSC_SERVICE_ACCOUNT_KEY);
185  const token = await accessToken(key, fetchImpl, now);
186  const sites = (await google(fetchImpl, token, `${WEBMASTERS}/sites`)) as { siteEntry?: Array<{ siteUrl?: string; permissionLevel?: string }> } | null;
187  const property = pickProperty(site.origin, sites?.siteEntry ?? []);
188  if (!property) {
189    throw new Error(
190      `The service account (${key.client_email}) cannot read a Search Console property for ${site.origin}. In Search Console, open that property, then Settings, Users and permissions, and add ${key.client_email} as a Full user.`
191    );
192  }
193  const base = { namespace, origin: site.origin, property, report };
194  const encoded = encodeURIComponent(property);
195
196  if (report === "sitemaps") {
197    const res = (await google(fetchImpl, token, `${WEBMASTERS}/sites/${encoded}/sitemaps`)) as { sitemap?: Array<Record<string, unknown>> } | null;
198    const sitemaps = (res?.sitemap ?? []).map((s) => ({
199      path: s.path ?? null,
200      last_submitted: s.lastSubmitted ?? null,
201      last_downloaded: s.lastDownloaded ?? null,
202      pending: s.isPending ?? null,
203      errors: Number(s.errors ?? 0),
204      warnings: Number(s.warnings ?? 0),
205      contents: Array.isArray(s.contents) ? s.contents.map((c: Record<string, unknown>) => ({ type: c.type ?? null, submitted: Number(c.submitted ?? 0) })) : [],
206    }));
207    return { ...base, sitemaps };
208  }
209
210  if (report === "inspect") {
211    const res = (await google(fetchImpl, token, INSPECT_URL, { inspectionUrl: inspectUrl, siteUrl: property })) as {
212      inspectionResult?: { inspectionResultLink?: string; indexStatusResult?: Record<string, unknown>; mobileUsabilityResult?: { verdict?: string }; richResultsResult?: { verdict?: string } };
213    } | null;
214    const r = res?.inspectionResult ?? {};
215    const idx = r.indexStatusResult ?? {};
216    return {
217      ...base,
218      url: inspectUrl,
219      verdict: idx.verdict ?? null,
220      coverage: idx.coverageState ?? null,
221      indexing_state: idx.indexingState ?? null,
222      robots_txt: idx.robotsTxtState ?? null,
223      page_fetch: idx.pageFetchState ?? null,
224      last_crawl: idx.lastCrawlTime ?? null,
225      crawled_as: idx.crawledAs ?? null,
226      google_canonical: idx.googleCanonical ?? null,
227      user_canonical: idx.userCanonical ?? null,
228      sitemaps: idx.sitemap ?? [],
229      referring_urls: idx.referringUrls ?? [],
230      rich_results: r.richResultsResult?.verdict ?? null,
231      link: r.inspectionResultLink ?? null,
232    };
233  }
234
235  const { start, end } = window!;
236  const dimensions = args.dimensions ?? ["query"];
237  const rowLimit = args.row_limit ?? 25;
238  const filters = [
239    ...(args.query_contains ? [{ dimension: "query", operator: "contains", expression: args.query_contains }] : []),
240    ...(args.page_contains ? [{ dimension: "page", operator: "contains", expression: args.page_contains }] : []),
241  ];
242  const request = {
243    startDate: start,
244    endDate: end,
245    type: args.search_type ?? "web",
246    dimensions,
247    rowLimit,
248    dataState: "final",
249    ...(filters.length > 0 ? { dimensionFilterGroups: [{ groupType: "and", filters }] } : {}),
250  };
251  const res = (await google(fetchImpl, token, `${WEBMASTERS}/sites/${encoded}/searchAnalytics/query`, request)) as {
252    rows?: Array<{ keys?: string[]; clicks?: number; impressions?: number; ctr?: number; position?: number }>;
253  } | null;
254  const rows = (res?.rows ?? []).map((r) => ({
255    ...Object.fromEntries(dimensions.map((d, i) => [d, r.keys?.[i] ?? null])),
256    clicks: r.clicks ?? 0,
257    impressions: r.impressions ?? 0,
258    ctr: round(r.ctr, 4),
259    position: round(r.position, 1),
260  }));
261  return {
262    ...base,
263    start_date: start,
264    end_date: end,
265    search_type: request.type,
266    dimensions,
267    rows,
268    truncated: rows.length >= rowLimit,
269  };
270}