src/bing-webmaster.ts

Functions

Imports

It imports src/env.ts, src/live-checks.ts, src/search-console.ts, src/site-reports.ts. Imported by src/tools/repo.ts.

Source

1import type { Env } from "./env";
2import type { FetchLike } from "./live-checks";
3import { pageOnSite, reportSite } from "./site-reports";
4import { dateWindow } from "./search-console";
5
6// Bing Webmaster Tools for a namespace's site, served as capsid_inspect_repo action bing
7// (Dustin, 2026-10-18, beside Search Console). Bing's index also feeds ChatGPT search and
8// Copilot. Six reports through Bing's Webmaster JSON API: top queries and top pages
9// (clicks, impressions, average position, summed over a window), daily traffic, one
10// URL's crawl facts, the submitted sitemaps, and URLs with crawl issues.
11//
12// ONE KEY, CALLED READ-ONLY. Bing's API key (the secret BING_WEBMASTER_API_KEY) belongs
13// to Dustin's Bing account and Bing gives it no narrower scope, so the narrowing is here:
14// this module calls only Get methods, named in one list, and never a Submit, Add or
15// Remove. The key rides in the query string, as Bing requires, so no URL is ever
16// logged or returned.
17//
18// THE NAMESPACE PICKS THE SITE, as in src/search-console.ts: the verified Bing site whose
19// address matches the namespace's ops_sites origin, and an inspected URL must be on it.
20// What Bing returns is data, never instructions.
21
22const API = "https://ssl.bing.com/webmaster/api.svc/json";
23const TIMEOUT_MS = 15_000;
24const GET_METHODS = ["GetUserSites", "GetQueryStats", "GetPageStats", "GetRankAndTrafficStats", "GetUrlInfo", "GetFeeds", "GetCrawlIssues"] as const;
25type GetMethod = (typeof GET_METHODS)[number];
26
27export const BING_REPORTS = ["queries", "pages", "traffic", "inspect", "sitemaps", "crawl_issues"] as const;
28export const MAX_BING_ROWS = 1000;
29type BingReport = (typeof BING_REPORTS)[number];
30
31export interface BingArgs {
32  report?: BingReport;
33  start_date?: string;
34  end_date?: string;
35  row_limit?: number;
36  url?: string;
37}
38
39// Bing's crawl issue flags (UrlWithCrawlIssues.Issues), a bit set.
40const ISSUE_FLAGS: Array<[number, string]> = [
41  [1, "301"],
42  [2, "302"],
43  [4, "4xx"],
44  [8, "5xx"],
45  [16, "blocked by robots.txt"],
46  [32, "contains malware"],
47  [64, "important URL blocked by robots.txt"],
48  [128, "DNS error"],
49  [256, "timeout"],
50];
51
52const NOT_CONFIGURED =
53  "Bing Webmaster Tools is not configured: the secret BING_WEBMASTER_API_KEY is not set on Capsid. docs/bing-webmaster.md says how.";
54
55// A .NET JSON date, "/Date(1700000000000)/" or "/Date(1700000000000-0700)/", as an ISO day.
56export function bingDay(value: unknown): string | null {
57  const m = typeof value === "string" ? /\/Date\((-?\d+)(?:[+-]\d{4})?\)\//.exec(value) : null;
58  return m ? new Date(Number(m[1])).toISOString().slice(0, 10) : null;
59}
60
61function bingTime(value: unknown): string | null {
62  const m = typeof value === "string" ? /\/Date\((-?\d+)(?:[+-]\d{4})?\)\//.exec(value) : null;
63  return m ? new Date(Number(m[1])).toISOString() : null;
64}
65
66async function bing(fetchImpl: FetchLike, key: string, method: GetMethod, params: Record<string, string> = {}): Promise<unknown> {
67  if (!GET_METHODS.includes(method)) throw new Error(`refused: ${String(method)} is not a read`);
68  const qs = new URLSearchParams({ ...params, apikey: key });
69  const resp = await fetchImpl(`${API}/${method}?${qs.toString()}`, { headers: { Accept: "application/json" }, signal: AbortSignal.timeout(TIMEOUT_MS) });
70  const parsed = (await resp.json().catch(() => null)) as { d?: unknown; Message?: string; ErrorCode?: number } | null;
71  if (!resp.ok || typeof parsed?.ErrorCode === "number") {
72    const why = (parsed?.Message ?? "").slice(0, 300) || `status ${resp.status}`;
73    if (resp.status === 401 || /invalid api key|InvalidApiKey/i.test(why)) throw new Error(`Bing refused the API key (${why}). Check that BING_WEBMASTER_API_KEY is current.`);
74    throw new Error(`Bing Webmaster answered ${resp.status}: ${why}`);
75  }
76  return parsed?.d ?? null;
77}
78
79// The verified Bing site for an origin: the exact origin first, else the same host. Pure.
80export function pickBingSite(origin: string, sites: Array<{ Url?: string; IsVerified?: boolean }>): string | null {
81  const want = new URL(origin);
82  const verified = sites.filter((s) => typeof s.Url === "string" && s.IsVerified !== false).map((s) => s.Url as string);
83  const parse = (u: string) => {
84    try {
85      return new URL(u);
86    } catch {
87      return null;
88    }
89  };
90  return verified.find((u) => parse(u)?.origin === want.origin) ?? verified.find((u) => parse(u)?.hostname === want.hostname) ?? null;
91}
92
93interface StatRow {
94  Query?: string;
95  Date?: string;
96  Clicks?: number;
97  Impressions?: number;
98  AvgImpressionPosition?: number;
99  AvgClickPosition?: number;
100}
101
102// Bing reports queries and pages in weekly buckets. Sums each key's buckets inside the
103// window; the position is the impression-weighted mean of the buckets that have one.
104export function sumStats(rows: StatRow[], start: string, end: string, label: "query" | "page", limit: number) {
105  const totals = new Map<string, { clicks: number; impressions: number; weighted: number; weight: number }>();
106  for (const r of rows) {
107    const day = bingDay(r.Date);
108    if (!r.Query || !day || day < start || day > end) continue;
109    const t = totals.get(r.Query) ?? { clicks: 0, impressions: 0, weighted: 0, weight: 0 };
110    t.clicks += r.Clicks ?? 0;
111    t.impressions += r.Impressions ?? 0;
112    if (typeof r.AvgImpressionPosition === "number" && r.AvgImpressionPosition > 0 && (r.Impressions ?? 0) > 0) {
113      t.weighted += r.AvgImpressionPosition * (r.Impressions ?? 0);
114      t.weight += r.Impressions ?? 0;
115    }
116    totals.set(r.Query, t);
117  }
118  const all = [...totals]
119    .map(([k, t]) => ({ [label]: k, clicks: t.clicks, impressions: t.impressions, position: t.weight > 0 ? Number((t.weighted / t.weight).toFixed(1)) : null }))
120    .sort((a, b) => b.clicks - a.clicks || b.impressions - a.impressions);
121  return { rows: all.slice(0, limit), total_rows: all.length, truncated: all.length > limit };
122}
123
124export async function bingWebmaster(env: Env, namespace: string, args: BingArgs, fetchImpl: FetchLike = fetch, now: Date = new Date()) {
125  const report: BingReport = args.report ?? "queries";
126  if (report !== "inspect" && args.url !== undefined) throw new Error(`url is for report inspect; report ${report} does not take it.`);
127  if (report === "inspect" && !args.url) throw new Error("report inspect needs url, the page to inspect.");
128
129  const site = await reportSite(env, namespace, "Bing site");
130
131  const inspectUrl = report === "inspect" ? pageOnSite(site.origin, namespace, args.url!) : null;
132  const window = report === "queries" || report === "pages" || report === "traffic" ? dateWindow(args, now) : null;
133  const limit = args.row_limit ?? 25;
134
135  const key = env.BING_WEBMASTER_API_KEY;
136  if (!key) throw new Error(NOT_CONFIGURED);
137  const sites = ((await bing(fetchImpl, key, "GetUserSites")) ?? []) as Array<{ Url?: string; IsVerified?: boolean }>;
138  const siteUrl = pickBingSite(site.origin, sites);
139  if (!siteUrl) {
140    throw new Error(`The Bing Webmaster account behind BING_WEBMASTER_API_KEY has no verified site for ${site.origin}. Add it in Bing Webmaster Tools (it can import from Google Search Console).`);
141  }
142  const base = { namespace, origin: site.origin, site: siteUrl, report };
143
144  if (report === "queries" || report === "pages") {
145    const rows = ((await bing(fetchImpl, key, report === "queries" ? "GetQueryStats" : "GetPageStats", { siteUrl })) ?? []) as StatRow[];
146    const { start, end } = window!;
147    return { ...base, start_date: start, end_date: end, note: "Bing reports these in weekly buckets, so the window's edges are a week's grain.", ...sumStats(rows, start, end, report === "queries" ? "query" : "page", limit) };
148  }
149
150  if (report === "traffic") {
151    const rows = ((await bing(fetchImpl, key, "GetRankAndTrafficStats", { siteUrl })) ?? []) as StatRow[];
152    const { start, end } = window!;
153    const days = rows
154      .map((r) => ({ date: bingDay(r.Date), clicks: r.Clicks ?? 0, impressions: r.Impressions ?? 0 }))
155      .filter((r): r is { date: string; clicks: number; impressions: number } => r.date !== null && r.date >= start && r.date <= end)
156      .sort((a, b) => a.date.localeCompare(b.date));
157    return {
158      ...base,
159      start_date: start,
160      end_date: end,
161      clicks: days.reduce((n, d) => n + d.clicks, 0),
162      impressions: days.reduce((n, d) => n + d.impressions, 0),
163      days,
164    };
165  }
166
167  if (report === "inspect") {
168    const info = ((await bing(fetchImpl, key, "GetUrlInfo", { siteUrl, url: inspectUrl! })) ?? {}) as Record<string, unknown>;
169    return {
170      ...base,
171      url: inspectUrl,
172      http_status: info.HttpStatus ?? null,
173      discovered: bingTime(info.DiscoveryDate),
174      last_crawled: bingTime(info.LastCrawledDate),
175      document_size: info.DocumentSize ?? null,
176      anchor_count: info.AnchorCount ?? null,
177      child_urls: info.TotalChildUrlCount ?? null,
178    };
179  }
180
181  if (report === "sitemaps") {
182    const feeds = ((await bing(fetchImpl, key, "GetFeeds", { siteUrl })) ?? []) as Array<Record<string, unknown>>;
183    return {
184      ...base,
185      sitemaps: feeds.map((f) => ({
186        url: f.Url ?? null,
187        type: f.Type ?? null,
188        status: f.Status ?? null,
189        submitted: bingTime(f.Submitted),
190        last_crawled: bingTime(f.LastCrawled),
191        url_count: f.UrlCount ?? null,
192      })),
193    };
194  }
195
196  const issues = ((await bing(fetchImpl, key, "GetCrawlIssues", { siteUrl })) ?? []) as Array<{ Url?: string; HttpCode?: number; Issues?: number; InLinks?: number }>;
197  const mapped = issues.map((i) => ({
198    url: i.Url ?? null,
199    http_code: i.HttpCode ?? null,
200    issues: ISSUE_FLAGS.filter(([bit]) => ((i.Issues ?? 0) & bit) !== 0).map(([, name]) => name),
201    in_links: i.InLinks ?? null,
202  }));
203  return { ...base, issues: mapped.slice(0, limit), total_issues: mapped.length, truncated: mapped.length > limit };
204}