bin/conformance.mjs

Functions

Imports

It imports no other module of this repo. No module of this repo imports it.

Source

1#!/usr/bin/env node
2// The MCP conformance gate. Runs the official suite (@modelcontextprotocol/conformance) against a Worker's real protocol
3// layer under `wrangler dev`, per era, and against its authorization server, then pins the two client-identity paths
4// the suite treats as optional: CIMD advertised, and a /register that registers only an allowlisted callback.
5//
6// COUNTS, NOT PASS/FAIL. A scenario with a recorded `passed` must not regress, and a count that improves fails too, so
7// the baseline gets tightened; one with none must pass every check. Demanding 100% would be a permanently red gate on
8// upstream SDK gaps; asserting nothing would be a gate that cannot notice the protocol breaking. Judged on the parsed
9// count, not the exit code: on Windows the suite can abort in teardown after printing its results.
10//
11// Runs locally, never on Actions (a full sweep takes minutes against wrangler dev).
12//
13//   npx devkit-conformance                    reads .conformance.json
14//   npx devkit-conformance --config <file>
15//
16// The config (wrangler and @modelcontextprotocol/conformance come from the repository's own node_modules):
17//
18//   {
19//     "mcp": { "entry": "test/conformance/entry.ts", "config": "wrangler.jsonc", "port": 8789,
20//              "persist": ".wrangler/state-conformance", "migrate": "DB", "vars": { "NAME": "value" } },
21//     "auth": { "config": "wrangler.jsonc", "port": 8791, "persist": ".wrangler/state-conformance-auth",
22//               "localUpstream": true, "resourceMetadata": true },
23//     "stub": 8790,
24//     "scenarios": [{ "spec": "2026-07-28", "scenario": "server-stateless", "passed": 24, "total": 28, "note": "why" }],
25//     "registration": { "allowed": "https://example.com/callback" },
26//     "after": "scripts/conformance-extra.mjs"
27//   }
28//
29// `entry` is optional (wrangler.jsonc's main otherwise). `migrate` applies that D1 binding's migrations locally first.
30// `stub` answers every request with { ok: true } on that port, a stand-in for an API the tools call. `registration.allowed`
31// is a callback /register must accept, or { "var": "NAME" } to take the first from that var in the auth config (or its
32// .example); left out, only the refusal is checked. `resourceMetadata: false` accepts a 401 from /mcp whose challenge
33// lacks resource_metadata (RFC 9728), for a server that does not advertise it yet. `after` is a module whose default export runs last with
34// { origin, pass, fail }, for the site's own checks against the auth server.
35
36import { execFileSync, spawn } from "node:child_process";
37import { existsSync, readFileSync, realpathSync } from "node:fs";
38import { createServer } from "node:http";
39import { resolve } from "node:path";
40import { setTimeout as sleep } from "node:timers/promises";
41import { pathToFileURL } from "node:url";
42
43const AUTH_SCENARIOS = [{ spec: "2026-07-28", scenario: "authorization-server-metadata-endpoint" }];
44const OFF_LIST = "http://127.0.0.1:3000/callback";
45
46/**
47 * @typedef {{ spec: string, scenario: string, passed?: number | null, total?: number | null, note?: string }} Scenario
48 * @typedef {{ entry?: string, config?: string, port?: number, persist?: string, migrate?: string, vars?: Record<string, string>,
49 *   localUpstream?: boolean, resourceMetadata?: boolean, scenarios?: Scenario[] }} Server
50 */
51
52/** The pass count the suite printed, in either of its two summary forms; null when it printed neither. */
53export function summarize(/** @type {string} */ out) {
54  const m = out.match(/Passed:\s*(\d+)\/(\d+)/);
55  if (m) return { passed: Number(m[1]), total: Number(m[2]) };
56  const s = out.match(/Total:\s*(\d+)\s+passed,\s*(\d+)\s+failed/);
57  if (s) return { passed: Number(s[1]), total: Number(s[1]) + Number(s[2]) };
58  return null;
59}
60
61/**
62 * One scenario's verdict against its baseline: the line to print and whether it failed the gate.
63 * @param {Scenario} want
64 * @param {{ passed: number, total: number } | null} got
65 */
66export function judge(want, got) {
67  const label = `${want.spec}  ${want.scenario}`;
68  if (!got) return { failed: true, line: `  NO RESULT  ${label}  (could not parse a count from the suite output)` };
69  const n = `${got.passed}/${got.total}`;
70  if (want.passed == null) {
71    return got.passed === got.total ? { failed: false, line: `  PASS       ${label}  (${n})` } : { failed: true, line: `  FAIL       ${label}  (${n})` };
72  }
73  if (got.total !== want.total) return { failed: true, line: `  RECOUNT    ${label}  total moved ${want.total} -> ${got.total}; the suite changed, re-baseline` };
74  if (got.passed < want.passed) return { failed: true, line: `  REGRESSED  ${label}  ${n}, baseline ${want.passed}` };
75  if (got.passed > want.passed) return { failed: true, line: `  IMPROVED   ${label}  ${n} beats baseline ${want.passed}; tighten it` };
76  return { failed: false, line: `  BASELINE   ${label}  ${n}${want.note ? `  (${want.note})` : ""}` };
77}
78
79async function main() {
80  const argv = process.argv.slice(2);
81  const at = argv.indexOf("--config");
82  const root = process.cwd();
83  const config = JSON.parse(readFileSync(resolve(root, at >= 0 ? argv[at + 1] : ".conformance.json"), "utf8"));
84  const wranglerJs = resolve(root, "node_modules/wrangler/bin/wrangler.js");
85  const suiteJs = resolve(root, "node_modules/@modelcontextprotocol/conformance/dist/index.js");
86  /** @type {Server} */
87  const mcp = { port: 8789, config: "wrangler.jsonc", ...config.mcp };
88  /** @type {Server} */
89  const auth = { port: 8791, config: "wrangler.jsonc", scenarios: AUTH_SCENARIOS, ...config.auth };
90  const origin = `http://localhost:${auth.port}`;
91  let failures = 0;
92  const log = (/** @type {string} */ s) => process.stdout.write(`${s}\n`);
93  const pass = (/** @type {string} */ what) => log(`  PASS       ${what}`);
94  const fail = (/** @type {string} */ what) => (failures++, log(`  FAIL       ${what}`));
95
96  const wrangler = (/** @type {string[]} */ args) => execFileSync(process.execPath, [wranglerJs, ...args], { cwd: root, stdio: ["ignore", "pipe", "pipe"] });
97  const dev = (/** @type {Server} */ s) => {
98    const args = [wranglerJs, "dev", ...(s.entry ? [s.entry] : []), "-c", String(s.config), "--port", String(s.port)];
99    // Each process needs its own inspector port (both default to 9229) and, for a Durable Object, its own state
100    // directory: two processes on one die at SQLITE_BUSY.
101    args.push("--inspector-port", String(Number(s.port) + 1000));
102    if (s.persist) args.push("--persist-to", s.persist);
103    // A config routed to a Custom Domain makes wrangler dev rewrite each request to that host, so the issuer would stop
104    // matching the URL the suite fetched.
105    if (s.localUpstream) args.push("--local-upstream", `localhost:${s.port}`);
106    for (const [k, v] of Object.entries(s.vars ?? {})) args.push("--var", `${k}:${v}`);
107    const child = spawn(process.execPath, args, { cwd: root, stdio: ["ignore", "pipe", "pipe"] });
108    let output = "";
109    child.stdout.on("data", (d) => (output += d));
110    child.stderr.on("data", (d) => (output += d));
111    return { child, output: () => output.slice(-3000) };
112  };
113  const suite = (/** @type {string[]} */ args) => {
114    try {
115      // Per-scenario timeout: one hanging scenario must not hang the gate.
116      return { ok: true, out: execFileSync(process.execPath, [suiteJs, ...args], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 120_000 }) };
117    } catch (error) {
118      const e = /** @type {{ stdout?: string, stderr?: string }} */ (error);
119      return { ok: false, out: `${e.stdout ?? ""}${e.stderr ?? ""}` };
120    }
121  };
122  const runAll = (/** @type {"server" | "authorization"} */ kind, /** @type {string} */ url, /** @type {Scenario[]} */ scenarios) => {
123    for (const want of scenarios) {
124      const { ok, out } = suite([kind, "--url", url, "--scenario", want.scenario, "--spec-version", want.spec]);
125      const verdict = judge(want, summarize(out));
126      if (verdict.failed) failures++;
127      log(`${verdict.line}${ok ? "" : "  [suite exited non-zero]"}`);
128      // The failing checks, printed so a baseline's note stays a measurement rather than a memory.
129      for (const l of out.split("\n").filter((x) => /FAIL/.test(x)).slice(0, 8)) log(`               ${l.trim().slice(0, 170)}`);
130      if (!summarize(out)) log(out.slice(-1500));
131    }
132  };
133  const waitFor = async (/** @type {string} */ url, /** @type {RequestInit} */ init, ready = (/** @type {Response} */ r) => r.status > 0) => {
134    for (const deadline = Date.now() + 120_000; Date.now() < deadline; await sleep(1000)) {
135      try {
136        if (ready(await fetch(url, init))) return true;
137      } catch {
138        /* not up yet */
139      }
140    }
141    return false;
142  };
143
144  const stub = config.stub
145    ? await new Promise((done) => {
146        const server = createServer((req, res) => {
147          req.resume();
148          req.on("end", () => res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ ok: true, data: { stub: true } })));
149        });
150        server.listen(config.stub, () => done(server));
151      })
152    : null;
153  if (mcp.migrate) wrangler(["d1", "migrations", "apply", mcp.migrate, "--local", "-c", String(mcp.config), ...(mcp.persist ? ["--persist-to", mcp.persist] : [])]);
154  const servers = [dev(mcp), dev(auth)];
155  const shutdown = () => {
156    for (const { child } of servers) child.kill();
157    /** @type {import("node:http").Server | null} */ (stub)?.close();
158  };
159  process.on("exit", shutdown);
160
161  const initialize = JSON.stringify({ jsonrpc: "2.0", id: 1, method: "initialize", params: {} });
162  if (!(await waitFor(`http://localhost:${mcp.port}/mcp`, { method: "POST", headers: { "content-type": "application/json" }, body: initialize }))) {
163    log(`wrangler dev (protocol) did not come up. Output follows:\n${servers[0].output()}`);
164    process.exit(2);
165  }
166  log(`wrangler dev on :${mcp.port} against the real protocol layer\n`);
167  runAll("server", `http://localhost:${mcp.port}/mcp`, config.scenarios ?? []);
168
169  if (!(await waitFor(`${origin}/.well-known/oauth-authorization-server`, {}, (r) => r.ok))) {
170    fail(`authorization: wrangler dev on the auth entry did not come up. Output follows:\n${servers[1].output()}`);
171  } else {
172    runAll("authorization", origin, auth.scenarios ?? AUTH_SCENARIOS);
173    const meta = await (await fetch(`${origin}/.well-known/oauth-authorization-server`)).json();
174    if (meta.client_id_metadata_document_supported === true) pass("metadata advertises CIMD (the path claude.ai and Claude Code take)");
175    else fail("metadata does not advertise client_id_metadata_document_supported");
176    if (meta.registration_endpoint === `${origin}/register`) pass("metadata advertises the registration endpoint at /register");
177    else fail(`metadata advertises registration_endpoint ${meta.registration_endpoint}, not ${origin}/register`);
178
179    const register = (/** @type {string} */ uri) =>
180      fetch(`${origin}/register`, {
181        method: "POST",
182        headers: { "content-type": "application/json" },
183        body: JSON.stringify({ client_name: "devkit-conformance", redirect_uris: [uri], token_endpoint_auth_method: "none", grant_types: ["authorization_code", "refresh_token"], response_types: ["code"] }),
184      });
185    const allowed = allowedCallback(root, auth, config.registration?.allowed);
186    if (!allowed) log("  SKIP       no allowlisted callback configured, so only the refusal is checked");
187    else {
188      const r = await register(allowed);
189      const body = await r.json().catch(() => ({}));
190      if (r.status === 201 && typeof body.client_id === "string" && body.client_id && body.access_token === undefined) pass(`POST /register registers ${allowed}, and grants nothing (201, no token)`);
191      else fail(`POST /register answered ${r.status} for the allowlisted ${allowed}`);
192    }
193    const refused = await register(OFF_LIST);
194    const refusedBody = await refused.json().catch(() => ({}));
195    if (refused.status === 400 && refusedBody.error === "invalid_redirect_uri") pass("POST /register refuses a callback off the allowlist (400 invalid_redirect_uri)");
196    else fail(`POST /register answered ${refused.status} ${refusedBody.error ?? ""} for a callback off the allowlist`);
197    const unauth = await fetch(`${origin}/mcp`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" });
198    const challenge = unauth.headers.get("www-authenticate") ?? "";
199    if (unauth.status === 401 && (auth.resourceMetadata === false || challenge.includes("resource_metadata="))) {
200      pass(`/mcp without a token answers 401${auth.resourceMetadata === false ? " (resource_metadata not required by this config)" : " with a resource_metadata challenge"}`);
201    } else fail(`/mcp without a token answered ${unauth.status} (${challenge})`);
202
203    if (config.after) await (await import(pathToFileURL(resolve(root, config.after)).href)).default({ origin, pass, fail });
204  }
205
206  log("");
207  shutdown();
208  if (failures) {
209    log(`devkit-conformance: ${failures} check(s) failed.`);
210    process.exit(1);
211  }
212  log("devkit-conformance: no regression against the recorded baseline in any era, and the authorization server serves CIMD and the allowlisted registration only.");
213}
214
215/** The callback /register must accept: a literal, or the first entry of a var in the auth server's wrangler config. */
216function allowedCallback(/** @type {string} */ root, /** @type {Server} */ auth, /** @type {string | { var: string } | undefined} */ allowed) {
217  if (typeof allowed !== "object") return allowed;
218  const file = [auth.config, `${auth.config}.example`].map((f) => resolve(root, String(f))).find((f) => existsSync(f));
219  const text = file ? readFileSync(file, "utf8") : "";
220  const list = new RegExp(`"${allowed.var}":\\s*"([^"]*)"`).exec(text)?.[1] ?? "";
221  return list.split(",").map((u) => u.trim()).find(Boolean);
222}
223
224// Run through node_modules/.bin, argv[1] is the link and import.meta.url the file it points to.
225if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) await main();