src/github/search.ts

Functions

Imports

It imports src/encoding.ts, src/env.ts, src/github/client.ts, src/limits.ts. Imported by src/github.ts, src/github/contents.ts.

Source

1import { base64Decode } from "../encoding";
2import type { RepoEnv as Env } from "../env";
3import { DEFAULT_SCAN_RESULTS, MAX_SCAN_CAP, DEFAULT_SCAN_FILES } from "../limits";
4import { ghFetch, resolveRepo, getDefaultBranch } from "./client";
5
6// search_code is a server-side tree walk, not the REST search API. GET /search/code
7// returns HTTP 200 with total_count 0 for private repos under a GitHub App
8// installation token, even for terms read_repo_file confirms are present: the code
9// search index does not serve App-token requests on private repos.
10const SEARCH_EXCLUDE_DIRS = ["node_modules/", ".git/", "dist/"];
11const SEARCH_EXCLUDE_FILES = new Set(["package-lock.json", "pnpm-lock.yaml", "yarn.lock", "bun.lockb"]);
12const SEARCH_EXCLUDE_EXTS = new Set([
13  "png", "jpg", "jpeg", "gif", "webp", "ico", "pdf", "zip", "gz", "tgz", "tar", "bz2",
14  "woff", "woff2", "ttf", "otf", "eot", "mp4", "mov", "webm", "mp3", "wav", "wasm",
15  "bin", "exe", "dll", "so", "dylib", "class", "jar", "pyc", "lockb",
16]);
17const SEARCH_BLOB_LIMIT = 200 * 1024; // skip blobs over 200KB
18export const SEARCH_TREE_LIMIT = 5000; // refuse to scan a tree bigger than this whole
19
20type TreeEntry = { path: string; type: string; sha: string; size?: number };
21
22// The files a search reads: every blob under pathPrefix in the tree at ref, minus the
23// excluded directories, files and extensions and anything over the blob limit.
24async function searchCandidates(
25  env: Env,
26  target: { owner: string; repo: string; full: string },
27  ref: string,
28  pathPrefix: string
29): Promise<TreeEntry[]> {
30  const { owner, repo, full } = target;
31  // GitHub resolves a branch, tag, or sha for the tree sha here. recursive=1
32  // returns the whole tree in one call.
33  const treeResp = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/git/trees/${encodeURIComponent(ref)}?recursive=1`);
34  if (!treeResp.ok) throw new Error(`search_code tree fetch failed (${treeResp.status}): ${await treeResp.text()}`);
35  const tree = (await treeResp.json()) as {
36    tree: TreeEntry[];
37    truncated: boolean;
38  };
39  if (tree.truncated || tree.tree.length > SEARCH_TREE_LIMIT) {
40    throw new Error(
41      `search_code: ${full}@${ref} tree is too large to scan whole (${tree.tree.length} entries, truncated=${tree.truncated}). Narrow it with path_prefix.`
42    );
43  }
44
45  return tree.tree.filter((e) => {
46    if (e.type !== "blob") return false;
47    if (pathPrefix && !e.path.startsWith(pathPrefix)) return false;
48    if (SEARCH_EXCLUDE_DIRS.some((d) => e.path.startsWith(d) || e.path.includes(`/${d}`))) return false;
49    const base = e.path.split("/").pop() ?? e.path;
50    if (SEARCH_EXCLUDE_FILES.has(base)) return false;
51    const ext = base.includes(".") ? base.slice(base.lastIndexOf(".") + 1).toLowerCase() : "";
52    if (SEARCH_EXCLUDE_EXTS.has(ext)) return false;
53    if (typeof e.size === "number" && e.size > SEARCH_BLOB_LIMIT) return false;
54    return true;
55  });
56}
57
58interface BlobScan {
59  items: Array<{ path: string; line: number; text: string }>;
60  // Blobs that returned a survivable error. Reported so a zero-result scan cannot
61  // pass for one that read everything it counted.
62  unreadable: string[];
63  filesScanned: number;
64  // The index of the first candidate not searched.
65  index: number;
66  stoppedAtFileCap: boolean;
67}
68
69// Reads candidates from `start`, one blob at a time, until the file cap or the result
70// cap is reached, and collects the matching lines.
71async function scanBlobs(
72  env: Env,
73  target: { owner: string; repo: string },
74  candidates: TreeEntry[],
75  query: string,
76  limits: { start: number; maxFiles: number; maxResults: number }
77): Promise<BlobScan> {
78  const { owner, repo } = target;
79  const { start, maxFiles, maxResults } = limits;
80  const needle = query.toLowerCase();
81  const items: BlobScan["items"] = [];
82  const unreadable: string[] = [];
83  let filesScanned = 0;
84  let index = start;
85  let stoppedAtFileCap = false;
86  for (; index < candidates.length; index++) {
87    if (filesScanned >= maxFiles) {
88      // Cap reached before candidates[index] was scanned, so it is the first
89      // unsearched file and a caller can resume from here.
90      stoppedAtFileCap = true;
91      break;
92    }
93    filesScanned++;
94    const c = candidates[index];
95    const blob = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/git/blobs/${c.sha}`);
96    if (!blob.ok) {
97      // A blob this scan could not read is not a blob with no match. Skipping it
98      // silently turns an exhausted rate limit into a confident "0 results".
99      //
100      // Quota and auth failures abort the whole scan: 401 and 403 cover token expiry,
101      // revoked installation access, and GitHub's primary and secondary rate limits,
102      // and 429 is the explicit rate-limit status. Once the limit is hit every later
103      // fetch fails the same way. Anything else (a 404 on a raced deletion, a 5xx on
104      // one blob) is survivable, so it is counted and reported.
105      if (blob.status === 401 || blob.status === 403 || blob.status === 429) {
106        throw new Error(
107          `search_code aborted at ${filesScanned} of ${candidates.length} candidate files: GitHub returned ${blob.status} fetching ${c.path}. ` +
108            `This is NOT an empty result. The scan could not read the repository, so no conclusion about whether "${query}" is present is available. ` +
109            (blob.status === 429 || blob.status === 403
110              ? "The App installation's rate limit is the usual cause; a full-tree scan costs one request per candidate file. Wait for the window to reset, narrow with path_prefix, or verify against a local checkout."
111              : "Check the App installation's permissions for this repo.")
112        );
113      }
114      unreadable.push(c.path);
115      continue;
116    }
117    const blobData = (await blob.json()) as { content?: string; encoding?: string };
118    if (blobData.encoding !== "base64" || !blobData.content) continue;
119    let text: string;
120    try {
121      text = base64Decode(blobData.content);
122    } catch {
123      continue; // binary that slipped past the extension filter
124    }
125    // Match line by line, then let text and its lines fall out of scope, so only one
126    // blob is held in memory at a time.
127    const lines = text.split("\n");
128    for (let i = 0; i < lines.length && items.length < maxResults; i++) {
129      if (lines[i].toLowerCase().includes(needle)) {
130        items.push({ path: c.path, line: i + 1, text: lines[i].trim().slice(0, 200) });
131      }
132    }
133    if (items.length >= maxResults) {
134      index++;
135      break;
136    }
137  }
138  return { items, unreadable, filesScanned, index, stoppedAtFileCap };
139}
140
141export async function searchCode(
142  env: Env,
143  namespace: string | undefined,
144  query: string,
145  opts: { pathPrefix?: string; ref?: string; repoSelector?: string; maxResults?: number; maxFiles?: number; start?: number } = {}
146) {
147  if (!namespace) {
148    throw new Error("search_code needs a namespace: it walks one repo's tree. Pass namespace (and optional repo).");
149  }
150  const target = await resolveRepo(env, namespace, opts.repoSelector);
151  const ref = opts.ref || (await getDefaultBranch(env, target.owner, target.repo));
152  // Capped server-side, the same shape ci_status uses for its limit. Each scanned
153  // file costs one blob fetch against the App installation's hourly quota, which every
154  // later repo call shares. The tree-size refusal does not cover this: the cost is per
155  // file fetched, not per candidate listed. Over the cap it clamps rather than
156  // refusing, because the result reports truncation and carries a next_start.
157  const maxResults = Math.min(opts.maxResults && opts.maxResults > 0 ? opts.maxResults : DEFAULT_SCAN_RESULTS, MAX_SCAN_CAP);
158  const maxFiles = Math.min(opts.maxFiles && opts.maxFiles > 0 ? opts.maxFiles : DEFAULT_SCAN_FILES, MAX_SCAN_CAP);
159  const start = opts.start && opts.start > 0 ? Math.floor(opts.start) : 0;
160  const pathPrefix = (opts.pathPrefix ?? "").replace(/^\/+/, "");
161
162  const candidates = await searchCandidates(env, target, ref, pathPrefix);
163  const { items, unreadable, filesScanned, index, stoppedAtFileCap } = await scanBlobs(env, target, candidates, query, {
164    start,
165    maxFiles,
166    maxResults,
167  });
168
169  const remaining = candidates.length - index;
170  const result: {
171    repo: string;
172    ref: string;
173    query: string;
174    candidates: number;
175    start: number;
176    files_scanned: number;
177    total_results: number;
178    truncated: boolean;
179    next_start?: number;
180    note?: string;
181    unreadable_files?: number;
182    unreadable_sample?: string[];
183    items: typeof items;
184  } = {
185    repo: target.full,
186    ref,
187    query,
188    candidates: candidates.length,
189    start,
190    files_scanned: filesScanned,
191    total_results: items.length,
192    truncated: false,
193    items,
194  };
195
196  // Surfaced rather than swallowed: "0 results over 200 files, 12 of which were
197  // unreadable" is a different claim from "0 results over 200 files".
198  if (unreadable.length > 0) {
199    result.unreadable_files = unreadable.length;
200    result.unreadable_sample = unreadable.slice(0, 10);
201  }
202
203  // A boolean alone is not actionable: say WHY it stopped and what to do next.
204  if (stoppedAtFileCap && remaining > 0) {
205    result.truncated = true;
206    result.next_start = index;
207    result.note =
208      `Stopped at the max_files cap (${maxFiles}): ${remaining} of ${candidates.length} candidate files were not searched, so matches past this point are NOT included. ` +
209      `Narrow with path_prefix (a subdirectory), or pass start=${index} to continue this scan from where it left off.`;
210  } else if (items.length >= maxResults && remaining > 0) {
211    result.truncated = true;
212    result.note =
213      `Returned the first ${maxResults} matches (max_results cap) with files still unsearched; more matches may exist. ` +
214      `Raise max_results, or narrow with path_prefix.`;
215  }
216
217  return result;
218}