token-goat 2.9.13 → 2.9.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +28 -1
  2. package/dist/token-goat-chunk-2X2EBBC6.mjs +277 -0
  3. package/dist/token-goat-chunk-3BTK54F3.mjs +1733 -0
  4. package/dist/token-goat-chunk-3NSDDTGL.mjs +34 -0
  5. package/dist/token-goat-chunk-4EXFN2AW.mjs +29 -0
  6. package/dist/token-goat-chunk-5V7DAC7V.mjs +123 -0
  7. package/dist/token-goat-chunk-6DLVZDB6.mjs +34 -0
  8. package/dist/token-goat-chunk-7ZYK25AO.mjs +24 -0
  9. package/dist/token-goat-chunk-A4JYKD5H.mjs +144 -0
  10. package/dist/token-goat-chunk-AH6QILZM.mjs +26 -0
  11. package/dist/token-goat-chunk-ASYEPR3S.mjs +212 -0
  12. package/dist/{token-goat-chunk-RDITECDL.mjs → token-goat-chunk-ATIFTMRC.mjs} +31 -13
  13. package/dist/{token-goat-chunk-ZZI3IDQZ.mjs → token-goat-chunk-BL5LNGBG.mjs} +3935 -11949
  14. package/dist/{token-goat-chunk-4NXUKV7D.mjs → token-goat-chunk-C5JIO6HK.mjs} +8 -4
  15. package/dist/{token-goat-chunk-FZU7GMUS.mjs → token-goat-chunk-DQ4J5AFF.mjs} +50 -18
  16. package/dist/token-goat-chunk-ERTXEKB6.mjs +228 -0
  17. package/dist/{token-goat-chunk-6B44WLIF.mjs → token-goat-chunk-GIIHUSZX.mjs} +142 -21
  18. package/dist/token-goat-chunk-GMOUBOX4.mjs +386 -0
  19. package/dist/{token-goat-chunk-3ESRORNM.mjs → token-goat-chunk-GMQQA7E4.mjs} +12576 -12326
  20. package/dist/token-goat-chunk-IVUQLQWN.mjs +2046 -0
  21. package/dist/token-goat-chunk-K7F2BFIK.mjs +2430 -0
  22. package/dist/{token-goat-chunk-U7X6LQD2.mjs → token-goat-chunk-LCZBPOIN.mjs} +10197 -9717
  23. package/dist/token-goat-chunk-LMFO66YD.mjs +331 -0
  24. package/dist/token-goat-chunk-LT7JRU6K.mjs +22 -0
  25. package/dist/token-goat-chunk-MZDIJJ3R.mjs +420 -0
  26. package/dist/token-goat-chunk-NDPO7GAH.mjs +177 -0
  27. package/dist/token-goat-chunk-NDRP4KJQ.mjs +4371 -0
  28. package/dist/token-goat-chunk-NEI4NC54.mjs +424 -0
  29. package/dist/token-goat-chunk-NU7TLMQK.mjs +585 -0
  30. package/dist/token-goat-chunk-OSUFN2FV.mjs +326 -0
  31. package/dist/token-goat-chunk-OUGNPMDA.mjs +959 -0
  32. package/dist/token-goat-chunk-PM76YS22.mjs +1341 -0
  33. package/dist/token-goat-chunk-S4XRY446.mjs +2637 -0
  34. package/dist/{token-goat-chunk-YOA4N6WA.mjs → token-goat-chunk-SFAS46RE.mjs} +5 -3
  35. package/dist/{token-goat-chunk-QWSUZWFP.mjs → token-goat-chunk-SZWYESBS.mjs} +793 -650
  36. package/dist/{token-goat-chunk-B3CTCQTH.mjs → token-goat-chunk-XEPXYDPI.mjs} +3 -2
  37. package/dist/token-goat-chunk-XTQAOTSO.mjs +89 -0
  38. package/dist/token-goat-chunk-Y4AFKTHK.mjs +22 -0
  39. package/dist/token-goat-chunk-YKG35VHC.mjs +228 -0
  40. package/dist/{token-goat-chunk-JOXLE672.mjs → token-goat-chunk-YQ7WI2CO.mjs} +990 -106
  41. package/dist/{token-goat-chunk-2WC4ZUXN.mjs → token-goat-chunk-YZX7EFG4.mjs} +1 -1
  42. package/dist/token-goat-chunk-Z6UXPYJA.mjs +62 -0
  43. package/dist/token-goat-hook.mjs +16 -8
  44. package/dist/token-goat.core.mjs +27 -10
  45. package/docs/cli.md +9 -6
  46. package/package.json +4 -2
  47. package/dist/token-goat-chunk-FQCNJV4V.mjs +0 -693
  48. package/dist/token-goat-chunk-JVNPCQB7.mjs +0 -31
  49. package/dist/token-goat-chunk-P2PU4CR5.mjs +0 -26
  50. package/dist/token-goat-chunk-QKXBGBQR.mjs +0 -3653
  51. package/dist/token-goat-chunk-UM47DRD3.mjs +0 -242
  52. package/dist/token-goat-chunk-UMXJN7DI.mjs +0 -5521
@@ -0,0 +1,4371 @@
1
+ import { createRequire as __cjsRequire } from 'node:module';
2
+ const require = __cjsRequire(import.meta.url);
3
+ import {
4
+ pathEqClause,
5
+ projectScopeClause,
6
+ querySymbols
7
+ } from "./token-goat-chunk-2X2EBBC6.mjs";
8
+ import {
9
+ DocumentRefusedError,
10
+ MAX_DOCUMENT_WORK_MILLIS
11
+ } from "./token-goat-chunk-Y4AFKTHK.mjs";
12
+ import {
13
+ TREE_SITTER_LANGUAGES,
14
+ _detectOpenQuote,
15
+ _lineClosesQuote,
16
+ buildLineIndex,
17
+ detectLanguage,
18
+ findHtmlHeadingMatches,
19
+ getDb,
20
+ offsetToLine,
21
+ precedingDocComment,
22
+ redactIfDotenv,
23
+ refineLanguageByContent,
24
+ stripCstyleComments,
25
+ stripJsComments,
26
+ stripStringLiterals,
27
+ stripXmlComments
28
+ } from "./token-goat-chunk-K7F2BFIK.mjs";
29
+ import {
30
+ fingerprintContent,
31
+ loadConfig
32
+ } from "./token-goat-chunk-S4XRY446.mjs";
33
+ import {
34
+ registerReset
35
+ } from "./token-goat-chunk-EEIDFMEM.mjs";
36
+ import {
37
+ SYMBOL_BODY_CHAR_CAP,
38
+ countContentLines,
39
+ dataDir,
40
+ decodeSource,
41
+ ensureDataDirPrivate,
42
+ ensureDirSync,
43
+ foldPath,
44
+ globalDbPath,
45
+ isCaseInsensitiveFs
46
+ } from "./token-goat-chunk-PM76YS22.mjs";
47
+ import {
48
+ displaySafeText,
49
+ normalizePath
50
+ } from "./token-goat-chunk-ERTXEKB6.mjs";
51
+ import {
52
+ init_define_import_meta_env
53
+ } from "./token-goat-chunk-A37V4PBF.mjs";
54
+
55
+ // src/doc_embed_extract.ts
56
+ init_define_import_meta_env();
57
+ import * as path from "node:path";
58
+ var EMBEDDABLE_DOCUMENT_EXTENSIONS = /* @__PURE__ */ new Set([".pdf", ".docx", ".pptx", ".xlsx"]);
59
+ var XLSX_SHEET_ROW_CAP = 500;
60
+ function isEmbeddableDocument(filePath) {
61
+ return EMBEDDABLE_DOCUMENT_EXTENSIONS.has(path.extname(filePath).toLowerCase());
62
+ }
63
+ function isDocumentRefusal(err) {
64
+ return err instanceof DocumentRefusedError;
65
+ }
66
+ function isTransientDocumentRefusal(err) {
67
+ return err instanceof DocumentRefusedError && err.transient;
68
+ }
69
+ async function extractEmbeddableDocumentText(filePath) {
70
+ switch (path.extname(filePath).toLowerCase()) {
71
+ case ".pdf": {
72
+ const { extractPdfText, readPdfFileWithinBounds } = await import("./token-goat-chunk-Z6UXPYJA.mjs");
73
+ const { text } = await extractPdfText(await readPdfFileWithinBounds(filePath));
74
+ return text;
75
+ }
76
+ case ".docx":
77
+ return await (await import("./token-goat-chunk-LT7JRU6K.mjs")).docxText(filePath);
78
+ case ".pptx":
79
+ return await (await import("./token-goat-chunk-7ZYK25AO.mjs")).pptxAllSlidesText(filePath, true);
80
+ case ".xlsx":
81
+ return await (await import("./token-goat-chunk-3NSDDTGL.mjs")).allSheetsHeadText(filePath, XLSX_SHEET_ROW_CAP);
82
+ default:
83
+ return null;
84
+ }
85
+ }
86
+
87
+ // src/embed_model.ts
88
+ init_define_import_meta_env();
89
+ import { createHash } from "node:crypto";
90
+ import * as fs from "node:fs";
91
+ import { createRequire } from "node:module";
92
+ import * as path2 from "node:path";
93
+ import { pipeline } from "node:stream/promises";
94
+
95
+ // src/embed_tokenizer.ts
96
+ init_define_import_meta_env();
97
+ var MAX_SEQUENCE_TOKENS = 512;
98
+ var UnsupportedTokenizerError = class extends Error {
99
+ constructor(message) {
100
+ super(message);
101
+ this.name = "UnsupportedTokenizerError";
102
+ }
103
+ };
104
+ function fail(message) {
105
+ throw new UnsupportedTokenizerError(message);
106
+ }
107
+ function asRecord(value, path5) {
108
+ if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${path5} is not an object`);
109
+ return value;
110
+ }
111
+ function asString(value, path5) {
112
+ if (typeof value !== "string") fail(`${path5} is not a string`);
113
+ return value;
114
+ }
115
+ function asArray(value, path5) {
116
+ if (!Array.isArray(value)) fail(`${path5} is not an array`);
117
+ return value;
118
+ }
119
+ function expectValue(actual, wanted, path5) {
120
+ if (actual !== wanted) fail(`${path5} is ${JSON.stringify(actual)}, expected ${JSON.stringify(wanted)}`);
121
+ }
122
+ function isChinese(cp) {
123
+ return cp >= 19968 && cp <= 40959 || cp >= 13312 && cp <= 19903 || cp >= 131072 && cp <= 173791 || cp >= 173824 && cp <= 177983 || cp >= 177984 && cp <= 178207 || cp >= 178208 && cp <= 183983 || cp >= 63744 && cp <= 64255 || cp >= 194560 && cp <= 195103;
124
+ }
125
+ var PUNCTUATION = new RegExp("\\p{P}", "u");
126
+ function isPunctuation(ch) {
127
+ const cp = ch.codePointAt(0) ?? 0;
128
+ if (cp >= 33 && cp <= 47 || cp >= 58 && cp <= 64 || cp >= 91 && cp <= 96 || cp >= 123 && cp <= 126) {
129
+ return true;
130
+ }
131
+ return PUNCTUATION.test(ch);
132
+ }
133
+ var CONTROL = new RegExp("\\p{Cc}|\\p{Cf}|\\p{Co}|\\p{Cs}", "u");
134
+ function isControl(ch) {
135
+ if (ch === " " || ch === "\n" || ch === "\r") return false;
136
+ return CONTROL.test(ch);
137
+ }
138
+ var SPACE_SEPARATOR = new RegExp("\\p{Zs}", "u");
139
+ function isWhitespace(ch) {
140
+ return ch === " " || ch === " " || ch === "\n" || ch === "\r" || SPACE_SEPARATOR.test(ch);
141
+ }
142
+ var COMBINING_MARK = /[\u0300-\u036f]/gu;
143
+ function normalize(text) {
144
+ let out = "";
145
+ for (const ch of text) {
146
+ const cp = ch.codePointAt(0) ?? 0;
147
+ if (cp === 0 || cp === 65533 || isControl(ch)) continue;
148
+ if (isWhitespace(ch)) {
149
+ out += " ";
150
+ continue;
151
+ }
152
+ if (isChinese(cp)) {
153
+ out += ` ${ch} `;
154
+ continue;
155
+ }
156
+ out += ch;
157
+ }
158
+ return out.normalize("NFD").replace(COMBINING_MARK, "").toLowerCase();
159
+ }
160
+ function preTokenize(text) {
161
+ const words = [];
162
+ for (const piece of text.split(/\s+/)) {
163
+ if (!piece) continue;
164
+ let buf = "";
165
+ for (const ch of piece) {
166
+ if (isPunctuation(ch)) {
167
+ if (buf) {
168
+ words.push(buf);
169
+ buf = "";
170
+ }
171
+ words.push(ch);
172
+ } else {
173
+ buf += ch;
174
+ }
175
+ }
176
+ if (buf) words.push(buf);
177
+ }
178
+ return words;
179
+ }
180
+ function readSpec(raw) {
181
+ const spec = asRecord(raw, "tokenizer.json");
182
+ const normalizer = asRecord(spec["normalizer"], "normalizer");
183
+ expectValue(normalizer["type"], "BertNormalizer", "normalizer.type");
184
+ expectValue(normalizer["clean_text"], true, "normalizer.clean_text");
185
+ expectValue(normalizer["handle_chinese_chars"], true, "normalizer.handle_chinese_chars");
186
+ expectValue(normalizer["lowercase"], true, "normalizer.lowercase");
187
+ const stripAccents = normalizer["strip_accents"];
188
+ if (stripAccents !== null && stripAccents !== true) {
189
+ fail(`normalizer.strip_accents is ${JSON.stringify(stripAccents)}, expected null or true`);
190
+ }
191
+ expectValue(asRecord(spec["pre_tokenizer"], "pre_tokenizer")["type"], "BertPreTokenizer", "pre_tokenizer.type");
192
+ const model = asRecord(spec["model"], "model");
193
+ expectValue(model["type"], "WordPiece", "model.type");
194
+ expectValue(model["continuing_subword_prefix"], "##", "model.continuing_subword_prefix");
195
+ const post = asRecord(spec["post_processor"], "post_processor");
196
+ expectValue(post["type"], "TemplateProcessing", "post_processor.type");
197
+ const template = asArray(post["single"], "post_processor.single").map((part) => {
198
+ const special = asRecord(part, "post_processor.single[]")["SpecialToken"];
199
+ return special === void 0 ? "A" : asString(asRecord(special, "SpecialToken")["id"], "SpecialToken.id");
200
+ }).join(" ");
201
+ if (template !== "[CLS] A [SEP]") fail(`post_processor.single is "${template}", expected "[CLS] A [SEP]"`);
202
+ const vocab = /* @__PURE__ */ new Map();
203
+ for (const [token, id] of Object.entries(asRecord(model["vocab"], "model.vocab"))) {
204
+ if (typeof id !== "number" || !Number.isInteger(id)) fail(`model.vocab["${token}"] is not an integer id`);
205
+ vocab.set(token, id);
206
+ }
207
+ const maxChars = model["max_input_chars_per_word"];
208
+ if (maxChars !== void 0 && (typeof maxChars !== "number" || !Number.isInteger(maxChars) || maxChars < 1)) {
209
+ fail("model.max_input_chars_per_word is not a positive integer");
210
+ }
211
+ return {
212
+ vocab,
213
+ unkToken: asString(model["unk_token"], "model.unk_token"),
214
+ maxInputCharsPerWord: typeof maxChars === "number" ? maxChars : 100
215
+ };
216
+ }
217
+ var BertWordPiece = class _BertWordPiece {
218
+ vocab;
219
+ maxInputCharsPerWord;
220
+ clsId;
221
+ sepId;
222
+ padId;
223
+ unkId;
224
+ /** @param raw the parsed contents of a tokenizer.json. */
225
+ constructor(raw) {
226
+ const spec = readSpec(raw);
227
+ this.vocab = spec.vocab;
228
+ this.maxInputCharsPerWord = spec.maxInputCharsPerWord;
229
+ this.clsId = this.requireToken("[CLS]");
230
+ this.sepId = this.requireToken("[SEP]");
231
+ this.padId = this.requireToken("[PAD]");
232
+ this.unkId = this.requireToken(spec.unkToken);
233
+ }
234
+ /** Parse and validate in one step, for the common case of reading the file off disk. */
235
+ static fromJson(json) {
236
+ let parsed;
237
+ try {
238
+ parsed = JSON.parse(json);
239
+ } catch (err) {
240
+ throw new UnsupportedTokenizerError(`tokenizer.json is not valid JSON: ${err.message}`);
241
+ }
242
+ return new _BertWordPiece(parsed);
243
+ }
244
+ requireToken(token) {
245
+ const id = this.vocab.get(token);
246
+ if (id === void 0) fail(`model.vocab is missing the ${token} token`);
247
+ return id;
248
+ }
249
+ /** Greedy longest-match-first over one whitespace- and punctuation-free word. */
250
+ wordToIds(word, into) {
251
+ if (word.length > this.maxInputCharsPerWord) {
252
+ into.push(this.unkId);
253
+ return;
254
+ }
255
+ const pieces = [];
256
+ let start = 0;
257
+ while (start < word.length) {
258
+ let end = word.length;
259
+ let found = -1;
260
+ while (start < end) {
261
+ const sub = start === 0 ? word.slice(start, end) : `##${word.slice(start, end)}`;
262
+ const id = this.vocab.get(sub);
263
+ if (id !== void 0) {
264
+ found = id;
265
+ break;
266
+ }
267
+ end--;
268
+ }
269
+ if (found === -1) {
270
+ into.push(this.unkId);
271
+ return;
272
+ }
273
+ pieces.push(found);
274
+ start = end;
275
+ }
276
+ for (const id of pieces) into.push(id);
277
+ }
278
+ /**
279
+ * `[CLS] ... [SEP]`, cut to `maxLength` tokens.
280
+ *
281
+ * The cut is taken after the markers are added, not before, which means a sequence long enough to
282
+ * be truncated ends on an ordinary token and has no [SEP] at all. That is what the reference does
283
+ * and therefore what the model has been fed all along, so it is deliberate rather than an
284
+ * oversight: keeping the [SEP] would be the more defensible sequence and a different one, and a
285
+ * tokenizer whose whole justification is producing identical ids does not get to improve on them.
286
+ * The oracle carries two cases that reach the limit; both end mid-text.
287
+ */
288
+ encode(text, maxLength = MAX_SEQUENCE_TOKENS) {
289
+ if (!Number.isInteger(maxLength) || maxLength < 2) {
290
+ throw new RangeError(`maxLength must be an integer >= 2 (both markers), got ${maxLength}`);
291
+ }
292
+ const ids = [this.clsId];
293
+ for (const word of preTokenize(normalize(text))) {
294
+ this.wordToIds(word, ids);
295
+ if (ids.length >= maxLength) return ids.slice(0, maxLength);
296
+ }
297
+ ids.push(this.sepId);
298
+ return ids.length > maxLength ? ids.slice(0, maxLength) : ids;
299
+ }
300
+ };
301
+
302
+ // src/embed_model.ts
303
+ var _require = createRequire(import.meta.url);
304
+ var DEFAULT_MODEL = "Xenova/bge-small-en-v1.5";
305
+ var DEFAULT_DIM = 384;
306
+ var DEFAULT_EMBED_THREADS = 2;
307
+ var PINNED_MODEL_REVISION = "ea104dacec62c0de699686887e3f920caeb4f3e3";
308
+ var MODEL_FILES = [
309
+ {
310
+ name: "tokenizer.json",
311
+ sha256: "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66",
312
+ bytes: 711396
313
+ },
314
+ {
315
+ name: "onnx/model_quantized.onnx",
316
+ sha256: "6c9c6101a956d62dfb5e7190c538226c0c5bb9cb27b651234b6df063ee7dbfe4",
317
+ bytes: 34014426
318
+ }
319
+ ];
320
+ function modelDir() {
321
+ return path2.join(dataDir(), "models", ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
322
+ }
323
+ function modelFilesPresent() {
324
+ const roots = [modelDir(), sharedModelCacheDir()].filter((d) => d !== null);
325
+ return roots.some(
326
+ (root) => MODEL_FILES.every((file) => {
327
+ try {
328
+ return fs.statSync(path2.join(root, file.name)).size === file.bytes;
329
+ } catch {
330
+ return false;
331
+ }
332
+ })
333
+ );
334
+ }
335
+ function downloadUrl(file) {
336
+ return `https://huggingface.co/${DEFAULT_MODEL}/resolve/${PINNED_MODEL_REVISION}/${file.name}`;
337
+ }
338
+ function sharedModelCacheDir() {
339
+ const raw = process.env["TOKEN_GOAT_MODEL_CACHE_DIR"]?.trim();
340
+ if (!raw) return null;
341
+ return path2.join(raw, ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
342
+ }
343
+ async function copyFromSharedCache(shared, file, target) {
344
+ const source = path2.join(shared, file.name);
345
+ const temp = `${target}.${process.pid}.shared`;
346
+ try {
347
+ const info = fs.lstatSync(source, { throwIfNoEntry: false });
348
+ if (!info?.isFile() || info.size !== file.bytes) return false;
349
+ fs.copyFileSync(source, temp);
350
+ if (await sha256Of(temp) !== file.sha256) {
351
+ fs.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
352
+ fs.rmSync(source, { force: true, maxRetries: 20, retryDelay: 25 });
353
+ return false;
354
+ }
355
+ fs.renameSync(temp, target);
356
+ return true;
357
+ } catch {
358
+ try {
359
+ fs.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
360
+ } catch {
361
+ }
362
+ return false;
363
+ }
364
+ }
365
+ function publishToSharedCache(shared, file, target) {
366
+ const destination = path2.join(shared, file.name);
367
+ const temp = `${destination}.${process.pid}.partial`;
368
+ let created = false;
369
+ try {
370
+ if (fs.existsSync(destination)) return;
371
+ ensureDirSync(path2.dirname(destination));
372
+ fs.copyFileSync(target, temp, fs.constants.COPYFILE_EXCL);
373
+ created = true;
374
+ fs.renameSync(temp, destination);
375
+ } catch {
376
+ if (!created) return;
377
+ try {
378
+ fs.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
379
+ } catch {
380
+ }
381
+ }
382
+ }
383
+ var _ort = null;
384
+ var _ortError = null;
385
+ var _ortLoadAttempted = false;
386
+ function ensureRuntimeLoaded() {
387
+ if (_ortLoadAttempted) return;
388
+ _ortLoadAttempted = true;
389
+ try {
390
+ _ort = _require("onnxruntime-node");
391
+ } catch (e) {
392
+ _ortError = e instanceof Error ? e : new Error(String(e));
393
+ }
394
+ }
395
+ function isRuntimeAvailable() {
396
+ ensureRuntimeLoaded();
397
+ return _ort !== null && _ortError === null;
398
+ }
399
+ function runtimeLoadError() {
400
+ ensureRuntimeLoaded();
401
+ return _ortError;
402
+ }
403
+ function runtimeVersion() {
404
+ ensureRuntimeLoaded();
405
+ if (_ort === null) return "unknown";
406
+ try {
407
+ let dir = path2.dirname(_require.resolve("onnxruntime-node"));
408
+ for (let depth = 0; depth < 6; depth++) {
409
+ const manifest = path2.join(dir, "package.json");
410
+ if (fs.existsSync(manifest)) {
411
+ const parsed = JSON.parse(fs.readFileSync(manifest, "utf8"));
412
+ if (parsed.name === "onnxruntime-node" && typeof parsed.version === "string") return parsed.version;
413
+ }
414
+ const up = path2.dirname(dir);
415
+ if (up === dir) break;
416
+ dir = up;
417
+ }
418
+ } catch {
419
+ }
420
+ return "unknown";
421
+ }
422
+ function sha256Of(filePath) {
423
+ return new Promise((resolve2, reject) => {
424
+ const hash = createHash("sha256");
425
+ const stream = fs.createReadStream(filePath);
426
+ stream.on("error", reject);
427
+ stream.on("data", (chunk) => hash.update(chunk));
428
+ stream.on("end", () => resolve2(hash.digest("hex")));
429
+ });
430
+ }
431
+ async function download(file, target) {
432
+ const url = downloadUrl(file);
433
+ const response = await fetch(url, { redirect: "follow" });
434
+ if (!response.ok) throw new Error(`GET ${url} returned ${response.status} ${response.statusText}`);
435
+ if (!response.body) throw new Error(`GET ${url} returned no body`);
436
+ const temp = `${target}.${process.pid}.partial`;
437
+ const hash = createHash("sha256");
438
+ let written = 0;
439
+ const out = fs.createWriteStream(temp);
440
+ try {
441
+ await pipeline(async function* () {
442
+ for await (const chunk of response.body) {
443
+ written += chunk.byteLength;
444
+ if (written > file.bytes) throw new Error(`${file.name} is longer than the pinned ${file.bytes} bytes`);
445
+ hash.update(chunk);
446
+ yield chunk;
447
+ }
448
+ }, out);
449
+ if (written !== file.bytes) {
450
+ throw new Error(`${file.name} is ${written} bytes, expected the pinned ${file.bytes}`);
451
+ }
452
+ const digest = hash.digest("hex");
453
+ if (digest !== file.sha256) {
454
+ throw new Error(`${file.name} has sha256 ${digest}, expected the pinned ${file.sha256}`);
455
+ }
456
+ fs.renameSync(temp, target);
457
+ } catch (e) {
458
+ await new Promise((resolve2) => {
459
+ if (out.closed) resolve2();
460
+ else out.once("close", () => resolve2());
461
+ });
462
+ try {
463
+ fs.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
464
+ } catch {
465
+ }
466
+ throw e;
467
+ }
468
+ }
469
+ var _inFlightModelDownload = null;
470
+ function ensureModelFiles(modelName = DEFAULT_MODEL) {
471
+ if (modelName !== DEFAULT_MODEL) {
472
+ return Promise.reject(
473
+ new Error(
474
+ `Only ${DEFAULT_MODEL} is supported: its files are pinned to a revision and to a sha256 each, and "${modelName}" has neither, so there would be nothing to check the download against.`
475
+ )
476
+ );
477
+ }
478
+ if (_inFlightModelDownload !== null) {
479
+ return _inFlightModelDownload;
480
+ }
481
+ _inFlightModelDownload = (async () => {
482
+ try {
483
+ return await _ensureModelFilesInner(modelName);
484
+ } finally {
485
+ _inFlightModelDownload = null;
486
+ }
487
+ })();
488
+ return _inFlightModelDownload;
489
+ }
490
+ async function _ensureModelFilesInner(_modelName) {
491
+ ensureDataDirPrivate();
492
+ const dir = modelDir();
493
+ const offline = loadConfig().network.offline;
494
+ const shared = sharedModelCacheDir();
495
+ for (const file of MODEL_FILES) {
496
+ const target = path2.join(dir, file.name);
497
+ if (fs.existsSync(target)) {
498
+ const digest = await sha256Of(target);
499
+ if (digest === file.sha256) continue;
500
+ fs.rmSync(target, { force: true });
501
+ }
502
+ if (shared) {
503
+ ensureDirSync(path2.dirname(target));
504
+ if (await copyFromSharedCache(shared, file, target)) continue;
505
+ }
506
+ if (offline) {
507
+ throw new Error(
508
+ `Offline mode is on (network.offline): refusing to download ${file.name} for the embedding model. Copy the pinned files into ${dir} on a connected machine to use semantic search here.`
509
+ );
510
+ }
511
+ ensureDirSync(path2.dirname(target));
512
+ console.warn(
513
+ `Downloading the embedding model, once (${file.name}, ${Math.round(file.bytes / 1024 / 1024)} MB) into ${dir}`
514
+ );
515
+ await download(file, target);
516
+ if (shared) publishToSharedCache(shared, file, target);
517
+ }
518
+ return dir;
519
+ }
520
+ function poolAndNormalize(hidden, seq, dim) {
521
+ const pooled = new Float64Array(dim);
522
+ for (let t = 0; t < seq; t++) {
523
+ const base = t * dim;
524
+ for (let d = 0; d < dim; d++) pooled[d] = (pooled[d] ?? 0) + (hidden[base + d] ?? 0);
525
+ }
526
+ let sumOfSquares = 0;
527
+ for (let d = 0; d < dim; d++) {
528
+ const mean = (pooled[d] ?? 0) / seq;
529
+ pooled[d] = mean;
530
+ sumOfSquares += mean * mean;
531
+ }
532
+ const norm = Math.sqrt(sumOfSquares);
533
+ const out = new Float32Array(dim);
534
+ if (norm === 0 || !Number.isFinite(norm)) return out;
535
+ for (let d = 0; d < dim; d++) out[d] = (pooled[d] ?? 0) / norm;
536
+ return out;
537
+ }
538
+ var EmbeddingModel = class _EmbeddingModel {
539
+ constructor(tokenizer, session, tensorFactory) {
540
+ this.tokenizer = tokenizer;
541
+ this.session = session;
542
+ this.tensorFactory = tensorFactory;
543
+ }
544
+ tokenizer;
545
+ session;
546
+ tensorFactory;
547
+ static async load(modelName = DEFAULT_MODEL) {
548
+ if (!isRuntimeAvailable()) {
549
+ throw new Error(`onnxruntime-node is not available: ${_ortError?.message ?? "unknown error"}`);
550
+ }
551
+ const dir = await ensureModelFiles(modelName);
552
+ const tokenizer = BertWordPiece.fromJson(fs.readFileSync(path2.join(dir, "tokenizer.json"), "utf8"));
553
+ const ort = _ort;
554
+ const threads = loadConfig().worker.embed_threads ?? DEFAULT_EMBED_THREADS;
555
+ const session = await ort.InferenceSession.create(path2.join(dir, "onnx", "model_quantized.onnx"), {
556
+ intraOpNumThreads: threads,
557
+ interOpNumThreads: 1
558
+ });
559
+ return new _EmbeddingModel(tokenizer, session, ort.Tensor);
560
+ }
561
+ /** Embed one text. Sequences are run singly, so there is no padding and no mask to get wrong. */
562
+ async embed(text) {
563
+ const ids = this.tokenizer.encode(text);
564
+ const length = ids.length;
565
+ const feeds = {
566
+ input_ids: new this.tensorFactory("int64", BigInt64Array.from(ids, BigInt), [1, length]),
567
+ attention_mask: new this.tensorFactory("int64", new BigInt64Array(length).fill(1n), [1, length])
568
+ };
569
+ if (this.session.inputNames.includes("token_type_ids")) {
570
+ feeds["token_type_ids"] = new this.tensorFactory("int64", new BigInt64Array(length), [1, length]);
571
+ }
572
+ const outputName = this.session.outputNames[0];
573
+ if (outputName === void 0) throw new Error("the model declares no outputs");
574
+ const output = (await this.session.run(feeds))[outputName];
575
+ if (!output) throw new Error(`the model produced no ${outputName}`);
576
+ const [, seq, dim] = output.dims;
577
+ if (seq === void 0 || dim === void 0) {
578
+ throw new Error(`expected a [batch, sequence, dimension] output, got [${output.dims.join(", ")}]`);
579
+ }
580
+ if (dim !== DEFAULT_DIM) {
581
+ throw new Error(`the model produced ${dim}-dimension vectors, expected ${DEFAULT_DIM}`);
582
+ }
583
+ return poolAndNormalize(output.data, seq, dim);
584
+ }
585
+ };
586
+ async function checkEmbeddingPreflight(options) {
587
+ const modelName = options?.modelName ?? DEFAULT_MODEL;
588
+ const cfg = loadConfig();
589
+ const enabled = cfg.indexing?.embeddings_enabled ?? true;
590
+ const mDir = modelDir();
591
+ const rtVer = runtimeVersion();
592
+ const rtAvail = isRuntimeAvailable();
593
+ const filesPresent = modelFilesPresent();
594
+ let warmed = false;
595
+ const buildResult = (params) => {
596
+ const indexed = params.coverage?.indexedFiles ?? 0;
597
+ const embedded = params.coverage?.embeddedFiles ?? 0;
598
+ const pct = indexed > 0 ? Math.round(embedded / indexed * 100) : 0;
599
+ return {
600
+ status: params.status,
601
+ available: params.available,
602
+ message: params.message,
603
+ summary: params.message,
604
+ modelName,
605
+ runtimeVersion: rtVer,
606
+ runtimeAvailable: rtAvail,
607
+ configEnabled: enabled,
608
+ modelFilesPresent: filesPresent,
609
+ modelWarmed: warmed,
610
+ modelDir: mDir,
611
+ ...params.suggestion !== void 0 ? { suggestion: params.suggestion, actionRequired: params.suggestion } : {},
612
+ ...params.error !== void 0 ? { error: params.error } : {},
613
+ indexedFiles: indexed,
614
+ embeddedFiles: embedded,
615
+ coveragePercent: pct,
616
+ ...params.coverage !== void 0 ? { coverage: params.coverage } : {}
617
+ };
618
+ };
619
+ if (!enabled) {
620
+ return buildResult({
621
+ status: "disabled",
622
+ available: false,
623
+ message: "Matching on meaning is off (indexing.embeddings_enabled / TOKEN_GOAT_EMBEDDINGS_ENABLED is disabled)",
624
+ suggestion: "Enable in config: set indexing.embeddings_enabled = true or unset TOKEN_GOAT_EMBEDDINGS_ENABLED"
625
+ });
626
+ }
627
+ if (!rtAvail) {
628
+ const err = runtimeLoadError();
629
+ return buildResult({
630
+ status: "missing_runtime",
631
+ available: false,
632
+ message: `Inference runtime is not available: ${err?.message ?? "onnxruntime-node is not installed"}`,
633
+ suggestion: "Install it with: npm install -g onnxruntime-node (drop -g if token-goat is a project dependency)",
634
+ ...err?.message ? { error: err.message } : {}
635
+ });
636
+ }
637
+ if (!filesPresent && cfg?.network?.offline) {
638
+ return buildResult({
639
+ status: "missing_model_files",
640
+ available: false,
641
+ message: "Embedding model files are missing and offline mode (network.offline) prevents downloading them",
642
+ suggestion: `Download or copy the pinned model files into ${mDir}`
643
+ });
644
+ }
645
+ if (options?.warm === true) {
646
+ try {
647
+ await ensureModelFiles(modelName);
648
+ await EmbeddingModel.load(modelName);
649
+ warmed = true;
650
+ } catch (e) {
651
+ const err = e instanceof Error ? e.message : String(e);
652
+ return buildResult({
653
+ status: "load_error",
654
+ available: false,
655
+ message: `Failed to warm/load embedding model: ${err}`,
656
+ suggestion: "Check model integrity or run 'token-goat doctor'",
657
+ error: err
658
+ });
659
+ }
660
+ }
661
+ const coverage = options?.coverage;
662
+ if (coverage && coverage.indexedFiles > 0 && coverage.embeddedFiles === 0) {
663
+ return buildResult({
664
+ status: "no_embeddings",
665
+ available: false,
666
+ message: `Embedding model is available, but 0 of ${coverage.indexedFiles} indexed file(s) in this project have embeddings`,
667
+ suggestion: options?.projectRoot ? `Run 'token-goat index "${options.projectRoot}"' to generate embeddings` : "Run 'token-goat index' to generate embeddings",
668
+ coverage
669
+ });
670
+ }
671
+ return buildResult({
672
+ status: "ready",
673
+ available: true,
674
+ message: "Semantic embedding model is ready and available",
675
+ ...coverage !== void 0 ? { coverage } : {}
676
+ });
677
+ }
678
+ registerReset(() => {
679
+ _ort = null;
680
+ _ortError = null;
681
+ _ortLoadAttempted = false;
682
+ _inFlightModelDownload = null;
683
+ });
684
+
685
+ // src/embeddings.ts
686
+ init_define_import_meta_env();
687
+
688
+ // src/embed_fingerprint.ts
689
+ init_define_import_meta_env();
690
+ var EMBED_FINGERPRINT = "b1fd4a80b27f2ca1";
691
+
692
+ // src/embeddings.ts
693
+ var QUERY_INSTRUCTION_PREFIX = "Represent this sentence for searching relevant passages: ";
694
+ var _extractorCache = /* @__PURE__ */ new Map();
695
+ var inHousePipelineFn = async (_task, modelName) => {
696
+ const model = await EmbeddingModel.load(modelName);
697
+ return async (text) => ({ data: await model.embed(text) });
698
+ };
699
+ var _pipelineFnOverride = null;
700
+ registerReset(() => {
701
+ _extractorCache.clear();
702
+ _pipelineFnOverride = null;
703
+ });
704
+ var PIPELINE_RETRY_ATTEMPTS = 3;
705
+ var PIPELINE_RETRY_DELAY_MS = 250;
706
+ var DEFAULT_PIPELINE_RETRY_DELAY_MS = PIPELINE_RETRY_DELAY_MS;
707
+ registerReset(() => {
708
+ PIPELINE_RETRY_DELAY_MS = DEFAULT_PIPELINE_RETRY_DELAY_MS;
709
+ });
710
+ function sleep(ms) {
711
+ return new Promise((resolve2) => setTimeout(resolve2, ms));
712
+ }
713
+ async function buildExtractorWithRetry(pipelineFn, modelName) {
714
+ let lastError;
715
+ for (let attempt = 1; attempt <= PIPELINE_RETRY_ATTEMPTS; attempt++) {
716
+ try {
717
+ return await pipelineFn("feature-extraction", modelName);
718
+ } catch (e) {
719
+ lastError = e;
720
+ if (attempt < PIPELINE_RETRY_ATTEMPTS) await sleep(PIPELINE_RETRY_DELAY_MS * attempt);
721
+ }
722
+ }
723
+ throw lastError instanceof Error ? lastError : new Error(String(lastError));
724
+ }
725
+ var MIN_CHUNK_CHARS = 50;
726
+ var MAX_CHUNK_CHARS = 8e3;
727
+ var DEFAULT_DISTANCE_THRESHOLD = 1.2;
728
+ var _GENERATED_PATH_SEGMENTS = /* @__PURE__ */ new Set([
729
+ "node_modules",
730
+ "dist",
731
+ "build",
732
+ "__pycache__",
733
+ ".next",
734
+ ".nuxt",
735
+ ".turbo",
736
+ ".cache",
737
+ "coverage",
738
+ "out",
739
+ "target",
740
+ "vendor",
741
+ ".venv",
742
+ "venv",
743
+ ".tox",
744
+ "site-packages",
745
+ "bower_components",
746
+ ".pytest_cache",
747
+ ".mypy_cache",
748
+ ".ruff_cache"
749
+ ]);
750
+ var _GENERATED_PATH_PENALTY = 0.5;
751
+ var _ARCHIVE_PATH_SEGMENTS = /* @__PURE__ */ new Set([
752
+ "archive",
753
+ "archived",
754
+ "old",
755
+ "deprecated",
756
+ "plans",
757
+ "drafts"
758
+ ]);
759
+ var _ARCHIVE_FILE_RE = /(^changelog|\.bak$|\.orig$)/i;
760
+ var _DOCS_FILE_RE = /\.md$/i;
761
+ var _DOCS_DIR_SEGMENT = "docs";
762
+ var _VERBATIM_TOKEN_BOOST = 0.05;
763
+ var _MAX_VERBATIM_BOOST = 0.25;
764
+ var _TOKEN_RE = /\w+/g;
765
+ var _MIN_TOKEN_LEN = 3;
766
+ var OVER_FETCH_FACTOR = 4;
767
+ var MAX_OVER_FETCH = 100;
768
+ var VEC_MAX_K = 4096;
769
+ function isAvailable() {
770
+ return isRuntimeAvailable();
771
+ }
772
+ function embeddingBackendLoadError() {
773
+ return runtimeLoadError();
774
+ }
775
+ async function embedTexts(texts, modelName = DEFAULT_MODEL) {
776
+ if (!isAvailable()) {
777
+ throw new Error(
778
+ `Embedding backend not available: ${runtimeLoadError()?.message ?? "unknown error"}`
779
+ );
780
+ }
781
+ if (texts.length === 0) {
782
+ return [];
783
+ }
784
+ let extractorPromise = _extractorCache.get(modelName);
785
+ if (!extractorPromise) {
786
+ const pipelineFn = _pipelineFnOverride ?? inHousePipelineFn;
787
+ extractorPromise = buildExtractorWithRetry(pipelineFn, modelName);
788
+ _extractorCache.set(modelName, extractorPromise);
789
+ extractorPromise.catch(() => {
790
+ if (_extractorCache.get(modelName) === extractorPromise) _extractorCache.delete(modelName);
791
+ });
792
+ }
793
+ const extractor = await extractorPromise;
794
+ const vecs = [];
795
+ const expectedDim = DEFAULT_DIM;
796
+ try {
797
+ for (const text of texts) {
798
+ const embedding = await extractor(text, {
799
+ pooling: "mean",
800
+ normalize: true
801
+ });
802
+ let vec;
803
+ const embeddingObj = embedding;
804
+ const embData = embeddingObj?.["data"];
805
+ if (Array.isArray(embData)) {
806
+ vec = Array.from(embData);
807
+ } else if (embData instanceof Float32Array) {
808
+ vec = Array.from(embData);
809
+ } else {
810
+ throw new Error(
811
+ `Unexpected embedding type: ${typeof embData}`
812
+ );
813
+ }
814
+ if (vec.length !== expectedDim) {
815
+ throw new Error(
816
+ `Dimension mismatch: model returned ${vec.length}-dim vector, expected ${expectedDim}`
817
+ );
818
+ }
819
+ const badIndex = vec.findIndex((component) => !Number.isFinite(component));
820
+ if (badIndex !== -1) {
821
+ throw new Error(
822
+ `Non-finite embedding component at index ${badIndex}: model returned ${String(vec[badIndex])}`
823
+ );
824
+ }
825
+ vecs.push(vec);
826
+ }
827
+ } finally {
828
+ }
829
+ return vecs;
830
+ }
831
+ function packVec(vec) {
832
+ const view = new Float32Array(vec.length);
833
+ for (const [i, val] of vec.entries()) {
834
+ view[i] = val;
835
+ if (!Number.isFinite(view[i])) {
836
+ throw new Error(
837
+ `Non-finite embedding component at index ${i}: ${String(val)} is not representable as a 32-bit float`
838
+ );
839
+ }
840
+ }
841
+ return Buffer.from(view.buffer);
842
+ }
843
+ function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize, overlap, kind) {
844
+ const chunks = [];
845
+ let currentChunk = "";
846
+ let startLine = rangeStart;
847
+ let currentLine = rangeStart;
848
+ for (let lineNo = rangeStart; lineNo <= rangeEnd; lineNo++) {
849
+ const line = lines2[lineNo - 1] ?? "";
850
+ const lineWithNewline = line + "\n";
851
+ if (currentChunk.length + lineWithNewline.length > chunkSize && currentChunk.length > 0) {
852
+ const trimmedLength = currentChunk.trim().length;
853
+ const currentChunkTooSmall = trimmedLength < MIN_CHUNK_CHARS;
854
+ if (!currentChunkTooSmall) {
855
+ chunks.push({
856
+ filePath,
857
+ startLine,
858
+ endLine: currentLine - 1,
859
+ text: currentChunk.trim(),
860
+ kind
861
+ });
862
+ }
863
+ let overlapChars = 0;
864
+ let computedOverlapStart = currentLine;
865
+ while (computedOverlapStart > rangeStart) {
866
+ const candidateChars = (lines2[computedOverlapStart - 2] ?? "").length + 1;
867
+ if (overlapChars + candidateChars > overlap) break;
868
+ overlapChars += candidateChars;
869
+ computedOverlapStart--;
870
+ }
871
+ const droppedChunkHadContent = currentChunkTooSmall && trimmedLength > 0;
872
+ const overlapStart = droppedChunkHadContent ? Math.min(computedOverlapStart, startLine) : computedOverlapStart;
873
+ if (overlapStart !== startLine) {
874
+ const overlapText = lines2.slice(overlapStart - 1, currentLine - 1).join("\n");
875
+ currentChunk = overlapText + "\n";
876
+ startLine = overlapStart;
877
+ }
878
+ }
879
+ currentChunk += lineWithNewline;
880
+ currentLine++;
881
+ }
882
+ if (currentChunk.trim().length >= MIN_CHUNK_CHARS) {
883
+ chunks.push({
884
+ filePath,
885
+ startLine,
886
+ endLine: rangeEnd,
887
+ text: currentChunk.trim(),
888
+ kind
889
+ });
890
+ } else if (currentChunk.length > 0 && chunks.length > 0) {
891
+ const last = chunks[chunks.length - 1];
892
+ last.endLine = rangeEnd;
893
+ last.text = lines2.slice(last.startLine - 1, rangeEnd).join("\n").trim();
894
+ }
895
+ return chunks;
896
+ }
897
+ function chunkFile(filePath, content, chunkSize = MAX_CHUNK_CHARS, overlap = 200, boundaries = []) {
898
+ const lines2 = content.split(/\r?\n/);
899
+ if (lines2.length > 1 && lines2[lines2.length - 1] === "") lines2.pop();
900
+ const totalLines = lines2.length;
901
+ if (boundaries.length === 0) {
902
+ return splitRangeIntoChunks(filePath, lines2, 1, totalLines, chunkSize, overlap, "window");
903
+ }
904
+ const clipped = boundaries.map((b) => ({
905
+ start: Math.max(1, Math.min(b.start, totalLines)),
906
+ end: Math.max(1, Math.min(b.end, totalLines)),
907
+ kind: b.kind
908
+ })).filter((b) => b.end >= b.start).sort((a, b) => a.start - b.start || b.end - a.end);
909
+ if (clipped.length === 0) {
910
+ return splitRangeIntoChunks(filePath, lines2, 1, totalLines, chunkSize, overlap, "window");
911
+ }
912
+ const flattened = [];
913
+ let openEnd = 0;
914
+ for (const b of clipped) {
915
+ const start = Math.max(b.start, openEnd + 1);
916
+ if (start > b.end) continue;
917
+ flattened.push({ ...b, start });
918
+ openEnd = b.end;
919
+ }
920
+ const gapLength = (start, end) => {
921
+ let text = "";
922
+ for (let lineNo = start; lineNo <= end; lineNo++) text += (lines2[lineNo - 1] ?? "") + "\n";
923
+ return text.trim().length;
924
+ };
925
+ const ranges = [];
926
+ let cursor = 1;
927
+ for (const b of flattened) {
928
+ let boundaryStart = b.start;
929
+ let boundaryKind = b.kind;
930
+ const gapStart = cursor;
931
+ const gapEnd = b.start - 1;
932
+ if (gapEnd >= gapStart) {
933
+ if (gapLength(gapStart, gapEnd) < MIN_CHUNK_CHARS) {
934
+ const prev = ranges[ranges.length - 1];
935
+ if (prev !== void 0) {
936
+ prev.end = gapEnd;
937
+ } else {
938
+ boundaryStart = gapStart;
939
+ boundaryKind = "window";
940
+ }
941
+ } else {
942
+ ranges.push({ start: gapStart, end: gapEnd, kind: "window" });
943
+ }
944
+ }
945
+ ranges.push({ start: boundaryStart, end: b.end, kind: boundaryKind });
946
+ cursor = b.end + 1;
947
+ }
948
+ if (cursor <= totalLines) {
949
+ const gapStart = cursor;
950
+ const gapEnd = totalLines;
951
+ if (gapLength(gapStart, gapEnd) < MIN_CHUNK_CHARS) {
952
+ ranges[ranges.length - 1].end = gapEnd;
953
+ } else {
954
+ ranges.push({ start: gapStart, end: gapEnd, kind: "window" });
955
+ }
956
+ }
957
+ let i = 0;
958
+ while (i < ranges.length) {
959
+ const r = ranges[i];
960
+ if (gapLength(r.start, r.end) >= MIN_CHUNK_CHARS) {
961
+ i++;
962
+ continue;
963
+ }
964
+ if (i > 0) {
965
+ ranges[i - 1].end = r.end;
966
+ ranges.splice(i, 1);
967
+ i--;
968
+ } else if (ranges.length > 1) {
969
+ ranges[i + 1].start = r.start;
970
+ ranges.splice(i, 1);
971
+ } else {
972
+ i++;
973
+ }
974
+ }
975
+ const chunks = [];
976
+ for (const r of ranges) {
977
+ chunks.push(...splitRangeIntoChunks(filePath, lines2, r.start, r.end, chunkSize, overlap, r.kind));
978
+ }
979
+ return chunks;
980
+ }
981
+ function insertChunkVector(stmt, rowid, embedding) {
982
+ stmt.run(BigInt(rowid), packVec(embedding));
983
+ }
984
+ async function upsertChunks(db, chunks) {
985
+ if (chunks.length === 0) {
986
+ return "embedded";
987
+ }
988
+ const filePath = chunks[0].filePath;
989
+ if (!isAvailable()) {
990
+ console.warn("Embeddings not available; skipping semantic indexing");
991
+ deleteFileEmbeddings(db, filePath);
992
+ return "unavailable";
993
+ }
994
+ if (!chunkVectorsTableExists(db)) {
995
+ deleteFileEmbeddings(db, filePath);
996
+ return "unavailable";
997
+ }
998
+ ensureEmbeddingProvenance(db);
999
+ const texts = chunks.map((c) => c.text);
1000
+ const embeddings = await embedTexts(texts);
1001
+ const chunkInsertStmt = db.prepare(`
1002
+ INSERT INTO chunks (file_path, start_line, end_line, text, kind)
1003
+ VALUES (?, ?, ?, ?, ?)
1004
+ `);
1005
+ const vectorInsertStmt = db.prepare(`
1006
+ INSERT INTO chunk_vectors (rowid, embedding)
1007
+ VALUES (?, ?)
1008
+ `);
1009
+ const tx = db.transaction(() => {
1010
+ deleteFileEmbeddings(db, filePath);
1011
+ for (let i = 0; i < chunks.length; i++) {
1012
+ const chunk = chunks[i];
1013
+ const embedding = embeddings[i];
1014
+ if (!chunk || !embedding) {
1015
+ continue;
1016
+ }
1017
+ const chunkResult = chunkInsertStmt.run(
1018
+ chunk.filePath,
1019
+ chunk.startLine,
1020
+ chunk.endLine,
1021
+ chunk.text,
1022
+ chunk.kind
1023
+ );
1024
+ insertChunkVector(vectorInsertStmt, chunkResult.lastInsertRowid, embedding);
1025
+ }
1026
+ });
1027
+ tx.immediate();
1028
+ return "embedded";
1029
+ }
1030
+ var BACKFILL_MULTIPLIER = 3;
1031
+ function countStoredVectors(db) {
1032
+ const row = db.prepare("SELECT count(*) AS n FROM chunks").get();
1033
+ return typeof row?.n === "number" && Number.isFinite(row.n) ? row.n : 0;
1034
+ }
1035
+ function fetchScopedHits(db, queryVec, k, maxDistance, rootDir) {
1036
+ const stmt = db.prepare(`
1037
+ SELECT rowid, distance FROM chunk_vectors
1038
+ WHERE embedding MATCH ?
1039
+ AND k = ?
1040
+ ORDER BY distance ASC
1041
+ `);
1042
+ const rows = stmt.all(packVec(queryVec), k);
1043
+ if (!rows || rows.length === 0) {
1044
+ return { hits: [], candidateCount: 0 };
1045
+ }
1046
+ const scope = rootDir !== void 0 ? projectScopeClause("file_path") : void 0;
1047
+ const chunkSql = scope !== void 0 ? `SELECT file_path, start_line, end_line, text, kind FROM chunks WHERE id = ? AND ${scope.clause}` : `SELECT file_path, start_line, end_line, text, kind FROM chunks WHERE id = ?`;
1048
+ const chunkStmt = db.prepare(chunkSql);
1049
+ const scopeParams = scope !== void 0 && rootDir !== void 0 ? scope.params(rootDir) : void 0;
1050
+ const hits = [];
1051
+ for (const row of rows) {
1052
+ if (!row) {
1053
+ continue;
1054
+ }
1055
+ if (typeof row.distance === "number" && Number.isFinite(row.distance) && row.distance <= maxDistance) {
1056
+ const chunk = scopeParams !== void 0 ? chunkStmt.get(row.rowid, ...scopeParams) : chunkStmt.get(row.rowid);
1057
+ if (chunk) {
1058
+ hits.push({
1059
+ filePath: chunk.file_path,
1060
+ startLine: chunk.start_line,
1061
+ endLine: chunk.end_line,
1062
+ kind: chunk.kind,
1063
+ distance: row.distance,
1064
+ text: chunk.text
1065
+ });
1066
+ }
1067
+ }
1068
+ }
1069
+ return { hits, candidateCount: rows.length };
1070
+ }
1071
+ function fetchScopedExactHits(db, queryVec, limit, maxDistance, rootDir) {
1072
+ const scope = projectScopeClause("c.file_path");
1073
+ const rows = db.prepare(
1074
+ `SELECT file_path, start_line, end_line, text, kind, distance FROM (
1075
+ SELECT c.id AS id, c.file_path AS file_path, c.start_line AS start_line, c.end_line AS end_line, c.text AS text, c.kind AS kind, vec_distance_L2(v.embedding, ?) AS distance
1076
+ FROM chunks c JOIN chunk_vectors v ON v.rowid = c.id
1077
+ WHERE ${scope.clause}
1078
+ )
1079
+ WHERE distance IS NOT NULL AND distance <= ?
1080
+ ORDER BY distance ASC, file_path ASC, start_line ASC, id ASC
1081
+ LIMIT ?`
1082
+ ).all(packVec(queryVec), ...scope.params(rootDir), maxDistance, limit);
1083
+ const hits = [];
1084
+ for (const row of rows) {
1085
+ if (!row || typeof row.distance !== "number" || !Number.isFinite(row.distance) || row.distance > maxDistance) {
1086
+ continue;
1087
+ }
1088
+ hits.push({
1089
+ filePath: row.file_path,
1090
+ startLine: row.start_line,
1091
+ endLine: row.end_line,
1092
+ kind: row.kind,
1093
+ distance: row.distance,
1094
+ text: row.text
1095
+ });
1096
+ }
1097
+ return hits;
1098
+ }
1099
+ async function searchSemantic(db, query, topK = 8, modelName = DEFAULT_MODEL, maxDistance = DEFAULT_DISTANCE_THRESHOLD, rootDir) {
1100
+ if (!isAvailable()) {
1101
+ return [];
1102
+ }
1103
+ if (query.trim().length === 0) {
1104
+ return [];
1105
+ }
1106
+ if (!chunkVectorsTableExists(db)) {
1107
+ return [];
1108
+ }
1109
+ ensureEmbeddingProvenance(db, modelName);
1110
+ const queryEmbeddings = await embedTexts([`${QUERY_INSTRUCTION_PREFIX}${query}`], modelName);
1111
+ if (queryEmbeddings.length === 0) {
1112
+ return [];
1113
+ }
1114
+ const queryVec = queryEmbeddings[0];
1115
+ if (!queryVec) {
1116
+ return [];
1117
+ }
1118
+ const overFetchK = Math.min(
1119
+ MAX_OVER_FETCH,
1120
+ Math.ceil(topK * OVER_FETCH_FACTOR)
1121
+ );
1122
+ let k = Math.min(overFetchK, VEC_MAX_K);
1123
+ let pass = fetchScopedHits(db, queryVec, k, maxDistance, rootDir);
1124
+ let hits = pass.hits;
1125
+ if (rootDir !== void 0) {
1126
+ const ceiling = Math.min(countStoredVectors(db), VEC_MAX_K);
1127
+ while (hits.length < topK && pass.candidateCount === k && k < ceiling) {
1128
+ k = Math.min(ceiling, k * BACKFILL_MULTIPLIER);
1129
+ pass = fetchScopedHits(db, queryVec, k, maxDistance, rootDir);
1130
+ hits = pass.hits;
1131
+ }
1132
+ if (hits.length < topK && pass.candidateCount === k && k >= ceiling && countStoredVectors(db) > VEC_MAX_K) {
1133
+ hits = fetchScopedExactHits(db, queryVec, VEC_MAX_K, maxDistance, rootDir);
1134
+ }
1135
+ }
1136
+ return rerankHits(hits, query, topK);
1137
+ }
1138
+ function rerankHits(hits, query, topK) {
1139
+ const queryTokens = _extractQueryTokens(query);
1140
+ const scored = hits.map((hit, index) => {
1141
+ let boost = 0;
1142
+ if (queryTokens.size > 0) {
1143
+ const hitTokens = _extractQueryTokens(hit.text);
1144
+ let matches = 0;
1145
+ for (const token of queryTokens) {
1146
+ if (hitTokens.has(token)) {
1147
+ matches++;
1148
+ }
1149
+ }
1150
+ boost = Math.min(matches * _VERBATIM_TOKEN_BOOST, _MAX_VERBATIM_BOOST);
1151
+ }
1152
+ const penalty = _isGeneratedPath(hit.filePath) ? _GENERATED_PATH_PENALTY : 0;
1153
+ const pathPenalty = _pathPriorityPenalty(hit.filePath);
1154
+ return { hit, index, adjusted: hit.distance - boost + penalty + pathPenalty };
1155
+ });
1156
+ scored.sort((a, b) => a.adjusted - b.adjusted || a.index - b.index);
1157
+ return scored.slice(0, topK).map((entry) => ({ ...entry.hit, adjustedDistance: entry.adjusted }));
1158
+ }
1159
+ function mergeNearbyHits(hits, proximity = 20) {
1160
+ if (hits.length <= 1) {
1161
+ return hits;
1162
+ }
1163
+ const byFile = /* @__PURE__ */ new Map();
1164
+ for (const hit of hits) {
1165
+ const fileHits = byFile.get(hit.filePath);
1166
+ if (fileHits) {
1167
+ fileHits.push(hit);
1168
+ } else {
1169
+ byFile.set(hit.filePath, [hit]);
1170
+ }
1171
+ }
1172
+ const merged = [];
1173
+ for (const fileHits of byFile.values()) {
1174
+ fileHits.sort((a, b) => a.startLine - b.startLine);
1175
+ let current = fileHits[0];
1176
+ if (!current) {
1177
+ continue;
1178
+ }
1179
+ let curStart = current.startLine;
1180
+ let curEnd = current.endLine;
1181
+ let curDist = current.distance;
1182
+ let curAdjusted = current.adjustedDistance ?? current.distance;
1183
+ const mergedTexts = [current.text];
1184
+ for (let i = 1; i < fileHits.length; i++) {
1185
+ const hit = fileHits[i];
1186
+ if (!hit) {
1187
+ continue;
1188
+ }
1189
+ const gap = hit.startLine - curEnd - 1;
1190
+ if (gap <= proximity) {
1191
+ curEnd = Math.max(curEnd, hit.endLine);
1192
+ curDist = Math.min(curDist, hit.distance);
1193
+ curAdjusted = Math.min(curAdjusted, hit.adjustedDistance ?? hit.distance);
1194
+ mergedTexts.push(hit.text);
1195
+ } else {
1196
+ merged.push({
1197
+ filePath: current.filePath,
1198
+ startLine: curStart,
1199
+ endLine: curEnd,
1200
+ kind: current.kind,
1201
+ distance: curDist,
1202
+ adjustedDistance: curAdjusted,
1203
+ text: mergedTexts.join("\n---\n")
1204
+ });
1205
+ current = hit;
1206
+ curStart = hit.startLine;
1207
+ curEnd = hit.endLine;
1208
+ curDist = hit.distance;
1209
+ curAdjusted = hit.adjustedDistance ?? hit.distance;
1210
+ mergedTexts.length = 0;
1211
+ mergedTexts.push(hit.text);
1212
+ }
1213
+ }
1214
+ merged.push({
1215
+ filePath: current.filePath,
1216
+ startLine: curStart,
1217
+ endLine: curEnd,
1218
+ kind: current.kind,
1219
+ distance: curDist,
1220
+ adjustedDistance: curAdjusted,
1221
+ text: mergedTexts.join("\n---\n")
1222
+ });
1223
+ }
1224
+ merged.sort((a, b) => (a.adjustedDistance ?? a.distance) - (b.adjustedDistance ?? b.distance));
1225
+ return merged;
1226
+ }
1227
+ async function indexFile(db, filePath, content, boundaries = []) {
1228
+ const chunks = chunkFile(filePath, content, void 0, void 0, boundaries);
1229
+ if (chunks.length > 0) {
1230
+ return upsertChunks(db, chunks);
1231
+ }
1232
+ deleteFileEmbeddings(db, filePath);
1233
+ return "embedded";
1234
+ }
1235
+ var _chunkVectorsUsable = /* @__PURE__ */ new WeakMap();
1236
+ function chunkVectorsTableExists(db) {
1237
+ const cached = _chunkVectorsUsable.get(db);
1238
+ if (cached !== void 0) {
1239
+ return cached;
1240
+ }
1241
+ let usable;
1242
+ try {
1243
+ db.prepare("SELECT rowid FROM chunk_vectors LIMIT 1").get();
1244
+ usable = true;
1245
+ } catch {
1246
+ usable = false;
1247
+ }
1248
+ _chunkVectorsUsable.set(db, usable);
1249
+ return usable;
1250
+ }
1251
+ function embeddingsDepsAvailable(db) {
1252
+ return isAvailable() && chunkVectorsTableExists(db);
1253
+ }
1254
+ function deleteFileEmbeddings(db, filePath) {
1255
+ const folded = foldPath(filePath);
1256
+ if (chunkVectorsTableExists(db)) {
1257
+ const ids = db.prepare(`SELECT id FROM chunks WHERE ${pathEqClause("file_path")}`).pluck().all(folded);
1258
+ const deleteVector = db.prepare("DELETE FROM chunk_vectors WHERE rowid = ?");
1259
+ for (const id of ids) deleteVector.run(id);
1260
+ }
1261
+ db.prepare(`DELETE FROM chunks WHERE ${pathEqClause("file_path")}`).run(folded);
1262
+ }
1263
+ function resetAllEmbeddings(db) {
1264
+ const paths = db.prepare("SELECT DISTINCT file_path FROM chunks").pluck().all();
1265
+ const clearEmbedSha = db.prepare(`UPDATE files SET embed_sha = NULL WHERE ${pathEqClause("path")}`);
1266
+ const tx = db.transaction(() => {
1267
+ for (const p of paths) deleteFileEmbeddings(db, p);
1268
+ for (const p of paths) clearEmbedSha.run(foldPath(p));
1269
+ });
1270
+ tx.immediate();
1271
+ return paths.length;
1272
+ }
1273
+ function embeddingProvenance(modelName = DEFAULT_MODEL) {
1274
+ const revision = modelName === DEFAULT_MODEL ? PINNED_MODEL_REVISION.slice(0, 12) : "unpinned";
1275
+ return `${modelName}@${revision}/${backendId()}/embed-${EMBED_FINGERPRINT}`;
1276
+ }
1277
+ function backendId() {
1278
+ return `onnxruntime-node@${majorMinor(runtimeVersion())}`;
1279
+ }
1280
+ function majorMinor(version) {
1281
+ const parts = version.split(".");
1282
+ return parts.length >= 2 ? `${parts[0]}.${parts[1]}` : version;
1283
+ }
1284
+ var _provenanceChecked = /* @__PURE__ */ new WeakSet();
1285
+ function ensureEmbeddingProvenance(db, modelName = DEFAULT_MODEL) {
1286
+ if (_provenanceChecked.has(db)) return;
1287
+ _provenanceChecked.add(db);
1288
+ const current = embeddingProvenance(modelName);
1289
+ const stored = db.prepare("SELECT provenance FROM embedding_provenance WHERE id = 1").pluck().get();
1290
+ if (stored === current) return;
1291
+ const cleared = resetAllEmbeddings(db);
1292
+ db.prepare(
1293
+ "INSERT INTO embedding_provenance (id, provenance) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET provenance = excluded.provenance"
1294
+ ).run(current);
1295
+ if (cleared > 0) {
1296
+ console.warn(
1297
+ `Embedding stack changed (${stored ?? "unrecorded"} -> ${current}); discarded ${cleared} file${cleared === 1 ? "" : "s"} worth of vectors because they no longer reliably describe this file's current chunks. Run \`token-goat index\` to rebuild them.`
1298
+ );
1299
+ }
1300
+ }
1301
+ function _extractQueryTokens(query) {
1302
+ const tokens = /* @__PURE__ */ new Set();
1303
+ const matches = query.matchAll(_TOKEN_RE);
1304
+ for (const match of matches) {
1305
+ const token = match[0].toLowerCase();
1306
+ if (token.length >= _MIN_TOKEN_LEN) {
1307
+ tokens.add(token);
1308
+ }
1309
+ }
1310
+ return tokens;
1311
+ }
1312
+ function _isGeneratedPath(filePath) {
1313
+ const segments = filePath.split(/[/\\]+/);
1314
+ for (const seg of segments) {
1315
+ if (_GENERATED_PATH_SEGMENTS.has(seg.toLowerCase())) {
1316
+ return true;
1317
+ }
1318
+ }
1319
+ return false;
1320
+ }
1321
+ function _pathPriorityPenalty(filePath) {
1322
+ const segments = filePath.split(/[/\\]+/);
1323
+ const basename2 = segments[segments.length - 1] ?? filePath;
1324
+ const weights = loadConfig().semantic;
1325
+ const isArchive = _ARCHIVE_FILE_RE.test(basename2) || segments.some((seg) => _ARCHIVE_PATH_SEGMENTS.has(seg.toLowerCase()));
1326
+ if (isArchive) {
1327
+ return 1 - weights.archive_weight;
1328
+ }
1329
+ const isDocs = _DOCS_FILE_RE.test(basename2) || segments.some((seg) => seg.toLowerCase() === _DOCS_DIR_SEGMENT);
1330
+ if (isDocs) {
1331
+ return 1 - weights.docs_weight;
1332
+ }
1333
+ return 0;
1334
+ }
1335
+
1336
+ // src/parser_fingerprint.ts
1337
+ init_define_import_meta_env();
1338
+ var PARSER_FINGERPRINT = "ef41adb465538db6";
1339
+
1340
+ // src/parser.ts
1341
+ init_define_import_meta_env();
1342
+ import * as fs2 from "node:fs";
1343
+ import { createRequire as createRequire2 } from "node:module";
1344
+ import * as path4 from "node:path";
1345
+
1346
+ // src/embedding_boundaries.ts
1347
+ init_define_import_meta_env();
1348
+
1349
+ // src/hints/markdown_hints.ts
1350
+ init_define_import_meta_env();
1351
+
1352
+ // src/markdown_lines.ts
1353
+ init_define_import_meta_env();
1354
+ function* eachUnfencedLine(lines2) {
1355
+ let fence = null;
1356
+ for (let i = 0; i < lines2.length; i++) {
1357
+ const line = lines2[i];
1358
+ if (line === void 0) continue;
1359
+ const fm = /^\s*(`{3,}|~{3,})([^\n]*)$/.exec(line);
1360
+ if (fm !== null && fm[1] !== void 0) {
1361
+ const run = fm[1];
1362
+ const ch = run[0] ?? "";
1363
+ const rest = fm[2] ?? "";
1364
+ if (fence === null) {
1365
+ fence = { ch, len: run.length };
1366
+ } else if (ch === fence.ch && run.length >= fence.len && rest.trim() === "") {
1367
+ fence = null;
1368
+ }
1369
+ continue;
1370
+ }
1371
+ if (fence !== null) continue;
1372
+ yield [i, line];
1373
+ }
1374
+ }
1375
+
1376
+ // src/hints/markdown_hints.ts
1377
+ var MARKDOWN_SIZE_THRESHOLD = 8e3;
1378
+ var MAX_HEADINGS = 40;
1379
+ var MAX_OUTPUT_LINES = 60;
1380
+ function extractMarkdownHeadings(content, limit = MAX_HEADINGS) {
1381
+ const headings = [];
1382
+ const lines2 = content.split("\n");
1383
+ const unfenced = Array.from(eachUnfencedLine(lines2));
1384
+ for (let u = 0; u < unfenced.length; u++) {
1385
+ const [i, line] = unfenced[u];
1386
+ if (!line) continue;
1387
+ const match = /^(#+)\s+([^\r\n]+?)(?:\s+#+)?\s*$/.exec(line);
1388
+ if (match && match.length >= 3) {
1389
+ const hashes = match[1];
1390
+ const headingText = match[2];
1391
+ const level = hashes.length;
1392
+ const maxLevel = limit === Infinity ? 6 : 3;
1393
+ if (level <= maxLevel) {
1394
+ const text = headingText.trim();
1395
+ if (text) {
1396
+ headings.push({
1397
+ level,
1398
+ text,
1399
+ lineNumber: i + 1
1400
+ });
1401
+ if (headings.length >= limit) break;
1402
+ }
1403
+ }
1404
+ continue;
1405
+ }
1406
+ const trimmed = line.trim();
1407
+ if (trimmed !== "" && !trimmed.startsWith("#") && !trimmed.startsWith("|") && !trimmed.startsWith("```") && !trimmed.startsWith("~~~") && !/^([-*+]|\d+\.)\s/.test(trimmed) && u + 1 < unfenced.length) {
1408
+ const [nextIdx, nextLine] = unfenced[u + 1];
1409
+ if (nextIdx === i + 1) {
1410
+ let setextLevel = 0;
1411
+ if (/^\s*(=+)\s*$/.test(nextLine)) setextLevel = 1;
1412
+ else if (/^\s*(-+)\s*$/.test(nextLine)) setextLevel = 2;
1413
+ if (setextLevel > 0) {
1414
+ headings.push({
1415
+ level: setextLevel,
1416
+ text: trimmed,
1417
+ lineNumber: i + 1
1418
+ });
1419
+ u++;
1420
+ if (headings.length >= limit) break;
1421
+ }
1422
+ }
1423
+ }
1424
+ }
1425
+ return headings;
1426
+ }
1427
+ function formatHeadingTreeParts(headings, filePath) {
1428
+ if (headings.length === 0) return { guidance: "", sectionsList: "" };
1429
+ const seenTexts = /* @__PURE__ */ new Map();
1430
+ const dedupedHeadings = [];
1431
+ for (const h of headings) {
1432
+ const count = (seenTexts.get(h.text) ?? 0) + 1;
1433
+ seenTexts.set(h.text, count);
1434
+ const suffix = count > 1 ? ` #${count}` : "";
1435
+ dedupedHeadings.push({
1436
+ text: h.text + suffix,
1437
+ level: h.level
1438
+ });
1439
+ }
1440
+ const guidanceLines = [];
1441
+ guidanceLines.push(`Large markdown file (${headings.length} headings). Use token-goat section to read a specific section:`);
1442
+ guidanceLines.push(` token-goat section "${filePath}::Heading Name"`);
1443
+ guidanceLines.push(` Tip: an unambiguous heading prefix also resolves (e.g. "Lesson 16" instead of the full heading text) \u2014 shorter to type and avoids shell-quoting issues with punctuation in long headings.`);
1444
+ guidanceLines.push(``);
1445
+ guidanceLines.push(`Sections:`);
1446
+ const sectionLines = [];
1447
+ let headingsAdded = 0;
1448
+ for (const h of dedupedHeadings) {
1449
+ if (guidanceLines.length + sectionLines.length + 1 >= MAX_OUTPUT_LINES) {
1450
+ const remaining = dedupedHeadings.length - headingsAdded;
1451
+ sectionLines.push(` ... (${remaining} more headings)`);
1452
+ break;
1453
+ }
1454
+ const indent = h.level === 1 ? "" : h.level === 2 ? " " : " ";
1455
+ const marker = "#".repeat(h.level);
1456
+ sectionLines.push(` ${indent}${marker} ${displaySafeText(h.text)}`);
1457
+ headingsAdded++;
1458
+ }
1459
+ return { guidance: guidanceLines.join("\n"), sectionsList: sectionLines.join("\n") };
1460
+ }
1461
+ var WELL_KNOWN_SECTIONS = {
1462
+ "CHANGELOG.md": ["Unreleased"],
1463
+ "README.md": ["Install", "Usage", "API", "Configuration", "Getting Started"],
1464
+ "CONTRIBUTING.md": ["Setup", "Commands", "Testing", "Development"],
1465
+ "CLAUDE.md": ["Commands", "Architecture"],
1466
+ "CLAUDE.arch.md": ["Component Map", "Architecture"]
1467
+ };
1468
+ function getWellKnownSections(basename2) {
1469
+ return WELL_KNOWN_SECTIONS[basename2] ?? [];
1470
+ }
1471
+ function extractChangelogVersionHint(content, filePath) {
1472
+ const lines2 = content.split("\n");
1473
+ let foundUnreleased = false;
1474
+ let firstVersion = null;
1475
+ for (const line of lines2) {
1476
+ const m = /^##\s+(\[?[\d]+\.[\d]+\.[\d]+\]?)/.exec(line);
1477
+ if (m) {
1478
+ if (foundUnreleased) {
1479
+ const ver = m[1];
1480
+ return ` | token-goat section "${filePath}::${ver}"`;
1481
+ }
1482
+ if (firstVersion === null) firstVersion = m[1];
1483
+ }
1484
+ if (/^##\s+\[?unreleased\]?/i.test(line)) {
1485
+ foundUnreleased = true;
1486
+ }
1487
+ }
1488
+ if (!foundUnreleased && firstVersion !== null) {
1489
+ return ` | token-goat section "${filePath}::${firstVersion}"`;
1490
+ }
1491
+ return "";
1492
+ }
1493
+
1494
+ // src/embedding_boundaries.ts
1495
+ function buildEmbeddingBoundaries(filePath, content, dbPath) {
1496
+ if (detectLanguage(filePath) === "markdown") {
1497
+ const headings = extractMarkdownHeadings(content, Infinity);
1498
+ return headings.map((h, i) => ({
1499
+ start: h.lineNumber,
1500
+ // Runs to just before the next heading, or to end-of-file for the last one. chunkFile clips end values to the file's actual line count, so this sentinel is safe without re-deriving the file's line count here.
1501
+ end: headings[i + 1] !== void 0 ? headings[i + 1].lineNumber - 1 : Number.MAX_SAFE_INTEGER,
1502
+ kind: "section"
1503
+ }));
1504
+ }
1505
+ const symbols = querySymbols({ filePath, limit: Number.MAX_SAFE_INTEGER }, dbPath);
1506
+ return symbols.map((s) => ({ start: s.lineStart, end: s.lineEnd, kind: "symbol" }));
1507
+ }
1508
+
1509
+ // src/languages/salesforce_frontend.ts
1510
+ init_define_import_meta_env();
1511
+ import * as path3 from "node:path";
1512
+ function lines(content) {
1513
+ return content.split("\n");
1514
+ }
1515
+ function bundleName(filePath) {
1516
+ const normalized = filePath.replaceAll("\\", "/");
1517
+ const parent = path3.posix.basename(path3.posix.dirname(normalized));
1518
+ const base = path3.posix.basename(normalized).replace(/\.[^.]+$/, "");
1519
+ return parent === "lwc" || parent === "aura" ? base : parent;
1520
+ }
1521
+ function lwcTagAlias(name) {
1522
+ const kebab = name.replace(/([A-Z]+)([A-Z][a-z])/g, "$1-$2").replace(/([a-z\d])([A-Z])/g, "$1-$2").toLowerCase();
1523
+ return `c-${kebab}`;
1524
+ }
1525
+ function symbol(filePath, name, kind, lineStart, lineEnd = lineStart) {
1526
+ return { filePath, name, kind, lineStart, lineEnd, body: "", docstring: "", parent: "" };
1527
+ }
1528
+ function ref(filePath, name, line, col, context) {
1529
+ return { filePath, name, line, col, context };
1530
+ }
1531
+ function dedupe(values, key) {
1532
+ const seen = /* @__PURE__ */ new Set();
1533
+ return values.filter((value) => {
1534
+ const id = key(value);
1535
+ if (seen.has(id)) return false;
1536
+ seen.add(id);
1537
+ return true;
1538
+ });
1539
+ }
1540
+ function extractLwcJavaScript(content, filePath) {
1541
+ const sourceLines = lines(content);
1542
+ const bundle = bundleName(filePath);
1543
+ const symbols = [
1544
+ symbol(filePath, bundle, "lwc_bundle", 1, countContentLines(content)),
1545
+ symbol(filePath, lwcTagAlias(bundle), "lwc_component_alias", 1, countContentLines(content))
1546
+ ];
1547
+ const refs = [];
1548
+ const commentFree = stripJsComments(content);
1549
+ const apiRe = /@api\s*(?:\r?\n\s*)?(?:(get|set)\s+)?(?:async\s+)?([A-Za-z_$][\w$]*)\s*(\()?/g;
1550
+ for (const match of commentFree.matchAll(apiRe)) {
1551
+ const before = commentFree.slice(0, match.index ?? 0);
1552
+ const line = before.split("\n").length;
1553
+ const kind = match[3] && !match[1] ? "lwc_api_method" : "lwc_api_property";
1554
+ symbols.push(symbol(filePath, match[2] ?? "", kind, line));
1555
+ }
1556
+ const importRe = /from\s+['"]@salesforce\/(apex|schema|label|resourceUrl|messageChannel|customPermission|userPermission)\/([^'"]+)['"]/g;
1557
+ for (const match of commentFree.matchAll(importRe)) {
1558
+ const offset = match.index ?? 0;
1559
+ const line = commentFree.slice(0, offset).split("\n").length;
1560
+ const context = sourceLines[line - 1]?.trim() ?? "";
1561
+ const target = match[2] ?? "";
1562
+ const targetOffset = offset + (match[0]?.indexOf(target) ?? 0);
1563
+ const col = targetOffset - (commentFree.lastIndexOf("\n", targetOffset) + 1);
1564
+ if (match[1] === "apex") {
1565
+ const className = target.split(".")[0] ?? target;
1566
+ refs.push(ref(filePath, className, line, col, context));
1567
+ }
1568
+ refs.push(ref(filePath, target, line, col, context));
1569
+ }
1570
+ return {
1571
+ symbols: dedupe(symbols, (entry) => `${entry.name}\0${entry.kind}\0${entry.lineStart}`),
1572
+ refs: dedupe(refs, (entry) => `${entry.filePath}\0${entry.name}\0${entry.line}\0${entry.col}`)
1573
+ };
1574
+ }
1575
+ function matchLine(content, offset) {
1576
+ return content.slice(0, offset).split("\n").length;
1577
+ }
1578
+ function lineContext(content, line) {
1579
+ return lines(content)[line - 1]?.trim() ?? "";
1580
+ }
1581
+ function extractLwcTemplate(content, filePath) {
1582
+ const symbols = [];
1583
+ const refs = [];
1584
+ const markupNoComments = stripXmlComments(content);
1585
+ for (const match of markupNoComments.matchAll(/\blwc:ref\s*=\s*["']([^"']+)["']/gi)) {
1586
+ const line = matchLine(markupNoComments, match.index ?? 0);
1587
+ symbols.push(symbol(filePath, match[1] ?? "", "lwc_ref", line));
1588
+ }
1589
+ for (const match of markupNoComments.matchAll(/\bid\s*=\s*["']([^"'{}:]+)["']/gi)) {
1590
+ const line = matchLine(markupNoComments, match.index ?? 0);
1591
+ symbols.push(symbol(filePath, match[1] ?? "", "lwc_id", line));
1592
+ }
1593
+ for (const match of markupNoComments.matchAll(/\bon[a-z][\w-]*\s*=\s*\{\s*([A-Za-z_$][\w$]*)\s*\}/gi)) {
1594
+ const offset = match.index ?? 0;
1595
+ const line = matchLine(markupNoComments, offset);
1596
+ refs.push(ref(filePath, match[1] ?? "", line, 0, lineContext(content, line)));
1597
+ }
1598
+ for (const match of markupNoComments.matchAll(/<\s*(c-[a-z][\w-]*)\b/gi)) {
1599
+ const offset = match.index ?? 0;
1600
+ const line = matchLine(markupNoComments, offset);
1601
+ refs.push(ref(filePath, (match[1] ?? "").toLowerCase(), line, 0, lineContext(content, line)));
1602
+ }
1603
+ return {
1604
+ symbols: dedupe(symbols, (entry) => `${entry.name}\0${entry.kind}\0${entry.lineStart}`),
1605
+ refs: dedupe(refs, (entry) => `${entry.filePath}\0${entry.name}\0${entry.line}\0${entry.col}`)
1606
+ };
1607
+ }
1608
+ var MARKUP_KIND = {
1609
+ ".cmp": "aura_bundle",
1610
+ ".app": "aura_application",
1611
+ ".evt": "aura_event_bundle",
1612
+ ".intf": "aura_interface",
1613
+ ".design": "aura_design",
1614
+ ".auradoc": "aura_documentation",
1615
+ ".tokens": "aura_tokens",
1616
+ ".page": "visualforce_page",
1617
+ ".component": "visualforce_component",
1618
+ ".email": "visualforce_email_template"
1619
+ };
1620
+ function markupArtifactName(filePath, extension) {
1621
+ if (extension === ".page" || extension === ".component" || extension === ".email") {
1622
+ return path3.posix.basename(filePath.replaceAll("\\", "/")).replace(new RegExp(`${extension.replace(".", "\\.")}$`, "i"), "");
1623
+ }
1624
+ return bundleName(filePath);
1625
+ }
1626
+ function addAttributeSymbols(symbols, content, filePath, tag, kind) {
1627
+ const tagRe = new RegExp(`<\\s*${tag}\\b[^>]*\\bname\\s*=\\s*["']([^"']+)["'][^>]*>`, "gi");
1628
+ for (const match of content.matchAll(tagRe)) {
1629
+ symbols.push(symbol(filePath, match[1] ?? "", kind, matchLine(content, match.index ?? 0)));
1630
+ }
1631
+ }
1632
+ function attributeRefs(refs, content, filePath, attribute, split = false) {
1633
+ const attributeRe = new RegExp(`\\b${attribute}\\s*=\\s*["']([^"']+)["']`, "gi");
1634
+ for (const match of content.matchAll(attributeRe)) {
1635
+ const line = matchLine(content, match.index ?? 0);
1636
+ const values = split ? (match[1] ?? "").split(",").map((value) => value.trim()).filter(Boolean) : [match[1] ?? ""];
1637
+ for (const value of values) refs.push(ref(filePath, value, line, 0, lineContext(content, line)));
1638
+ }
1639
+ }
1640
+ function extractSalesforceMarkup(content, filePath) {
1641
+ const normalized = filePath.replaceAll("\\", "/");
1642
+ const extension = path3.posix.extname(normalized).toLowerCase();
1643
+ const kind = MARKUP_KIND[extension] ?? "salesforce_markup";
1644
+ const symbols = [
1645
+ symbol(filePath, markupArtifactName(normalized, extension), kind, 1, countContentLines(content))
1646
+ ];
1647
+ const refs = [];
1648
+ const isAura = [".cmp", ".app", ".evt", ".intf", ".design", ".auradoc", ".tokens"].includes(extension);
1649
+ const markupNoComments = stripXmlComments(content);
1650
+ if (isAura) {
1651
+ addAttributeSymbols(symbols, markupNoComments, filePath, "aura:attribute", "aura_attribute");
1652
+ addAttributeSymbols(symbols, markupNoComments, filePath, "aura:handler", "aura_handler");
1653
+ addAttributeSymbols(symbols, markupNoComments, filePath, "aura:registerEvent", "aura_event");
1654
+ addAttributeSymbols(symbols, markupNoComments, filePath, "design:attribute", "aura_design_attribute");
1655
+ }
1656
+ attributeRefs(refs, markupNoComments, filePath, "controller");
1657
+ attributeRefs(refs, markupNoComments, filePath, "extensions", true);
1658
+ const actionRe = isAura ? /\{!\s*c\.([A-Za-z_$][\w$]*)(?:[^}\w$][^}]*)?\}/gi : /\baction\s*=\s*["']\{!\s*(?:c\.)?([A-Za-z_$][\w$]*)(?:[^}\w$][^}]*)?\}["']/gi;
1659
+ for (const match of markupNoComments.matchAll(actionRe)) {
1660
+ const line = matchLine(markupNoComments, match.index ?? 0);
1661
+ refs.push(ref(filePath, match[1] ?? "", line, 0, lineContext(content, line)));
1662
+ }
1663
+ for (const match of markupNoComments.matchAll(/\bc:[A-Za-z_$][\w$]*/g)) {
1664
+ const line = matchLine(markupNoComments, match.index ?? 0);
1665
+ refs.push(ref(filePath, match[0], line, 0, lineContext(content, line)));
1666
+ }
1667
+ return {
1668
+ symbols: dedupe(symbols, (entry) => `${entry.name}\0${entry.kind}\0${entry.lineStart}`),
1669
+ refs: dedupe(refs, (entry) => `${entry.filePath}\0${entry.name}\0${entry.line}\0${entry.col}`)
1670
+ };
1671
+ }
1672
+
1673
+ // src/languages/ipynb_idx.ts
1674
+ init_define_import_meta_env();
1675
+ function normalizeSource(source) {
1676
+ if (typeof source === "string") return source;
1677
+ if (!Array.isArray(source)) return "";
1678
+ const lines2 = source.filter((s) => typeof s === "string");
1679
+ if (lines2.length === 0) return "";
1680
+ const noneEndWithNewline = lines2.every((l) => !l.endsWith("\n"));
1681
+ return noneEndWithNewline ? lines2.join("\n") : lines2.join("");
1682
+ }
1683
+ function resolveKernelLanguage(metadata) {
1684
+ const kernelspecLang = metadata?.kernelspec?.language;
1685
+ if (typeof kernelspecLang === "string" && kernelspecLang !== "") return kernelspecLang.toLowerCase();
1686
+ const infoLang = metadata?.language_info?.name;
1687
+ if (typeof infoLang === "string" && infoLang !== "") return infoLang.toLowerCase();
1688
+ return void 0;
1689
+ }
1690
+ function ipynbToVirtualSource(raw) {
1691
+ let data;
1692
+ try {
1693
+ data = JSON.parse(raw);
1694
+ } catch {
1695
+ return { content: "", cellLanguage: null };
1696
+ }
1697
+ if (typeof data !== "object" || data === null) return { content: "", cellLanguage: null };
1698
+ const nb = data;
1699
+ if (!Array.isArray(nb.cells)) return { content: "", cellLanguage: null };
1700
+ const declared = resolveKernelLanguage(nb.metadata);
1701
+ const isPython = declared === void 0 || declared === "python" || declared === "python3";
1702
+ if (!isPython) return { content: "", cellLanguage: null };
1703
+ const blocks = [];
1704
+ nb.cells.forEach((cell, index) => {
1705
+ if (typeof cell !== "object" || cell === null) return;
1706
+ const c = cell;
1707
+ if (c.cell_type === "code") {
1708
+ const src = normalizeSource(c.source).replace(/\n+$/, "");
1709
+ blocks.push(`# %% cell ${index}
1710
+ ${src}`);
1711
+ } else if (c.cell_type === "markdown") {
1712
+ const src = normalizeSource(c.source).replace(/\n+$/, "");
1713
+ const commented = src.split("\n").map((line) => line === "" ? "#" : `# ${line}`).join("\n");
1714
+ blocks.push(`# %% [markdown] cell ${index}
1715
+ ${commented}`);
1716
+ }
1717
+ });
1718
+ return { content: blocks.join("\n\n"), cellLanguage: "python" };
1719
+ }
1720
+
1721
+ // src/parser_treesitter.ts
1722
+ init_define_import_meta_env();
1723
+ var TSJS_KIND_BY_TYPE = /* @__PURE__ */ new Map([
1724
+ ["function_declaration", "function"],
1725
+ ["generator_function_declaration", "function"],
1726
+ ["class_declaration", "class"],
1727
+ ["abstract_class_declaration", "class"],
1728
+ ["method_definition", "method"],
1729
+ ["interface_declaration", "interface"],
1730
+ ["type_alias_declaration", "type"],
1731
+ ["enum_declaration", "enum"],
1732
+ // Interface members and abstract methods are distinct node types from concrete method_definition.
1733
+ ["method_signature", "method"],
1734
+ ["property_signature", "var"],
1735
+ ["abstract_method_signature", "method"],
1736
+ // Ambient function declarations in .d.ts files parse as function_signature.
1737
+ ["function_signature", "function"],
1738
+ // `namespace Foo { ... }` (and the legacy `module Foo { ... }` synonym) parses as
1739
+ // `internal_module`; `declare module "some-string" { ... }` (an ambient module declaration,
1740
+ // common in .d.ts files) parses as `module` -- a distinct node type from either. Neither had a
1741
+ // kind-map entry, so the namespace/module declaration itself was silently invisible to
1742
+ // `symbol`/`outline`/`skeleton`/`read`, even though everything nested inside it still indexed
1743
+ // fine (the walk recurses into every node's children regardless of the parent's kind-map
1744
+ // membership) -- the same container-drop shape already fixed for C++ `namespace_definition`
1745
+ // and Rust `mod_item`. Both node types expose their name on the standard `name` field
1746
+ // (an identifier, nested_identifier, or string), so `nodeName` resolves it without special-casing.
1747
+ ["internal_module", "namespace"],
1748
+ ["module", "namespace"]
1749
+ ]);
1750
+ var TSJS_TYPE_MEMBER_TYPES = /* @__PURE__ */ new Set(["property_signature", "method_signature"]);
1751
+ var TSJS_TYPE_WRAPPER_TYPES = /* @__PURE__ */ new Set([
1752
+ "parenthesized_type",
1753
+ "union_type",
1754
+ "intersection_type",
1755
+ "array_type",
1756
+ "readonly_type",
1757
+ "tuple_type",
1758
+ "optional_type",
1759
+ "rest_type",
1760
+ "generic_type",
1761
+ "type_arguments"
1762
+ ]);
1763
+ function isNamedTypeMember(node) {
1764
+ const parent = node.parent;
1765
+ if (parent === null) return false;
1766
+ if (parent.type === "interface_body" || parent.type === "class_body") return true;
1767
+ if (parent.type !== "object_type") return false;
1768
+ let owner = parent.parent;
1769
+ while (owner !== null && TSJS_TYPE_WRAPPER_TYPES.has(owner.type)) owner = owner.parent;
1770
+ return owner !== null && owner.type === "type_alias_declaration";
1771
+ }
1772
+ function leadingTsDecorators(node) {
1773
+ const decorators = [];
1774
+ let cur = node.previousNamedSibling;
1775
+ while (cur !== null && cur.type === "decorator") {
1776
+ decorators.unshift(cur);
1777
+ cur = cur.previousNamedSibling;
1778
+ }
1779
+ return decorators;
1780
+ }
1781
+ function nodeName(node) {
1782
+ const named = node.childForFieldName("name");
1783
+ if (named !== null) return named.text;
1784
+ return null;
1785
+ }
1786
+ var SPEC_DECLARATION_OWNER = /* @__PURE__ */ new Map([
1787
+ ["variable_declarator", /* @__PURE__ */ new Set(["lexical_declaration", "variable_declaration"])],
1788
+ ["var_spec", /* @__PURE__ */ new Set(["var_declaration"])],
1789
+ ["const_spec", /* @__PURE__ */ new Set(["const_declaration"])],
1790
+ ["type_spec", /* @__PURE__ */ new Set(["type_declaration"])]
1791
+ ]);
1792
+ var PREFIX_WRAPPER_TYPES = /* @__PURE__ */ new Set(["export_statement", "ambient_declaration"]);
1793
+ function widenToDeclaration(node) {
1794
+ let widened = node;
1795
+ const owner = SPEC_DECLARATION_OWNER.get(widened.type);
1796
+ if (owner !== void 0) {
1797
+ const decl = widened.parent;
1798
+ if (decl === null || !owner.has(decl.type)) return widened;
1799
+ let specs = 0;
1800
+ for (const c of decl.namedChildren) if (c.type === widened.type) specs++;
1801
+ if (specs !== 1) return widened;
1802
+ if (decl.startPosition.row !== widened.startPosition.row) return widened;
1803
+ widened = decl;
1804
+ }
1805
+ for (; ; ) {
1806
+ const parent = widened.parent;
1807
+ if (parent === null || !PREFIX_WRAPPER_TYPES.has(parent.type)) return widened;
1808
+ widened = parent;
1809
+ }
1810
+ }
1811
+ function makeSymbol(filePath, name, kind, node, lines2, style) {
1812
+ const ranged = widenToDeclaration(node);
1813
+ const lineStart = ranged.startPosition.row + 1;
1814
+ return {
1815
+ filePath,
1816
+ name,
1817
+ kind,
1818
+ lineStart,
1819
+ lineEnd: ranged.endPosition.row + 1,
1820
+ body: ranged.text,
1821
+ docstring: lines2 !== void 0 && style !== void 0 ? precedingDocComment(lines2, lineStart, style) : "",
1822
+ parent: ""
1823
+ };
1824
+ }
1825
+ function fanOutElidesBodies(nameCount, declarationChars) {
1826
+ return nameCount > 1 && nameCount * declarationChars > SYMBOL_BODY_CHAR_CAP;
1827
+ }
1828
+ function collectPatternBindings(node) {
1829
+ const names = [];
1830
+ const walk = (n) => {
1831
+ if (n.type === "identifier" || n.type === "shorthand_property_identifier_pattern") {
1832
+ if (n.text !== "") names.push(n.text);
1833
+ return;
1834
+ }
1835
+ if (n.type === "assignment_pattern" || n.type === "object_assignment_pattern") {
1836
+ const left = n.childForFieldName("left");
1837
+ if (left !== null) walk(left);
1838
+ return;
1839
+ }
1840
+ if (n.type === "pair_pattern") {
1841
+ const value = n.childForFieldName("value");
1842
+ if (value !== null) walk(value);
1843
+ return;
1844
+ }
1845
+ for (const child of n.namedChildren) walk(child);
1846
+ };
1847
+ walk(node);
1848
+ return names;
1849
+ }
1850
+ var TSJS_FN_SCOPE_TYPES = /* @__PURE__ */ new Set([
1851
+ "function_declaration",
1852
+ "function_expression",
1853
+ "arrow_function",
1854
+ "method_definition",
1855
+ "generator_function",
1856
+ "generator_function_declaration"
1857
+ ]);
1858
+ function extractTsJsSymbols(root, filePath, lines2) {
1859
+ const out = [];
1860
+ const visit = (node, insideFunction) => {
1861
+ const kind = TSJS_KIND_BY_TYPE.get(node.type);
1862
+ const isLocalFunction = insideFunction && (node.type === "function_declaration" || node.type === "generator_function_declaration");
1863
+ const isAnonymousTypeMember = TSJS_TYPE_MEMBER_TYPES.has(node.type) && !isNamedTypeMember(node);
1864
+ if (kind !== void 0 && !isLocalFunction && !isAnonymousTypeMember) {
1865
+ const name = nodeName(node);
1866
+ if (name !== null && name !== "") {
1867
+ const decorators = leadingTsDecorators(node);
1868
+ if (decorators.length === 0) {
1869
+ out.push(makeSymbol(filePath, name, kind, node, lines2, "c"));
1870
+ } else {
1871
+ const lineStart = decorators[0].startPosition.row + 1;
1872
+ const decoratedEnd = widenToDeclaration(node).endPosition.row + 1;
1873
+ out.push({
1874
+ filePath,
1875
+ name,
1876
+ kind,
1877
+ lineStart,
1878
+ lineEnd: decoratedEnd,
1879
+ // Read the body off the file rather than gluing the decorator and the node
1880
+ // together with a newline: `@dec export class X {}` has no newline between
1881
+ // them, and the glued form both invents one and drops the `export` that sits
1882
+ // between the two nodes. A decorated declaration yields one symbol, so taking
1883
+ // its whole span cannot fan out.
1884
+ // The trailing replace keeps a convention the rest of the index follows: a
1885
+ // tree-sitter node never carries the indentation of its own first line,
1886
+ // because it starts at the first real character. Reading the span off the
1887
+ // file would otherwise make decorated symbols the one shape that does.
1888
+ body: lines2.slice(lineStart - 1, decoratedEnd).join("\n").replace(/^[ \t]+/, ""),
1889
+ docstring: precedingDocComment(lines2, lineStart, "c"),
1890
+ parent: ""
1891
+ });
1892
+ }
1893
+ }
1894
+ }
1895
+ if (!insideFunction && (node.type === "lexical_declaration" || node.type === "variable_declaration")) {
1896
+ for (const child of node.namedChildren) {
1897
+ if (child.type !== "variable_declarator") continue;
1898
+ const name = child.childForFieldName("name");
1899
+ const value = child.childForFieldName("value");
1900
+ if (name === null) continue;
1901
+ if (name.type === "identifier") {
1902
+ const isFn = value !== null && (value.type === "arrow_function" || value.type === "function_expression" || value.type === "function");
1903
+ out.push(makeSymbol(filePath, name.text, isFn ? "function" : "variable", child, lines2, "c"));
1904
+ } else {
1905
+ const bindings = collectPatternBindings(name);
1906
+ const elideBodies = fanOutElidesBodies(bindings.length, child.text.length);
1907
+ for (const bound of bindings) {
1908
+ const sym = makeSymbol(filePath, bound, "variable", child, lines2, "c");
1909
+ out.push(elideBodies ? { ...sym, body: "" } : sym);
1910
+ }
1911
+ }
1912
+ }
1913
+ }
1914
+ if (node.type === "public_field_definition" || node.type === "field_definition") {
1915
+ const fieldName = node.childForFieldName("name") ?? node.childForFieldName("property");
1916
+ const value = node.childForFieldName("value");
1917
+ if (fieldName !== null && value !== null && (value.type === "arrow_function" || value.type === "function_expression" || value.type === "function")) {
1918
+ out.push(makeSymbol(filePath, fieldName.text, "method", node, lines2, "c"));
1919
+ }
1920
+ }
1921
+ const childInside = insideFunction || TSJS_FN_SCOPE_TYPES.has(node.type);
1922
+ for (const child of node.namedChildren) {
1923
+ visit(child, childInside);
1924
+ }
1925
+ };
1926
+ visit(root, false);
1927
+ return out;
1928
+ }
1929
+ var PY_KIND_BY_TYPE = /* @__PURE__ */ new Map([
1930
+ ["function_definition", "function"],
1931
+ ["class_definition", "class"],
1932
+ // PEP 695 (Python 3.12) `type X = ...` / `type X[T] = ...` statement. Its node carries no
1933
+ // `name` field (only `left`/`right`, both wrapping a `type` node — see
1934
+ // pythonTypeAliasName below), so nodeName() alone can never resolve it; without this entry
1935
+ // every PEP 695 type alias in a 3.12+ codebase was silently invisible to symbol/read/outline.
1936
+ ["type_alias_statement", "type"]
1937
+ ]);
1938
+ function pythonTypeAliasName(node) {
1939
+ const left = node.childForFieldName("left");
1940
+ if (left === null) return null;
1941
+ let cur = left;
1942
+ while (cur !== null) {
1943
+ if (cur.type === "identifier") return cur.text;
1944
+ cur = cur.namedChildren[0] ?? null;
1945
+ }
1946
+ return null;
1947
+ }
1948
+ function extractPythonSymbols(root, filePath) {
1949
+ const out = [];
1950
+ const visit = (node, insideClass) => {
1951
+ const baseKind = PY_KIND_BY_TYPE.get(node.type);
1952
+ if (baseKind !== void 0) {
1953
+ const name = node.type === "type_alias_statement" ? pythonTypeAliasName(node) : nodeName(node);
1954
+ if (name !== null && name !== "") {
1955
+ const kind = node.type === "function_definition" && insideClass ? "method" : baseKind;
1956
+ const rangeNode = node.parent?.type === "decorated_definition" ? node.parent : node;
1957
+ const docstring = node.type === "type_alias_statement" ? "" : pythonDocstring(node);
1958
+ out.push({
1959
+ ...makeSymbol(filePath, name, kind, rangeNode),
1960
+ docstring
1961
+ });
1962
+ }
1963
+ }
1964
+ for (const child of node.namedChildren) {
1965
+ const childInsideClass = node.type === "class_definition" ? true : node.type === "function_definition" ? false : insideClass;
1966
+ visit(child, childInsideClass);
1967
+ }
1968
+ };
1969
+ visit(root, false);
1970
+ return out;
1971
+ }
1972
+ function pythonDocstring(node) {
1973
+ const block = node.childForFieldName("body");
1974
+ if (block === null) return "";
1975
+ const first = block.namedChildren[0];
1976
+ if (first === void 0 || first.type !== "expression_statement") return "";
1977
+ const str = first.namedChildren[0];
1978
+ if (str === void 0 || str.type !== "string") return "";
1979
+ return stripPythonStringQuotes(str.text);
1980
+ }
1981
+ function stripPythonStringQuotes(raw) {
1982
+ let s = raw.trim();
1983
+ s = s.replace(/^[A-Za-z]+/, "");
1984
+ for (const q of ['"""', "'''", '"', "'"]) {
1985
+ if (s.startsWith(q) && s.endsWith(q) && s.length >= q.length * 2) {
1986
+ return s.slice(q.length, s.length - q.length).trim();
1987
+ }
1988
+ }
1989
+ return s.trim();
1990
+ }
1991
+ var GO_KIND_BY_TYPE = /* @__PURE__ */ new Map([
1992
+ ["function_declaration", "function"],
1993
+ ["method_declaration", "method"],
1994
+ // An interface's declared method set (`type Reader interface { Read(...) (int, error) }`) is
1995
+ // parsed as `method_elem` -- a distinct node type from `method_declaration` (a concrete method
1996
+ // with a receiver and body). `method_elem` exposes its own `name` field (a `field_identifier`),
1997
+ // exactly like `method_declaration` does, but had no map entry: every method signature declared
1998
+ // inside a Go interface -- the entire point of the interface -- was silently invisible to
1999
+ // `symbol`/`outline`/`skeleton`/`read`, even though the interface type itself indexed fine via
2000
+ // `type_spec` below.
2001
+ ["method_elem", "method"],
2002
+ // Go type/const/var names live on the nested *_spec node, not the *_declaration wrapper (which exposes no `name` field). A grouped `type (...)` / `const (...)` / `var (...)` block holds several specs, each reached by the namedChildren recursion in extractGoSymbols, so keying on the spec node yields one symbol per declared name. `type X = Y` parses as type_alias, which also carries the name field.
2003
+ ["type_spec", "type"],
2004
+ ["type_alias", "type"],
2005
+ ["const_spec", "const"],
2006
+ ["var_spec", "variable"]
2007
+ ]);
2008
+ var GO_FN_SCOPE_TYPES = /* @__PURE__ */ new Set([
2009
+ "function_declaration",
2010
+ "method_declaration",
2011
+ "func_literal"
2012
+ ]);
2013
+ var GO_LOCAL_KINDS = /* @__PURE__ */ new Set([
2014
+ "var_spec",
2015
+ "const_spec",
2016
+ "type_spec",
2017
+ "type_alias",
2018
+ // A local interface type (`type Reader interface { Read(...) }` declared inside a func body) is
2019
+ // itself excluded via `type_spec` above; its nested `method_elem` signatures must be excluded
2020
+ // the same way, or a function-local interface's methods would leak into the index even though
2021
+ // the interface type declaring them does not.
2022
+ "method_elem"
2023
+ ]);
2024
+ var GO_MULTI_NAME_SPECS = /* @__PURE__ */ new Set(["var_spec", "const_spec"]);
2025
+ function extractGoSymbols(root, filePath, lines2) {
2026
+ const out = [];
2027
+ const visit = (node, insideFunction) => {
2028
+ const kind = GO_KIND_BY_TYPE.get(node.type);
2029
+ if (kind !== void 0 && !(insideFunction && GO_LOCAL_KINDS.has(node.type))) {
2030
+ if (GO_MULTI_NAME_SPECS.has(node.type)) {
2031
+ const declared = node.namedChildren.filter(
2032
+ (c) => c.type === "identifier" && c.text !== "" && c.text !== "_"
2033
+ );
2034
+ const elideBodies = fanOutElidesBodies(declared.length, node.text.length);
2035
+ for (const child of declared) {
2036
+ const sym = makeSymbol(filePath, child.text, kind, node, lines2, "c");
2037
+ out.push(elideBodies ? { ...sym, body: "" } : sym);
2038
+ }
2039
+ } else {
2040
+ const name = nodeName(node);
2041
+ if (name !== null && name !== "") {
2042
+ out.push(makeSymbol(filePath, name, kind, node, lines2, "c"));
2043
+ }
2044
+ }
2045
+ }
2046
+ const childInside = insideFunction || GO_FN_SCOPE_TYPES.has(node.type);
2047
+ for (const child of node.namedChildren) {
2048
+ visit(child, childInside);
2049
+ }
2050
+ };
2051
+ visit(root, false);
2052
+ return out;
2053
+ }
2054
+ var RUST_KIND_BY_TYPE = /* @__PURE__ */ new Map([
2055
+ ["function_item", "function"],
2056
+ ["struct_item", "struct"],
2057
+ ["enum_item", "enum"],
2058
+ ["impl_item", "impl"],
2059
+ ["trait_item", "trait"],
2060
+ ["type_item", "type"],
2061
+ ["const_item", "const"],
2062
+ // `mod foo { ... }` / `mod foo;` — Rust modules are ubiquitous (submodule trees, `#[cfg(test)]
2063
+ // mod tests`) and parse as `mod_item`, which was absent here, so every module declaration was
2064
+ // silently dropped from the index. Reuses the 'module' kind already used by the Ruby extractor.
2065
+ // Not added to RUST_LOCAL_KINDS below: like a nested struct/fn, a mod declared inside a function
2066
+ // body stays indexed (only value bindings — `const` — are treated as function-local noise).
2067
+ ["mod_item", "module"],
2068
+ // Unbodied `fn` signatures — trait required methods (`fn find(&self) -> u32;` inside a `trait`
2069
+ // block) and `extern "C" { ... }` foreign-function declarations — parse as `function_signature_item`,
2070
+ // NOT `function_item` (which requires a body). Absent here, so every trait interface method without
2071
+ // a default body and every FFI declaration was silently dropped from the index: `token-goat symbol`
2072
+ // / `read` returned nothing for them even though the trait/extern block itself indexed. Mapped to
2073
+ // 'function', matching bodied `function_item`, so a trait's methods index whether or not they carry
2074
+ // a default body. Not a value binding, so it never appears as a function-local — no RUST_LOCAL_KINDS
2075
+ // entry needed.
2076
+ ["function_signature_item", "function"],
2077
+ // `extern "C" { ... }` / `extern "system" { ... }` foreign-module blocks -- unlike `trait_item`
2078
+ // (which IS indexed, so a trait method rendered standalone still has its enclosing `trait Foo`
2079
+ // symbol nearby for context), `foreign_mod_item` was entirely absent here, so an FFI declaration's
2080
+ // `function_signature_item` was the ONLY trace of the block in the index -- its ABI string
2081
+ // (`"C"` vs `"system"`, real calling-convention information) and any `#[link(name = "...")]`
2082
+ // attribute naming the linked library were both invisible with no other symbol to find them on.
2083
+ // Kept as its own entry (not folded into the child fn's body) to match how every other container
2084
+ // in this extractor works: the parent supplies context, the child stays standalone.
2085
+ ["foreign_mod_item", "extern"],
2086
+ // `macro_rules! foo { ... }` declarative macros parse as `macro_definition`, which was absent
2087
+ // here, so every `macro_rules!` definition was silently dropped from the index. Macros are
2088
+ // uniquely painful to lose: an invocation site (`foo!(...)`) carries no path back to the
2089
+ // definition, so without a name index there is no cheap way to jump from a call to the
2090
+ // `macro_rules!` block. The name lives on the standard `name` field (an `identifier`), so
2091
+ // `nodeName` resolves it like any other item. Like `mod`/`fn`/`trait` — a definition, not a
2092
+ // value binding — so it is NOT added to RUST_LOCAL_KINDS: a macro nested in a function stays
2093
+ // indexed, matching how nested fns/structs are kept and only `const`/`static` value bindings
2094
+ // are treated as function-local noise.
2095
+ ["macro_definition", "macro"],
2096
+ // `static FOO: T = ...;` / `pub static mut COUNTER: T = ...;` bindings parse as `static_item`,
2097
+ // which was absent here, so every `static` was silently dropped from the index — including the
2098
+ // ubiquitous top-level `static` tables and `static mut` globals real Rust code carries. Like
2099
+ // `const_item`, a `static` is a value binding, so it is ALSO added to RUST_LOCAL_KINDS below: a
2100
+ // `static` declared inside a function body is function-local noise (it has `'static` lifetime but
2101
+ // function scope) and must not pollute the global symbol index, matching how function-local
2102
+ // `const` is excluded. Its name lives on the standard `name` field, so `nodeName` resolves it.
2103
+ ["static_item", "static"],
2104
+ // `union Foo { ... }` (C-style untagged unions, mostly FFI/unsafe code) parse as `union_item`,
2105
+ // which was absent here, so every union was silently dropped from the index. A union is a type
2106
+ // definition like `struct`/`enum` — NOT a value binding — so it stays indexed even when nested,
2107
+ // and gets no RUST_LOCAL_KINDS entry (mirroring how nested structs/enums stay indexed). Its name
2108
+ // lives on the standard `name` field.
2109
+ ["union_item", "union"],
2110
+ // `type Item;` (unbodied) / `type Item = Foo;` (with a default) declared inside a `trait { ... }`
2111
+ // block — an associated type, the mechanism behind `Iterator::Item`, `Deref::Target`, and every
2112
+ // other trait with a type member — parses as its OWN node type, `associated_type`, which is
2113
+ // distinct from the free-standing `type_item` already mapped above (`type Alias = Foo;` at module
2114
+ // scope). `associated_type` was absent here, so every trait associated-type declaration was
2115
+ // silently dropped from the index, same failure shape as the `function_signature_item` gap fixed
2116
+ // above for unbodied trait methods. Its name lives on the standard `name` field, so `nodeName`
2117
+ // resolves it. Mapped to 'type', matching free-standing `type_item`, so both forms of "this is a
2118
+ // type declaration" land under one kind. Not a value binding, so no RUST_LOCAL_KINDS entry —
2119
+ // though in practice `associated_type` only ever appears inside a `trait`/`impl` body, never a fn.
2120
+ ["associated_type", "type"]
2121
+ ]);
2122
+ var RUST_FN_SCOPE_TYPES = /* @__PURE__ */ new Set(["function_item", "closure_expression"]);
2123
+ var RUST_LOCAL_KINDS = /* @__PURE__ */ new Set(["const_item", "static_item"]);
2124
+ function leadingRustAttributes(node) {
2125
+ const attrs = [];
2126
+ let cur = node.previousNamedSibling;
2127
+ while (cur !== null && cur.type === "attribute_item") {
2128
+ attrs.unshift(cur);
2129
+ cur = cur.previousNamedSibling;
2130
+ }
2131
+ return attrs;
2132
+ }
2133
+ function extractRustSymbols(root, filePath, lines2) {
2134
+ const out = [];
2135
+ const visit = (node, insideFunction) => {
2136
+ const kind = RUST_KIND_BY_TYPE.get(node.type);
2137
+ if (kind !== void 0 && !(insideFunction && RUST_LOCAL_KINDS.has(node.type))) {
2138
+ const name = node.type === "impl_item" ? node.childForFieldName("type")?.text ?? null : node.type === "foreign_mod_item" ? node.namedChildren.find((c) => c.type === "extern_modifier")?.text ?? "extern" : nodeName(node);
2139
+ if (name !== null && name !== "") {
2140
+ const attrs = leadingRustAttributes(node);
2141
+ if (attrs.length === 0) {
2142
+ out.push(makeSymbol(filePath, name, kind, node, lines2, "c"));
2143
+ } else {
2144
+ const lineStart = attrs[0].startPosition.row + 1;
2145
+ out.push({
2146
+ filePath,
2147
+ name,
2148
+ kind,
2149
+ lineStart,
2150
+ lineEnd: node.endPosition.row + 1,
2151
+ body: [...attrs, node].map((n) => n.text).join("\n"),
2152
+ docstring: precedingDocComment(lines2, lineStart, "c"),
2153
+ parent: ""
2154
+ });
2155
+ }
2156
+ }
2157
+ }
2158
+ const childInside = insideFunction || RUST_FN_SCOPE_TYPES.has(node.type);
2159
+ for (const child of node.namedChildren) {
2160
+ visit(child, childInside);
2161
+ }
2162
+ };
2163
+ visit(root, false);
2164
+ return out;
2165
+ }
2166
+ var RUBY_KIND_BY_TYPE = /* @__PURE__ */ new Map([
2167
+ ["method", "method"],
2168
+ ["singleton_method", "method"],
2169
+ ["class", "class"],
2170
+ ["module", "module"]
2171
+ ]);
2172
+ function extractSimpleSymbols(root, filePath, kindByType, lines2, style, nameFor = nodeName) {
2173
+ const out = [];
2174
+ const visit = (node) => {
2175
+ const kind = kindByType.get(node.type);
2176
+ if (kind !== void 0) {
2177
+ const name = nameFor(node);
2178
+ if (name !== null && name !== "") {
2179
+ out.push(makeSymbol(filePath, name, kind, node, lines2, style));
2180
+ }
2181
+ }
2182
+ for (const child of node.namedChildren) {
2183
+ visit(child);
2184
+ }
2185
+ };
2186
+ visit(root);
2187
+ return out;
2188
+ }
2189
+ function extractRubySymbols(root, filePath, lines2) {
2190
+ return extractSimpleSymbols(root, filePath, RUBY_KIND_BY_TYPE, lines2, "hash");
2191
+ }
2192
+ var JAVA_KIND_BY_TYPE = /* @__PURE__ */ new Map([
2193
+ ["method_declaration", "method"],
2194
+ ["class_declaration", "class"],
2195
+ ["interface_declaration", "interface"],
2196
+ ["enum_declaration", "enum"],
2197
+ ["constructor_declaration", "method"],
2198
+ ["record_declaration", "class"],
2199
+ ["annotation_type_declaration", "interface"],
2200
+ // An annotation type's members (`String value() default "";`, `int count();` inside an
2201
+ // `@interface` body) parse as `annotation_type_element_declaration` -- a distinct node type
2202
+ // from `method_declaration`, even though it is the exact same "signature-shaped declaration"
2203
+ // as an interface method. It exposes its own `name` field (an `identifier`), same shape as
2204
+ // `method_declaration`, but had no map entry here: every annotation member was silently
2205
+ // invisible to `symbol`/`outline`/`skeleton`/`read`, even though the annotation type itself
2206
+ // indexed fine via `annotation_type_declaration` above. Mapped to 'method' to match how a
2207
+ // Go interface's `method_elem` and a Rust trait's `function_signature_item` are folded into
2208
+ // the same kind as their bodied counterparts.
2209
+ ["annotation_type_element_declaration", "method"]
2210
+ ]);
2211
+ function extractJavaSymbols(root, filePath, lines2) {
2212
+ return extractSimpleSymbols(root, filePath, JAVA_KIND_BY_TYPE, lines2, "c");
2213
+ }
2214
+ var CPP_KIND_BY_TYPE = /* @__PURE__ */ new Map([
2215
+ ["function_definition", "function"],
2216
+ ["class_specifier", "class"],
2217
+ ["struct_specifier", "struct"],
2218
+ ["enum_specifier", "enum"],
2219
+ // `union_specifier` exposes the same `name` field (a `type_identifier`) as struct/enum in both
2220
+ // the C and C++ grammars, so a named union indexes as kind 'union' and is visible to `types`.
2221
+ ["union_specifier", "union"],
2222
+ // A `typedef ... Alias;` parses as `type_definition`; its aliased name lives on the nested
2223
+ // `declarator` chain, not a `name` field. The dominant real-world form `typedef struct { ... }
2224
+ // Alias;` (anonymous tag) otherwise indexes nothing at all: the inner struct/enum/union
2225
+ // specifier has no `name`, and the alias itself was never reached. Kind 'type' matches how the
2226
+ // TS/Go/Rust type aliases are indexed and is in `types`' TYPE_KINDS.
2227
+ ["type_definition", "type"],
2228
+ // `using Alias = Type;` (the C++11 alias-declaration form) parses as `alias_declaration`, a
2229
+ // distinct node type from `type_definition` above -- it had no entry here, so every C++11-style
2230
+ // type alias was silently invisible to symbol/outline/skeleton/types even though its `name`
2231
+ // field (a `type_identifier`) resolves fine via the default nodeName() lookup, unlike typedef's
2232
+ // declarator-chain descent (cTypedefAliasName). Kind 'type' matches type_definition's convention.
2233
+ ["alias_declaration", "type"],
2234
+ // `namespace Foo { ... }` (including the C++17 nested `namespace A::B { ... }` shorthand) parses
2235
+ // as `namespace_definition`, which had no entry here, so the namespace itself was silently
2236
+ // dropped from the index -- `symbol`/`outline`/`skeleton` never showed the declaration line,
2237
+ // even though everything nested inside it still indexed (extractSimpleSymbols always recurses
2238
+ // into children regardless of the parent's kind-map membership). The default `nodeName` lookup
2239
+ // (childForFieldName('name')) resolves both the simple `namespace_identifier` case and the
2240
+ // nested `nested_namespace_specifier` case (whose `.text` is the full `A::B` path) without any
2241
+ // special-casing. An anonymous `namespace { ... }` has no `name` field, so `nodeName` returns
2242
+ // null and it is correctly skipped -- matching how an anonymous struct/enum/union tag is only
2243
+ // ever indexed via its typedef alias, never as a bare symbol of its own. Kind 'namespace', not
2244
+ // 'module' (already used for Rust `mod`/Ruby `module`), since C++ namespaces are reopenable and
2245
+ // additive rather than a single owning declaration -- and NOT added to graph_commands.ts's
2246
+ // TYPE_KINDS, since a namespace is a container, not a type declaration (mirrors 'module' being
2247
+ // absent from TYPE_KINDS for the same reason).
2248
+ ["namespace_definition", "namespace"],
2249
+ // A bodiless function prototype (`int add(int a, int b);`) parses as a plain `declaration`, NOT
2250
+ // `function_definition` (which requires a `{ ... }` body) -- the dominant content of any C/C++
2251
+ // header file, which is almost entirely prototypes forward-declaring functions defined
2252
+ // elsewhere. Pre-fix, every one of these was silently dropped: `symbol`/`read`/`outline` on a
2253
+ // header returned nothing for its declared API surface. `declaration` is also the node type for
2254
+ // every plain variable/extern declaration (`int x;`, `extern int y;`) and for a function-pointer
2255
+ // *variable* (`int (*fp)(int);`, whose declarator, confusingly, ALSO nests a `function_declarator`
2256
+ // around a `parenthesized_declarator`), so this can't be a blanket kind-map entry the way
2257
+ // struct/enum/union are -- `cFunctionPrototypeName` below does the real filtering by inspecting
2258
+ // the declarator shape, and returns null (silently skipped, matching how an unnamed struct/enum
2259
+ // tag is skipped) for anything that isn't a genuine function prototype.
2260
+ ["declaration", "function"]
2261
+ ]);
2262
+ function extractCppSymbols(root, filePath, lines2) {
2263
+ return extractSimpleSymbols(
2264
+ root,
2265
+ filePath,
2266
+ CPP_KIND_BY_TYPE,
2267
+ lines2,
2268
+ "c",
2269
+ (node) => node.type === "function_definition" ? cFunctionName(node) : node.type === "type_definition" ? cTypedefAliasName(node) : node.type === "declaration" ? cFunctionPrototypeName(node) : nodeName(node)
2270
+ );
2271
+ }
2272
+ function cFunctionPrototypeName(node) {
2273
+ let cur = node.childForFieldName("declarator");
2274
+ for (let i = 0; cur !== null && i < 16; i++) {
2275
+ if (cur.type === "function_declarator") {
2276
+ const inner = cur.childForFieldName("declarator");
2277
+ if (inner === null) return null;
2278
+ if (inner.type === "identifier" || inner.type === "field_identifier") return inner.text;
2279
+ if (inner.type === "qualified_identifier") return lastSegment(inner.text);
2280
+ return null;
2281
+ }
2282
+ if (cur.type === "pointer_declarator" || cur.type === "reference_declarator") {
2283
+ cur = cur.childForFieldName("declarator");
2284
+ continue;
2285
+ }
2286
+ return null;
2287
+ }
2288
+ return null;
2289
+ }
2290
+ function cTypedefAliasName(node) {
2291
+ let cur = node.childForFieldName("declarator");
2292
+ for (let i = 0; cur !== null && i < 16; i++) {
2293
+ if (cur.type === "type_identifier") return cur.text;
2294
+ const next = cur.childForFieldName("declarator");
2295
+ cur = next ?? (cur.type === "parenthesized_declarator" ? cur.namedChildren[0] ?? null : null);
2296
+ }
2297
+ return null;
2298
+ }
2299
+ function lastSegment(text) {
2300
+ const parts = text.split(/::|\./);
2301
+ return parts[parts.length - 1] ?? text;
2302
+ }
2303
+ function cFunctionName(node) {
2304
+ let cur = node.childForFieldName("declarator");
2305
+ for (let i = 0; cur !== null && i < 16; i++) {
2306
+ if (cur.type === "identifier" || cur.type === "field_identifier") return cur.text;
2307
+ if (cur.type === "qualified_identifier") return lastSegment(cur.text);
2308
+ cur = cur.childForFieldName("declarator");
2309
+ }
2310
+ return null;
2311
+ }
2312
+
2313
+ // src/parser_refs.ts
2314
+ init_define_import_meta_env();
2315
+ var SCOPE_TYPES_BY_LANG = /* @__PURE__ */ new Map([
2316
+ [
2317
+ "typescript",
2318
+ /* @__PURE__ */ new Set([
2319
+ "function_declaration",
2320
+ "generator_function_declaration",
2321
+ "class_declaration",
2322
+ "abstract_class_declaration",
2323
+ "method_definition"
2324
+ ])
2325
+ ],
2326
+ [
2327
+ "javascript",
2328
+ /* @__PURE__ */ new Set([
2329
+ "function_declaration",
2330
+ "generator_function_declaration",
2331
+ "class_declaration",
2332
+ "method_definition"
2333
+ ])
2334
+ ],
2335
+ ["python", /* @__PURE__ */ new Set(["function_definition", "class_definition"])],
2336
+ ["go", /* @__PURE__ */ new Set(["function_declaration", "method_declaration"])],
2337
+ ["rust", /* @__PURE__ */ new Set(["function_item"])],
2338
+ [
2339
+ "java",
2340
+ /* @__PURE__ */ new Set(["method_declaration", "constructor_declaration", "class_declaration"])
2341
+ ],
2342
+ ["c", /* @__PURE__ */ new Set(["function_definition"])],
2343
+ ["cpp", /* @__PURE__ */ new Set(["function_definition"])],
2344
+ ["ruby", /* @__PURE__ */ new Set(["method", "singleton_method", "class", "module"])]
2345
+ ]);
2346
+ var CALL_TYPES_BY_LANG = /* @__PURE__ */ new Map([
2347
+ ["typescript", /* @__PURE__ */ new Set(["call_expression", "new_expression"])],
2348
+ ["javascript", /* @__PURE__ */ new Set(["call_expression", "new_expression"])],
2349
+ ["python", /* @__PURE__ */ new Set(["call"])],
2350
+ ["go", /* @__PURE__ */ new Set(["call_expression"])],
2351
+ ["rust", /* @__PURE__ */ new Set(["call_expression", "macro_invocation"])],
2352
+ ["java", /* @__PURE__ */ new Set(["method_invocation", "object_creation_expression"])],
2353
+ ["c", /* @__PURE__ */ new Set(["call_expression"])],
2354
+ ["cpp", /* @__PURE__ */ new Set(["call_expression", "new_expression"])],
2355
+ ["ruby", /* @__PURE__ */ new Set(["call"])]
2356
+ ]);
2357
+ var REF_NOISE_BY_LANG = /* @__PURE__ */ new Map([
2358
+ [
2359
+ "typescript",
2360
+ /* @__PURE__ */ new Set([
2361
+ "require",
2362
+ "Boolean",
2363
+ "Number",
2364
+ "String",
2365
+ "Array",
2366
+ "Object",
2367
+ "Symbol",
2368
+ "BigInt",
2369
+ "parseInt",
2370
+ "parseFloat",
2371
+ "isNaN",
2372
+ "isFinite",
2373
+ "setTimeout",
2374
+ "setInterval",
2375
+ "clearTimeout",
2376
+ "clearInterval"
2377
+ ])
2378
+ ],
2379
+ [
2380
+ "python",
2381
+ /* @__PURE__ */ new Set([
2382
+ "print",
2383
+ "len",
2384
+ "range",
2385
+ "str",
2386
+ "int",
2387
+ "float",
2388
+ "bool",
2389
+ "list",
2390
+ "dict",
2391
+ "set",
2392
+ "tuple",
2393
+ "type",
2394
+ "isinstance",
2395
+ "issubclass",
2396
+ "hasattr",
2397
+ "getattr",
2398
+ "setattr",
2399
+ "enumerate",
2400
+ "zip",
2401
+ "sorted",
2402
+ "reversed",
2403
+ "min",
2404
+ "max",
2405
+ "sum",
2406
+ "abs",
2407
+ "open",
2408
+ "repr",
2409
+ "super"
2410
+ ])
2411
+ ],
2412
+ // go/rust/c/cpp/ruby below cover only genuinely bare-identifier builtins (Go's predeclared functions, Rust macros, C/POSIX libc, Ruby's Kernel methods) -- the same restriction the comment above REF_NOISE_BY_LANG already documents for every language. java is deliberately left unpopulated: Java has no unqualified global builtin equivalent to these (stdout access always goes through System.out.*, a member call captured by its property name, not a bare identifier), so a bare `name` field from method_invocation is far more likely to be a real same-class helper call than a builtin -- adding entries here would risk false-negative "unreferenced" callers reports for real user methods that happen to share a common name.
2413
+ [
2414
+ "go",
2415
+ /* @__PURE__ */ new Set([
2416
+ "len",
2417
+ "cap",
2418
+ "make",
2419
+ "new",
2420
+ "append",
2421
+ "copy",
2422
+ "delete",
2423
+ "panic",
2424
+ "recover",
2425
+ "print",
2426
+ "println"
2427
+ ])
2428
+ ],
2429
+ [
2430
+ "rust",
2431
+ /* @__PURE__ */ new Set([
2432
+ "println",
2433
+ "print",
2434
+ "eprintln",
2435
+ "eprint",
2436
+ "format",
2437
+ "vec",
2438
+ "write",
2439
+ "writeln",
2440
+ "assert",
2441
+ "assert_eq",
2442
+ "assert_ne",
2443
+ "debug_assert",
2444
+ "panic",
2445
+ "todo",
2446
+ "unimplemented",
2447
+ "dbg"
2448
+ ])
2449
+ ],
2450
+ [
2451
+ "c",
2452
+ /* @__PURE__ */ new Set([
2453
+ "printf",
2454
+ "sprintf",
2455
+ "snprintf",
2456
+ "scanf",
2457
+ "malloc",
2458
+ "calloc",
2459
+ "realloc",
2460
+ "free",
2461
+ "memcpy",
2462
+ "memset",
2463
+ "memmove",
2464
+ "strlen",
2465
+ "strcpy",
2466
+ "strcmp",
2467
+ "exit",
2468
+ "abort",
2469
+ "assert"
2470
+ ])
2471
+ ],
2472
+ [
2473
+ "cpp",
2474
+ /* @__PURE__ */ new Set([
2475
+ "printf",
2476
+ "sprintf",
2477
+ "snprintf",
2478
+ "scanf",
2479
+ "malloc",
2480
+ "calloc",
2481
+ "realloc",
2482
+ "free",
2483
+ "memcpy",
2484
+ "memset",
2485
+ "memmove",
2486
+ "strlen",
2487
+ "strcpy",
2488
+ "strcmp",
2489
+ "exit",
2490
+ "abort",
2491
+ "assert"
2492
+ ])
2493
+ ],
2494
+ [
2495
+ "ruby",
2496
+ /* @__PURE__ */ new Set([
2497
+ "puts",
2498
+ "print",
2499
+ "p",
2500
+ "pp",
2501
+ "gets",
2502
+ "require",
2503
+ "require_relative",
2504
+ "raise",
2505
+ "loop",
2506
+ "lambda",
2507
+ "proc",
2508
+ "sleep",
2509
+ "exit",
2510
+ "freeze"
2511
+ ])
2512
+ ]
2513
+ ]);
2514
+ var JS_NOISE = REF_NOISE_BY_LANG.get("typescript") ?? /* @__PURE__ */ new Set();
2515
+ var EMPTY_STRING_SET = /* @__PURE__ */ new Set();
2516
+ function lastSegment2(text) {
2517
+ const parts = text.split(/::|\./);
2518
+ return parts[parts.length - 1] ?? text;
2519
+ }
2520
+ function scopeName(node, language) {
2521
+ if ((language === "typescript" || language === "javascript") && node.type === "variable_declarator") {
2522
+ const value = node.childForFieldName("value");
2523
+ if (value !== null && (value.type === "arrow_function" || value.type === "function_expression" || value.type === "function")) {
2524
+ return node.childForFieldName("name")?.text ?? null;
2525
+ }
2526
+ return null;
2527
+ }
2528
+ const scopeTypes = SCOPE_TYPES_BY_LANG.get(language);
2529
+ if (scopeTypes !== void 0 && scopeTypes.has(node.type)) {
2530
+ if ((language === "c" || language === "cpp") && node.type === "function_definition") {
2531
+ return cFunctionName2(node);
2532
+ }
2533
+ return node.childForFieldName("name")?.text ?? null;
2534
+ }
2535
+ return null;
2536
+ }
2537
+ function cFunctionName2(node) {
2538
+ let cur = node.childForFieldName("declarator");
2539
+ for (let i = 0; cur !== null && i < 16; i++) {
2540
+ if (cur.type === "identifier" || cur.type === "field_identifier") return cur.text;
2541
+ if (cur.type === "qualified_identifier") return lastSegment2(cur.text);
2542
+ cur = cur.childForFieldName("declarator");
2543
+ }
2544
+ return null;
2545
+ }
2546
+ function cppCalleeName(node) {
2547
+ switch (node.type) {
2548
+ case "identifier":
2549
+ case "field_identifier":
2550
+ case "type_identifier":
2551
+ return node.text;
2552
+ // An overloaded operator's name is its own node type, and it is the spelling the extractor gives the operator's definition too, so returning the text here is what makes the two match. Measured: `r.template operator()<int>(3)` parses as `dependent_name(template, template_method(operator_name "operator()", template_argument_list "<int>"))`, so without this case the name resolved to null and the field_expression fallback below recorded `template operator()<int>` -- a spelling no symbol row ever holds, leaving the call permanently unresolvable.
2553
+ case "operator_name":
2554
+ return node.text;
2555
+ case "template_function":
2556
+ case "template_method":
2557
+ case "template_type": {
2558
+ const name = node.childForFieldName("name");
2559
+ return name !== null ? cppCalleeName(name) : null;
2560
+ }
2561
+ case "qualified_identifier": {
2562
+ const name = node.childForFieldName("name");
2563
+ return name !== null ? cppCalleeName(name) : lastSegment2(node.text);
2564
+ }
2565
+ case "dependent_name": {
2566
+ const last = node.namedChildren[node.namedChildren.length - 1];
2567
+ return last !== void 0 ? cppCalleeName(last) : null;
2568
+ }
2569
+ case "field_expression": {
2570
+ const field = node.childForFieldName("field");
2571
+ if (field === null) return null;
2572
+ return cppCalleeName(field) ?? field.text;
2573
+ }
2574
+ default:
2575
+ return null;
2576
+ }
2577
+ }
2578
+ function calleeName(call, language) {
2579
+ switch (language) {
2580
+ case "typescript":
2581
+ case "javascript": {
2582
+ if (call.type === "new_expression") {
2583
+ const c = call.childForFieldName("constructor");
2584
+ if (c === null) return null;
2585
+ if (c.type === "identifier") return c.text;
2586
+ if (c.type === "member_expression") return c.childForFieldName("property")?.text ?? null;
2587
+ return null;
2588
+ }
2589
+ const fn = call.childForFieldName("function");
2590
+ if (fn === null) return null;
2591
+ if (fn.type === "identifier") return fn.text;
2592
+ if (fn.type === "member_expression") return fn.childForFieldName("property")?.text ?? null;
2593
+ return null;
2594
+ }
2595
+ case "python": {
2596
+ const fn = call.childForFieldName("function");
2597
+ if (fn === null) return null;
2598
+ if (fn.type === "identifier") return fn.text;
2599
+ if (fn.type === "attribute") return fn.childForFieldName("attribute")?.text ?? null;
2600
+ return null;
2601
+ }
2602
+ case "go": {
2603
+ const fn = call.childForFieldName("function");
2604
+ if (fn === null) return null;
2605
+ if (fn.type === "identifier") return fn.text;
2606
+ if (fn.type === "selector_expression") return fn.childForFieldName("field")?.text ?? null;
2607
+ return null;
2608
+ }
2609
+ case "rust": {
2610
+ if (call.type === "macro_invocation") {
2611
+ const m = call.childForFieldName("macro");
2612
+ return m !== null ? lastSegment2(m.text) : null;
2613
+ }
2614
+ const fn = call.childForFieldName("function");
2615
+ if (fn === null) return null;
2616
+ if (fn.type === "identifier") return fn.text;
2617
+ if (fn.type === "field_expression") return fn.childForFieldName("field")?.text ?? null;
2618
+ if (fn.type === "scoped_identifier") {
2619
+ return fn.childForFieldName("name")?.text ?? lastSegment2(fn.text);
2620
+ }
2621
+ return null;
2622
+ }
2623
+ case "java": {
2624
+ const n = call.childForFieldName("name") ?? call.childForFieldName("type");
2625
+ return n !== null ? lastSegment2(n.text) : null;
2626
+ }
2627
+ case "c":
2628
+ case "cpp": {
2629
+ const fn = call.type === "new_expression" ? call.childForFieldName("type") : call.childForFieldName("function");
2630
+ if (fn === null) return null;
2631
+ return cppCalleeName(fn);
2632
+ }
2633
+ case "ruby": {
2634
+ const m = call.childForFieldName("method");
2635
+ return m?.text ?? null;
2636
+ }
2637
+ default:
2638
+ return null;
2639
+ }
2640
+ }
2641
+ var VALUE_WRAPPER_TYPES = /* @__PURE__ */ new Set([
2642
+ "parenthesized_expression",
2643
+ "as_expression",
2644
+ "satisfies_expression",
2645
+ "non_null_expression",
2646
+ "type_assertion",
2647
+ "instantiation_expression",
2648
+ "await_expression",
2649
+ "unary_expression",
2650
+ // Python spells this as a bare `await` rather than with an `_expression` suffix.
2651
+ "await"
2652
+ ]);
2653
+ function unwrapValueIdentifier(node) {
2654
+ if (node === null || node === void 0) return null;
2655
+ if (node.type === "identifier") return node;
2656
+ if (!VALUE_WRAPPER_TYPES.has(node.type)) return null;
2657
+ for (const child of node.namedChildren) {
2658
+ const inner = unwrapValueIdentifier(child);
2659
+ if (inner !== null) return inner;
2660
+ }
2661
+ return null;
2662
+ }
2663
+ function pushValueIdentifier(result, node) {
2664
+ const identifier = unwrapValueIdentifier(node);
2665
+ if (identifier !== null) result.push(identifier);
2666
+ }
2667
+ function valueRefIdentifiers(node, language) {
2668
+ const isJs = language === "typescript" || language === "javascript";
2669
+ const isPy = language === "python";
2670
+ if (!isJs && !isPy) return [];
2671
+ const result = [];
2672
+ if (isJs && node.type === "arguments" || isPy && node.type === "argument_list") {
2673
+ for (const child of node.namedChildren) {
2674
+ pushValueIdentifier(result, child);
2675
+ }
2676
+ }
2677
+ if (isPy && node.type === "keyword_argument") {
2678
+ const value = node.childForFieldName("value");
2679
+ pushValueIdentifier(result, value);
2680
+ }
2681
+ if (isJs && node.type === "binary_expression") {
2682
+ const operator = node.childForFieldName("operator")?.text;
2683
+ if (operator === "??" || operator === "||" || operator === "&&") {
2684
+ const left = node.childForFieldName("left");
2685
+ const right = node.childForFieldName("right");
2686
+ pushValueIdentifier(result, left);
2687
+ pushValueIdentifier(result, right);
2688
+ }
2689
+ }
2690
+ if (isJs && (node.type === "variable_declarator" || node.type === "assignment_expression")) {
2691
+ const value = node.childForFieldName(node.type === "variable_declarator" ? "value" : "right");
2692
+ pushValueIdentifier(result, value);
2693
+ }
2694
+ if (isPy && node.type === "assignment") {
2695
+ const value = node.childForFieldName("right");
2696
+ pushValueIdentifier(result, value);
2697
+ }
2698
+ if (isJs && node.type === "pair") {
2699
+ const value = node.childForFieldName("value");
2700
+ pushValueIdentifier(result, value);
2701
+ }
2702
+ if (isJs && (node.type === "required_parameter" || node.type === "optional_parameter")) {
2703
+ const value = node.childForFieldName("value");
2704
+ pushValueIdentifier(result, value);
2705
+ }
2706
+ if (isJs && node.type === "array" || isPy && (node.type === "list" || node.type === "tuple" || node.type === "set")) {
2707
+ for (const child of node.namedChildren) {
2708
+ pushValueIdentifier(result, child);
2709
+ }
2710
+ }
2711
+ if (isJs && node.type === "ternary_expression") {
2712
+ const consequence = node.childForFieldName("consequence");
2713
+ const alternative = node.childForFieldName("alternative");
2714
+ pushValueIdentifier(result, consequence);
2715
+ pushValueIdentifier(result, alternative);
2716
+ }
2717
+ if (isPy && node.type === "conditional_expression" && node.namedChildren.length === 3) {
2718
+ const consequence = node.namedChildren[0];
2719
+ const alternative = node.namedChildren[2];
2720
+ pushValueIdentifier(result, consequence);
2721
+ pushValueIdentifier(result, alternative);
2722
+ }
2723
+ if (isJs && (node.type === "public_field_definition" || node.type === "field_definition")) {
2724
+ const value = node.childForFieldName("value");
2725
+ pushValueIdentifier(result, value);
2726
+ }
2727
+ if (node.type === "return_statement" && node.namedChildren.length === 1) {
2728
+ const value = node.namedChildren[0];
2729
+ pushValueIdentifier(result, value);
2730
+ }
2731
+ if (isJs && (node.type === "assignment_pattern" || node.type === "object_assignment_pattern")) {
2732
+ const value = node.childForFieldName("right");
2733
+ pushValueIdentifier(result, value);
2734
+ }
2735
+ if (isJs && node.type === "template_substitution" && node.namedChildren.length === 1) {
2736
+ const value = node.namedChildren[0];
2737
+ pushValueIdentifier(result, value);
2738
+ }
2739
+ if (isJs && node.type === "object") {
2740
+ for (const child of node.namedChildren) {
2741
+ if (child.type === "shorthand_property_identifier") result.push(child);
2742
+ }
2743
+ }
2744
+ if (node.type === "decorator") pushValueIdentifier(result, node.namedChildren[0]);
2745
+ if (isPy && node.type === "pair") pushValueIdentifier(result, node.childForFieldName("value"));
2746
+ if (isPy && node.type === "interpolation") pushValueIdentifier(result, node.childForFieldName("expression"));
2747
+ if (isJs && node.type === "augmented_assignment_expression") pushValueIdentifier(result, node.childForFieldName("right"));
2748
+ if (isPy && node.type === "augmented_assignment") pushValueIdentifier(result, node.childForFieldName("right"));
2749
+ if (isJs && node.type === "sequence_expression") {
2750
+ for (const child of node.namedChildren) pushValueIdentifier(result, child);
2751
+ }
2752
+ if (isJs && node.type === "spread_element") pushValueIdentifier(result, node.namedChildren[0]);
2753
+ if (isPy && (node.type === "list_splat" || node.type === "dictionary_splat")) pushValueIdentifier(result, node.namedChildren[0]);
2754
+ if (isJs && node.type === "computed_property_name") pushValueIdentifier(result, node.namedChildren[0]);
2755
+ if (isPy && (node.type === "list_comprehension" || node.type === "set_comprehension" || node.type === "generator_expression")) {
2756
+ pushValueIdentifier(result, node.namedChildren[0]);
2757
+ }
2758
+ if (isPy && node.type === "for_in_clause") pushValueIdentifier(result, node.childForFieldName("right"));
2759
+ if (isPy && node.type === "lambda") pushValueIdentifier(result, node.childForFieldName("body"));
2760
+ if (isJs && node.type === "yield_expression") pushValueIdentifier(result, node.namedChildren[0]);
2761
+ if (isPy && node.type === "yield") pushValueIdentifier(result, node.namedChildren[0]);
2762
+ if (isPy && node.type === "raise_statement") pushValueIdentifier(result, node.namedChildren[0]);
2763
+ if (isJs && (node.type === "extends_clause" || node.type === "class_heritage")) {
2764
+ const base = node.namedChildren[0];
2765
+ if (base !== void 0) {
2766
+ if (base.type === "identifier") result.push(base);
2767
+ else if (base.type === "member_expression") {
2768
+ const object = base.childForFieldName("object");
2769
+ if (object !== null && object.type === "identifier") result.push(object);
2770
+ }
2771
+ }
2772
+ }
2773
+ return result;
2774
+ }
2775
+ function extractRefs(root, filePath, language) {
2776
+ const out = [];
2777
+ const seen = /* @__PURE__ */ new Set();
2778
+ const callTypes = CALL_TYPES_BY_LANG.get(language) ?? EMPTY_STRING_SET;
2779
+ const noise = language === "javascript" ? JS_NOISE : REF_NOISE_BY_LANG.get(language) ?? EMPTY_STRING_SET;
2780
+ const stack = [];
2781
+ const record = (name, node) => {
2782
+ if (name.length <= 1 || noise.has(name)) return;
2783
+ const line = node.startPosition.row + 1;
2784
+ const key = `${name}\0${line}`;
2785
+ if (seen.has(key)) return;
2786
+ seen.add(key);
2787
+ out.push({
2788
+ filePath,
2789
+ name,
2790
+ line,
2791
+ col: node.startPosition.column,
2792
+ context: stack.length > 0 ? stack[stack.length - 1] ?? "" : ""
2793
+ });
2794
+ };
2795
+ const visit = (node) => {
2796
+ const enclosing = scopeName(node, language);
2797
+ if (enclosing !== null && enclosing !== "") stack.push(enclosing);
2798
+ if (callTypes.has(node.type)) {
2799
+ const callee = calleeName(node, language);
2800
+ if (callee !== null) record(callee, node);
2801
+ }
2802
+ for (const idNode of valueRefIdentifiers(node, language)) {
2803
+ record(idNode.text, idNode);
2804
+ }
2805
+ for (const child of node.namedChildren) visit(child);
2806
+ if (enclosing !== null && enclosing !== "") stack.pop();
2807
+ };
2808
+ visit(root);
2809
+ return out;
2810
+ }
2811
+
2812
+ // src/parser_structured.ts
2813
+ init_define_import_meta_env();
2814
+ function scanJsonValueEnd(content, start) {
2815
+ const first = content[start];
2816
+ if (first === void 0) return content.length;
2817
+ if (first === '"') {
2818
+ let escaping = false;
2819
+ for (let j = start + 1; j < content.length; j++) {
2820
+ const c = content[j];
2821
+ if (escaping) {
2822
+ escaping = false;
2823
+ continue;
2824
+ }
2825
+ if (c === "\\") {
2826
+ escaping = true;
2827
+ continue;
2828
+ }
2829
+ if (c === '"') return j + 1;
2830
+ }
2831
+ return content.length;
2832
+ }
2833
+ if (first === "{" || first === "[") {
2834
+ const stack = [];
2835
+ let inStr = false;
2836
+ let escaping = false;
2837
+ for (let j = start; j < content.length; j++) {
2838
+ const c = content[j];
2839
+ if (inStr) {
2840
+ if (escaping) {
2841
+ escaping = false;
2842
+ continue;
2843
+ }
2844
+ if (c === "\\") {
2845
+ escaping = true;
2846
+ continue;
2847
+ }
2848
+ if (c === '"') inStr = false;
2849
+ continue;
2850
+ }
2851
+ if (c === '"') {
2852
+ inStr = true;
2853
+ continue;
2854
+ }
2855
+ if (c === "{" || c === "[") stack.push(c);
2856
+ else if (c === "}" || c === "]") {
2857
+ const open = stack.pop();
2858
+ if (open !== (c === "}" ? "{" : "[")) return j;
2859
+ if (stack.length === 0) return j + 1;
2860
+ }
2861
+ }
2862
+ return content.length;
2863
+ }
2864
+ for (let j = start; j < content.length; j++) {
2865
+ const c = content[j];
2866
+ if (c === "," || c === "}" || c === "]" || c === "\n" || c === "\r") return j;
2867
+ }
2868
+ return content.length;
2869
+ }
2870
+ function countNewlines(s) {
2871
+ let n = 0;
2872
+ for (let i = 0; i < s.length; i++) if (s[i] === "\n") n++;
2873
+ return n;
2874
+ }
2875
+ function extractMarkdownSymbols(content, filePath) {
2876
+ const out = [];
2877
+ const lines2 = content.split(/\r?\n/);
2878
+ const unfenced = Array.from(eachUnfencedLine(lines2));
2879
+ for (let u = 0; u < unfenced.length; u++) {
2880
+ const [i, line] = unfenced[u];
2881
+ const atxMatch = /^(#{1,6})\s+(.+?)(?:\s+#+\s*)?$/.exec(line);
2882
+ if (atxMatch !== null && atxMatch[2] !== void 0) {
2883
+ const name = atxMatch[2].trim();
2884
+ if (name !== "") {
2885
+ out.push({
2886
+ filePath,
2887
+ name,
2888
+ kind: "heading",
2889
+ lineStart: i + 1,
2890
+ lineEnd: i + 1,
2891
+ body: line.trim(),
2892
+ docstring: "",
2893
+ parent: ""
2894
+ });
2895
+ }
2896
+ continue;
2897
+ }
2898
+ const trimmed = line.trim();
2899
+ if (trimmed !== "" && !trimmed.startsWith("#") && !trimmed.startsWith("|") && !trimmed.startsWith("```") && !trimmed.startsWith("~~~") && !/^([-*+]|\d+\.)\s/.test(trimmed) && u + 1 < unfenced.length) {
2900
+ const [nextIdx, nextLine] = unfenced[u + 1];
2901
+ if (nextIdx === i + 1) {
2902
+ const isUnderline = /^\s*(=+|-+)\s*$/.test(nextLine);
2903
+ if (isUnderline) {
2904
+ out.push({
2905
+ filePath,
2906
+ name: trimmed,
2907
+ kind: "heading",
2908
+ lineStart: i + 1,
2909
+ lineEnd: i + 2,
2910
+ body: `${trimmed}
2911
+ ${nextLine.trim()}`,
2912
+ docstring: "",
2913
+ parent: ""
2914
+ });
2915
+ u++;
2916
+ }
2917
+ }
2918
+ }
2919
+ }
2920
+ return out;
2921
+ }
2922
+ function blankJsonComments(raw) {
2923
+ const out = [];
2924
+ let inStr = false;
2925
+ let esc = false;
2926
+ for (let i = 0; i < raw.length; i++) {
2927
+ const ch = raw[i] ?? "";
2928
+ if (inStr) {
2929
+ out.push(ch);
2930
+ if (esc) esc = false;
2931
+ else if (ch === "\\") esc = true;
2932
+ else if (ch === '"') inStr = false;
2933
+ continue;
2934
+ }
2935
+ if (ch === '"') {
2936
+ inStr = true;
2937
+ out.push(ch);
2938
+ continue;
2939
+ }
2940
+ if (ch === "/" && raw[i + 1] === "/") {
2941
+ while (i < raw.length && raw[i] !== "\n") {
2942
+ out.push(" ");
2943
+ i++;
2944
+ }
2945
+ if (i < raw.length) out.push("\n");
2946
+ continue;
2947
+ }
2948
+ if (ch === "/" && raw[i + 1] === "*") {
2949
+ out.push(" ");
2950
+ i += 2;
2951
+ while (i < raw.length && !(raw[i] === "*" && raw[i + 1] === "/")) {
2952
+ out.push(raw[i] === "\n" ? "\n" : " ");
2953
+ i++;
2954
+ }
2955
+ if (i < raw.length) {
2956
+ out.push(" ");
2957
+ i++;
2958
+ }
2959
+ continue;
2960
+ }
2961
+ out.push(ch);
2962
+ }
2963
+ return out.join("");
2964
+ }
2965
+ function extractJsonSymbols(raw, filePath) {
2966
+ const out = [];
2967
+ const content = raw.includes("/") ? blankJsonComments(raw) : raw;
2968
+ try {
2969
+ let depth = 0;
2970
+ let inString = false;
2971
+ let escaping = false;
2972
+ let strChars = [];
2973
+ let strStartLine = 1;
2974
+ let strStartOffset = 0;
2975
+ let depthWhenStringOpened = 0;
2976
+ let line = 1;
2977
+ for (let i = 0; i < content.length; i++) {
2978
+ const ch = content[i];
2979
+ if (ch === void 0) continue;
2980
+ if (ch === "\n") line++;
2981
+ if (escaping) {
2982
+ escaping = false;
2983
+ if (inString) strChars.push(ch);
2984
+ continue;
2985
+ }
2986
+ if (ch === "\\" && inString) {
2987
+ escaping = true;
2988
+ continue;
2989
+ }
2990
+ if (ch === '"') {
2991
+ if (!inString) {
2992
+ inString = true;
2993
+ strChars = [];
2994
+ strStartLine = line;
2995
+ strStartOffset = i;
2996
+ depthWhenStringOpened = depth;
2997
+ } else {
2998
+ inString = false;
2999
+ let k = i + 1;
3000
+ while (k < content.length && /\s/.test(content[k] ?? "")) {
3001
+ k++;
3002
+ }
3003
+ if (content[k] === ":" && depthWhenStringOpened === 1) {
3004
+ let v = k + 1;
3005
+ while (v < content.length && /\s/.test(content[v] ?? "")) {
3006
+ v++;
3007
+ }
3008
+ const valueEnd = scanJsonValueEnd(content, v);
3009
+ const body = raw.slice(strStartOffset, valueEnd);
3010
+ const lineEnd = strStartLine + countNewlines(body);
3011
+ out.push({
3012
+ filePath,
3013
+ name: strChars.join(""),
3014
+ kind: "property",
3015
+ lineStart: strStartLine,
3016
+ lineEnd,
3017
+ body,
3018
+ docstring: "",
3019
+ parent: ""
3020
+ });
3021
+ }
3022
+ }
3023
+ continue;
3024
+ }
3025
+ if (inString) {
3026
+ strChars.push(ch);
3027
+ continue;
3028
+ }
3029
+ if (ch === "{" || ch === "[") depth++;
3030
+ else if (ch === "}" || ch === "]") depth--;
3031
+ }
3032
+ } catch {
3033
+ }
3034
+ return out;
3035
+ }
3036
+ function yamlOpenQuoteAfter(line, startIdx) {
3037
+ const value = line.slice(startIdx);
3038
+ const trimmed = value.replace(/^\s+/, "");
3039
+ const q = trimmed[0];
3040
+ if (q !== '"' && q !== "'") return null;
3041
+ if (q === '"') {
3042
+ let j2 = 1;
3043
+ while (j2 < trimmed.length) {
3044
+ if (trimmed[j2] === "\\") {
3045
+ j2 += 2;
3046
+ continue;
3047
+ }
3048
+ if (trimmed[j2] === '"') return null;
3049
+ j2++;
3050
+ }
3051
+ return '"';
3052
+ }
3053
+ let j = 1;
3054
+ while (j < trimmed.length) {
3055
+ if (trimmed[j] === "'" && trimmed[j + 1] === "'") {
3056
+ j += 2;
3057
+ continue;
3058
+ }
3059
+ if (trimmed[j] === "'") return null;
3060
+ j++;
3061
+ }
3062
+ return "'";
3063
+ }
3064
+ function yamlLineClosesQuote(line, quote) {
3065
+ let i = 0;
3066
+ while (i < line.length) {
3067
+ if (quote === '"') {
3068
+ if (line[i] === "\\") {
3069
+ i += 2;
3070
+ continue;
3071
+ }
3072
+ if (line[i] === '"') return true;
3073
+ } else {
3074
+ if (line[i] === "'" && line[i + 1] === "'") {
3075
+ i += 2;
3076
+ continue;
3077
+ }
3078
+ if (line[i] === "'") return true;
3079
+ }
3080
+ i++;
3081
+ }
3082
+ return false;
3083
+ }
3084
+ function extractYamlSymbols(content, filePath) {
3085
+ const out = [];
3086
+ const lines2 = content.split(/\r?\n/);
3087
+ let openQuote = null;
3088
+ for (let i = 0; i < lines2.length; i++) {
3089
+ const line = lines2[i];
3090
+ if (line === void 0) continue;
3091
+ if (openQuote !== null) {
3092
+ if (yamlLineClosesQuote(line, openQuote)) openQuote = null;
3093
+ continue;
3094
+ }
3095
+ const match = /^([a-zA-Z_][\w.-]*)\s*:(?!\/\/)/.exec(line);
3096
+ if (match !== null && match[1] !== void 0) {
3097
+ out.push({
3098
+ filePath,
3099
+ name: match[1],
3100
+ kind: "key",
3101
+ lineStart: i + 1,
3102
+ lineEnd: i + 1,
3103
+ body: line.trim(),
3104
+ docstring: "",
3105
+ parent: ""
3106
+ });
3107
+ openQuote = yamlOpenQuoteAfter(line, match[0].length);
3108
+ }
3109
+ }
3110
+ return out;
3111
+ }
3112
+ function lineOpenDelimiterAfter(line, startIdx) {
3113
+ let pos = startIdx;
3114
+ let open = null;
3115
+ for (; ; ) {
3116
+ if (open === null) {
3117
+ const dIdx = line.indexOf('"""', pos);
3118
+ const sIdx = line.indexOf("'''", pos);
3119
+ if (dIdx === -1 && sIdx === -1) return null;
3120
+ if (dIdx !== -1 && (sIdx === -1 || dIdx <= sIdx)) {
3121
+ open = '"""';
3122
+ pos = dIdx + 3;
3123
+ } else {
3124
+ open = "'''";
3125
+ pos = sIdx + 3;
3126
+ }
3127
+ } else {
3128
+ const closeIdx = line.indexOf(open, pos);
3129
+ if (closeIdx === -1) return open;
3130
+ open = null;
3131
+ pos = closeIdx + 3;
3132
+ }
3133
+ }
3134
+ }
3135
+ function stripTomlComment(line) {
3136
+ let inBasic = false;
3137
+ let inLiteral = false;
3138
+ for (let i = 0; i < line.length; i++) {
3139
+ const ch = line[i];
3140
+ if (inBasic) {
3141
+ if (ch === "\\") i++;
3142
+ else if (ch === '"') inBasic = false;
3143
+ continue;
3144
+ }
3145
+ if (inLiteral) {
3146
+ if (ch === "'") inLiteral = false;
3147
+ continue;
3148
+ }
3149
+ if (ch === '"') inBasic = true;
3150
+ else if (ch === "'") inLiteral = true;
3151
+ else if (ch === "#") return line.slice(0, i);
3152
+ }
3153
+ return line;
3154
+ }
3155
+ function tomlBracketDelta(line) {
3156
+ const stripped = stripStringLiterals(line);
3157
+ let delta = 0;
3158
+ for (const ch of stripped) {
3159
+ if (ch === "[") delta++;
3160
+ else if (ch === "]") delta--;
3161
+ }
3162
+ return delta;
3163
+ }
3164
+ var TOML_SIMPLE_KEY = `(?:[A-Za-z0-9_-]+|"(?:[^"\\\\]|\\\\.)*"|'[^']*')`;
3165
+ var TOML_KEY_RE = new RegExp(`^\\s*(${TOML_SIMPLE_KEY}(?:\\s*\\.\\s*${TOML_SIMPLE_KEY})*)\\s*=`);
3166
+ function extractTomlSymbols(content, filePath) {
3167
+ const out = [];
3168
+ const lines2 = content.split(/\r?\n/);
3169
+ function matchLine2(line, lineNum) {
3170
+ const sectionMatch = /^\s*\[\[?\s*([^\]]+)\s*\]/.exec(line);
3171
+ if (sectionMatch !== null && sectionMatch[1] !== void 0) {
3172
+ out.push({
3173
+ filePath,
3174
+ name: sectionMatch[1].trim(),
3175
+ kind: "section",
3176
+ lineStart: lineNum + 1,
3177
+ lineEnd: lineNum + 1,
3178
+ body: line.trim(),
3179
+ docstring: "",
3180
+ parent: ""
3181
+ });
3182
+ }
3183
+ const keyMatch = TOML_KEY_RE.exec(line);
3184
+ if (keyMatch !== null && keyMatch[1] !== void 0) {
3185
+ out.push({
3186
+ filePath,
3187
+ name: keyMatch[1],
3188
+ kind: "key",
3189
+ lineStart: lineNum + 1,
3190
+ lineEnd: lineNum + 1,
3191
+ body: line.trim(),
3192
+ docstring: "",
3193
+ parent: ""
3194
+ });
3195
+ }
3196
+ }
3197
+ let openDelim = null;
3198
+ let arrayDepth = 0;
3199
+ for (let i = 0; i < lines2.length; i++) {
3200
+ const line = lines2[i];
3201
+ if (line === void 0) continue;
3202
+ if (openDelim !== null) {
3203
+ const closeIdx = line.indexOf(openDelim);
3204
+ if (closeIdx === -1) continue;
3205
+ const restStart = closeIdx + openDelim.length;
3206
+ matchLine2(line.slice(restStart), i);
3207
+ openDelim = lineOpenDelimiterAfter(stripTomlComment(line.slice(restStart)), 0);
3208
+ continue;
3209
+ }
3210
+ if (arrayDepth > 0) {
3211
+ arrayDepth = Math.max(0, arrayDepth + tomlBracketDelta(stripTomlComment(line)));
3212
+ continue;
3213
+ }
3214
+ matchLine2(line, i);
3215
+ const code = stripTomlComment(line);
3216
+ openDelim = lineOpenDelimiterAfter(code, 0);
3217
+ if (openDelim === null) arrayDepth = Math.max(0, tomlBracketDelta(code));
3218
+ }
3219
+ return out;
3220
+ }
3221
+ function splitTopLevelSelectors(rawCapture, strippedCapture) {
3222
+ const parts = [];
3223
+ let depth = 0;
3224
+ let start = 0;
3225
+ for (let i = 0; i < strippedCapture.length; i++) {
3226
+ const ch = strippedCapture[i];
3227
+ if (ch === "(") depth += 1;
3228
+ else if (ch === ")") depth = Math.max(0, depth - 1);
3229
+ else if (ch === "," && depth === 0) {
3230
+ parts.push(rawCapture.slice(start, i));
3231
+ start = i + 1;
3232
+ }
3233
+ }
3234
+ parts.push(rawCapture.slice(start));
3235
+ return parts;
3236
+ }
3237
+ function nextContentLineOpensBrace(lines2, i) {
3238
+ for (let j = i + 1; j < lines2.length; j++) {
3239
+ const next = lines2[j]?.trim() ?? "";
3240
+ if (next.length === 0) continue;
3241
+ return next === "{";
3242
+ }
3243
+ return false;
3244
+ }
3245
+ function extractCssSymbols(content, filePath) {
3246
+ const out = [];
3247
+ const lines2 = stripCstyleComments(content).split(/\r?\n/);
3248
+ const rawLines = content.split(/\r?\n/);
3249
+ let pending = [];
3250
+ for (let i = 0; i < lines2.length; i++) {
3251
+ const line = lines2[i];
3252
+ if (line === void 0) continue;
3253
+ const trimmed = line.trim();
3254
+ if (trimmed.length === 0 && (rawLines[i]?.trim().length ?? 0) > 0) {
3255
+ continue;
3256
+ }
3257
+ const strippedLine = stripStringLiterals(line);
3258
+ const selectorLineMatch = /^[ \t]*([^{}@][^{]*)\{/d.exec(strippedLine);
3259
+ if (selectorLineMatch !== null && selectorLineMatch[1] !== void 0) {
3260
+ for (const p of pending) {
3261
+ out.push({
3262
+ filePath,
3263
+ name: p.name,
3264
+ kind: "selector",
3265
+ lineStart: p.line,
3266
+ lineEnd: p.line,
3267
+ body: p.body,
3268
+ docstring: "",
3269
+ parent: ""
3270
+ });
3271
+ }
3272
+ pending = [];
3273
+ const captureRange = selectorLineMatch.indices?.[1];
3274
+ const rawCapture = captureRange ? line.slice(captureRange[0], captureRange[1]) : selectorLineMatch[1];
3275
+ const strippedCapture = captureRange ? strippedLine.slice(captureRange[0], captureRange[1]) : selectorLineMatch[1];
3276
+ for (const part of splitTopLevelSelectors(rawCapture, strippedCapture)) {
3277
+ const name = part.trim();
3278
+ if (name) {
3279
+ out.push({
3280
+ filePath,
3281
+ name,
3282
+ kind: "selector",
3283
+ lineStart: i + 1,
3284
+ lineEnd: i + 1,
3285
+ body: line.trim(),
3286
+ docstring: "",
3287
+ parent: ""
3288
+ });
3289
+ }
3290
+ }
3291
+ continue;
3292
+ }
3293
+ if (trimmed === "{") {
3294
+ for (const p of pending) {
3295
+ out.push({
3296
+ filePath,
3297
+ name: p.name,
3298
+ kind: "selector",
3299
+ lineStart: p.line,
3300
+ lineEnd: p.line,
3301
+ body: p.body,
3302
+ docstring: "",
3303
+ parent: ""
3304
+ });
3305
+ }
3306
+ pending = [];
3307
+ continue;
3308
+ }
3309
+ if (trimmed.length > 0 && !trimmed.startsWith("@") && !trimmed.includes("{") && !trimmed.includes("}") && !trimmed.includes(";")) {
3310
+ const endsWithComma = trimmed.endsWith(",");
3311
+ if (endsWithComma || pending.length > 0 || nextContentLineOpensBrace(lines2, i)) {
3312
+ const name = endsWithComma ? trimmed.slice(0, -1).trim() : trimmed;
3313
+ if (name) pending.push({ name, line: i + 1, body: trimmed });
3314
+ continue;
3315
+ }
3316
+ }
3317
+ pending = [];
3318
+ }
3319
+ return out;
3320
+ }
3321
+ function extractDockerfileSymbols(content, filePath) {
3322
+ const out = [];
3323
+ const lines2 = content.split(/\r?\n/);
3324
+ let continuing = false;
3325
+ for (let i = 0; i < lines2.length; i++) {
3326
+ const line = lines2[i];
3327
+ if (line === void 0) continue;
3328
+ if (continuing) {
3329
+ continuing = line.trimEnd().endsWith("\\");
3330
+ continue;
3331
+ }
3332
+ const isComment = line.trim().startsWith("#");
3333
+ const match = isComment ? null : /^\s*(FROM|RUN|COPY|ADD|EXPOSE|ENV|WORKDIR|CMD|ENTRYPOINT|ARG|LABEL|VOLUME|USER|HEALTHCHECK|ONBUILD|SHELL|STOPSIGNAL|MAINTAINER)\s+(.+)/i.exec(
3334
+ line
3335
+ );
3336
+ if (match !== null && match[1] !== void 0) {
3337
+ const cmd = match[1];
3338
+ const arg = (match[2] ?? "").substring(0, 40);
3339
+ const name = `${cmd} ${arg}`.trim();
3340
+ out.push({
3341
+ filePath,
3342
+ name,
3343
+ kind: "directive",
3344
+ lineStart: i + 1,
3345
+ lineEnd: i + 1,
3346
+ body: line.trim(),
3347
+ docstring: "",
3348
+ parent: ""
3349
+ });
3350
+ }
3351
+ continuing = !isComment && line.trimEnd().endsWith("\\");
3352
+ }
3353
+ return out;
3354
+ }
3355
+ var FALLBACK_PATTERNS = [
3356
+ // Python
3357
+ { re: /^[ \t]*(?:async\s+)?def\s+([A-Za-z_]\w*)/, kind: "function", style: "hash" },
3358
+ { re: /^[ \t]*class\s+([A-Za-z_]\w*)/, kind: "class", style: "hash" },
3359
+ // TS/JS function & class declarations (optionally exported/async)
3360
+ {
3361
+ re: /^[ \t]*(?:export\s+)?(?:default\s+)?(?:async\s+)?function\s*(?:\*\s*)?([A-Za-z_$][\w$]*)/,
3362
+ kind: "function",
3363
+ style: "c"
3364
+ },
3365
+ {
3366
+ re: /^[ \t]*(?:export\s+)?(?:default\s+)?(?:abstract\s+)?class\s+([A-Za-z_$][\w$]*)/,
3367
+ kind: "class",
3368
+ style: "c"
3369
+ },
3370
+ { re: /^[ \t]*(?:export\s+)?interface\s+([A-Za-z_$][\w$]*)/, kind: "interface", style: "c" },
3371
+ { re: /^[ \t]*(?:export\s+)?type\s+([A-Za-z_$][\w$]*)\s*=/, kind: "type", style: "c" },
3372
+ // const/let/var bound to an arrow or function expression
3373
+ {
3374
+ re: /^[ \t]*(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>/,
3375
+ kind: "function",
3376
+ style: "c"
3377
+ },
3378
+ // Rust / Go function & struct/type patterns
3379
+ { re: /^[ \t]*(?:pub\s+)?(?:async\s+)?fn\s+([A-Za-z_]\w*)/, kind: "function", style: "c" },
3380
+ { re: /^[ \t]*(?:pub\s+)?struct\s+([A-Za-z_]\w*)/, kind: "struct", style: "c" },
3381
+ { re: /^[ \t]*func\s+(?:\([^)]*\)\s*)?([A-Za-z_]\w*)/, kind: "function", style: "c" }
3382
+ ];
3383
+ function extractWithRegex(content, filePath) {
3384
+ const out = [];
3385
+ const lines2 = content.split(/\r?\n/);
3386
+ for (let i = 0; i < lines2.length; i++) {
3387
+ const line = lines2[i];
3388
+ if (line === void 0) continue;
3389
+ for (const { re, kind, style } of FALLBACK_PATTERNS) {
3390
+ const m = re.exec(line);
3391
+ if (m !== null && m[1] !== void 0) {
3392
+ out.push({
3393
+ filePath,
3394
+ name: m[1],
3395
+ kind,
3396
+ lineStart: i + 1,
3397
+ lineEnd: i + 1,
3398
+ body: line.trim(),
3399
+ docstring: precedingDocComment(lines2, i + 1, style),
3400
+ parent: ""
3401
+ });
3402
+ break;
3403
+ }
3404
+ }
3405
+ }
3406
+ return out;
3407
+ }
3408
+
3409
+ // src/parser.ts
3410
+ var _require2 = createRequire2(import.meta.url);
3411
+ var COMPILER_ARTIFACT_REF_NAMES = /* @__PURE__ */ new Set(["__name"]);
3412
+ var REF_LANGUAGES = /* @__PURE__ */ new Set([
3413
+ "typescript",
3414
+ "javascript",
3415
+ "python",
3416
+ "go",
3417
+ "rust",
3418
+ "java",
3419
+ "c",
3420
+ "cpp",
3421
+ "ruby"
3422
+ ]);
3423
+ var _parserCtor;
3424
+ var _parserCtorError = null;
3425
+ var _parserCtorOverride = void 0;
3426
+ var _parserCtorErrorOverride = void 0;
3427
+ var _grammarCache = /* @__PURE__ */ new Map();
3428
+ function loadParserCtor() {
3429
+ if (_parserCtorOverride !== void 0) return _parserCtorOverride;
3430
+ if (_parserCtor !== void 0) return _parserCtor;
3431
+ try {
3432
+ _parserCtor = _require2("tree-sitter");
3433
+ } catch (e) {
3434
+ _parserCtor = null;
3435
+ _parserCtorError = e instanceof Error ? e : new Error(String(e));
3436
+ }
3437
+ return _parserCtor;
3438
+ }
3439
+ function treeSitterCoreAvailable() {
3440
+ return loadParserCtor() !== null;
3441
+ }
3442
+ function treeSitterCoreLoadError() {
3443
+ return _parserCtorErrorOverride !== void 0 ? _parserCtorErrorOverride : _parserCtorError;
3444
+ }
3445
+ var CPP_HEADER_SNIFF_RE = /\bclass\s+\w|\bnamespace\s+\w|\btemplate\s*<|::\s*\w|\b(?:public|private|protected)\s*:/;
3446
+ var MAX_SYMBOL_BODY_CHARS = SYMBOL_BODY_CHAR_CAP;
3447
+ function boundSymbolBody(body) {
3448
+ return body.length > MAX_SYMBOL_BODY_CHARS ? "" : body;
3449
+ }
3450
+ var MAX_SYMBOL_DOCSTRING_CHARS = 16 * 1024;
3451
+ var DOCSTRING_TRUNCATION_MARKER = "\n[... docstring truncated by token-goat ...]";
3452
+ function boundSymbolDocstring(docstring) {
3453
+ if (docstring.length <= MAX_SYMBOL_DOCSTRING_CHARS) return docstring;
3454
+ let end = MAX_SYMBOL_DOCSTRING_CHARS - DOCSTRING_TRUNCATION_MARKER.length;
3455
+ const cutsSurrogatePair = end > 0 && docstring.charCodeAt(end - 1) >= 55296 && docstring.charCodeAt(end - 1) <= 56319;
3456
+ if (cutsSurrogatePair) end -= 1;
3457
+ return docstring.slice(0, Math.max(0, end)) + DOCSTRING_TRUNCATION_MARKER;
3458
+ }
3459
+ function loadGrammar(lang, filePath, content) {
3460
+ const useTsx = lang === "typescript" && filePath !== void 0 && path4.extname(filePath).toLowerCase() === ".tsx";
3461
+ const useCppHeader = lang === "c" && filePath !== void 0 && path4.extname(filePath).toLowerCase() === ".h" && content !== void 0 && CPP_HEADER_SNIFF_RE.test(content);
3462
+ const cacheKey = useTsx ? "typescript:tsx" : useCppHeader ? "c:cpp-header" : lang;
3463
+ const cached = _grammarCache.get(cacheKey);
3464
+ if (cached !== void 0) return cached;
3465
+ let grammar = null;
3466
+ try {
3467
+ if (lang === "typescript") {
3468
+ const mod = _require2("tree-sitter-typescript");
3469
+ grammar = useTsx ? mod.tsx : mod.typescript;
3470
+ } else if (lang === "javascript") {
3471
+ grammar = _require2("tree-sitter-javascript");
3472
+ } else if (lang === "python") {
3473
+ grammar = _require2("tree-sitter-python");
3474
+ } else if (lang === "go") {
3475
+ grammar = _require2("tree-sitter-go");
3476
+ } else if (lang === "rust") {
3477
+ grammar = _require2("tree-sitter-rust");
3478
+ } else if (lang === "ruby") {
3479
+ grammar = _require2("tree-sitter-ruby");
3480
+ } else if (lang === "java") {
3481
+ grammar = _require2("tree-sitter-java");
3482
+ } else if (lang === "c") {
3483
+ grammar = useCppHeader ? _require2("tree-sitter-cpp") : _require2("tree-sitter-c");
3484
+ } else if (lang === "cpp") {
3485
+ grammar = _require2("tree-sitter-cpp");
3486
+ }
3487
+ } catch {
3488
+ grammar = null;
3489
+ }
3490
+ _grammarCache.set(cacheKey, grammar);
3491
+ return grammar;
3492
+ }
3493
+ function isTreeSitterAvailable(lang) {
3494
+ if (!TREE_SITTER_LANGUAGES.includes(lang)) return false;
3495
+ return loadParserCtor() !== null && loadGrammar(lang) !== null;
3496
+ }
3497
+ var TREE_SITTER_GRAMMAR_PACKAGES = [
3498
+ "tree-sitter-typescript",
3499
+ "tree-sitter-javascript",
3500
+ "tree-sitter-python",
3501
+ "tree-sitter-go",
3502
+ "tree-sitter-rust",
3503
+ "tree-sitter-ruby",
3504
+ "tree-sitter-java",
3505
+ "tree-sitter-c",
3506
+ "tree-sitter-cpp"
3507
+ ];
3508
+ function missingTreeSitterGrammarPackages() {
3509
+ return TREE_SITTER_GRAMMAR_PACKAGES.filter((pkg) => {
3510
+ try {
3511
+ _require2.resolve(pkg);
3512
+ return false;
3513
+ } catch {
3514
+ return true;
3515
+ }
3516
+ });
3517
+ }
3518
+ function isLwcFile(filePath, extension) {
3519
+ const normalized = filePath.replace(/\\/g, "/");
3520
+ return /\/lwc\/[^/]+\/[^/]+$/i.test(normalized) && normalized.toLowerCase().endsWith(extension);
3521
+ }
3522
+ function mergeParseResults(...results) {
3523
+ const symbols = [];
3524
+ const refs = [];
3525
+ const seenSymbols = /* @__PURE__ */ new Set();
3526
+ const seenRefs = /* @__PURE__ */ new Set();
3527
+ for (const result of results) {
3528
+ for (const entry of result.symbols) {
3529
+ const key = `${entry.filePath}\0${entry.name}\0${entry.kind}\0${entry.lineStart}`;
3530
+ if (seenSymbols.has(key)) continue;
3531
+ seenSymbols.add(key);
3532
+ symbols.push(entry);
3533
+ }
3534
+ for (const entry of result.refs) {
3535
+ const key = `${entry.filePath}\0${entry.name}\0${entry.line}\0${entry.col}`;
3536
+ if (seenRefs.has(key)) continue;
3537
+ seenRefs.add(key);
3538
+ refs.push(entry);
3539
+ }
3540
+ }
3541
+ return { symbols, refs };
3542
+ }
3543
+ function parseContent(content, filePath, language) {
3544
+ if (content.charCodeAt(0) === 65279) {
3545
+ content = content.slice(1);
3546
+ }
3547
+ if (language === "ipynb") {
3548
+ const virtual = ipynbToVirtualSource(content);
3549
+ if (virtual.cellLanguage === null) return { symbols: [], refs: [] };
3550
+ return parseContent(virtual.content, filePath, "python");
3551
+ }
3552
+ const viaTreeSitter = parseWithTreeSitter(content, filePath, language);
3553
+ if (viaTreeSitter !== null) return viaTreeSitter;
3554
+ return extractNoTreeSitter(content, filePath, language);
3555
+ }
3556
+ function parseWithTreeSitter(content, filePath, language) {
3557
+ if (!isTreeSitterAvailable(language)) return null;
3558
+ try {
3559
+ const Ctor = loadParserCtor();
3560
+ const grammar = loadGrammar(language, filePath, content);
3561
+ if (Ctor === null || grammar === null) return null;
3562
+ const parser = new Ctor();
3563
+ parser.setLanguage(grammar);
3564
+ const tree = parser.parse(content);
3565
+ const root = tree.rootNode;
3566
+ let symbols;
3567
+ if (language === "python") {
3568
+ symbols = extractPythonSymbols(root, filePath);
3569
+ } else if (language === "go") {
3570
+ symbols = extractGoSymbols(root, filePath, content.split(/\r?\n/));
3571
+ } else if (language === "rust") {
3572
+ symbols = extractRustSymbols(root, filePath, content.split(/\r?\n/));
3573
+ } else if (language === "ruby") {
3574
+ symbols = extractRubySymbols(root, filePath, content.split(/\r?\n/));
3575
+ } else if (language === "java") {
3576
+ symbols = extractJavaSymbols(root, filePath, content.split(/\r?\n/));
3577
+ } else if (language === "cpp" || language === "c") {
3578
+ symbols = extractCppSymbols(root, filePath, content.split(/\r?\n/));
3579
+ } else {
3580
+ symbols = extractTsJsSymbols(root, filePath, content.split(/\r?\n/));
3581
+ }
3582
+ const refs = REF_LANGUAGES.has(language) ? extractRefs(root, filePath, language) : [];
3583
+ const parsed = { symbols, refs };
3584
+ return language === "javascript" && isLwcFile(filePath, ".js") ? mergeParseResults(parsed, extractLwcJavaScript(content, filePath)) : parsed;
3585
+ } catch {
3586
+ return null;
3587
+ }
3588
+ }
3589
+ function parseSourceSymbolsTreeSitterOnly(content, filePath, language) {
3590
+ const text = content.charCodeAt(0) === 65279 ? content.slice(1) : content;
3591
+ const parsed = parseWithTreeSitter(text, filePath, language);
3592
+ return parsed === null ? null : parsed.symbols;
3593
+ }
3594
+ var REGEX_ADAPTERS_SLOT = /* @__PURE__ */ Symbol.for("token-goat.regex-adapters");
3595
+ var adapterSlot = globalThis;
3596
+ async function loadRegexExtractors() {
3597
+ adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-GIIHUSZX.mjs");
3598
+ }
3599
+ function regexAdapters() {
3600
+ const loaded = adapterSlot[REGEX_ADAPTERS_SLOT];
3601
+ if (loaded === void 0) throw new Error("token-goat: the regex language adapters are not loaded; await loadRegexExtractors() before parsing");
3602
+ return loaded;
3603
+ }
3604
+ var noTreeSitterTable;
3605
+ function noTreeSitterExtractors() {
3606
+ noTreeSitterTable ??= {
3607
+ markdown: extractMarkdownSymbols,
3608
+ json: extractJsonSymbols,
3609
+ yaml: extractYamlSymbols,
3610
+ toml: extractTomlSymbols,
3611
+ css: extractCssSymbols,
3612
+ dockerfile: extractDockerfileSymbols,
3613
+ ...regexAdapters().ADAPTER_EXTRACTORS
3614
+ };
3615
+ return noTreeSitterTable;
3616
+ }
3617
+ function extractNoTreeSitter(content, filePath, language) {
3618
+ const adapters = regexAdapters();
3619
+ if (language === "salesforce_metadata") return adapters.extractSalesforceMetadata(content, filePath);
3620
+ if (language === "salesforce_markup") return extractSalesforceMarkup(content, filePath);
3621
+ if (language === "html" && isLwcFile(filePath, ".html")) {
3622
+ const base = { symbols: adapters.ADAPTER_EXTRACTORS.html(content, filePath), refs: [] };
3623
+ return mergeParseResults(base, extractLwcTemplate(content, filePath));
3624
+ }
3625
+ if (language === "vue") return adapters.extractVue(content, filePath);
3626
+ if (language === "svelte") return adapters.extractSvelte(content, filePath);
3627
+ if (language === "astro") return adapters.extractAstro(content, filePath);
3628
+ if (language === "cobol") {
3629
+ const r = adapters.extractCobol(content, filePath);
3630
+ return { symbols: r.symbols, refs: r.refs };
3631
+ }
3632
+ if (language === "natural") {
3633
+ const r = adapters.extractNatural(content, filePath);
3634
+ return { symbols: r.symbols, refs: r.refs };
3635
+ }
3636
+ const parsed = {
3637
+ symbols: extractSymbolsNoTreeSitter(content, filePath, language),
3638
+ refs: []
3639
+ };
3640
+ return language === "javascript" && isLwcFile(filePath, ".js") ? mergeParseResults(parsed, extractLwcJavaScript(content, filePath)) : parsed;
3641
+ }
3642
+ function extractSymbolsNoTreeSitter(content, filePath, language) {
3643
+ if (language === "unknown") return [];
3644
+ return (noTreeSitterExtractors()[language] ?? extractWithRegex)(content, filePath);
3645
+ }
3646
+ function deleteFileRows(db, filePath) {
3647
+ const folded = foldPath(filePath);
3648
+ db.prepare(`DELETE FROM symbols WHERE ${pathEqClause("file_path")}`).run(folded);
3649
+ db.prepare(`DELETE FROM refs WHERE ${pathEqClause("file_path")}`).run(folded);
3650
+ db.prepare(`DELETE FROM files WHERE ${pathEqClause("path")}`).run(folded);
3651
+ }
3652
+ function isUnderSkipDir(filePath, skipDirs) {
3653
+ if (skipDirs.length === 0) return false;
3654
+ const segments = filePath.split(/[/\\]/);
3655
+ return segments.slice(0, -1).some((seg) => skipDirs.includes(seg));
3656
+ }
3657
+ function isParseSkipEligible(filePath, cfg) {
3658
+ if (isUnderSkipDir(filePath, cfg.skip_dirs)) return true;
3659
+ if (cfg.skip_files.includes(path4.basename(filePath))) return true;
3660
+ try {
3661
+ const stat = fs2.statSync(filePath);
3662
+ if (stat.size > cfg.large_file_skip_kb * 1024) return true;
3663
+ } catch {
3664
+ }
3665
+ return false;
3666
+ }
3667
+ function writeParseResult(filePath, content, result, dbPath) {
3668
+ const db = getDb(dbPath);
3669
+ const sha = fingerprintContent(content);
3670
+ const mtime = safeMtime(filePath);
3671
+ const now = Date.now() / 1e3;
3672
+ const priorRow = db.prepare(`SELECT sha, embed_sha FROM files WHERE ${pathEqClause("path")}`).get(foldPath(filePath));
3673
+ const embedShaToCarry = priorRow !== void 0 && priorRow.sha === sha ? priorRow.embed_sha : null;
3674
+ const writeAll = db.transaction(() => {
3675
+ deleteFileRows(db, filePath);
3676
+ db.prepare(
3677
+ "INSERT INTO files (path, sha, mtime, language, indexed_at, parser_sha, embed_sha) VALUES (?, ?, ?, ?, ?, ?, ?)"
3678
+ ).run(filePath, sha, mtime, result.language, now, PARSER_FINGERPRINT, embedShaToCarry);
3679
+ const insSym = db.prepare(
3680
+ "INSERT INTO symbols (file_path, name, kind, line_start, line_end, body, docstring, parent) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
3681
+ );
3682
+ for (const s of result.symbols) {
3683
+ if (s.name === "" || s.kind === "") continue;
3684
+ insSym.run(
3685
+ s.filePath,
3686
+ s.name,
3687
+ s.kind,
3688
+ s.lineStart,
3689
+ s.lineEnd,
3690
+ boundSymbolBody(s.body),
3691
+ boundSymbolDocstring(s.docstring),
3692
+ s.parent
3693
+ );
3694
+ }
3695
+ const insRef = db.prepare(
3696
+ "INSERT INTO refs (file_path, name, line, col, context) VALUES (?, ?, ?, ?, ?)"
3697
+ );
3698
+ for (const r of result.refs) {
3699
+ if (r.name === "" || COMPILER_ARTIFACT_REF_NAMES.has(r.name)) continue;
3700
+ insRef.run(r.filePath, r.name, r.line, r.col, r.context);
3701
+ }
3702
+ });
3703
+ writeAll.immediate();
3704
+ }
3705
+ function indexFileSync(rawPath, dbPath = globalDbPath(), preReadBytes) {
3706
+ const filePath = canonicalizeIndexPath(rawPath);
3707
+ const ixCfg = loadConfig().indexing;
3708
+ if (ixCfg !== void 0 && isParseSkipEligible(filePath, ixCfg)) {
3709
+ const db = getDb(dbPath);
3710
+ deleteFileRows(db, filePath);
3711
+ deleteFileEmbeddings(db, filePath);
3712
+ return;
3713
+ }
3714
+ const pathLanguage = detectLanguage(filePath);
3715
+ let raw;
3716
+ if (preReadBytes !== void 0) {
3717
+ raw = preReadBytes;
3718
+ } else {
3719
+ try {
3720
+ raw = fs2.readFileSync(filePath);
3721
+ } catch (err) {
3722
+ if (err.code === "ENOENT") return;
3723
+ throw err;
3724
+ }
3725
+ }
3726
+ const content = decodeSource(raw);
3727
+ const language = refineLanguageByContent(filePath, pathLanguage, content);
3728
+ const { symbols, refs } = parseContent(content, filePath, language);
3729
+ writeParseResult(filePath, raw, { symbols, refs, language, duration: 0 }, dbPath);
3730
+ }
3731
+ var DISABLED_EMBED_SHA_PREFIX = "disabled:";
3732
+ function disabledEmbedSha(sha) {
3733
+ return DISABLED_EMBED_SHA_PREFIX + sha;
3734
+ }
3735
+ var UNAVAILABLE_EMBED_SHA_PREFIX = "unavailable:";
3736
+ function unavailableEmbedSha(sha) {
3737
+ return UNAVAILABLE_EMBED_SHA_PREFIX + sha;
3738
+ }
3739
+ var OVERSIZE_EMBED_SHA_PREFIX = "oversize:";
3740
+ function oversizeEmbedSha(sha, symbolOnlyKb) {
3741
+ return `${OVERSIZE_EMBED_SHA_PREFIX}${symbolOnlyKb}:${sha}`;
3742
+ }
3743
+ var TIMEOUT_EMBED_SHA_PREFIX = "timeout:";
3744
+ function timeoutEmbedSha(sha, workMillis) {
3745
+ return `${TIMEOUT_EMBED_SHA_PREFIX}${workMillis}:${sha}`;
3746
+ }
3747
+ function canonicalizeIndexPath(absPath) {
3748
+ if (!isCaseInsensitiveFs()) return absPath;
3749
+ let real;
3750
+ try {
3751
+ real = fs2.realpathSync.native(absPath);
3752
+ } catch {
3753
+ return absPath;
3754
+ }
3755
+ const cut = Math.max(absPath.lastIndexOf("/"), absPath.lastIndexOf("\\"));
3756
+ const base = absPath.slice(cut + 1);
3757
+ const realNorm = normalizePath(real);
3758
+ const realBase = path4.basename(realNorm);
3759
+ if (base === realBase) return absPath;
3760
+ if (foldPath(base) !== foldPath(realBase)) return absPath;
3761
+ const callerDir = cut < 0 ? normalizePath(path4.resolve(".")) : normalizePath(absPath.slice(0, cut));
3762
+ if (foldPath(callerDir) !== foldPath(path4.dirname(realNorm))) return absPath;
3763
+ return absPath.slice(0, cut + 1) + realBase;
3764
+ }
3765
+ function indexedPathSpellingIsStale(storedPath, absPath) {
3766
+ if (!isCaseInsensitiveFs()) return false;
3767
+ const stored = normalizePath(storedPath);
3768
+ const candidate = normalizePath(path4.resolve(absPath));
3769
+ if (foldPath(stored) !== foldPath(candidate)) return false;
3770
+ let real;
3771
+ try {
3772
+ real = normalizePath(fs2.realpathSync.native(absPath));
3773
+ } catch {
3774
+ return false;
3775
+ }
3776
+ if (real === stored) return false;
3777
+ if (foldPath(real) !== foldPath(stored)) return false;
3778
+ const storedSegments = stored.split("/");
3779
+ const realSegments = real.split("/");
3780
+ if (storedSegments.length !== realSegments.length) return real !== stored;
3781
+ const storedBase = storedSegments[storedSegments.length - 1];
3782
+ const realBase = realSegments[realSegments.length - 1];
3783
+ if (storedBase !== realBase) return true;
3784
+ for (let i = storedSegments.length - 2; i >= 0; i--) {
3785
+ if (storedSegments[i] !== realSegments[i]) {
3786
+ if (storedSegments.slice(0, i + 1).join("/") === candidate.split("/").slice(0, i + 1).join("/")) {
3787
+ continue;
3788
+ }
3789
+ return true;
3790
+ }
3791
+ }
3792
+ return false;
3793
+ }
3794
+ function isEmbedFresh(storedEmbedSha, sha, embeddingsEnabled, depsAvailable, symbolOnlyKb) {
3795
+ if (storedEmbedSha === void 0) return false;
3796
+ if (!embeddingsEnabled) return storedEmbedSha === disabledEmbedSha(sha);
3797
+ if (storedEmbedSha === sha) return true;
3798
+ if (!depsAvailable && storedEmbedSha === unavailableEmbedSha(sha)) return true;
3799
+ if (storedEmbedSha === oversizeEmbedSha(sha, symbolOnlyKb)) return true;
3800
+ if (storedEmbedSha === timeoutEmbedSha(sha, MAX_DOCUMENT_WORK_MILLIS)) return true;
3801
+ return false;
3802
+ }
3803
+ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onError) {
3804
+ const filePath = canonicalizeIndexPath(rawPath);
3805
+ const ixCfg = loadConfig().indexing;
3806
+ if (!ixCfg.embeddings_enabled) {
3807
+ stampEmbedSha(getDb(dbPath), filePath, sha, disabledEmbedSha);
3808
+ return;
3809
+ }
3810
+ if (filePath.toLowerCase().endsWith(".profile-meta.xml")) {
3811
+ const db = getDb(dbPath);
3812
+ deleteFileEmbeddings(db, filePath);
3813
+ stampEmbedSha(db, filePath, sha, (s) => s);
3814
+ return;
3815
+ }
3816
+ if (isEmbeddableDocument(filePath)) {
3817
+ let extracted;
3818
+ let refusedOnTheClock = false;
3819
+ try {
3820
+ extracted = await extractEmbeddableDocumentText(filePath);
3821
+ } catch (err) {
3822
+ if (!isDocumentRefusal(err)) {
3823
+ onError?.(err);
3824
+ return;
3825
+ }
3826
+ refusedOnTheClock = isTransientDocumentRefusal(err);
3827
+ extracted = null;
3828
+ }
3829
+ if (extracted === null || extracted.trim().length === 0) {
3830
+ const db = getDb(dbPath);
3831
+ deleteFileEmbeddings(db, filePath);
3832
+ stampEmbedSha(db, filePath, sha, (s) => refusedOnTheClock ? timeoutEmbedSha(s, MAX_DOCUMENT_WORK_MILLIS) : s);
3833
+ return;
3834
+ }
3835
+ if (extracted.length > ixCfg.large_file_symbol_only_kb * 1024) {
3836
+ const db = getDb(dbPath);
3837
+ deleteFileEmbeddings(db, filePath);
3838
+ stampEmbedSha(db, filePath, sha, (s) => oversizeEmbedSha(s, ixCfg.large_file_symbol_only_kb));
3839
+ return;
3840
+ }
3841
+ try {
3842
+ const db = getDb(dbPath);
3843
+ const outcome = await indexFile(db, filePath, extracted, []);
3844
+ stampEmbedSha(db, filePath, sha, (s) => outcome === "unavailable" ? unavailableEmbedSha(s) : s);
3845
+ } catch (err) {
3846
+ onError?.(err);
3847
+ }
3848
+ return;
3849
+ }
3850
+ let content;
3851
+ try {
3852
+ content = decodeSource(await fs2.promises.readFile(filePath));
3853
+ } catch {
3854
+ return;
3855
+ }
3856
+ content = redactIfDotenv(filePath, content);
3857
+ if (detectLanguage(filePath) === "ipynb") {
3858
+ const virtual = ipynbToVirtualSource(content);
3859
+ if (virtual.cellLanguage === null) {
3860
+ const db = getDb(dbPath);
3861
+ deleteFileEmbeddings(db, filePath);
3862
+ stampEmbedSha(db, filePath, sha, (s) => s);
3863
+ return;
3864
+ }
3865
+ content = virtual.content;
3866
+ }
3867
+ if (content.length > ixCfg.large_file_symbol_only_kb * 1024) {
3868
+ const db = getDb(dbPath);
3869
+ deleteFileEmbeddings(db, filePath);
3870
+ stampEmbedSha(db, filePath, sha, (s) => oversizeEmbedSha(s, ixCfg.large_file_symbol_only_kb));
3871
+ return;
3872
+ }
3873
+ if (detectLanguage(filePath) === "salesforce_metadata" && content.length > 512 * 1024) {
3874
+ const db = getDb(dbPath);
3875
+ deleteFileEmbeddings(db, filePath);
3876
+ stampEmbedSha(db, filePath, sha, (s) => s);
3877
+ return;
3878
+ }
3879
+ try {
3880
+ const db = getDb(dbPath);
3881
+ const boundaries = buildEmbeddingBoundaries(filePath, content, dbPath);
3882
+ const outcome = await indexFile(db, filePath, content, boundaries);
3883
+ stampEmbedSha(db, filePath, sha, (s) => outcome === "unavailable" ? unavailableEmbedSha(s) : s);
3884
+ } catch (err) {
3885
+ onError?.(err);
3886
+ }
3887
+ }
3888
+ function stampEmbedSha(db, filePath, sha, makeValue) {
3889
+ if (sha === void 0) return;
3890
+ db.prepare(`UPDATE files SET embed_sha = ? WHERE ${pathEqClause("path")} AND sha = ?`).run(
3891
+ makeValue(sha),
3892
+ foldPath(filePath),
3893
+ sha
3894
+ );
3895
+ }
3896
+ function safeMtime(filePath) {
3897
+ try {
3898
+ return fs2.statSync(filePath).mtimeMs / 1e3;
3899
+ } catch {
3900
+ return 0;
3901
+ }
3902
+ }
3903
+
3904
+ // src/section_reader.ts
3905
+ init_define_import_meta_env();
3906
+ import { readFileSync as readFileSync3 } from "node:fs";
3907
+ function parseHeadingSpec(spec, headers) {
3908
+ const m = /^([^#\r\n]+)#(\d+)$/.exec(spec);
3909
+ if (m !== null && m[1] !== void 0 && m[2] !== void 0) {
3910
+ const specLower = spec.trim().toLowerCase();
3911
+ const isLiteralHeading = headers?.some((h) => h.heading.trim().toLowerCase() === specLower) ?? false;
3912
+ if (isLiteralHeading) {
3913
+ return { base: spec.trim(), ordinal: null };
3914
+ }
3915
+ return { base: m[1].trim(), ordinal: Number.parseInt(m[2], 10) };
3916
+ }
3917
+ return { base: spec.trim(), ordinal: null };
3918
+ }
3919
+ function normalizeHeading(s) {
3920
+ let n = s.replace(/[—–]/g, "-");
3921
+ n = n.replace(/\s*\([^)]+\)\s*$/, "");
3922
+ n = n.replace(/^\d+\.\s+/, "");
3923
+ return n.replace(/\s+/g, " ").trim();
3924
+ }
3925
+ function normalizeHeadingStrip(s) {
3926
+ let n = s.replace(/\s*[—–].*$/, "");
3927
+ n = n.replace(/\s*\([^)]+\)\s*$/, "");
3928
+ n = n.replace(/^\d+\.\s+/, "");
3929
+ return n.replace(/\s+/g, " ").trim();
3930
+ }
3931
+ var MIN_WIDEN_WORD_LEN = 3;
3932
+ var MARKDOWN_HEADER_RE = /^(#{1,6})\s+([^\r\n]+?)(?:\s+#+)?\s*$/;
3933
+ var TABLE_HEADER_RE = /^\s*\[+\s*([^\]]+?)\s*\]+\s*(?:[#;].*)?$/;
3934
+ var PYTHON_HEADER_RE = /^(\s*)(?:async\s+)?(?:def|class)\s+([A-Za-z_]\w*)/;
3935
+ var KEYVALUE_HEADER_RE = /^([A-Za-z_][\w.-]*)\s*(?:=|:(?!\/\/))/;
3936
+ var ENV_KEYVALUE_HEADER_RE = /^(?:export\s+)?([A-Za-z_][\w.-]*)\s*(?:=|:(?!\/\/))/;
3937
+ function findMarkdownHeaders(lines2) {
3938
+ const headers = [];
3939
+ const unfenced = Array.from(eachUnfencedLine(lines2));
3940
+ for (let u = 0; u < unfenced.length; u++) {
3941
+ const [i, line] = unfenced[u];
3942
+ const m = MARKDOWN_HEADER_RE.exec(line);
3943
+ if (m !== null && m[1] !== void 0 && m[2] !== void 0) {
3944
+ headers.push({ heading: m[2].trim(), level: m[1].length, index: i });
3945
+ continue;
3946
+ }
3947
+ const trimmed = line.trim();
3948
+ if (trimmed !== "" && !trimmed.startsWith("#") && !trimmed.startsWith("|") && !trimmed.startsWith("```") && !trimmed.startsWith("~~~") && !/^([-*+]|\d+\.)\s/.test(trimmed) && u + 1 < unfenced.length) {
3949
+ const [nextIdx, nextLine] = unfenced[u + 1];
3950
+ if (nextIdx === i + 1) {
3951
+ const eqMatch = /^\s*(=+)\s*$/.exec(nextLine);
3952
+ const dashMatch = /^\s*(-+)\s*$/.exec(nextLine);
3953
+ if (eqMatch !== null) {
3954
+ headers.push({ heading: trimmed, level: 1, index: i });
3955
+ u++;
3956
+ continue;
3957
+ } else if (dashMatch !== null) {
3958
+ headers.push({ heading: trimmed, level: 2, index: i });
3959
+ u++;
3960
+ continue;
3961
+ }
3962
+ }
3963
+ }
3964
+ }
3965
+ return headers;
3966
+ }
3967
+ function findTableHeaders(lines2, isToml) {
3968
+ const headers = [];
3969
+ let openDelim = null;
3970
+ let arrayDepth = 0;
3971
+ for (let i = 0; i < lines2.length; i++) {
3972
+ const line = lines2[i];
3973
+ if (line === void 0) continue;
3974
+ if (openDelim !== null) {
3975
+ const closeIdx = line.indexOf(openDelim);
3976
+ if (closeIdx === -1) continue;
3977
+ const restStart = closeIdx + openDelim.length;
3978
+ const m2 = TABLE_HEADER_RE.exec(line.slice(restStart));
3979
+ if (m2 !== null && m2[1] !== void 0) headers.push({ heading: m2[1].trim(), level: 1, index: i });
3980
+ openDelim = lineOpenDelimiterAfter(stripTomlComment(line.slice(restStart)), 0);
3981
+ continue;
3982
+ }
3983
+ if (isToml && arrayDepth > 0) {
3984
+ arrayDepth = Math.max(0, arrayDepth + tomlBracketDelta(stripTomlComment(line)));
3985
+ continue;
3986
+ }
3987
+ const m = TABLE_HEADER_RE.exec(line);
3988
+ if (m !== null && m[1] !== void 0) headers.push({ heading: m[1].trim(), level: 1, index: i });
3989
+ if (isToml) {
3990
+ const code = stripTomlComment(line);
3991
+ openDelim = lineOpenDelimiterAfter(code, 0);
3992
+ if (openDelim === null) arrayDepth = Math.max(0, tomlBracketDelta(code));
3993
+ }
3994
+ }
3995
+ return headers;
3996
+ }
3997
+ var VB_REGION_RE = /^\s*#\s*Region\s+"((?:[^"]|"")*)"/i;
3998
+ var VB_END_REGION_RE = /^\s*#\s*End\s+Region\b/i;
3999
+ function extractShellBannerHeading(line) {
4000
+ const trimmed = line.trim();
4001
+ if (!trimmed.startsWith("#")) return null;
4002
+ if (trimmed.startsWith("#!")) return null;
4003
+ const mdMatch = /^(#{2,6})[^\S\r\n]+(\S(?:.*?\S)?)(?:[^\S\r\n]+#+)?$/.exec(trimmed);
4004
+ if (mdMatch && mdMatch[1] && mdMatch[2]) {
4005
+ const text = mdMatch[2].trim();
4006
+ if (text && !/^[-=]+$/.test(text)) {
4007
+ return { heading: text, level: mdMatch[1].length };
4008
+ }
4009
+ }
4010
+ const ruleMatch = /^#\s*[-=]{2,}\s*(\S(?:.*?\S)?)\s*[-=]{2,}$/.exec(trimmed);
4011
+ if (ruleMatch && ruleMatch[1]) {
4012
+ const text = ruleMatch[1].trim();
4013
+ if (text && !/^[-=]+$/.test(text)) {
4014
+ return { heading: text, level: 1 };
4015
+ }
4016
+ }
4017
+ const bracketMatch = /^#\s*\[([a-zA-Z0-9_.\-\s]+)\]$/.exec(trimmed);
4018
+ if (bracketMatch && bracketMatch[1]) {
4019
+ const text = bracketMatch[1].trim();
4020
+ if (text) {
4021
+ return { heading: text, level: 1 };
4022
+ }
4023
+ }
4024
+ const regionMatch = /^#\s*(?:REGION|SECTION):\s*(\S.*)$/i.exec(trimmed);
4025
+ if (regionMatch && regionMatch[1]) {
4026
+ const text = regionMatch[1].trim();
4027
+ if (text) {
4028
+ return { heading: text, level: 1 };
4029
+ }
4030
+ }
4031
+ return null;
4032
+ }
4033
+ function findShellBannerHeaders(lines2) {
4034
+ const headers = [];
4035
+ for (let i = 0; i < lines2.length; i++) {
4036
+ const line = lines2[i] ?? "";
4037
+ const banner = extractShellBannerHeading(line);
4038
+ if (banner !== null) {
4039
+ headers.push({ heading: banner.heading, level: banner.level, index: i });
4040
+ }
4041
+ }
4042
+ return headers;
4043
+ }
4044
+ function findVbRegionHeaders(lines2) {
4045
+ const headers = [];
4046
+ const open = [];
4047
+ for (let i = 0; i < lines2.length; i++) {
4048
+ const line = lines2[i] ?? "";
4049
+ const m = VB_REGION_RE.exec(line);
4050
+ if (m !== null) {
4051
+ open.push(headers.length);
4052
+ headers.push({ heading: (m[1] ?? "").replace(/""/g, '"'), level: open.length, index: i });
4053
+ continue;
4054
+ }
4055
+ if (VB_END_REGION_RE.test(line)) {
4056
+ const pos = open.pop();
4057
+ const h = pos === void 0 ? void 0 : headers[pos];
4058
+ if (pos !== void 0 && h !== void 0) headers[pos] = { ...h, endIndex: i + 1 };
4059
+ }
4060
+ }
4061
+ return headers;
4062
+ }
4063
+ function headerEndIndex(headers, kind, headerPos, totalLines) {
4064
+ const explicit = headers[headerPos]?.endIndex;
4065
+ if (explicit !== void 0) return explicit;
4066
+ return kind === "table-toml" ? tableSectionEndIndex(headers, headerPos, totalLines) : sectionEndIndex(headers, headerPos, totalLines);
4067
+ }
4068
+ function findPythonHeaders(lines2) {
4069
+ const headers = [];
4070
+ const indentStack = [];
4071
+ let openDelim = null;
4072
+ for (let i = 0; i < lines2.length; i++) {
4073
+ const line = lines2[i];
4074
+ if (line === void 0) continue;
4075
+ if (openDelim !== null) {
4076
+ const closeIdx = line.indexOf(openDelim);
4077
+ if (closeIdx === -1) continue;
4078
+ openDelim = lineOpenDelimiterAfter(line, closeIdx + openDelim.length);
4079
+ continue;
4080
+ }
4081
+ const m = PYTHON_HEADER_RE.exec(line);
4082
+ if (m === null || m[1] === void 0 || m[2] === void 0) {
4083
+ openDelim = lineOpenDelimiterAfter(line, 0);
4084
+ continue;
4085
+ }
4086
+ const indent = m[1].replace(/\t/g, " ").length;
4087
+ while (indentStack.length > 0 && indent <= (indentStack[indentStack.length - 1] ?? -1)) {
4088
+ indentStack.pop();
4089
+ }
4090
+ const level = indentStack.length + 1;
4091
+ indentStack.push(indent);
4092
+ headers.push({ heading: m[2], level, index: i });
4093
+ openDelim = lineOpenDelimiterAfter(line, 0);
4094
+ }
4095
+ return headers;
4096
+ }
4097
+ function findKeyValueHeaders(lines2, language) {
4098
+ const headers = [];
4099
+ const isEnv = language === "env_file";
4100
+ let openQuote = null;
4101
+ for (let i = 0; i < lines2.length; i++) {
4102
+ const line = lines2[i];
4103
+ if (line === void 0) continue;
4104
+ if (openQuote !== null) {
4105
+ const closed = isEnv ? _lineClosesQuote(line, openQuote) : yamlLineClosesQuote(line, openQuote);
4106
+ if (closed) openQuote = null;
4107
+ continue;
4108
+ }
4109
+ const scanned = isEnv ? line.replace(/^[ \t]+/, "") : line;
4110
+ const m = (isEnv ? ENV_KEYVALUE_HEADER_RE : KEYVALUE_HEADER_RE).exec(scanned);
4111
+ if (m === null || m[1] === void 0) continue;
4112
+ headers.push({ heading: m[1], level: 1, index: i });
4113
+ openQuote = isEnv ? _detectOpenQuote(scanned.slice(m[0].length)) : yamlOpenQuoteAfter(line, m[0].length);
4114
+ }
4115
+ return headers;
4116
+ }
4117
+ function findHtmlHeaders(text) {
4118
+ const lineIndex = buildLineIndex(text);
4119
+ const headers = [];
4120
+ for (const hm of findHtmlHeadingMatches(text)) {
4121
+ if (!hm.heading) continue;
4122
+ const index = offsetToLine(lineIndex, hm.offset) - 1;
4123
+ headers.push({ heading: hm.heading, level: hm.level, index });
4124
+ }
4125
+ return headers;
4126
+ }
4127
+ function findHeaders(text, language) {
4128
+ const lines2 = text.split("\n");
4129
+ if (language === "markdown") return { headers: findMarkdownHeaders(lines2), kind: "markdown" };
4130
+ if (language === "html" || language === "liquid") return { headers: findHtmlHeaders(text), kind: "markdown" };
4131
+ if (language === "toml") return { headers: findTableHeaders(lines2, true), kind: "table-toml" };
4132
+ if (language === "python") return { headers: findPythonHeaders(lines2), kind: "python" };
4133
+ if (language === "bash") {
4134
+ const banners = findShellBannerHeaders(lines2);
4135
+ if (banners.length > 0) return { headers: banners, kind: "banner" };
4136
+ }
4137
+ if (language === "vb") return { headers: findVbRegionHeaders(lines2), kind: "region" };
4138
+ if (language === "ini") return { headers: findTableHeaders(lines2, false), kind: "table-flat" };
4139
+ if (language === "yaml" || language === "env_file")
4140
+ return { headers: findKeyValueHeaders(lines2, language), kind: "keyvalue" };
4141
+ const md = findMarkdownHeaders(lines2);
4142
+ if (md.length > 0) return { headers: md, kind: "markdown" };
4143
+ const tbl = findTableHeaders(lines2, false);
4144
+ if (tbl.length > 0) return { headers: tbl, kind: "table-toml" };
4145
+ return { headers: findKeyValueHeaders(lines2, language), kind: "keyvalue" };
4146
+ }
4147
+ function sectionEndIndex(headers, headerPos, totalLines) {
4148
+ const current = headers[headerPos];
4149
+ if (current === void 0) return totalLines;
4150
+ for (let j = headerPos + 1; j < headers.length; j++) {
4151
+ const next = headers[j];
4152
+ if (next === void 0) continue;
4153
+ if (next.level <= current.level) return next.index;
4154
+ }
4155
+ return totalLines;
4156
+ }
4157
+ function tableSectionEndIndex(headers, headerPos, totalLines) {
4158
+ const current = headers[headerPos];
4159
+ if (current === void 0) return totalLines;
4160
+ const prefix = current.heading + ".";
4161
+ for (let j = headerPos + 1; j < headers.length; j++) {
4162
+ const next = headers[j];
4163
+ if (next === void 0) continue;
4164
+ if (next.heading.startsWith(prefix)) continue;
4165
+ return next.index;
4166
+ }
4167
+ return totalLines;
4168
+ }
4169
+ function resolveHeaderPos(headers, base, ordinal) {
4170
+ const target = base.toLowerCase();
4171
+ const normalizedTarget = normalizeHeading(base).toLowerCase();
4172
+ const strippedTarget = normalizeHeadingStrip(base).toLowerCase();
4173
+ const exactMatches = [];
4174
+ const normalizedMatches = [];
4175
+ const strippedMatches = [];
4176
+ for (let i = 0; i < headers.length; i++) {
4177
+ const h = headers[i];
4178
+ if (h === void 0) continue;
4179
+ const headingLower = h.heading.toLowerCase();
4180
+ if (headingLower === target) {
4181
+ exactMatches.push(i);
4182
+ } else if (normalizeHeading(h.heading).toLowerCase() === normalizedTarget) {
4183
+ normalizedMatches.push(i);
4184
+ } else if (normalizeHeadingStrip(h.heading).toLowerCase() === strippedTarget) {
4185
+ strippedMatches.push(i);
4186
+ }
4187
+ }
4188
+ const matches = exactMatches.length > 0 ? exactMatches : normalizedMatches.length > 0 ? normalizedMatches : strippedMatches;
4189
+ if (matches.length > 0) {
4190
+ const pick = ordinal === null ? 0 : ordinal - 1;
4191
+ const headerPos = matches[pick];
4192
+ if (headerPos === void 0) return null;
4193
+ let occurrences = null;
4194
+ if (ordinal === null && matches.length > 1) {
4195
+ occurrences = [];
4196
+ for (const i of matches) {
4197
+ const h = headers[i];
4198
+ if (h !== void 0) occurrences.push(h.index + 1);
4199
+ }
4200
+ }
4201
+ return { headerPos, redirectedFrom: null, occurrences };
4202
+ }
4203
+ if (ordinal !== null || normalizedTarget.length === 0) return null;
4204
+ let prefixPos = -1;
4205
+ const distinct = /* @__PURE__ */ new Set();
4206
+ for (let i = 0; i < headers.length; i++) {
4207
+ const h = headers[i];
4208
+ if (h === void 0) continue;
4209
+ const norm = normalizeHeading(h.heading).toLowerCase();
4210
+ if (norm.startsWith(normalizedTarget)) {
4211
+ distinct.add(norm);
4212
+ if (prefixPos === -1) prefixPos = i;
4213
+ }
4214
+ }
4215
+ if (distinct.size === 1 && prefixPos !== -1) {
4216
+ const chosen = headers[prefixPos];
4217
+ if (chosen === void 0) return null;
4218
+ return { headerPos: prefixPos, redirectedFrom: base, occurrences: null };
4219
+ }
4220
+ const queryWords = normalizedTarget.split(/\s+/).filter((w) => w.length >= MIN_WIDEN_WORD_LEN);
4221
+ if (queryWords.length === 0) return null;
4222
+ const widenedMatches = [];
4223
+ for (let i = 0; i < headers.length; i++) {
4224
+ const h = headers[i];
4225
+ if (h === void 0) continue;
4226
+ const headingWords = normalizeHeading(h.heading).toLowerCase().split(/\s+/);
4227
+ const allWordsMatch = queryWords.every((qw) => headingWords.some((hw) => hw.includes(qw)));
4228
+ if (allWordsMatch) widenedMatches.push(i);
4229
+ }
4230
+ if (widenedMatches.length !== 1) return null;
4231
+ const widenedPos = widenedMatches[0];
4232
+ if (widenedPos === void 0) return null;
4233
+ return { headerPos: widenedPos, redirectedFrom: base, occurrences: null };
4234
+ }
4235
+ function buildSectionResult(headers, kind, lines2, headerPos, redirectedFrom) {
4236
+ const header = headers[headerPos];
4237
+ if (header === void 0) return null;
4238
+ const endIndex = headerEndIndex(headers, kind, headerPos, lines2.length);
4239
+ let endExclusive = endIndex;
4240
+ while (endExclusive > header.index + 1 && (lines2[endExclusive - 1] === "" || lines2[endExclusive - 1] === "\r")) {
4241
+ endExclusive--;
4242
+ }
4243
+ const content = lines2.slice(header.index, endExclusive).join("\n");
4244
+ const result = {
4245
+ heading: header.heading,
4246
+ content,
4247
+ lineStart: header.index + 1,
4248
+ lineEnd: endExclusive
4249
+ };
4250
+ return redirectedFrom === null ? result : { ...result, redirectedFrom };
4251
+ }
4252
+ function resolveSectionFromText(text, headingSpec, language) {
4253
+ const { headers, kind } = findHeaders(text, language);
4254
+ const { base, ordinal } = parseHeadingSpec(headingSpec, headers);
4255
+ if (base.length === 0) return null;
4256
+ const lines2 = text.split("\n");
4257
+ const resolved = resolveHeaderPos(headers, base, ordinal);
4258
+ if (resolved === null) return null;
4259
+ const built = buildSectionResult(headers, kind, lines2, resolved.headerPos, resolved.redirectedFrom);
4260
+ if (built === null || resolved.occurrences === null) return built;
4261
+ return { ...built, occurrences: resolved.occurrences };
4262
+ }
4263
+ function extractSection(text, headingSpec) {
4264
+ return resolveSectionFromText(text, headingSpec, "unknown");
4265
+ }
4266
+ function readTextForSections(filePath, readFn) {
4267
+ let text;
4268
+ if (readFn !== void 0) {
4269
+ const read = readFn(filePath);
4270
+ if (read === null) return null;
4271
+ text = read;
4272
+ } else {
4273
+ try {
4274
+ text = decodeSource(readFileSync3(filePath));
4275
+ } catch {
4276
+ return null;
4277
+ }
4278
+ }
4279
+ if (text.charCodeAt(0) === 65279) {
4280
+ text = text.slice(1);
4281
+ }
4282
+ return redactIfDotenv(filePath, text);
4283
+ }
4284
+ function readSection(filePath, headingSpec, readFn) {
4285
+ const text = readTextForSections(filePath, readFn);
4286
+ if (text === null) return null;
4287
+ return resolveSectionFromText(text, headingSpec, refineLanguageByContent(filePath, detectLanguage(filePath), text));
4288
+ }
4289
+ function findContainingSection(filePath, lineStart, lineEnd, readFn) {
4290
+ const text = readTextForSections(filePath, readFn);
4291
+ if (text === null) return null;
4292
+ const language = refineLanguageByContent(filePath, detectLanguage(filePath), text);
4293
+ const { headers, kind } = findHeaders(text, language);
4294
+ if (headers.length === 0) return null;
4295
+ const lines2 = text.split("\n");
4296
+ let bestPos = -1;
4297
+ let bestHeaderLine = -1;
4298
+ for (let i = 0; i < headers.length; i++) {
4299
+ const header = headers[i];
4300
+ if (header === void 0) continue;
4301
+ const endIndex = headerEndIndex(headers, kind, i, lines2.length);
4302
+ const sectionLineStart = header.index + 1;
4303
+ if (sectionLineStart <= lineStart && lineEnd <= endIndex) {
4304
+ if (header.index > bestHeaderLine) {
4305
+ bestHeaderLine = header.index;
4306
+ bestPos = i;
4307
+ }
4308
+ }
4309
+ }
4310
+ if (bestPos === -1) return null;
4311
+ const result = buildSectionResult(headers, kind, lines2, bestPos, null);
4312
+ if (result === null) return null;
4313
+ const bestHeader = headers[bestPos];
4314
+ if (bestHeader === void 0) return result;
4315
+ const target = bestHeader.heading.toLowerCase();
4316
+ const matches = [];
4317
+ for (let i = 0; i < headers.length; i++) {
4318
+ const h = headers[i];
4319
+ if (h !== void 0 && h.heading.toLowerCase() === target) matches.push(i);
4320
+ }
4321
+ if (matches.length <= 1) return result;
4322
+ const ordinal = matches.indexOf(bestPos) + 1;
4323
+ return { ...result, heading: `${bestHeader.heading}#${ordinal}` };
4324
+ }
4325
+ function listSections(filePath, readFn) {
4326
+ const text = readTextForSections(filePath, readFn);
4327
+ if (text === null) return [];
4328
+ const language = refineLanguageByContent(filePath, detectLanguage(filePath), text);
4329
+ const { headers } = findHeaders(text, language);
4330
+ return headers.map((h) => h.heading);
4331
+ }
4332
+
4333
+ export {
4334
+ isEmbeddableDocument,
4335
+ ipynbToVirtualSource,
4336
+ eachUnfencedLine,
4337
+ checkEmbeddingPreflight,
4338
+ OVER_FETCH_FACTOR,
4339
+ MAX_OVER_FETCH,
4340
+ isAvailable,
4341
+ embeddingBackendLoadError,
4342
+ embedTexts,
4343
+ searchSemantic,
4344
+ mergeNearbyHits,
4345
+ embeddingsDepsAvailable,
4346
+ deleteFileEmbeddings,
4347
+ ensureEmbeddingProvenance,
4348
+ MARKDOWN_SIZE_THRESHOLD,
4349
+ extractMarkdownHeadings,
4350
+ formatHeadingTreeParts,
4351
+ getWellKnownSections,
4352
+ extractChangelogVersionHint,
4353
+ PARSER_FINGERPRINT,
4354
+ treeSitterCoreAvailable,
4355
+ treeSitterCoreLoadError,
4356
+ isTreeSitterAvailable,
4357
+ missingTreeSitterGrammarPackages,
4358
+ parseSourceSymbolsTreeSitterOnly,
4359
+ loadRegexExtractors,
4360
+ deleteFileRows,
4361
+ isParseSkipEligible,
4362
+ indexFileSync,
4363
+ indexedPathSpellingIsStale,
4364
+ isEmbedFresh,
4365
+ indexFileEmbeddings,
4366
+ extractShellBannerHeading,
4367
+ extractSection,
4368
+ readSection,
4369
+ findContainingSection,
4370
+ listSections
4371
+ };