token-goat 2.9.29 → 2.9.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -40
- package/SECURITY.md +28 -10
- package/THIRD_PARTY_NOTICES.md +40 -3
- package/dist/native/linux-arm64/tg-hook +0 -0
- package/dist/native/linux-x64/tg-hook +0 -0
- package/dist/ort-wasm-simd-threaded.mjs +59 -0
- package/dist/token-goat-chunk-3CHUNJ2O.mjs +53 -0
- package/dist/token-goat-chunk-3Z5S4KGM.mjs +918 -0
- package/dist/token-goat-chunk-4FVIAHSI.mjs +211 -0
- package/dist/{token-goat-chunk-PGTH2U4C.mjs → token-goat-chunk-4JJS54TR.mjs} +4 -3
- package/dist/token-goat-chunk-4LCCNRTG.mjs +313 -0
- package/dist/{token-goat-chunk-L7H3U27L.mjs → token-goat-chunk-54K3EUV3.mjs} +1 -1
- package/dist/token-goat-chunk-56EIMS3E.mjs +159 -0
- package/dist/token-goat-chunk-5BCIJXMO.mjs +609 -0
- package/dist/token-goat-chunk-5KDMCMF2.mjs +362 -0
- package/dist/token-goat-chunk-5RG5SBL6.mjs +221 -0
- package/dist/{token-goat-chunk-RRNZMM3A.mjs → token-goat-chunk-5Y6ZYL6U.mjs} +21 -2
- package/dist/{token-goat-chunk-GLXRTTLE.mjs → token-goat-chunk-64FBPDTK.mjs} +12 -8
- package/dist/{token-goat-chunk-G23DWHV4.mjs → token-goat-chunk-664RTTUV.mjs} +28 -15
- package/dist/token-goat-chunk-67TQJ76B.mjs +57 -0
- package/dist/token-goat-chunk-6QMAS7AB.mjs +42 -0
- package/dist/token-goat-chunk-7BXSNBY5.mjs +253 -0
- package/dist/{token-goat-chunk-IVWI2OCK.mjs → token-goat-chunk-7F2UBPD3.mjs} +40 -11
- package/dist/{token-goat-chunk-TGOXNAHW.mjs → token-goat-chunk-7IN2MKDS.mjs} +682 -501
- package/dist/token-goat-chunk-7NADEDKJ.mjs +124 -0
- package/dist/{token-goat-chunk-KOPIR3KZ.mjs → token-goat-chunk-A3CK6N5Z.mjs} +618 -72
- package/dist/{token-goat-chunk-O6I2XU4P.mjs → token-goat-chunk-AFLYLHDH.mjs} +44 -31
- package/dist/{token-goat-chunk-TCQANXYC.mjs → token-goat-chunk-AM5V6GSX.mjs} +18 -2
- package/dist/{token-goat-chunk-E4ZOJIZK.mjs → token-goat-chunk-APGZXSPS.mjs} +39 -146
- package/dist/{token-goat-chunk-D4ETHJD6.mjs → token-goat-chunk-ARDNEC2U.mjs} +9 -7
- package/dist/token-goat-chunk-ATCPY7EQ.mjs +134 -0
- package/dist/token-goat-chunk-B2RZZCSS.mjs +354 -0
- package/dist/{token-goat-chunk-BGYOZ6DO.mjs → token-goat-chunk-B45VMIIH.mjs} +6 -7
- package/dist/token-goat-chunk-BPGCUMR7.mjs +30 -0
- package/dist/{token-goat-chunk-TDUT2CL6.mjs → token-goat-chunk-C63JQV5K.mjs} +45 -6
- package/dist/{token-goat-chunk-ZER6EAXZ.mjs → token-goat-chunk-COBQGL5S.mjs} +2661 -2440
- package/dist/token-goat-chunk-COTXZHTS.mjs +39 -0
- package/dist/{token-goat-chunk-RQGGMPCR.mjs → token-goat-chunk-CYCEYAEO.mjs} +76 -16
- package/dist/token-goat-chunk-CZ3LUXYZ.mjs +4115 -0
- package/dist/{token-goat-chunk-ASLWOQ3I.mjs → token-goat-chunk-DZX2YDPT.mjs} +2 -1
- package/dist/token-goat-chunk-E3OQX2XT.mjs +824 -0
- package/dist/token-goat-chunk-EFHMMIGP.mjs +68 -0
- package/dist/token-goat-chunk-ETBHVDB6.mjs +328 -0
- package/dist/token-goat-chunk-EVMZJ27Z.mjs +147 -0
- package/dist/token-goat-chunk-EXKGNHEG.mjs +2096 -0
- package/dist/{token-goat-chunk-RTMVLHWX.mjs → token-goat-chunk-FBNQNSJK.mjs} +2555 -5127
- package/dist/{token-goat-chunk-NMTKNYGF.mjs → token-goat-chunk-FNFBKLRD.mjs} +74 -9
- package/dist/{token-goat-chunk-RLP6J7WR.mjs → token-goat-chunk-FZ3NEW5O.mjs} +33 -19
- package/dist/token-goat-chunk-G5PJ7AGM.mjs +36 -0
- package/dist/{token-goat-chunk-DNRPXHYH.mjs → token-goat-chunk-GFBSN7BD.mjs} +29 -401
- package/dist/{token-goat-chunk-XYWPOSCN.mjs → token-goat-chunk-GOLEOXZP.mjs} +763 -211
- package/dist/token-goat-chunk-H2K3QP6H.mjs +24 -0
- package/dist/token-goat-chunk-H2PHHGM5.mjs +188 -0
- package/dist/token-goat-chunk-HAVAOB3S.mjs +87 -0
- package/dist/token-goat-chunk-HG45AF2B.mjs +2228 -0
- package/dist/{token-goat-chunk-HUYUHODZ.mjs → token-goat-chunk-HOJLCGDP.mjs} +12965 -11748
- package/dist/{token-goat-chunk-435UQTKD.mjs → token-goat-chunk-HYBFA5X3.mjs} +118 -19
- package/dist/token-goat-chunk-IVALLT3S.mjs +48 -0
- package/dist/{token-goat-chunk-UXSXQQIY.mjs → token-goat-chunk-IVKIZWAB.mjs} +3 -3
- package/dist/{token-goat-chunk-52NSDWYY.mjs → token-goat-chunk-JB6QN6VF.mjs} +6 -4
- package/dist/{token-goat-chunk-HNFZZHFW.mjs → token-goat-chunk-JDHE26RP.mjs} +10 -3
- package/dist/{token-goat-chunk-FTP4U4QA.mjs → token-goat-chunk-JDU3EXSU.mjs} +12 -11
- package/dist/token-goat-chunk-JDWCRD6M.mjs +253 -0
- package/dist/{token-goat-chunk-6AK46ZT7.mjs → token-goat-chunk-JN4XMLSD.mjs} +5 -1
- package/dist/token-goat-chunk-JTQQGXZH.mjs +733 -0
- package/dist/token-goat-chunk-KKUFXAPD.mjs +25 -0
- package/dist/token-goat-chunk-KQNZDCBD.mjs +47 -0
- package/dist/{token-goat-chunk-2WOUYWLM.mjs → token-goat-chunk-KWPNJBDH.mjs} +27 -3
- package/dist/{token-goat-chunk-DPNLUJWB.mjs → token-goat-chunk-LOBGHXPB.mjs} +30 -8
- package/dist/token-goat-chunk-MKPF4CL5.mjs +37 -0
- package/dist/token-goat-chunk-MNWKPRAL.mjs +13 -0
- package/dist/{token-goat-chunk-DNSPXQXQ.mjs → token-goat-chunk-N2NMER56.mjs} +25 -15
- package/dist/token-goat-chunk-NYKZLXMK.mjs +218 -0
- package/dist/token-goat-chunk-OJBNYTJP.mjs +3087 -0
- package/dist/token-goat-chunk-P4QMM4WK.mjs +295 -0
- package/dist/token-goat-chunk-PD5VKH46.mjs +184 -0
- package/dist/token-goat-chunk-PEPTHRFF.mjs +129 -0
- package/dist/{token-goat-chunk-NMBWLRHW.mjs → token-goat-chunk-PU7V3FD6.mjs} +8721 -11104
- package/dist/token-goat-chunk-PY533T7F.mjs +46 -0
- package/dist/token-goat-chunk-Q5VTGRH7.mjs +761 -0
- package/dist/{token-goat-chunk-67BNIKKO.mjs → token-goat-chunk-QCLDAQTS.mjs} +31 -20
- package/dist/{token-goat-chunk-SE6E4BKJ.mjs → token-goat-chunk-QSBRTGM2.mjs} +9 -2
- package/dist/{token-goat-chunk-VZQEWRVH.mjs → token-goat-chunk-RC35RBUL.mjs} +4 -4
- package/dist/{token-goat-chunk-VMFWNHGR.mjs → token-goat-chunk-RHXKCZX3.mjs} +873 -504
- package/dist/{token-goat-chunk-PYSUADX4.mjs → token-goat-chunk-RZRTDST7.mjs} +17 -7
- package/dist/{token-goat-chunk-ONXH6NZ5.mjs → token-goat-chunk-SJHWVNDQ.mjs} +15 -21
- package/dist/token-goat-chunk-SMVRN2HU.mjs +188 -0
- package/dist/{token-goat-chunk-QO3LXVVR.mjs → token-goat-chunk-SQSSNQPE.mjs} +2295 -1718
- package/dist/{token-goat-chunk-2AWQ2R5X.mjs → token-goat-chunk-T4FK7VOV.mjs} +2 -2
- package/dist/token-goat-chunk-U4N3BN2W.mjs +92 -0
- package/dist/token-goat-chunk-UA5V6P3P.mjs +122 -0
- package/dist/{token-goat-chunk-UA5BKWWQ.mjs → token-goat-chunk-UQ5UK2T7.mjs} +15 -4
- package/dist/token-goat-chunk-USILZ3T3.mjs +53 -0
- package/dist/{token-goat-chunk-MELOKFDJ.mjs → token-goat-chunk-VES7EL7O.mjs} +7 -1
- package/dist/{token-goat-chunk-7CPMDFGB.mjs → token-goat-chunk-VL3YIFHZ.mjs} +389 -14
- package/dist/{token-goat-chunk-E5TK5DNO.mjs → token-goat-chunk-VPBBTSJR.mjs} +356 -129
- package/dist/{token-goat-chunk-YOOVFPTP.mjs → token-goat-chunk-W3QCJS6G.mjs} +10 -2
- package/dist/{token-goat-chunk-LZNBQTDK.mjs → token-goat-chunk-WY3L4DVT.mjs} +11 -7
- package/dist/token-goat-chunk-XKI4SEMX.mjs +66 -0
- package/dist/token-goat-chunk-XU4P6Z24.mjs +133 -0
- package/dist/{token-goat-chunk-H5J3C56Q.mjs → token-goat-chunk-Y776AENC.mjs} +112 -26
- package/dist/token-goat-chunk-YARZFCZS.mjs +359 -0
- package/dist/{token-goat-chunk-S4VSXURL.mjs → token-goat-chunk-Z44YO3QZ.mjs} +4 -0
- package/dist/token-goat-chunk-Z57YODA2.mjs +35 -0
- package/dist/{token-goat-chunk-PQ53BSCF.mjs → token-goat-chunk-ZCDTCVIS.mjs} +48 -29
- package/dist/token-goat-chunk-ZEJVMUQH.mjs +48 -0
- package/dist/{token-goat-chunk-BGHSSJUA.mjs → token-goat-chunk-ZFOBKUWH.mjs} +1 -1
- package/dist/token-goat-chunk-ZMBL6UVV.mjs +110 -0
- package/dist/{token-goat-chunk-PKRWMHVP.mjs → token-goat-chunk-ZQSJMO2R.mjs} +338 -170
- package/dist/token-goat-hook-client.cjs +87 -23
- package/dist/token-goat-hook-client.mjs +3 -3
- package/dist/token-goat-hook.mjs +46 -30
- package/dist/token-goat.core.mjs +6 -6
- package/docs/C4_RUNTIME_ARCHITECTURE.md +1 -1
- package/docs/cli.md +80 -46
- package/docs/install.md +47 -14
- package/docs/security.md +7 -7
- package/package.json +17 -7
- package/scripts/verify-native-dist.mjs +386 -0
- package/dist/token-goat-chunk-2NNUA2LR.mjs +0 -143
- package/dist/token-goat-chunk-44BM7DDQ.mjs +0 -294
- package/dist/token-goat-chunk-C5YPNZKQ.mjs +0 -318
- package/dist/token-goat-chunk-CIQ2FTY3.mjs +0 -1990
- package/dist/token-goat-chunk-EUSDXRXU.mjs +0 -1429
- package/dist/token-goat-chunk-G7IWXCFF.mjs +0 -456
- package/dist/token-goat-chunk-JTKCQMQA.mjs +0 -418
- package/dist/token-goat-chunk-ORNFMIER.mjs +0 -295
- package/dist/token-goat-chunk-VQ6F6T7X.mjs +0 -36
- package/dist/token-goat-chunk-YAGLESS3.mjs +0 -23
- package/dist/token-goat-chunk-YKSRAWUU.mjs +0 -33
- package/dist/token-goat-chunk-Z6KBMOBI.mjs +0 -53
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
import { createRequire as __cjsRequire } from 'node:module';
|
|
2
2
|
const require = __cjsRequire(import.meta.url);
|
|
3
3
|
import {
|
|
4
|
+
ATX_CLASS_BREAK_RE,
|
|
5
|
+
ATX_DOT_BREAK_RE,
|
|
6
|
+
matchAtxHeading,
|
|
7
|
+
matchCssSelectorSpan,
|
|
8
|
+
matchRuleBannerTitle,
|
|
9
|
+
matchTableHeaderName,
|
|
10
|
+
matchTomlSectionName
|
|
11
|
+
} from "./token-goat-chunk-EVMZJ27Z.mjs";
|
|
12
|
+
import {
|
|
13
|
+
ownProjectScope,
|
|
4
14
|
pathEqClause,
|
|
5
|
-
projectScopeClause,
|
|
6
15
|
querySymbols
|
|
7
|
-
} from "./token-goat-chunk-
|
|
16
|
+
} from "./token-goat-chunk-HYBFA5X3.mjs";
|
|
8
17
|
import {
|
|
9
18
|
TREE_SITTER_LANGUAGES,
|
|
10
19
|
_detectOpenQuote,
|
|
@@ -22,37 +31,38 @@ import {
|
|
|
22
31
|
stripJsComments,
|
|
23
32
|
stripStringLiterals,
|
|
24
33
|
stripXmlComments
|
|
25
|
-
} from "./token-goat-chunk-
|
|
34
|
+
} from "./token-goat-chunk-VL3YIFHZ.mjs";
|
|
26
35
|
import {
|
|
36
|
+
findProject,
|
|
27
37
|
fingerprintContent,
|
|
28
38
|
loadConfig
|
|
29
|
-
} from "./token-goat-chunk-
|
|
39
|
+
} from "./token-goat-chunk-7IN2MKDS.mjs";
|
|
30
40
|
import {
|
|
31
41
|
registerReset
|
|
32
42
|
} from "./token-goat-chunk-YZAAZU2S.mjs";
|
|
43
|
+
import {
|
|
44
|
+
DocumentRefusedError,
|
|
45
|
+
MAX_DOCUMENT_WORK_MILLIS
|
|
46
|
+
} from "./token-goat-chunk-Y4AFKTHK.mjs";
|
|
33
47
|
import {
|
|
34
48
|
countContentLines,
|
|
35
49
|
decodeSource,
|
|
36
50
|
ensureDirSync
|
|
37
|
-
} from "./token-goat-chunk-
|
|
51
|
+
} from "./token-goat-chunk-GOLEOXZP.mjs";
|
|
38
52
|
import {
|
|
39
53
|
foldPath,
|
|
40
54
|
isCaseInsensitiveFs
|
|
41
|
-
} from "./token-goat-chunk-
|
|
55
|
+
} from "./token-goat-chunk-AM5V6GSX.mjs";
|
|
42
56
|
import {
|
|
43
57
|
displaySafeText,
|
|
44
58
|
normalizePath
|
|
45
|
-
} from "./token-goat-chunk-
|
|
59
|
+
} from "./token-goat-chunk-FNFBKLRD.mjs";
|
|
46
60
|
import {
|
|
47
61
|
SYMBOL_BODY_CHAR_CAP,
|
|
48
62
|
dataDir,
|
|
49
63
|
ensureDataDirPrivate,
|
|
50
64
|
globalDbPath
|
|
51
|
-
} from "./token-goat-chunk-
|
|
52
|
-
import {
|
|
53
|
-
DocumentRefusedError,
|
|
54
|
-
MAX_DOCUMENT_WORK_MILLIS
|
|
55
|
-
} from "./token-goat-chunk-Y4AFKTHK.mjs";
|
|
65
|
+
} from "./token-goat-chunk-VES7EL7O.mjs";
|
|
56
66
|
import {
|
|
57
67
|
init_define_import_meta_env
|
|
58
68
|
} from "./token-goat-chunk-A37V4PBF.mjs";
|
|
@@ -74,28 +84,28 @@ function isTransientDocumentRefusal(err) {
|
|
|
74
84
|
async function extractEmbeddableDocumentText(filePath) {
|
|
75
85
|
switch (path.extname(filePath).toLowerCase()) {
|
|
76
86
|
case ".pdf": {
|
|
77
|
-
const { extractPdfText, readPdfFileWithinBounds } = await import("./token-goat-chunk-
|
|
87
|
+
const { extractPdfText, readPdfFileWithinBounds } = await import("./token-goat-chunk-W3QCJS6G.mjs");
|
|
78
88
|
const { text } = await extractPdfText(await readPdfFileWithinBounds(filePath));
|
|
79
89
|
return text;
|
|
80
90
|
}
|
|
81
91
|
case ".docx":
|
|
82
|
-
return await (await import("./token-goat-chunk-
|
|
92
|
+
return await (await import("./token-goat-chunk-B45VMIIH.mjs")).docxText(filePath);
|
|
83
93
|
case ".pptx":
|
|
84
|
-
return await (await import("./token-goat-chunk-
|
|
94
|
+
return await (await import("./token-goat-chunk-WY3L4DVT.mjs")).pptxAllSlidesText(filePath, true);
|
|
85
95
|
case ".xlsx":
|
|
86
|
-
return await (await import("./token-goat-chunk-
|
|
96
|
+
return await (await import("./token-goat-chunk-64FBPDTK.mjs")).allSheetsHeadText(filePath, XLSX_SHEET_ROW_CAP);
|
|
87
97
|
default:
|
|
88
98
|
return null;
|
|
89
99
|
}
|
|
90
100
|
}
|
|
91
101
|
|
|
102
|
+
// src/embeddings.ts
|
|
103
|
+
init_define_import_meta_env();
|
|
104
|
+
|
|
92
105
|
// src/embed_model.ts
|
|
93
106
|
init_define_import_meta_env();
|
|
94
|
-
import
|
|
95
|
-
import * as
|
|
96
|
-
import { createRequire } from "node:module";
|
|
97
|
-
import * as path2 from "node:path";
|
|
98
|
-
import { pipeline } from "node:stream/promises";
|
|
107
|
+
import * as fs3 from "node:fs";
|
|
108
|
+
import * as path4 from "node:path";
|
|
99
109
|
|
|
100
110
|
// src/embed_tokenizer.ts
|
|
101
111
|
init_define_import_meta_env();
|
|
@@ -109,20 +119,20 @@ var UnsupportedTokenizerError = class extends Error {
|
|
|
109
119
|
function fail(message) {
|
|
110
120
|
throw new UnsupportedTokenizerError(message);
|
|
111
121
|
}
|
|
112
|
-
function asRecord(value,
|
|
113
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${
|
|
122
|
+
function asRecord(value, path10) {
|
|
123
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${path10} is not an object`);
|
|
114
124
|
return value;
|
|
115
125
|
}
|
|
116
|
-
function asString(value,
|
|
117
|
-
if (typeof value !== "string") fail(`${
|
|
126
|
+
function asString(value, path10) {
|
|
127
|
+
if (typeof value !== "string") fail(`${path10} is not a string`);
|
|
118
128
|
return value;
|
|
119
129
|
}
|
|
120
|
-
function asArray(value,
|
|
121
|
-
if (!Array.isArray(value)) fail(`${
|
|
130
|
+
function asArray(value, path10) {
|
|
131
|
+
if (!Array.isArray(value)) fail(`${path10} is not an array`);
|
|
122
132
|
return value;
|
|
123
133
|
}
|
|
124
|
-
function expectValue(actual, wanted,
|
|
125
|
-
if (actual !== wanted) fail(`${
|
|
134
|
+
function expectValue(actual, wanted, path10) {
|
|
135
|
+
if (actual !== wanted) fail(`${path10} is ${JSON.stringify(actual)}, expected ${JSON.stringify(wanted)}`);
|
|
126
136
|
}
|
|
127
137
|
function isChinese(cp) {
|
|
128
138
|
return cp >= 19968 && cp <= 40959 || cp >= 13312 && cp <= 19903 || cp >= 131072 && cp <= 173791 || cp >= 173824 && cp <= 177983 || cp >= 177984 && cp <= 178207 || cp >= 178208 && cp <= 183983 || cp >= 63744 && cp <= 64255 || cp >= 194560 && cp <= 195103;
|
|
@@ -310,68 +320,25 @@ var BertWordPiece = class _BertWordPiece {
|
|
|
310
320
|
}
|
|
311
321
|
};
|
|
312
322
|
|
|
313
|
-
// src/
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
name: "onnx/model_quantized.onnx",
|
|
327
|
-
sha256: "6c9c6101a956d62dfb5e7190c538226c0c5bb9cb27b651234b6df063ee7dbfe4",
|
|
328
|
-
bytes: 34014426
|
|
329
|
-
}
|
|
330
|
-
];
|
|
331
|
-
function modelDir() {
|
|
332
|
-
return path2.join(dataDir(), "models", ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
333
|
-
}
|
|
334
|
-
function modelFilesPresent() {
|
|
335
|
-
const roots = [modelDir(), sharedModelCacheDir()].filter((d) => d !== null);
|
|
336
|
-
return roots.some(
|
|
337
|
-
(root) => MODEL_FILES.every((file) => {
|
|
338
|
-
try {
|
|
339
|
-
return fs.statSync(path2.join(root, file.name)).size === file.bytes;
|
|
340
|
-
} catch {
|
|
341
|
-
return false;
|
|
342
|
-
}
|
|
343
|
-
})
|
|
344
|
-
);
|
|
345
|
-
}
|
|
346
|
-
function downloadUrl(file) {
|
|
347
|
-
return `https://huggingface.co/${DEFAULT_MODEL}/resolve/${PINNED_MODEL_REVISION}/${file.name}`;
|
|
348
|
-
}
|
|
349
|
-
function sharedModelCacheDir() {
|
|
350
|
-
const raw = process.env["TOKEN_GOAT_MODEL_CACHE_DIR"]?.trim();
|
|
351
|
-
if (!raw) return null;
|
|
352
|
-
return path2.join(raw, ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
323
|
+
// src/embed_runtime.ts
|
|
324
|
+
init_define_import_meta_env();
|
|
325
|
+
import * as fs2 from "node:fs";
|
|
326
|
+
import { createRequire } from "node:module";
|
|
327
|
+
import * as path3 from "node:path";
|
|
328
|
+
import { fileURLToPath } from "node:url";
|
|
329
|
+
|
|
330
|
+
// src/pinned_file.ts
|
|
331
|
+
init_define_import_meta_env();
|
|
332
|
+
import * as fs from "node:fs";
|
|
333
|
+
import * as path2 from "node:path";
|
|
334
|
+
async function sha256Of(filePath) {
|
|
335
|
+
return (await import("./token-goat-chunk-H2K3QP6H.mjs")).sha256Of(filePath);
|
|
353
336
|
}
|
|
354
337
|
async function copyFromSharedCache(shared, file, target) {
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
if (!info?.isFile() || info.size !== file.bytes) return false;
|
|
360
|
-
fs.copyFileSync(source, temp);
|
|
361
|
-
if (await sha256Of(temp) !== file.sha256) {
|
|
362
|
-
fs.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
363
|
-
fs.rmSync(source, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
364
|
-
return false;
|
|
365
|
-
}
|
|
366
|
-
fs.renameSync(temp, target);
|
|
367
|
-
return true;
|
|
368
|
-
} catch {
|
|
369
|
-
try {
|
|
370
|
-
fs.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
371
|
-
} catch {
|
|
372
|
-
}
|
|
373
|
-
return false;
|
|
374
|
-
}
|
|
338
|
+
return (await import("./token-goat-chunk-H2K3QP6H.mjs")).copyFromSharedCache(shared, file, target);
|
|
339
|
+
}
|
|
340
|
+
async function downloadPinned(url, file, target) {
|
|
341
|
+
return (await import("./token-goat-chunk-H2K3QP6H.mjs")).downloadPinned(url, file, target);
|
|
375
342
|
}
|
|
376
343
|
function publishToSharedCache(shared, file, target) {
|
|
377
344
|
const destination = path2.join(shared, file.name);
|
|
@@ -391,38 +358,104 @@ function publishToSharedCache(shared, file, target) {
|
|
|
391
358
|
}
|
|
392
359
|
}
|
|
393
360
|
}
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
var
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
361
|
+
|
|
362
|
+
// src/embed_runtime.ts
|
|
363
|
+
var _require = createRequire(import.meta.url);
|
|
364
|
+
var ORT_WEB_VERSION = "1.30.0";
|
|
365
|
+
var ORT_WEB_WASM = {
|
|
366
|
+
name: "ort-wasm-simd-threaded.wasm",
|
|
367
|
+
sha256: "3398c10d07d229bd91b364548e130e0e51a8e5704b88c7c083ebbeb78842dee2",
|
|
368
|
+
bytes: 14239897
|
|
369
|
+
};
|
|
370
|
+
var WEB_RETRY_AFTER_MS = 10 * 60 * 1e3;
|
|
371
|
+
var START_FAILURE_HOLD_MS = 24 * 60 * 60 * 1e3;
|
|
372
|
+
var NATIVE_RUNTIME_INSTALL = "install the native runtime with: npm install -g onnxruntime-node (drop -g if token-goat is a project dependency)";
|
|
373
|
+
var RUNTIME_UNAVAILABLE_ADVICE = `The WebAssembly runtime is downloaded once from registry.npmjs.org (onnxruntime-web ${ORT_WEB_VERSION}), so check network access and network.offline; or ${NATIVE_RUNTIME_INSTALL}`;
|
|
374
|
+
var _chosen = null;
|
|
375
|
+
var _nodeOrt = null;
|
|
376
|
+
var _nodeError = null;
|
|
377
|
+
var _webFailure = null;
|
|
378
|
+
var _recordChecked = false;
|
|
379
|
+
function asError(e) {
|
|
380
|
+
return e instanceof Error ? e : new Error(String(e));
|
|
381
|
+
}
|
|
382
|
+
function chooseRuntime() {
|
|
383
|
+
if (_chosen !== null) return _chosen;
|
|
400
384
|
try {
|
|
401
|
-
|
|
385
|
+
_nodeOrt = _require("onnxruntime-node");
|
|
386
|
+
_chosen = "onnxruntime-node";
|
|
402
387
|
} catch (e) {
|
|
403
|
-
|
|
388
|
+
_nodeError = asError(e);
|
|
389
|
+
_chosen = "onnxruntime-web";
|
|
390
|
+
}
|
|
391
|
+
return _chosen;
|
|
392
|
+
}
|
|
393
|
+
function recentWebFailure() {
|
|
394
|
+
if (_webFailure === null && !_recordChecked) {
|
|
395
|
+
_recordChecked = true;
|
|
396
|
+
_webFailure = recordedStartFailure();
|
|
397
|
+
}
|
|
398
|
+
if (_webFailure === null) return null;
|
|
399
|
+
if (Date.now() - _webFailure.at >= (_webFailure.started ? START_FAILURE_HOLD_MS : WEB_RETRY_AFTER_MS) || _webFailure.started && !fs2.existsSync(startFailurePath())) {
|
|
400
|
+
_webFailure = null;
|
|
401
|
+
return null;
|
|
404
402
|
}
|
|
403
|
+
return _webFailure.error;
|
|
404
|
+
}
|
|
405
|
+
function startFailurePath() {
|
|
406
|
+
return path3.join(wasmDir(), "start-failure.json");
|
|
407
|
+
}
|
|
408
|
+
var NODE_OPTIONS_AT_START = process.env["NODE_OPTIONS"] ?? "";
|
|
409
|
+
function startFailureKey() {
|
|
410
|
+
let build = "";
|
|
411
|
+
try {
|
|
412
|
+
const self = fs2.statSync(fileURLToPath(import.meta.url));
|
|
413
|
+
build = `${self.size}:${self.mtimeMs}`;
|
|
414
|
+
} catch {
|
|
415
|
+
}
|
|
416
|
+
return JSON.stringify([ORT_WEB_VERSION, ORT_WEB_WASM.sha256, build, process.execPath, process.version, process.platform, process.arch, process.execArgv, NODE_OPTIONS_AT_START]);
|
|
417
|
+
}
|
|
418
|
+
function recordedStartFailure() {
|
|
419
|
+
const file = startFailurePath();
|
|
420
|
+
try {
|
|
421
|
+
const parsed = JSON.parse(fs2.readFileSync(file, "utf8"));
|
|
422
|
+
if (parsed.key !== startFailureKey() || typeof parsed.message !== "string" || typeof parsed.at !== "number") return null;
|
|
423
|
+
const when = new Date(parsed.at).toISOString();
|
|
424
|
+
return {
|
|
425
|
+
error: new Error(`${parsed.message} (in an earlier run, at ${when}; it is tried again once Node, its flags or token-goat change, or a day after, and deleting ${file} tries it at once)`),
|
|
426
|
+
at: parsed.at,
|
|
427
|
+
started: true
|
|
428
|
+
};
|
|
429
|
+
} catch {
|
|
430
|
+
return null;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
function activeRuntime() {
|
|
434
|
+
return chooseRuntime();
|
|
435
|
+
}
|
|
436
|
+
function nativeRuntimeLoadError() {
|
|
437
|
+
chooseRuntime();
|
|
438
|
+
return _nodeError;
|
|
405
439
|
}
|
|
406
440
|
function isRuntimeAvailable() {
|
|
407
|
-
|
|
408
|
-
return
|
|
441
|
+
if (chooseRuntime() === "onnxruntime-node") return true;
|
|
442
|
+
return recentWebFailure() === null;
|
|
409
443
|
}
|
|
410
444
|
function runtimeLoadError() {
|
|
411
|
-
|
|
412
|
-
return
|
|
445
|
+
if (chooseRuntime() === "onnxruntime-node") return null;
|
|
446
|
+
return recentWebFailure();
|
|
413
447
|
}
|
|
414
448
|
function runtimeVersion() {
|
|
415
|
-
|
|
416
|
-
if (_ort === null) return "unknown";
|
|
449
|
+
if (chooseRuntime() === "onnxruntime-web") return ORT_WEB_VERSION;
|
|
417
450
|
try {
|
|
418
|
-
let dir =
|
|
451
|
+
let dir = path3.dirname(_require.resolve("onnxruntime-node"));
|
|
419
452
|
for (let depth = 0; depth < 6; depth++) {
|
|
420
|
-
const manifest =
|
|
421
|
-
if (
|
|
422
|
-
const parsed = JSON.parse(
|
|
453
|
+
const manifest = path3.join(dir, "package.json");
|
|
454
|
+
if (fs2.existsSync(manifest)) {
|
|
455
|
+
const parsed = JSON.parse(fs2.readFileSync(manifest, "utf8"));
|
|
423
456
|
if (parsed.name === "onnxruntime-node" && typeof parsed.version === "string") return parsed.version;
|
|
424
457
|
}
|
|
425
|
-
const up =
|
|
458
|
+
const up = path3.dirname(dir);
|
|
426
459
|
if (up === dir) break;
|
|
427
460
|
dir = up;
|
|
428
461
|
}
|
|
@@ -430,52 +463,101 @@ function runtimeVersion() {
|
|
|
430
463
|
}
|
|
431
464
|
return "unknown";
|
|
432
465
|
}
|
|
433
|
-
function
|
|
434
|
-
return
|
|
435
|
-
const hash = createHash("sha256");
|
|
436
|
-
const stream = fs.createReadStream(filePath);
|
|
437
|
-
stream.on("error", reject);
|
|
438
|
-
stream.on("data", (chunk) => hash.update(chunk));
|
|
439
|
-
stream.on("end", () => resolve2(hash.digest("hex")));
|
|
440
|
-
});
|
|
466
|
+
function wasmDir() {
|
|
467
|
+
return path3.join(dataDir(), "models", "onnxruntime-web", ORT_WEB_VERSION);
|
|
441
468
|
}
|
|
442
|
-
|
|
443
|
-
const
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
const
|
|
449
|
-
|
|
450
|
-
const out = fs.createWriteStream(temp);
|
|
451
|
-
try {
|
|
452
|
-
await pipeline(async function* () {
|
|
453
|
-
for await (const chunk of response.body) {
|
|
454
|
-
written += chunk.byteLength;
|
|
455
|
-
if (written > file.bytes) throw new Error(`${file.name} is longer than the pinned ${file.bytes} bytes`);
|
|
456
|
-
hash.update(chunk);
|
|
457
|
-
yield chunk;
|
|
458
|
-
}
|
|
459
|
-
}, out);
|
|
460
|
-
if (written !== file.bytes) {
|
|
461
|
-
throw new Error(`${file.name} is ${written} bytes, expected the pinned ${file.bytes}`);
|
|
462
|
-
}
|
|
463
|
-
const digest = hash.digest("hex");
|
|
464
|
-
if (digest !== file.sha256) {
|
|
465
|
-
throw new Error(`${file.name} has sha256 ${digest}, expected the pinned ${file.sha256}`);
|
|
466
|
-
}
|
|
467
|
-
fs.renameSync(temp, target);
|
|
468
|
-
} catch (e) {
|
|
469
|
-
await new Promise((resolve2) => {
|
|
470
|
-
if (out.closed) resolve2();
|
|
471
|
-
else out.once("close", () => resolve2());
|
|
472
|
-
});
|
|
469
|
+
function sharedWasmCacheDir() {
|
|
470
|
+
const raw = process.env["TOKEN_GOAT_MODEL_CACHE_DIR"]?.trim();
|
|
471
|
+
if (!raw) return null;
|
|
472
|
+
return path3.join(raw, "onnxruntime-web", ORT_WEB_VERSION);
|
|
473
|
+
}
|
|
474
|
+
function wasmBinaryPresent() {
|
|
475
|
+
const roots = [wasmDir(), sharedWasmCacheDir()].filter((d) => d !== null);
|
|
476
|
+
return roots.some((root) => {
|
|
473
477
|
try {
|
|
474
|
-
|
|
478
|
+
return fs2.statSync(path3.join(root, ORT_WEB_WASM.name)).size === ORT_WEB_WASM.bytes;
|
|
475
479
|
} catch {
|
|
480
|
+
return false;
|
|
481
|
+
}
|
|
482
|
+
});
|
|
483
|
+
}
|
|
484
|
+
async function ensureWasmBinary() {
|
|
485
|
+
return (await import("./token-goat-chunk-4LCCNRTG.mjs")).ensureWasmBinary(webHost());
|
|
486
|
+
}
|
|
487
|
+
function webHost() {
|
|
488
|
+
return {
|
|
489
|
+
version: ORT_WEB_VERSION,
|
|
490
|
+
wasm: ORT_WEB_WASM,
|
|
491
|
+
dir: wasmDir(),
|
|
492
|
+
sharedDir: sharedWasmCacheDir(),
|
|
493
|
+
publish: publishToSharedCache,
|
|
494
|
+
startFailure: { path: startFailurePath(), key: startFailureKey() },
|
|
495
|
+
failed(e, started) {
|
|
496
|
+
const error = asError(e);
|
|
497
|
+
_webFailure = { error, at: Date.now(), started };
|
|
498
|
+
return error;
|
|
476
499
|
}
|
|
477
|
-
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
async function createInferenceSession(modelPath, threads) {
|
|
503
|
+
if (chooseRuntime() === "onnxruntime-node") {
|
|
504
|
+
const ort = _nodeOrt;
|
|
505
|
+
return { session: await ort.InferenceSession.create(modelPath, { intraOpNumThreads: threads, interOpNumThreads: 1 }), Tensor: ort.Tensor };
|
|
478
506
|
}
|
|
507
|
+
const remembered = recentWebFailure();
|
|
508
|
+
if (remembered) throw remembered;
|
|
509
|
+
return (await import("./token-goat-chunk-4LCCNRTG.mjs")).createWebSession(modelPath, threads, webHost());
|
|
510
|
+
}
|
|
511
|
+
registerReset(() => {
|
|
512
|
+
_chosen = null;
|
|
513
|
+
_nodeOrt = null;
|
|
514
|
+
_nodeError = null;
|
|
515
|
+
_webFailure = null;
|
|
516
|
+
_recordChecked = false;
|
|
517
|
+
});
|
|
518
|
+
|
|
519
|
+
// src/embed_model.ts
|
|
520
|
+
var DEFAULT_MODEL = "Xenova/bge-small-en-v1.5";
|
|
521
|
+
var DEFAULT_DIM = 384;
|
|
522
|
+
var DEFAULT_EMBED_THREADS = 4;
|
|
523
|
+
var PINNED_MODEL_REVISION = "ea104dacec62c0de699686887e3f920caeb4f3e3";
|
|
524
|
+
var MODEL_FILES = [
|
|
525
|
+
{
|
|
526
|
+
name: "tokenizer.json",
|
|
527
|
+
sha256: "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66",
|
|
528
|
+
bytes: 711396
|
|
529
|
+
},
|
|
530
|
+
{
|
|
531
|
+
name: "onnx/model_quantized.onnx",
|
|
532
|
+
sha256: "6c9c6101a956d62dfb5e7190c538226c0c5bb9cb27b651234b6df063ee7dbfe4",
|
|
533
|
+
bytes: 34014426
|
|
534
|
+
}
|
|
535
|
+
];
|
|
536
|
+
function modelDir() {
|
|
537
|
+
return path4.join(dataDir(), "models", ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
538
|
+
}
|
|
539
|
+
function modelFilesPresent() {
|
|
540
|
+
const roots = [modelDir(), sharedModelCacheDir()].filter((d) => d !== null);
|
|
541
|
+
return roots.some(
|
|
542
|
+
(root) => MODEL_FILES.every((file) => {
|
|
543
|
+
try {
|
|
544
|
+
return fs3.statSync(path4.join(root, file.name)).size === file.bytes;
|
|
545
|
+
} catch {
|
|
546
|
+
return false;
|
|
547
|
+
}
|
|
548
|
+
})
|
|
549
|
+
);
|
|
550
|
+
}
|
|
551
|
+
function downloadUrl(file) {
|
|
552
|
+
return `https://huggingface.co/${DEFAULT_MODEL}/resolve/${PINNED_MODEL_REVISION}/${file.name}`;
|
|
553
|
+
}
|
|
554
|
+
function sharedModelCacheDir() {
|
|
555
|
+
const raw = process.env["TOKEN_GOAT_MODEL_CACHE_DIR"]?.trim();
|
|
556
|
+
if (!raw) return null;
|
|
557
|
+
return path4.join(raw, ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
558
|
+
}
|
|
559
|
+
function download(file, target) {
|
|
560
|
+
return downloadPinned(downloadUrl(file), file, target);
|
|
479
561
|
}
|
|
480
562
|
var _inFlightModelDownload = null;
|
|
481
563
|
function ensureModelFiles(modelName = DEFAULT_MODEL) {
|
|
@@ -504,14 +586,14 @@ async function _ensureModelFilesInner(_modelName) {
|
|
|
504
586
|
const offline = loadConfig().network.offline;
|
|
505
587
|
const shared = sharedModelCacheDir();
|
|
506
588
|
for (const file of MODEL_FILES) {
|
|
507
|
-
const target =
|
|
508
|
-
if (
|
|
589
|
+
const target = path4.join(dir, file.name);
|
|
590
|
+
if (fs3.existsSync(target)) {
|
|
509
591
|
const digest = await sha256Of(target);
|
|
510
592
|
if (digest === file.sha256) continue;
|
|
511
|
-
|
|
593
|
+
fs3.rmSync(target, { force: true });
|
|
512
594
|
}
|
|
513
595
|
if (shared) {
|
|
514
|
-
ensureDirSync(
|
|
596
|
+
ensureDirSync(path4.dirname(target));
|
|
515
597
|
if (await copyFromSharedCache(shared, file, target)) continue;
|
|
516
598
|
}
|
|
517
599
|
if (offline) {
|
|
@@ -519,7 +601,7 @@ async function _ensureModelFilesInner(_modelName) {
|
|
|
519
601
|
`Offline mode is on (network.offline): refusing to download ${file.name} for the embedding model. Copy the pinned files into ${dir} on a connected machine to use semantic search here.`
|
|
520
602
|
);
|
|
521
603
|
}
|
|
522
|
-
ensureDirSync(
|
|
604
|
+
ensureDirSync(path4.dirname(target));
|
|
523
605
|
console.warn(
|
|
524
606
|
`Downloading the embedding model, once (${file.name}, ${Math.round(file.bytes / 1024 / 1024)} MB) into ${dir}`
|
|
525
607
|
);
|
|
@@ -557,17 +639,13 @@ var EmbeddingModel = class _EmbeddingModel {
|
|
|
557
639
|
tensorFactory;
|
|
558
640
|
static async load(modelName = DEFAULT_MODEL) {
|
|
559
641
|
if (!isRuntimeAvailable()) {
|
|
560
|
-
throw new Error(`
|
|
642
|
+
throw new Error(`the inference runtime is not available: ${runtimeLoadError()?.message ?? "unknown error"}`);
|
|
561
643
|
}
|
|
562
644
|
const dir = await ensureModelFiles(modelName);
|
|
563
|
-
const tokenizer = BertWordPiece.fromJson(
|
|
564
|
-
const ort = _ort;
|
|
645
|
+
const tokenizer = BertWordPiece.fromJson(fs3.readFileSync(path4.join(dir, "tokenizer.json"), "utf8"));
|
|
565
646
|
const threads = loadConfig().worker.embed_threads ?? DEFAULT_EMBED_THREADS;
|
|
566
|
-
const session = await
|
|
567
|
-
|
|
568
|
-
interOpNumThreads: 1
|
|
569
|
-
});
|
|
570
|
-
return new _EmbeddingModel(tokenizer, session, ort.Tensor);
|
|
647
|
+
const { session, Tensor } = await createInferenceSession(path4.join(dir, "onnx", "model_quantized.onnx"), threads);
|
|
648
|
+
return new _EmbeddingModel(tokenizer, session, Tensor);
|
|
571
649
|
}
|
|
572
650
|
/** Wordpieces `text` becomes under this model's own tokenizer, markers excluded. */
|
|
573
651
|
countTokens(text) {
|
|
@@ -603,8 +681,10 @@ async function checkEmbeddingPreflight(options) {
|
|
|
603
681
|
const cfg = loadConfig();
|
|
604
682
|
const enabled = cfg.indexing?.embeddings_enabled ?? true;
|
|
605
683
|
const mDir = modelDir();
|
|
684
|
+
const rt = activeRuntime();
|
|
606
685
|
const rtVer = runtimeVersion();
|
|
607
686
|
const rtAvail = isRuntimeAvailable();
|
|
687
|
+
const binaryPresent = rt === "onnxruntime-web" ? wasmBinaryPresent() : null;
|
|
608
688
|
const filesPresent = modelFilesPresent();
|
|
609
689
|
let warmed = false;
|
|
610
690
|
const buildResult = (params) => {
|
|
@@ -617,8 +697,10 @@ async function checkEmbeddingPreflight(options) {
|
|
|
617
697
|
message: params.message,
|
|
618
698
|
summary: params.message,
|
|
619
699
|
modelName,
|
|
700
|
+
runtime: rt,
|
|
620
701
|
runtimeVersion: rtVer,
|
|
621
702
|
runtimeAvailable: rtAvail,
|
|
703
|
+
runtimeBinaryPresent: binaryPresent,
|
|
622
704
|
configEnabled: enabled,
|
|
623
705
|
modelFilesPresent: filesPresent,
|
|
624
706
|
modelWarmed: warmed,
|
|
@@ -644,11 +726,19 @@ async function checkEmbeddingPreflight(options) {
|
|
|
644
726
|
return buildResult({
|
|
645
727
|
status: "missing_runtime",
|
|
646
728
|
available: false,
|
|
647
|
-
message: `Inference runtime is not available: ${err?.message ?? "
|
|
648
|
-
suggestion:
|
|
729
|
+
message: `Inference runtime is not available: ${err?.message ?? "unknown error"}`,
|
|
730
|
+
suggestion: RUNTIME_UNAVAILABLE_ADVICE,
|
|
649
731
|
...err?.message ? { error: err.message } : {}
|
|
650
732
|
});
|
|
651
733
|
}
|
|
734
|
+
if (binaryPresent === false && cfg?.network?.offline) {
|
|
735
|
+
return buildResult({
|
|
736
|
+
status: "missing_runtime",
|
|
737
|
+
available: false,
|
|
738
|
+
message: `The WebAssembly inference runtime's ${ORT_WEB_WASM.name} is not downloaded yet and offline mode (network.offline) prevents fetching it`,
|
|
739
|
+
suggestion: `Copy ${ORT_WEB_WASM.name} from the onnxruntime-web package's dist/ into ${wasmDir()}, or ${NATIVE_RUNTIME_INSTALL}`
|
|
740
|
+
});
|
|
741
|
+
}
|
|
652
742
|
if (!filesPresent && cfg?.network?.offline) {
|
|
653
743
|
return buildResult({
|
|
654
744
|
status: "missing_model_files",
|
|
@@ -691,33 +781,27 @@ async function checkEmbeddingPreflight(options) {
|
|
|
691
781
|
});
|
|
692
782
|
}
|
|
693
783
|
registerReset(() => {
|
|
694
|
-
_ort = null;
|
|
695
|
-
_ortError = null;
|
|
696
|
-
_ortLoadAttempted = false;
|
|
697
784
|
_inFlightModelDownload = null;
|
|
698
785
|
});
|
|
699
786
|
|
|
700
|
-
// src/embeddings.ts
|
|
701
|
-
init_define_import_meta_env();
|
|
702
|
-
|
|
703
787
|
// src/embed_fingerprint.ts
|
|
704
788
|
init_define_import_meta_env();
|
|
705
|
-
var EMBED_FINGERPRINT = "
|
|
789
|
+
var EMBED_FINGERPRINT = "98753e3639c3c9e1";
|
|
706
790
|
var EMBED_KIND_FINGERPRINTS = /* @__PURE__ */ new Map([
|
|
707
|
-
["docx", "
|
|
708
|
-
["markdown", "
|
|
709
|
-
["pdf", "
|
|
710
|
-
["pptx", "
|
|
711
|
-
["xlsx", "
|
|
791
|
+
["docx", "ea76a9e656cab4a7"],
|
|
792
|
+
["markdown", "7c9b8d7beb37ac60"],
|
|
793
|
+
["pdf", "ef7a41b66bc04bc1"],
|
|
794
|
+
["pptx", "8b18b531b42106e7"],
|
|
795
|
+
["xlsx", "67a12e733136381e"]
|
|
712
796
|
]);
|
|
713
797
|
var PRE_KIND_EMBED_FINGERPRINT = "b7b2ff71de288d13";
|
|
714
798
|
var SPLIT_EMBED_FINGERPRINT = "2310db32d9bc8a2b";
|
|
715
799
|
|
|
716
800
|
// src/embed_stamp.ts
|
|
717
801
|
init_define_import_meta_env();
|
|
718
|
-
import * as
|
|
802
|
+
import * as path5 from "node:path";
|
|
719
803
|
function embedKindForPath(filePath) {
|
|
720
|
-
if (isEmbeddableDocument(filePath)) return
|
|
804
|
+
if (isEmbeddableDocument(filePath)) return path5.extname(filePath).toLowerCase().slice(1);
|
|
721
805
|
if (detectLanguage(filePath) === "markdown") return "markdown";
|
|
722
806
|
return null;
|
|
723
807
|
}
|
|
@@ -749,6 +833,8 @@ async function buildExtractorWithRetry(pipelineFn, modelName) {
|
|
|
749
833
|
try {
|
|
750
834
|
return await pipelineFn("feature-extraction", modelName);
|
|
751
835
|
} catch (e) {
|
|
836
|
+
const { failedAtOf } = await import("./token-goat-chunk-COTXZHTS.mjs");
|
|
837
|
+
if (failedAtOf(e) !== null) throw e;
|
|
752
838
|
lastError = e;
|
|
753
839
|
if (attempt < PIPELINE_RETRY_ATTEMPTS) await sleep(PIPELINE_RETRY_DELAY_MS * attempt);
|
|
754
840
|
}
|
|
@@ -1073,7 +1159,7 @@ function chunkFile(filePath, content, chunkSize = MAX_CHUNK_CHARS, overlap = 200
|
|
|
1073
1159
|
function insertChunkVector(stmt, rowid, embedding) {
|
|
1074
1160
|
stmt.run(BigInt(rowid), packVec(embedding));
|
|
1075
1161
|
}
|
|
1076
|
-
async function upsertChunks(db, chunks) {
|
|
1162
|
+
async function upsertChunks(db, chunks, expectedSha) {
|
|
1077
1163
|
if (chunks.length === 0) {
|
|
1078
1164
|
return "embedded";
|
|
1079
1165
|
}
|
|
@@ -1099,6 +1185,10 @@ async function upsertChunks(db, chunks) {
|
|
|
1099
1185
|
VALUES (?, ?)
|
|
1100
1186
|
`);
|
|
1101
1187
|
const tx = db.transaction(() => {
|
|
1188
|
+
if (expectedSha !== void 0) {
|
|
1189
|
+
const current = db.prepare(`SELECT sha FROM files WHERE ${pathEqClause("path")}`).pluck().get(foldPath(filePath));
|
|
1190
|
+
if (current !== expectedSha) return "stale";
|
|
1191
|
+
}
|
|
1102
1192
|
deleteFileEmbeddings(db, filePath);
|
|
1103
1193
|
for (let i = 0; i < chunks.length; i++) {
|
|
1104
1194
|
const chunk = chunks[i];
|
|
@@ -1115,9 +1205,9 @@ async function upsertChunks(db, chunks) {
|
|
|
1115
1205
|
);
|
|
1116
1206
|
insertChunkVector(vectorInsertStmt, chunkResult.lastInsertRowid, embedding);
|
|
1117
1207
|
}
|
|
1208
|
+
return "embedded";
|
|
1118
1209
|
});
|
|
1119
|
-
tx.immediate();
|
|
1120
|
-
return "embedded";
|
|
1210
|
+
return tx.immediate();
|
|
1121
1211
|
}
|
|
1122
1212
|
var BACKFILL_MULTIPLIER = 3;
|
|
1123
1213
|
function countStoredVectors(db) {
|
|
@@ -1135,10 +1225,10 @@ function fetchScopedHits(db, queryVec, k, maxDistance, rootDir) {
|
|
|
1135
1225
|
if (!rows || rows.length === 0) {
|
|
1136
1226
|
return { hits: [], candidateCount: 0 };
|
|
1137
1227
|
}
|
|
1138
|
-
const scope = rootDir !== void 0 ?
|
|
1228
|
+
const scope = rootDir !== void 0 ? ownProjectScope("file_path", rootDir) : void 0;
|
|
1139
1229
|
const chunkSql = scope !== void 0 ? `SELECT file_path, start_line, end_line, text, kind FROM chunks WHERE id = ? AND ${scope.clause}` : `SELECT file_path, start_line, end_line, text, kind FROM chunks WHERE id = ?`;
|
|
1140
1230
|
const chunkStmt = db.prepare(chunkSql);
|
|
1141
|
-
const scopeParams = scope
|
|
1231
|
+
const scopeParams = scope?.params;
|
|
1142
1232
|
const hits = [];
|
|
1143
1233
|
for (const row of rows) {
|
|
1144
1234
|
if (!row) {
|
|
@@ -1161,7 +1251,7 @@ function fetchScopedHits(db, queryVec, k, maxDistance, rootDir) {
|
|
|
1161
1251
|
return { hits, candidateCount: rows.length };
|
|
1162
1252
|
}
|
|
1163
1253
|
function fetchScopedExactHits(db, queryVec, limit, maxDistance, rootDir) {
|
|
1164
|
-
const scope =
|
|
1254
|
+
const scope = ownProjectScope("c.file_path", rootDir);
|
|
1165
1255
|
const rows = db.prepare(
|
|
1166
1256
|
`SELECT file_path, start_line, end_line, text, kind, distance FROM (
|
|
1167
1257
|
SELECT c.id AS id, c.file_path AS file_path, c.start_line AS start_line, c.end_line AS end_line, c.text AS text, c.kind AS kind, vec_distance_L2(v.embedding, ?) AS distance
|
|
@@ -1171,7 +1261,7 @@ function fetchScopedExactHits(db, queryVec, limit, maxDistance, rootDir) {
|
|
|
1171
1261
|
WHERE distance IS NOT NULL AND distance <= ?
|
|
1172
1262
|
ORDER BY distance ASC, file_path ASC, start_line ASC, id ASC
|
|
1173
1263
|
LIMIT ?`
|
|
1174
|
-
).all(packVec(queryVec), ...scope.params
|
|
1264
|
+
).all(packVec(queryVec), ...scope.params, maxDistance, limit);
|
|
1175
1265
|
const hits = [];
|
|
1176
1266
|
for (const row of rows) {
|
|
1177
1267
|
if (!row || typeof row.distance !== "number" || !Number.isFinite(row.distance) || row.distance > maxDistance) {
|
|
@@ -1248,82 +1338,14 @@ function rerankHits(hits, query, topK) {
|
|
|
1248
1338
|
scored.sort((a, b) => a.adjusted - b.adjusted || a.index - b.index);
|
|
1249
1339
|
return scored.slice(0, topK).map((entry) => ({ ...entry.hit, adjustedDistance: entry.adjusted }));
|
|
1250
1340
|
}
|
|
1251
|
-
function
|
|
1252
|
-
if (hits.length <= 1) {
|
|
1253
|
-
return hits;
|
|
1254
|
-
}
|
|
1255
|
-
const byFile = /* @__PURE__ */ new Map();
|
|
1256
|
-
for (const hit of hits) {
|
|
1257
|
-
const fileHits = byFile.get(hit.filePath);
|
|
1258
|
-
if (fileHits) {
|
|
1259
|
-
fileHits.push(hit);
|
|
1260
|
-
} else {
|
|
1261
|
-
byFile.set(hit.filePath, [hit]);
|
|
1262
|
-
}
|
|
1263
|
-
}
|
|
1264
|
-
const merged = [];
|
|
1265
|
-
for (const fileHits of byFile.values()) {
|
|
1266
|
-
fileHits.sort((a, b) => a.startLine - b.startLine);
|
|
1267
|
-
let current = fileHits[0];
|
|
1268
|
-
if (!current) {
|
|
1269
|
-
continue;
|
|
1270
|
-
}
|
|
1271
|
-
let curStart = current.startLine;
|
|
1272
|
-
let curEnd = current.endLine;
|
|
1273
|
-
let curDist = current.distance;
|
|
1274
|
-
let curAdjusted = current.adjustedDistance ?? current.distance;
|
|
1275
|
-
const mergedTexts = [current.text];
|
|
1276
|
-
for (let i = 1; i < fileHits.length; i++) {
|
|
1277
|
-
const hit = fileHits[i];
|
|
1278
|
-
if (!hit) {
|
|
1279
|
-
continue;
|
|
1280
|
-
}
|
|
1281
|
-
const gap = hit.startLine - curEnd - 1;
|
|
1282
|
-
if (gap <= proximity) {
|
|
1283
|
-
curEnd = Math.max(curEnd, hit.endLine);
|
|
1284
|
-
curDist = Math.min(curDist, hit.distance);
|
|
1285
|
-
curAdjusted = Math.min(curAdjusted, hit.adjustedDistance ?? hit.distance);
|
|
1286
|
-
mergedTexts.push(hit.text);
|
|
1287
|
-
} else {
|
|
1288
|
-
merged.push({
|
|
1289
|
-
filePath: current.filePath,
|
|
1290
|
-
startLine: curStart,
|
|
1291
|
-
endLine: curEnd,
|
|
1292
|
-
kind: current.kind,
|
|
1293
|
-
distance: curDist,
|
|
1294
|
-
adjustedDistance: curAdjusted,
|
|
1295
|
-
text: mergedTexts.join("\n---\n")
|
|
1296
|
-
});
|
|
1297
|
-
current = hit;
|
|
1298
|
-
curStart = hit.startLine;
|
|
1299
|
-
curEnd = hit.endLine;
|
|
1300
|
-
curDist = hit.distance;
|
|
1301
|
-
curAdjusted = hit.adjustedDistance ?? hit.distance;
|
|
1302
|
-
mergedTexts.length = 0;
|
|
1303
|
-
mergedTexts.push(hit.text);
|
|
1304
|
-
}
|
|
1305
|
-
}
|
|
1306
|
-
merged.push({
|
|
1307
|
-
filePath: current.filePath,
|
|
1308
|
-
startLine: curStart,
|
|
1309
|
-
endLine: curEnd,
|
|
1310
|
-
kind: current.kind,
|
|
1311
|
-
distance: curDist,
|
|
1312
|
-
adjustedDistance: curAdjusted,
|
|
1313
|
-
text: mergedTexts.join("\n---\n")
|
|
1314
|
-
});
|
|
1315
|
-
}
|
|
1316
|
-
merged.sort((a, b) => (a.adjustedDistance ?? a.distance) - (b.adjustedDistance ?? b.distance));
|
|
1317
|
-
return merged;
|
|
1318
|
-
}
|
|
1319
|
-
async function indexFile(db, filePath, content, boundaries = []) {
|
|
1341
|
+
async function indexFile(db, filePath, content, boundaries = [], expectedSha) {
|
|
1320
1342
|
let chunks = chunkFile(filePath, content, void 0, void 0, boundaries);
|
|
1321
1343
|
if (chunks.length > 0) {
|
|
1322
1344
|
if (embeddingsDepsAvailable(db)) {
|
|
1323
1345
|
const { countTokens } = await getExtractor(DEFAULT_MODEL);
|
|
1324
1346
|
if (countTokens !== void 0) chunks = chunkFile(filePath, content, void 0, void 0, boundaries, countTokens);
|
|
1325
1347
|
}
|
|
1326
|
-
return upsertChunks(db, chunks);
|
|
1348
|
+
return upsertChunks(db, chunks, expectedSha);
|
|
1327
1349
|
}
|
|
1328
1350
|
deleteFileEmbeddings(db, filePath);
|
|
1329
1351
|
return "embedded";
|
|
@@ -1406,7 +1428,7 @@ function resetEmbeddingsForKinds(db, kinds) {
|
|
|
1406
1428
|
return paths.length;
|
|
1407
1429
|
}
|
|
1408
1430
|
function backendId() {
|
|
1409
|
-
return
|
|
1431
|
+
return `${activeRuntime()}@${majorMinor(runtimeVersion())}`;
|
|
1410
1432
|
}
|
|
1411
1433
|
function majorMinor(version) {
|
|
1412
1434
|
const parts = version.split(".");
|
|
@@ -1419,15 +1441,21 @@ function vectorSpaceOf(provenance) {
|
|
|
1419
1441
|
}
|
|
1420
1442
|
function ensureEmbeddingProvenance(db, modelName = DEFAULT_MODEL) {
|
|
1421
1443
|
if (_provenanceChecked.has(db)) return;
|
|
1422
|
-
_provenanceChecked.add(db);
|
|
1423
1444
|
const current = embeddingProvenance(modelName);
|
|
1424
1445
|
const stored = db.prepare("SELECT provenance FROM embedding_provenance WHERE id = 1").pluck().get();
|
|
1425
|
-
if (stored === current)
|
|
1446
|
+
if (stored === current) {
|
|
1447
|
+
_provenanceChecked.add(db);
|
|
1448
|
+
return;
|
|
1449
|
+
}
|
|
1426
1450
|
const keepVectors = stored !== void 0 && vectorSpaceOf(stored) === vectorSpaceOf(current);
|
|
1427
|
-
const cleared =
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1451
|
+
const cleared = db.transaction(() => {
|
|
1452
|
+
const count = stored !== void 0 && keepVectors ? resetStaleChunking(db, stored, current) : resetAllEmbeddings(db, false);
|
|
1453
|
+
db.prepare(
|
|
1454
|
+
"INSERT INTO embedding_provenance (id, provenance) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET provenance = excluded.provenance"
|
|
1455
|
+
).run(current);
|
|
1456
|
+
return count;
|
|
1457
|
+
}).immediate();
|
|
1458
|
+
_provenanceChecked.add(db);
|
|
1431
1459
|
if (cleared > 0 && !keepVectors) {
|
|
1432
1460
|
console.warn(
|
|
1433
1461
|
`Embedding stack changed (${stored ?? "unrecorded"} -> ${current}); discarded ${cleared} file${cleared === 1 ? "" : "s"} worth of vectors because they no longer reliably describe this file's current chunks. Run \`token-goat index\` to rebuild them.`
|
|
@@ -1469,65 +1497,110 @@ function _pathPriorityPenalty(filePath) {
|
|
|
1469
1497
|
return 0;
|
|
1470
1498
|
}
|
|
1471
1499
|
|
|
1500
|
+
// src/skip_scope.ts
|
|
1501
|
+
init_define_import_meta_env();
|
|
1502
|
+
import * as os from "node:os";
|
|
1503
|
+
import * as path6 from "node:path";
|
|
1504
|
+
var declaredRoot;
|
|
1505
|
+
function withDeclaredSkipScopeRoot(root, fn) {
|
|
1506
|
+
const previous = declaredRoot;
|
|
1507
|
+
declaredRoot = root;
|
|
1508
|
+
try {
|
|
1509
|
+
return fn();
|
|
1510
|
+
} finally {
|
|
1511
|
+
declaredRoot = previous;
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
function pathSegments(p) {
|
|
1515
|
+
return p.split(/[/\\]/).filter((s) => s.length > 0);
|
|
1516
|
+
}
|
|
1517
|
+
function outermostProjectRoot(filePath) {
|
|
1518
|
+
const first = findProject(path6.dirname(filePath));
|
|
1519
|
+
if (first === null) return void 0;
|
|
1520
|
+
const home = foldPath(os.homedir()).replace(/\\/g, "/");
|
|
1521
|
+
let root = first.root;
|
|
1522
|
+
for (; ; ) {
|
|
1523
|
+
const parent = path6.dirname(root);
|
|
1524
|
+
if (parent === root) return root;
|
|
1525
|
+
const outer = findProject(parent);
|
|
1526
|
+
if (outer === null || outer.root === root) return root;
|
|
1527
|
+
const outerFolded = foldPath(outer.root).replace(/\\/g, "/").replace(/\/+$/, "");
|
|
1528
|
+
if (home === outerFolded || home.startsWith(`${outerFolded}/`)) return root;
|
|
1529
|
+
root = outer.root;
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
function hasSkipSegmentBelowRoot(filePath, projectRoot, isSkipSegment) {
|
|
1533
|
+
const dirSegments = pathSegments(filePath).slice(0, -1);
|
|
1534
|
+
if (!dirSegments.some(isSkipSegment)) return false;
|
|
1535
|
+
const root = projectRoot ?? declaredRoot ?? outermostProjectRoot(filePath);
|
|
1536
|
+
if (root === void 0) return true;
|
|
1537
|
+
const rootSegments = pathSegments(root);
|
|
1538
|
+
if (rootSegments.length > dirSegments.length) return true;
|
|
1539
|
+
for (let i = 0; i < rootSegments.length; i++) {
|
|
1540
|
+
if (foldPath(rootSegments[i] ?? "") !== foldPath(dirSegments[i] ?? "")) return true;
|
|
1541
|
+
}
|
|
1542
|
+
return dirSegments.slice(rootSegments.length).some(isSkipSegment);
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1472
1545
|
// src/parser_stamp.ts
|
|
1473
1546
|
init_define_import_meta_env();
|
|
1474
1547
|
|
|
1475
1548
|
// src/parser_fingerprint.ts
|
|
1476
1549
|
init_define_import_meta_env();
|
|
1477
|
-
var PARSER_FINGERPRINT = "
|
|
1550
|
+
var PARSER_FINGERPRINT = "c8a3aa41909be8c0";
|
|
1478
1551
|
var LANGUAGE_PARSER_FINGERPRINTS = /* @__PURE__ */ new Map([
|
|
1479
|
-
["abap", "
|
|
1480
|
-
["abl", "
|
|
1481
|
-
["apache", "
|
|
1482
|
-
["apex", "
|
|
1483
|
-
["asm", "
|
|
1484
|
-
["bash", "
|
|
1485
|
-
["batch", "
|
|
1486
|
-
["caddy", "
|
|
1487
|
-
["clojure", "
|
|
1488
|
-
["cmake", "
|
|
1489
|
-
["common_lisp", "
|
|
1490
|
-
["csharp", "
|
|
1491
|
-
["dart", "
|
|
1492
|
-
["elixir", "
|
|
1493
|
-
["emacs_lisp", "
|
|
1494
|
-
["erlang", "
|
|
1495
|
-
["fortran", "
|
|
1496
|
-
["fsharp", "
|
|
1497
|
-
["graphql", "
|
|
1498
|
-
["groovy", "
|
|
1499
|
-
["haskell", "
|
|
1500
|
-
["jcl", "
|
|
1501
|
-
["kotlin", "
|
|
1502
|
-
["liquid", "
|
|
1503
|
-
["lua", "
|
|
1504
|
-
["makefile", "
|
|
1505
|
-
["matlab", "
|
|
1506
|
-
["nginx", "
|
|
1507
|
-
["nix", "
|
|
1508
|
-
["objc", "
|
|
1509
|
-
["ocaml", "
|
|
1510
|
-
["pascal", "
|
|
1511
|
-
["perl", "
|
|
1512
|
-
["php", "
|
|
1513
|
-
["pli", "
|
|
1514
|
-
["powershell", "
|
|
1515
|
-
["proto", "
|
|
1516
|
-
["r", "
|
|
1517
|
-
["racket", "
|
|
1518
|
-
["rpg", "
|
|
1519
|
-
["salesforce_metadata", "
|
|
1520
|
-
["sas", "
|
|
1521
|
-
["scala", "
|
|
1522
|
-
["scheme", "
|
|
1523
|
-
["solidity", "
|
|
1524
|
-
["sql", "
|
|
1525
|
-
["swift", "
|
|
1526
|
-
["terraform", "
|
|
1527
|
-
["thrift", "
|
|
1528
|
-
["vb", "
|
|
1529
|
-
["vhdl", "
|
|
1530
|
-
["zig", "
|
|
1552
|
+
["abap", "accec34183d6fc45"],
|
|
1553
|
+
["abl", "b721e43e5720a56d"],
|
|
1554
|
+
["apache", "6e840e4ac38d88c1"],
|
|
1555
|
+
["apex", "6e621f41fce0d396"],
|
|
1556
|
+
["asm", "845db656efdc0fca"],
|
|
1557
|
+
["bash", "656e307df22168bb"],
|
|
1558
|
+
["batch", "d20823bbe2a8eec4"],
|
|
1559
|
+
["caddy", "b0f8fdf4ea685095"],
|
|
1560
|
+
["clojure", "85b583da30a1a44e"],
|
|
1561
|
+
["cmake", "1e0b257e06742425"],
|
|
1562
|
+
["common_lisp", "b6c3bc62b3a255a5"],
|
|
1563
|
+
["csharp", "68d9c4bb7fb18e1c"],
|
|
1564
|
+
["dart", "40a3921e9143f0f4"],
|
|
1565
|
+
["elixir", "719739d859fb5a25"],
|
|
1566
|
+
["emacs_lisp", "463153969879ead0"],
|
|
1567
|
+
["erlang", "3aaebe53622499d3"],
|
|
1568
|
+
["fortran", "7ecf0ab7c3f52b6f"],
|
|
1569
|
+
["fsharp", "b9c219bd5fbdc8b5"],
|
|
1570
|
+
["graphql", "34c5923656df5406"],
|
|
1571
|
+
["groovy", "780274c5b5c7d79a"],
|
|
1572
|
+
["haskell", "942d08d94feba789"],
|
|
1573
|
+
["jcl", "d0868f4420b941ea"],
|
|
1574
|
+
["kotlin", "df0ce105cba0fc0b"],
|
|
1575
|
+
["liquid", "436055b0a6b64ceb"],
|
|
1576
|
+
["lua", "ded20290e6d00d06"],
|
|
1577
|
+
["makefile", "dbbd6b32d5cc62eb"],
|
|
1578
|
+
["matlab", "d8a2f8c83f070614"],
|
|
1579
|
+
["nginx", "63248e67cc521def"],
|
|
1580
|
+
["nix", "056a50108db861a6"],
|
|
1581
|
+
["objc", "c21c0d7c13775c0f"],
|
|
1582
|
+
["ocaml", "22bf5a0eaa2b5f80"],
|
|
1583
|
+
["pascal", "24579492a16ad7e4"],
|
|
1584
|
+
["perl", "bd8ac6c679b29abe"],
|
|
1585
|
+
["php", "b823c3df1138d2b7"],
|
|
1586
|
+
["pli", "38f7027519e34e39"],
|
|
1587
|
+
["powershell", "70dd31bfbcf69a68"],
|
|
1588
|
+
["proto", "9c42c66e3fee093f"],
|
|
1589
|
+
["r", "5db6f150582b5550"],
|
|
1590
|
+
["racket", "e900071a3d70d0b3"],
|
|
1591
|
+
["rpg", "5839e0f9c099c11b"],
|
|
1592
|
+
["salesforce_metadata", "a4408c65eb712933"],
|
|
1593
|
+
["sas", "7e8a6772363c76a1"],
|
|
1594
|
+
["scala", "72c64544a81c0c98"],
|
|
1595
|
+
["scheme", "592b9b37b5494f0e"],
|
|
1596
|
+
["solidity", "3f0947056f925fad"],
|
|
1597
|
+
["sql", "5d423d87e7a0c2b3"],
|
|
1598
|
+
["swift", "3c7306bc8c96279d"],
|
|
1599
|
+
["terraform", "ead711fa99e96a85"],
|
|
1600
|
+
["thrift", "a773492ef26655c7"],
|
|
1601
|
+
["vb", "97ade1ad9cd06689"],
|
|
1602
|
+
["vhdl", "cd0e1ee6c5da9bd1"],
|
|
1603
|
+
["zig", "9034a79cede0b52f"]
|
|
1531
1604
|
]);
|
|
1532
1605
|
|
|
1533
1606
|
// src/parser_stamp.ts
|
|
@@ -1537,13 +1610,13 @@ function parserFingerprintForLanguage(language) {
|
|
|
1537
1610
|
|
|
1538
1611
|
// src/parser.ts
|
|
1539
1612
|
init_define_import_meta_env();
|
|
1540
|
-
import * as
|
|
1613
|
+
import * as fs4 from "node:fs";
|
|
1541
1614
|
import { createRequire as createRequire2 } from "node:module";
|
|
1542
|
-
import * as
|
|
1615
|
+
import * as path9 from "node:path";
|
|
1543
1616
|
|
|
1544
1617
|
// src/asset_extensions.ts
|
|
1545
1618
|
init_define_import_meta_env();
|
|
1546
|
-
import * as
|
|
1619
|
+
import * as path7 from "node:path";
|
|
1547
1620
|
var IMAGE_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
1548
1621
|
".png",
|
|
1549
1622
|
".jpg",
|
|
@@ -1558,7 +1631,7 @@ var IMAGE_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
|
1558
1631
|
".heif"
|
|
1559
1632
|
]);
|
|
1560
1633
|
function isImagePath(p) {
|
|
1561
|
-
return IMAGE_EXTENSIONS.has(
|
|
1634
|
+
return IMAGE_EXTENSIONS.has(path7.extname(p).toLowerCase());
|
|
1562
1635
|
}
|
|
1563
1636
|
var NON_TEXT_ASSET_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
1564
1637
|
...IMAGE_EXTENSIONS,
|
|
@@ -1606,7 +1679,7 @@ var NON_TEXT_ASSET_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
|
1606
1679
|
".sqlite3"
|
|
1607
1680
|
]);
|
|
1608
1681
|
function isNonTextAsset(p) {
|
|
1609
|
-
return NON_TEXT_ASSET_EXTENSIONS.has(
|
|
1682
|
+
return NON_TEXT_ASSET_EXTENSIONS.has(path7.extname(p).toLowerCase());
|
|
1610
1683
|
}
|
|
1611
1684
|
var NON_TEXT_ASSET_SET_ID = (() => {
|
|
1612
1685
|
const joined = [...NON_TEXT_ASSET_EXTENSIONS].sort().join(",");
|
|
@@ -1628,9 +1701,14 @@ init_define_import_meta_env();
|
|
|
1628
1701
|
init_define_import_meta_env();
|
|
1629
1702
|
function* eachUnfencedLine(lines) {
|
|
1630
1703
|
let fence = null;
|
|
1704
|
+
let inComment = false;
|
|
1631
1705
|
for (let i = 0; i < lines.length; i++) {
|
|
1632
1706
|
const line = lines[i];
|
|
1633
1707
|
if (line === void 0) continue;
|
|
1708
|
+
if (inComment) {
|
|
1709
|
+
if (line.includes("-->")) inComment = false;
|
|
1710
|
+
continue;
|
|
1711
|
+
}
|
|
1634
1712
|
const fm = /^\s*(`{3,}|~{3,})([^\n]*)$/.exec(line);
|
|
1635
1713
|
if (fm !== null && fm[1] !== void 0) {
|
|
1636
1714
|
const run = fm[1];
|
|
@@ -1644,10 +1722,28 @@ function* eachUnfencedLine(lines) {
|
|
|
1644
1722
|
continue;
|
|
1645
1723
|
}
|
|
1646
1724
|
if (fence !== null) continue;
|
|
1725
|
+
if (/^ {0,3}<!--/.test(line)) {
|
|
1726
|
+
inComment = !line.includes("-->");
|
|
1727
|
+
continue;
|
|
1728
|
+
}
|
|
1647
1729
|
yield [i, line];
|
|
1648
1730
|
}
|
|
1649
1731
|
}
|
|
1650
1732
|
|
|
1733
|
+
// src/markdown_frontmatter.ts
|
|
1734
|
+
init_define_import_meta_env();
|
|
1735
|
+
function frontMatterEndIndex(lines) {
|
|
1736
|
+
const first = lines[0];
|
|
1737
|
+
if (first === void 0) return 0;
|
|
1738
|
+
const open = first.replace(/^\uFEFF/, "").trimEnd();
|
|
1739
|
+
if (open !== "---" && open !== "+++") return 0;
|
|
1740
|
+
for (let i = 1; i < lines.length; i++) {
|
|
1741
|
+
const t = (lines[i] ?? "").trimEnd();
|
|
1742
|
+
if (t === open || open === "---" && t === "...") return i + 1;
|
|
1743
|
+
}
|
|
1744
|
+
return 0;
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1651
1747
|
// src/hints/markdown_hints.ts
|
|
1652
1748
|
var MARKDOWN_SIZE_THRESHOLD = 4e3;
|
|
1653
1749
|
var MAX_HEADINGS = 40;
|
|
@@ -1655,47 +1751,44 @@ var MAX_OUTPUT_LINES = 60;
|
|
|
1655
1751
|
function extractMarkdownHeadings(content, limit = MAX_HEADINGS) {
|
|
1656
1752
|
const headings = [];
|
|
1657
1753
|
const lines = content.split("\n");
|
|
1658
|
-
const
|
|
1754
|
+
const fmEnd = frontMatterEndIndex(lines);
|
|
1755
|
+
const unfenced = Array.from(eachUnfencedLine(lines)).filter(([i]) => i >= fmEnd);
|
|
1756
|
+
let paraStart = -1;
|
|
1659
1757
|
for (let u = 0; u < unfenced.length; u++) {
|
|
1660
1758
|
const [i, line] = unfenced[u];
|
|
1661
|
-
if (!line)
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1759
|
+
if (!line) {
|
|
1760
|
+
paraStart = -1;
|
|
1761
|
+
continue;
|
|
1762
|
+
}
|
|
1763
|
+
const atx = matchAtxHeading(line, ATX_CLASS_BREAK_RE, true);
|
|
1764
|
+
if (atx !== null) {
|
|
1667
1765
|
const maxLevel = limit === Infinity ? 6 : 3;
|
|
1668
|
-
if (level <= maxLevel) {
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
});
|
|
1676
|
-
if (headings.length >= limit) break;
|
|
1677
|
-
}
|
|
1766
|
+
if (atx.level <= maxLevel && atx.name) {
|
|
1767
|
+
headings.push({
|
|
1768
|
+
level: atx.level,
|
|
1769
|
+
text: atx.name,
|
|
1770
|
+
lineNumber: i + 1
|
|
1771
|
+
});
|
|
1772
|
+
if (headings.length >= limit) break;
|
|
1678
1773
|
}
|
|
1774
|
+
paraStart = -1;
|
|
1679
1775
|
continue;
|
|
1680
1776
|
}
|
|
1777
|
+
if (paraStart !== -1 && i !== unfenced[u - 1][0] + 1) paraStart = -1;
|
|
1681
1778
|
const trimmed = line.trim();
|
|
1682
|
-
if (
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
text: trimmed,
|
|
1692
|
-
lineNumber: i + 1
|
|
1693
|
-
});
|
|
1694
|
-
u++;
|
|
1695
|
-
if (headings.length >= limit) break;
|
|
1696
|
-
}
|
|
1697
|
-
}
|
|
1779
|
+
if (paraStart !== -1 && /^(=+|-+)$/.test(trimmed)) {
|
|
1780
|
+
headings.push({
|
|
1781
|
+
level: trimmed.startsWith("=") ? 1 : 2,
|
|
1782
|
+
text: unfenced.slice(paraStart, u).map(([, text]) => text.trim()).join(" "),
|
|
1783
|
+
lineNumber: unfenced[paraStart][0] + 1
|
|
1784
|
+
});
|
|
1785
|
+
paraStart = -1;
|
|
1786
|
+
if (headings.length >= limit) break;
|
|
1787
|
+
continue;
|
|
1698
1788
|
}
|
|
1789
|
+
const isText = trimmed !== "" && !trimmed.startsWith("#") && !trimmed.startsWith("|") && !trimmed.startsWith("```") && !trimmed.startsWith("~~~") && !/^([-*+]|\d+\.)\s/.test(trimmed) && !/^(=+|-+)$/.test(trimmed);
|
|
1790
|
+
if (!isText) paraStart = -1;
|
|
1791
|
+
else if (paraStart === -1) paraStart = u;
|
|
1699
1792
|
}
|
|
1700
1793
|
return headings;
|
|
1701
1794
|
}
|
|
@@ -1783,11 +1876,11 @@ function buildEmbeddingBoundaries(filePath, content, dbPath) {
|
|
|
1783
1876
|
|
|
1784
1877
|
// src/languages/salesforce_frontend.ts
|
|
1785
1878
|
init_define_import_meta_env();
|
|
1786
|
-
import * as
|
|
1879
|
+
import * as path8 from "node:path";
|
|
1787
1880
|
function bundleName(filePath) {
|
|
1788
1881
|
const normalized = filePath.replaceAll("\\", "/");
|
|
1789
|
-
const parent =
|
|
1790
|
-
const base =
|
|
1882
|
+
const parent = path8.posix.basename(path8.posix.dirname(normalized));
|
|
1883
|
+
const base = path8.posix.basename(normalized).replace(/\.[^.]+$/, "");
|
|
1791
1884
|
return parent === "lwc" || parent === "aura" ? base : parent;
|
|
1792
1885
|
}
|
|
1793
1886
|
function lwcTagAlias(name) {
|
|
@@ -1885,7 +1978,7 @@ var MARKUP_KIND = {
|
|
|
1885
1978
|
};
|
|
1886
1979
|
function markupArtifactName(filePath, extension) {
|
|
1887
1980
|
if (extension === ".page" || extension === ".component" || extension === ".email") {
|
|
1888
|
-
return
|
|
1981
|
+
return path8.posix.basename(filePath.replaceAll("\\", "/")).replace(new RegExp(`${extension.replace(".", "\\.")}$`, "i"), "");
|
|
1889
1982
|
}
|
|
1890
1983
|
return bundleName(filePath);
|
|
1891
1984
|
}
|
|
@@ -1905,7 +1998,7 @@ function attributeRefs(refs, content, lineIndex, filePath, attribute, split = fa
|
|
|
1905
1998
|
}
|
|
1906
1999
|
function extractSalesforceMarkup(content, filePath) {
|
|
1907
2000
|
const normalized = filePath.replaceAll("\\", "/");
|
|
1908
|
-
const extension =
|
|
2001
|
+
const extension = path8.posix.extname(normalized).toLowerCase();
|
|
1909
2002
|
const kind = MARKUP_KIND[extension] ?? "salesforce_markup";
|
|
1910
2003
|
const symbols = [
|
|
1911
2004
|
symbol(filePath, markupArtifactName(normalized, extension), kind, 1, countContentLines(content))
|
|
@@ -2157,6 +2250,39 @@ function isModuleWrapperCallback(node) {
|
|
|
2157
2250
|
}
|
|
2158
2251
|
return false;
|
|
2159
2252
|
}
|
|
2253
|
+
var TSJS_FN_VALUE_TYPES = /* @__PURE__ */ new Set(["arrow_function", "function_expression", "function"]);
|
|
2254
|
+
function isModuleExports(node) {
|
|
2255
|
+
if (node.type !== "member_expression") return false;
|
|
2256
|
+
const obj = node.childForFieldName("object");
|
|
2257
|
+
const prop = node.childForFieldName("property");
|
|
2258
|
+
return obj !== null && prop !== null && obj.type === "identifier" && obj.text === "module" && prop.text === "exports";
|
|
2259
|
+
}
|
|
2260
|
+
function commonJsAssignmentSymbol(stmt, filePath, lines) {
|
|
2261
|
+
const assign = stmt.namedChildren[0];
|
|
2262
|
+
if (assign === void 0 || assign.type !== "assignment_expression") return null;
|
|
2263
|
+
const left = assign.childForFieldName("left");
|
|
2264
|
+
const right = assign.childForFieldName("right");
|
|
2265
|
+
if (left === null || right === null || left.type !== "member_expression") return null;
|
|
2266
|
+
const isFn = TSJS_FN_VALUE_TYPES.has(right.type);
|
|
2267
|
+
if (isModuleExports(left)) {
|
|
2268
|
+
const own = right.type === "function_expression" || right.type === "class" ? right.childForFieldName("name") : null;
|
|
2269
|
+
if (own === null || own.text === "") return null;
|
|
2270
|
+
return makeSymbol(filePath, own.text, right.type === "class" ? "class" : "function", stmt, lines, "c");
|
|
2271
|
+
}
|
|
2272
|
+
if (!isFn) return null;
|
|
2273
|
+
const obj = left.childForFieldName("object");
|
|
2274
|
+
const prop = left.childForFieldName("property");
|
|
2275
|
+
if (obj === null || prop === null || prop.type !== "property_identifier") return null;
|
|
2276
|
+
if (obj.type === "identifier" && obj.text === "exports") return makeSymbol(filePath, prop.text, "function", stmt, lines, "c");
|
|
2277
|
+
if (isModuleExports(obj)) return makeSymbol(filePath, prop.text, "function", stmt, lines, "c");
|
|
2278
|
+
if (obj.type === "member_expression" && obj.childForFieldName("property")?.text === "prototype") {
|
|
2279
|
+
const owner = obj.childForFieldName("object");
|
|
2280
|
+
if (owner === null) return null;
|
|
2281
|
+
const sym = makeSymbol(filePath, prop.text, "method", stmt, lines, "c");
|
|
2282
|
+
return { ...sym, parent: owner.text };
|
|
2283
|
+
}
|
|
2284
|
+
return null;
|
|
2285
|
+
}
|
|
2160
2286
|
function extractTsJsSymbols(root, filePath, lines) {
|
|
2161
2287
|
const out = [];
|
|
2162
2288
|
const visit = (node, insideFunction) => {
|
|
@@ -2213,6 +2339,10 @@ function extractTsJsSymbols(root, filePath, lines) {
|
|
|
2213
2339
|
}
|
|
2214
2340
|
}
|
|
2215
2341
|
}
|
|
2342
|
+
if (!insideFunction && node.type === "expression_statement") {
|
|
2343
|
+
const assigned = commonJsAssignmentSymbol(node, filePath, lines);
|
|
2344
|
+
if (assigned !== null) out.push(assigned);
|
|
2345
|
+
}
|
|
2216
2346
|
if (node.type === "public_field_definition" || node.type === "field_definition") {
|
|
2217
2347
|
const fieldName = node.childForFieldName("name") ?? node.childForFieldName("property");
|
|
2218
2348
|
const value = node.childForFieldName("value");
|
|
@@ -2342,6 +2472,12 @@ var GO_LOCAL_KINDS = /* @__PURE__ */ new Set([
|
|
|
2342
2472
|
"method_elem"
|
|
2343
2473
|
]);
|
|
2344
2474
|
var GO_MULTI_NAME_SPECS = /* @__PURE__ */ new Set(["var_spec", "const_spec"]);
|
|
2475
|
+
function goReceiverType(node) {
|
|
2476
|
+
const receiver = node.childForFieldName("receiver");
|
|
2477
|
+
const param = receiver?.namedChildren.find((c) => c.type === "parameter_declaration");
|
|
2478
|
+
const typeText = param?.childForFieldName("type")?.text ?? "";
|
|
2479
|
+
return typeText.replace(/\[[^\]]*\]/g, "").replace(/[*()\s]/g, "");
|
|
2480
|
+
}
|
|
2345
2481
|
function extractGoSymbols(root, filePath, lines) {
|
|
2346
2482
|
const out = [];
|
|
2347
2483
|
const visit = (node, insideFunction) => {
|
|
@@ -2359,7 +2495,9 @@ function extractGoSymbols(root, filePath, lines) {
|
|
|
2359
2495
|
} else {
|
|
2360
2496
|
const name = nodeName(node);
|
|
2361
2497
|
if (name !== null && name !== "") {
|
|
2362
|
-
|
|
2498
|
+
const sym = makeSymbol(filePath, name, kind, node, lines, "c");
|
|
2499
|
+
const receiverType = node.type === "method_declaration" ? goReceiverType(node) : "";
|
|
2500
|
+
out.push(receiverType !== "" ? { ...sym, parent: receiverType } : sym);
|
|
2363
2501
|
}
|
|
2364
2502
|
}
|
|
2365
2503
|
}
|
|
@@ -2450,12 +2588,18 @@ function leadingRustAttributes(node) {
|
|
|
2450
2588
|
}
|
|
2451
2589
|
return attrs;
|
|
2452
2590
|
}
|
|
2591
|
+
function rustImplTypeName(type) {
|
|
2592
|
+
if (type === null) return null;
|
|
2593
|
+
const inner = type.type === "generic_type" || type.type === "reference_type" ? type.childForFieldName("type") : null;
|
|
2594
|
+
if (inner !== null) return rustImplTypeName(inner);
|
|
2595
|
+
return type.type === "scoped_type_identifier" ? type.childForFieldName("name")?.text ?? type.text : type.text;
|
|
2596
|
+
}
|
|
2453
2597
|
function extractRustSymbols(root, filePath, lines) {
|
|
2454
2598
|
const out = [];
|
|
2455
2599
|
const visit = (node, insideFunction) => {
|
|
2456
2600
|
const kind = RUST_KIND_BY_TYPE.get(node.type);
|
|
2457
2601
|
if (kind !== void 0 && !(insideFunction && RUST_LOCAL_KINDS.has(node.type))) {
|
|
2458
|
-
const name = node.type === "impl_item" ? node.childForFieldName("type")
|
|
2602
|
+
const name = node.type === "impl_item" ? rustImplTypeName(node.childForFieldName("type")) : node.type === "foreign_mod_item" ? node.namedChildren.find((c) => c.type === "extern_modifier")?.text ?? "extern" : nodeName(node);
|
|
2459
2603
|
if (name !== null && name !== "") {
|
|
2460
2604
|
const attrs = leadingRustAttributes(node);
|
|
2461
2605
|
if (attrs.length === 0) {
|
|
@@ -2489,14 +2633,16 @@ var RUBY_KIND_BY_TYPE = /* @__PURE__ */ new Map([
|
|
|
2489
2633
|
["class", "class"],
|
|
2490
2634
|
["module", "module"]
|
|
2491
2635
|
]);
|
|
2492
|
-
function extractSimpleSymbols(root, filePath, kindByType, lines, style, nameFor = nodeName) {
|
|
2636
|
+
function extractSimpleSymbols(root, filePath, kindByType, lines, style, nameFor = nodeName, parentFor) {
|
|
2493
2637
|
const out = [];
|
|
2494
2638
|
const visit = (node) => {
|
|
2495
2639
|
const kind = kindByType.get(node.type);
|
|
2496
2640
|
if (kind !== void 0) {
|
|
2497
2641
|
const name = nameFor(node);
|
|
2498
2642
|
if (name !== null && name !== "") {
|
|
2499
|
-
|
|
2643
|
+
const sym = makeSymbol(filePath, name, kind, node, lines, style);
|
|
2644
|
+
const parent = parentFor === void 0 ? "" : parentFor(node);
|
|
2645
|
+
out.push(parent !== "" ? { ...sym, parent } : sym);
|
|
2500
2646
|
}
|
|
2501
2647
|
}
|
|
2502
2648
|
for (const child of node.namedChildren) {
|
|
@@ -2586,21 +2732,46 @@ function extractCppSymbols(root, filePath, lines) {
|
|
|
2586
2732
|
CPP_KIND_BY_TYPE,
|
|
2587
2733
|
lines,
|
|
2588
2734
|
"c",
|
|
2589
|
-
(node) => node.type === "function_definition" ? cFunctionName(node) : node.type === "type_definition" ? cTypedefAliasName(node) : node.type === "declaration" ? cFunctionPrototypeName(node) : nodeName(node)
|
|
2735
|
+
(node) => node.type === "function_definition" ? cFunctionName(node) : node.type === "type_definition" ? cTypedefAliasName(node) : node.type === "declaration" ? cFunctionPrototypeName(node) : C_TAG_SPECIFIER_TYPES.has(node.type) && node.childForFieldName("body") === null ? null : nodeName(node),
|
|
2736
|
+
cMethodParent
|
|
2590
2737
|
);
|
|
2591
2738
|
}
|
|
2739
|
+
var C_TAG_SPECIFIER_TYPES = /* @__PURE__ */ new Set(["struct_specifier", "class_specifier", "union_specifier", "enum_specifier"]);
|
|
2740
|
+
function cMethodParent(node) {
|
|
2741
|
+
if (node.type !== "function_definition" && node.type !== "declaration") return "";
|
|
2742
|
+
const body = node.parent;
|
|
2743
|
+
const owner = body?.parent ?? null;
|
|
2744
|
+
if (body !== null && body.type === "field_declaration_list" && owner !== null && (owner.type === "class_specifier" || owner.type === "struct_specifier" || owner.type === "union_specifier")) {
|
|
2745
|
+
return owner.childForFieldName("name")?.text ?? "";
|
|
2746
|
+
}
|
|
2747
|
+
let cur = node.childForFieldName("declarator");
|
|
2748
|
+
for (let i = 0; cur !== null && i < 16; i++) {
|
|
2749
|
+
if (cur.type === "qualified_identifier") {
|
|
2750
|
+
let inner = cur;
|
|
2751
|
+
for (let j = 0; j < 16; j++) {
|
|
2752
|
+
const next = inner.childForFieldName("name");
|
|
2753
|
+
if (next === null || next.type !== "qualified_identifier") break;
|
|
2754
|
+
inner = next;
|
|
2755
|
+
}
|
|
2756
|
+
const scope = inner.childForFieldName("scope");
|
|
2757
|
+
return scope === null ? "" : scope.text.replace(/<.*$/s, "").trim();
|
|
2758
|
+
}
|
|
2759
|
+
cur = innerDeclarator(cur);
|
|
2760
|
+
}
|
|
2761
|
+
return "";
|
|
2762
|
+
}
|
|
2592
2763
|
function cFunctionPrototypeName(node) {
|
|
2593
2764
|
let cur = node.childForFieldName("declarator");
|
|
2594
2765
|
for (let i = 0; cur !== null && i < 16; i++) {
|
|
2595
2766
|
if (cur.type === "function_declarator") {
|
|
2596
2767
|
const inner = cur.childForFieldName("declarator");
|
|
2597
2768
|
if (inner === null) return null;
|
|
2598
|
-
if (inner.type === "identifier" || inner.type === "field_identifier") return inner.text;
|
|
2769
|
+
if (inner.type === "identifier" || inner.type === "field_identifier" || inner.type === "destructor_name" || inner.type === "operator_name") return inner.text;
|
|
2599
2770
|
if (inner.type === "qualified_identifier") return lastSegment(inner.text);
|
|
2600
2771
|
return null;
|
|
2601
2772
|
}
|
|
2602
2773
|
if (cur.type === "pointer_declarator" || cur.type === "reference_declarator") {
|
|
2603
|
-
cur = cur
|
|
2774
|
+
cur = innerDeclarator(cur);
|
|
2604
2775
|
continue;
|
|
2605
2776
|
}
|
|
2606
2777
|
return null;
|
|
@@ -2620,12 +2791,18 @@ function lastSegment(text) {
|
|
|
2620
2791
|
const parts = text.split(/::|\./);
|
|
2621
2792
|
return parts[parts.length - 1] ?? text;
|
|
2622
2793
|
}
|
|
2794
|
+
function innerDeclarator(node) {
|
|
2795
|
+
const named = node.childForFieldName("declarator");
|
|
2796
|
+
if (named !== null) return named;
|
|
2797
|
+
return node.type === "reference_declarator" ? node.namedChildren[0] ?? null : null;
|
|
2798
|
+
}
|
|
2623
2799
|
function cFunctionName(node) {
|
|
2624
2800
|
let cur = node.childForFieldName("declarator");
|
|
2625
2801
|
for (let i = 0; cur !== null && i < 16; i++) {
|
|
2626
2802
|
if (cur.type === "identifier" || cur.type === "field_identifier") return cur.text;
|
|
2803
|
+
if (cur.type === "destructor_name" || cur.type === "operator_name") return cur.text;
|
|
2627
2804
|
if (cur.type === "qualified_identifier") return lastSegment(cur.text);
|
|
2628
|
-
cur = cur
|
|
2805
|
+
cur = innerDeclarator(cur);
|
|
2629
2806
|
}
|
|
2630
2807
|
return null;
|
|
2631
2808
|
}
|
|
@@ -2664,8 +2841,8 @@ var SCOPE_TYPES_BY_LANG = /* @__PURE__ */ new Map([
|
|
|
2664
2841
|
["ruby", /* @__PURE__ */ new Set(["method", "singleton_method", "class", "module"])]
|
|
2665
2842
|
]);
|
|
2666
2843
|
var CALL_TYPES_BY_LANG = /* @__PURE__ */ new Map([
|
|
2667
|
-
["typescript", /* @__PURE__ */ new Set(["call_expression", "new_expression"])],
|
|
2668
|
-
["javascript", /* @__PURE__ */ new Set(["call_expression", "new_expression"])],
|
|
2844
|
+
["typescript", /* @__PURE__ */ new Set(["call_expression", "new_expression", "jsx_self_closing_element", "jsx_opening_element"])],
|
|
2845
|
+
["javascript", /* @__PURE__ */ new Set(["call_expression", "new_expression", "jsx_self_closing_element", "jsx_opening_element"])],
|
|
2669
2846
|
["python", /* @__PURE__ */ new Set(["call"])],
|
|
2670
2847
|
["go", /* @__PURE__ */ new Set(["call_expression"])],
|
|
2671
2848
|
["rust", /* @__PURE__ */ new Set(["call_expression", "macro_invocation"])],
|
|
@@ -2827,7 +3004,13 @@ var REF_NOISE_BY_LANG = /* @__PURE__ */ new Map([
|
|
|
2827
3004
|
"proc",
|
|
2828
3005
|
"sleep",
|
|
2829
3006
|
"exit",
|
|
2830
|
-
"freeze"
|
|
3007
|
+
"freeze",
|
|
3008
|
+
"private",
|
|
3009
|
+
"protected",
|
|
3010
|
+
"public",
|
|
3011
|
+
"module_function",
|
|
3012
|
+
"block_given?",
|
|
3013
|
+
"binding"
|
|
2831
3014
|
])
|
|
2832
3015
|
]
|
|
2833
3016
|
]);
|
|
@@ -2837,6 +3020,19 @@ function lastSegment2(text) {
|
|
|
2837
3020
|
const parts = text.split(/::|\./);
|
|
2838
3021
|
return parts[parts.length - 1] ?? text;
|
|
2839
3022
|
}
|
|
3023
|
+
var TSJS_FUNCTION_VALUE_TYPES = /* @__PURE__ */ new Set(["arrow_function", "function_expression", "function"]);
|
|
3024
|
+
function tsJsFunctionValuedMemberName(node) {
|
|
3025
|
+
const isField = node.type === "public_field_definition" || node.type === "field_definition";
|
|
3026
|
+
const isPair = node.type === "pair" && node.parent?.type === "object";
|
|
3027
|
+
if (!isField && !isPair) return void 0;
|
|
3028
|
+
const value = node.childForFieldName("value");
|
|
3029
|
+
if (value === null || !TSJS_FUNCTION_VALUE_TYPES.has(value.type)) return void 0;
|
|
3030
|
+
if (isField) return (node.childForFieldName("name") ?? node.childForFieldName("property"))?.text;
|
|
3031
|
+
const key = node.childForFieldName("key");
|
|
3032
|
+
if (key === null || key.type !== "identifier" && key.type !== "property_identifier" && key.type !== "string") return void 0;
|
|
3033
|
+
const name = key.text.replace(/^['"]|['"]$/g, "");
|
|
3034
|
+
return name === "" ? void 0 : name;
|
|
3035
|
+
}
|
|
2840
3036
|
function scopeName(node, language) {
|
|
2841
3037
|
if ((language === "typescript" || language === "javascript") && node.type === "variable_declarator") {
|
|
2842
3038
|
const value = node.childForFieldName("value");
|
|
@@ -2845,6 +3041,10 @@ function scopeName(node, language) {
|
|
|
2845
3041
|
}
|
|
2846
3042
|
return null;
|
|
2847
3043
|
}
|
|
3044
|
+
if (language === "typescript" || language === "javascript") {
|
|
3045
|
+
const member = tsJsFunctionValuedMemberName(node);
|
|
3046
|
+
if (member !== void 0) return member;
|
|
3047
|
+
}
|
|
2848
3048
|
const scopeTypes = SCOPE_TYPES_BY_LANG.get(language);
|
|
2849
3049
|
if (scopeTypes !== void 0 && scopeTypes.has(node.type)) {
|
|
2850
3050
|
if ((language === "c" || language === "cpp") && node.type === "function_definition") {
|
|
@@ -2899,6 +3099,14 @@ function calleeName(call, language) {
|
|
|
2899
3099
|
switch (language) {
|
|
2900
3100
|
case "typescript":
|
|
2901
3101
|
case "javascript": {
|
|
3102
|
+
if (call.type === "jsx_self_closing_element" || call.type === "jsx_opening_element") {
|
|
3103
|
+
const tag = call.childForFieldName("name");
|
|
3104
|
+
if (tag === null) return null;
|
|
3105
|
+
if (tag.type === "identifier") return /^[A-Z]/.test(tag.text) ? tag.text : null;
|
|
3106
|
+
if (tag.type === "member_expression") return tag.childForFieldName("property")?.text ?? null;
|
|
3107
|
+
if (tag.type === "nested_identifier") return lastSegment2(tag.text);
|
|
3108
|
+
return null;
|
|
3109
|
+
}
|
|
2902
3110
|
if (call.type === "new_expression") {
|
|
2903
3111
|
const c = call.childForFieldName("constructor");
|
|
2904
3112
|
if (c === null) return null;
|
|
@@ -2942,7 +3150,9 @@ function calleeName(call, language) {
|
|
|
2942
3150
|
}
|
|
2943
3151
|
case "java": {
|
|
2944
3152
|
const n = call.childForFieldName("name") ?? call.childForFieldName("type");
|
|
2945
|
-
|
|
3153
|
+
if (n === null) return null;
|
|
3154
|
+
const named = n.type === "generic_type" ? n.namedChildren[0] ?? n : n;
|
|
3155
|
+
return lastSegment2(named.text);
|
|
2946
3156
|
}
|
|
2947
3157
|
case "c":
|
|
2948
3158
|
case "cpp": {
|
|
@@ -3092,12 +3302,43 @@ function valueRefIdentifiers(node, language) {
|
|
|
3092
3302
|
}
|
|
3093
3303
|
return result;
|
|
3094
3304
|
}
|
|
3305
|
+
var RUBY_FRESH_SCOPES = /* @__PURE__ */ new Set(["program", "method", "singleton_method", "class", "singleton_class", "module"]);
|
|
3306
|
+
var RUBY_BLOCK_SCOPES = /* @__PURE__ */ new Set(["block", "do_block", "lambda"]);
|
|
3307
|
+
var RUBY_IMPLICIT_PARAM = /^(?:it|_[1-9])$/;
|
|
3308
|
+
function sameNode2(a, b) {
|
|
3309
|
+
return a !== null && a.type === b.type && a.startPosition.row === b.startPosition.row && a.startPosition.column === b.startPosition.column && a.endPosition.column === b.endPosition.column;
|
|
3310
|
+
}
|
|
3311
|
+
function rubyIsLocal(scopes, name) {
|
|
3312
|
+
for (let i = scopes.length - 1; i >= 0; i--) {
|
|
3313
|
+
const scope = scopes[i];
|
|
3314
|
+
if (scope.vars.has(name)) return true;
|
|
3315
|
+
if (!scope.inherits) return false;
|
|
3316
|
+
}
|
|
3317
|
+
return false;
|
|
3318
|
+
}
|
|
3319
|
+
function rubyIdentifierRole(node) {
|
|
3320
|
+
const parent = node.parent;
|
|
3321
|
+
if (parent === null) return "use";
|
|
3322
|
+
const t = parent.type;
|
|
3323
|
+
if (t === "method" || t === "singleton_method") return sameNode2(parent.childForFieldName("name"), node) ? "skip" : "use";
|
|
3324
|
+
if (t === "call") return sameNode2(parent.childForFieldName("method"), node) ? "skip" : "use";
|
|
3325
|
+
if (t === "alias" || t === "undef" || t === "scope_resolution") return "skip";
|
|
3326
|
+
if (t.endsWith("_parameters") || t === "destructured_parameter") return "declare";
|
|
3327
|
+
if (t.endsWith("_parameter")) return sameNode2(parent.childForFieldName("name"), node) ? "declare" : "use";
|
|
3328
|
+
if (t === "assignment" || t === "operator_assignment") return sameNode2(parent.childForFieldName("left"), node) ? "declare" : "use";
|
|
3329
|
+
if (t === "left_assignment_list" || t === "rest_assignment" || t === "destructured_left_assignment" || t === "exception_variable") return "declare";
|
|
3330
|
+
if (t === "for") return sameNode2(parent.childForFieldName("pattern"), node) ? "declare" : "use";
|
|
3331
|
+
if (t === "in_clause") return sameNode2(parent.childForFieldName("pattern"), node) ? "declare" : "use";
|
|
3332
|
+
if (t.endsWith("_pattern")) return "declare";
|
|
3333
|
+
return "use";
|
|
3334
|
+
}
|
|
3095
3335
|
function extractRefs(root, filePath, language) {
|
|
3096
3336
|
const out = [];
|
|
3097
3337
|
const seen = /* @__PURE__ */ new Set();
|
|
3098
3338
|
const callTypes = CALL_TYPES_BY_LANG.get(language) ?? EMPTY_STRING_SET;
|
|
3099
3339
|
const noise = language === "javascript" ? JS_NOISE : REF_NOISE_BY_LANG.get(language) ?? EMPTY_STRING_SET;
|
|
3100
3340
|
const stack = [];
|
|
3341
|
+
const rubyScopes = [];
|
|
3101
3342
|
const record = (name, node) => {
|
|
3102
3343
|
if (name.length <= 1 || noise.has(name)) return;
|
|
3103
3344
|
const line = node.startPosition.row + 1;
|
|
@@ -3115,6 +3356,12 @@ function extractRefs(root, filePath, language) {
|
|
|
3115
3356
|
const visit = (node) => {
|
|
3116
3357
|
const enclosing = scopeName(node, language);
|
|
3117
3358
|
if (enclosing !== null && enclosing !== "") stack.push(enclosing);
|
|
3359
|
+
let rubyScoped = false;
|
|
3360
|
+
if (language === "ruby") {
|
|
3361
|
+
rubyScoped = RUBY_FRESH_SCOPES.has(node.type) || RUBY_BLOCK_SCOPES.has(node.type);
|
|
3362
|
+
if (rubyScoped) rubyScopes.push({ vars: /* @__PURE__ */ new Set(), inherits: RUBY_BLOCK_SCOPES.has(node.type) });
|
|
3363
|
+
visitRubyBinding(node);
|
|
3364
|
+
}
|
|
3118
3365
|
if (callTypes.has(node.type)) {
|
|
3119
3366
|
const callee = calleeName(node, language);
|
|
3120
3367
|
if (callee !== null) record(callee, node);
|
|
@@ -3123,8 +3370,22 @@ function extractRefs(root, filePath, language) {
|
|
|
3123
3370
|
record(idNode.text, idNode);
|
|
3124
3371
|
}
|
|
3125
3372
|
for (const child of node.namedChildren) visit(child);
|
|
3373
|
+
if (rubyScoped) rubyScopes.pop();
|
|
3126
3374
|
if (enclosing !== null && enclosing !== "") stack.pop();
|
|
3127
3375
|
};
|
|
3376
|
+
const visitRubyBinding = (node) => {
|
|
3377
|
+
const scope = rubyScopes[rubyScopes.length - 1];
|
|
3378
|
+
if (scope === void 0) return;
|
|
3379
|
+
if (node.type === "keyword_pattern" && node.childForFieldName("value") === null) {
|
|
3380
|
+
const key = node.childForFieldName("key");
|
|
3381
|
+
if (key !== null) scope.vars.add(key.text);
|
|
3382
|
+
return;
|
|
3383
|
+
}
|
|
3384
|
+
if (node.type !== "identifier") return;
|
|
3385
|
+
const role = rubyIdentifierRole(node);
|
|
3386
|
+
if (role === "declare") scope.vars.add(node.text);
|
|
3387
|
+
else if (role === "use" && !RUBY_IMPLICIT_PARAM.test(node.text) && !rubyIsLocal(rubyScopes, node.text)) record(node.text, node);
|
|
3388
|
+
};
|
|
3128
3389
|
visit(root);
|
|
3129
3390
|
return out;
|
|
3130
3391
|
}
|
|
@@ -3195,13 +3456,18 @@ function countNewlines(s) {
|
|
|
3195
3456
|
function extractMarkdownSymbols(content, filePath) {
|
|
3196
3457
|
const out = [];
|
|
3197
3458
|
const lines = content.split(/\r?\n/);
|
|
3198
|
-
const
|
|
3459
|
+
const fmEnd = frontMatterEndIndex(lines);
|
|
3460
|
+
const unfenced = Array.from(eachUnfencedLine(lines)).filter(([i]) => i >= fmEnd);
|
|
3461
|
+
const bounds = [];
|
|
3462
|
+
let paraStart = -1;
|
|
3199
3463
|
for (let u = 0; u < unfenced.length; u++) {
|
|
3200
3464
|
const [i, line] = unfenced[u];
|
|
3201
|
-
const
|
|
3202
|
-
if (
|
|
3203
|
-
const name =
|
|
3465
|
+
const atx = matchAtxHeading(line, ATX_DOT_BREAK_RE, false);
|
|
3466
|
+
if (atx !== null) {
|
|
3467
|
+
const name = atx.name;
|
|
3468
|
+
let at = -1;
|
|
3204
3469
|
if (name !== "") {
|
|
3470
|
+
at = out.length;
|
|
3205
3471
|
out.push({
|
|
3206
3472
|
filePath,
|
|
3207
3473
|
name,
|
|
@@ -3213,29 +3479,47 @@ function extractMarkdownSymbols(content, filePath) {
|
|
|
3213
3479
|
parent: ""
|
|
3214
3480
|
});
|
|
3215
3481
|
}
|
|
3482
|
+
bounds.push({ line: i + 1, level: atx.level, at });
|
|
3483
|
+
paraStart = -1;
|
|
3216
3484
|
continue;
|
|
3217
3485
|
}
|
|
3486
|
+
if (paraStart !== -1 && i !== unfenced[u - 1][0] + 1) paraStart = -1;
|
|
3218
3487
|
const trimmed = line.trim();
|
|
3219
|
-
if (
|
|
3220
|
-
const
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3488
|
+
if (paraStart !== -1 && /^(=+|-+)$/.test(trimmed)) {
|
|
3489
|
+
const para = unfenced.slice(paraStart, u).map(([, text]) => text.trim());
|
|
3490
|
+
const at = out.length;
|
|
3491
|
+
out.push({
|
|
3492
|
+
filePath,
|
|
3493
|
+
name: para.join(" "),
|
|
3494
|
+
kind: "heading",
|
|
3495
|
+
lineStart: unfenced[paraStart][0] + 1,
|
|
3496
|
+
lineEnd: i + 1,
|
|
3497
|
+
body: `${para.join("\n")}
|
|
3498
|
+
${trimmed}`,
|
|
3499
|
+
docstring: "",
|
|
3500
|
+
parent: ""
|
|
3501
|
+
});
|
|
3502
|
+
bounds.push({ line: unfenced[paraStart][0] + 1, level: trimmed.startsWith("=") ? 1 : 2, at });
|
|
3503
|
+
paraStart = -1;
|
|
3504
|
+
continue;
|
|
3505
|
+
}
|
|
3506
|
+
const isText = trimmed !== "" && !trimmed.startsWith("#") && !trimmed.startsWith("|") && !trimmed.startsWith("```") && !trimmed.startsWith("~~~") && !/^([-*+]|\d+\.)\s/.test(trimmed) && !/^(=+|-+)$/.test(trimmed);
|
|
3507
|
+
if (!isText) paraStart = -1;
|
|
3508
|
+
else if (paraStart === -1) paraStart = u;
|
|
3509
|
+
}
|
|
3510
|
+
for (let k = 0; k < bounds.length; k++) {
|
|
3511
|
+
const cur = bounds[k];
|
|
3512
|
+
const sym = out[cur.at];
|
|
3513
|
+
if (sym === void 0) continue;
|
|
3514
|
+
let end = lines.length;
|
|
3515
|
+
for (let j = k + 1; j < bounds.length; j++) {
|
|
3516
|
+
if (bounds[j].level <= cur.level) {
|
|
3517
|
+
end = bounds[j].line - 1;
|
|
3518
|
+
break;
|
|
3237
3519
|
}
|
|
3238
3520
|
}
|
|
3521
|
+
while (end > sym.lineEnd && (lines[end - 1] ?? "").trim() === "") end--;
|
|
3522
|
+
out[cur.at] = { ...sym, lineEnd: Math.max(sym.lineEnd, end) };
|
|
3239
3523
|
}
|
|
3240
3524
|
return out;
|
|
3241
3525
|
}
|
|
@@ -3401,10 +3685,13 @@ function yamlLineClosesQuote(line, quote) {
|
|
|
3401
3685
|
}
|
|
3402
3686
|
return false;
|
|
3403
3687
|
}
|
|
3688
|
+
var YAML_TOP_KEY_RE = /^(?:([a-zA-Z_0-9][\w.-]*)|"((?:[^"\\]|\\.)*)"|'((?:[^']|'')*)')\s*:(?!\/\/)/;
|
|
3404
3689
|
function extractYamlSymbols(content, filePath) {
|
|
3405
3690
|
const out = [];
|
|
3406
3691
|
const lines = content.split(/\r?\n/);
|
|
3407
3692
|
let openQuote = null;
|
|
3693
|
+
const boundaries = [];
|
|
3694
|
+
const keyAt = /* @__PURE__ */ new Map();
|
|
3408
3695
|
for (let i = 0; i < lines.length; i++) {
|
|
3409
3696
|
const line = lines[i];
|
|
3410
3697
|
if (line === void 0) continue;
|
|
@@ -3412,11 +3699,18 @@ function extractYamlSymbols(content, filePath) {
|
|
|
3412
3699
|
if (yamlLineClosesQuote(line, openQuote)) openQuote = null;
|
|
3413
3700
|
continue;
|
|
3414
3701
|
}
|
|
3415
|
-
|
|
3416
|
-
|
|
3702
|
+
if (/^(?:---|\.\.\.)(?:\s|$)/.test(line)) {
|
|
3703
|
+
boundaries.push(i);
|
|
3704
|
+
continue;
|
|
3705
|
+
}
|
|
3706
|
+
const match = YAML_TOP_KEY_RE.exec(line);
|
|
3707
|
+
if (match !== null) {
|
|
3708
|
+
const name = match[2] !== void 0 ? match[2].replace(/\\(["\\/])/g, "$1") : match[3] !== void 0 ? match[3].replace(/''/g, "'") : match[1] ?? "";
|
|
3709
|
+
boundaries.push(i);
|
|
3710
|
+
keyAt.set(i, out.length);
|
|
3417
3711
|
out.push({
|
|
3418
3712
|
filePath,
|
|
3419
|
-
name
|
|
3713
|
+
name,
|
|
3420
3714
|
kind: "key",
|
|
3421
3715
|
lineStart: i + 1,
|
|
3422
3716
|
lineEnd: i + 1,
|
|
@@ -3427,6 +3721,12 @@ function extractYamlSymbols(content, filePath) {
|
|
|
3427
3721
|
openQuote = yamlOpenQuoteAfter(line, match[0].length);
|
|
3428
3722
|
}
|
|
3429
3723
|
}
|
|
3724
|
+
for (const [start, at] of keyAt) {
|
|
3725
|
+
const sym = out[at];
|
|
3726
|
+
let end = (boundaries.find((b) => b > start) ?? lines.length) - 1;
|
|
3727
|
+
while (end > start && (lines[end].trim() === "" || lines[end].startsWith("#"))) end--;
|
|
3728
|
+
out[at] = { ...sym, lineEnd: end + 1, body: lines.slice(start, end + 1).join("\n").trim() };
|
|
3729
|
+
}
|
|
3430
3730
|
return out;
|
|
3431
3731
|
}
|
|
3432
3732
|
function lineOpenDelimiterAfter(line, startIdx) {
|
|
@@ -3487,11 +3787,11 @@ function extractTomlSymbols(content, filePath) {
|
|
|
3487
3787
|
const out = [];
|
|
3488
3788
|
const lines = content.split(/\r?\n/);
|
|
3489
3789
|
function matchLine(line, lineNum) {
|
|
3490
|
-
const
|
|
3491
|
-
if (
|
|
3790
|
+
const sectionName = matchTomlSectionName(line);
|
|
3791
|
+
if (sectionName !== null) {
|
|
3492
3792
|
out.push({
|
|
3493
3793
|
filePath,
|
|
3494
|
-
name:
|
|
3794
|
+
name: sectionName.trim(),
|
|
3495
3795
|
kind: "section",
|
|
3496
3796
|
lineStart: lineNum + 1,
|
|
3497
3797
|
lineEnd: lineNum + 1,
|
|
@@ -3536,6 +3836,14 @@ function extractTomlSymbols(content, filePath) {
|
|
|
3536
3836
|
openDelim = lineOpenDelimiterAfter(code, 0);
|
|
3537
3837
|
if (openDelim === null) arrayDepth = Math.max(0, tomlBracketDelta(code));
|
|
3538
3838
|
}
|
|
3839
|
+
const headers = out.filter((sym) => sym.kind === "section");
|
|
3840
|
+
headers.forEach((sym, h) => {
|
|
3841
|
+
const next = headers[h + 1];
|
|
3842
|
+
let end = (next === void 0 ? lines.length : next.lineStart - 1) - 1;
|
|
3843
|
+
while (end > sym.lineStart - 1 && (lines[end].trim() === "" || lines[end].trim().startsWith("#"))) end--;
|
|
3844
|
+
const at = out.indexOf(sym);
|
|
3845
|
+
out[at] = { ...sym, lineEnd: end + 1, body: lines.slice(sym.lineStart - 1, end + 1).join("\n").trim() };
|
|
3846
|
+
});
|
|
3539
3847
|
return out;
|
|
3540
3848
|
}
|
|
3541
3849
|
function splitTopLevelSelectors(rawCapture, strippedCapture) {
|
|
@@ -3562,68 +3870,75 @@ function nextContentLineOpensBrace(lines, i) {
|
|
|
3562
3870
|
}
|
|
3563
3871
|
return false;
|
|
3564
3872
|
}
|
|
3873
|
+
function ruleEndIdx(blanked, startIdx, braceCol) {
|
|
3874
|
+
let depth = 0;
|
|
3875
|
+
for (let i = startIdx; i < blanked.length; i++) {
|
|
3876
|
+
const text = blanked[i] ?? "";
|
|
3877
|
+
for (let c = i === startIdx ? braceCol : 0; c < text.length; c++) {
|
|
3878
|
+
if (text[c] === "{") depth++;
|
|
3879
|
+
else if (text[c] === "}" && --depth === 0) return i;
|
|
3880
|
+
}
|
|
3881
|
+
}
|
|
3882
|
+
return startIdx;
|
|
3883
|
+
}
|
|
3884
|
+
function stripCssLineComment(line) {
|
|
3885
|
+
let inUrl = false;
|
|
3886
|
+
for (let c = 0; c < line.length; c++) {
|
|
3887
|
+
if (inUrl) {
|
|
3888
|
+
if (line[c] === ")") inUrl = false;
|
|
3889
|
+
} else if (line[c] === "/" && line[c + 1] === "/") {
|
|
3890
|
+
return line.slice(0, c);
|
|
3891
|
+
} else if (line[c] === "(" && /url$/i.test(line.slice(Math.max(0, c - 3), c))) {
|
|
3892
|
+
inUrl = true;
|
|
3893
|
+
}
|
|
3894
|
+
}
|
|
3895
|
+
return line;
|
|
3896
|
+
}
|
|
3565
3897
|
function extractCssSymbols(content, filePath) {
|
|
3566
3898
|
const out = [];
|
|
3567
3899
|
const lines = stripCstyleComments(content).split(/\r?\n/);
|
|
3568
3900
|
const rawLines = content.split(/\r?\n/);
|
|
3569
3901
|
let pending = [];
|
|
3902
|
+
const lineComments = /\.(?:scss|sass|less)$/i.test(filePath);
|
|
3903
|
+
const blanked = lines.map((l) => lineComments ? stripCssLineComment(stripStringLiterals(l)) : stripStringLiterals(l));
|
|
3904
|
+
const spanEntry = (name, startIdx, endIdx) => ({
|
|
3905
|
+
filePath,
|
|
3906
|
+
name,
|
|
3907
|
+
kind: "selector",
|
|
3908
|
+
lineStart: startIdx + 1,
|
|
3909
|
+
lineEnd: endIdx + 1,
|
|
3910
|
+
body: endIdx > startIdx ? rawLines.slice(startIdx, endIdx + 1).join("\n") : (lines[startIdx] ?? "").trim(),
|
|
3911
|
+
docstring: "",
|
|
3912
|
+
parent: ""
|
|
3913
|
+
});
|
|
3914
|
+
const flushPending = (endIdx) => {
|
|
3915
|
+
for (const p of pending) out.push(spanEntry(p.name, p.line - 1, endIdx));
|
|
3916
|
+
pending = [];
|
|
3917
|
+
};
|
|
3570
3918
|
for (let i = 0; i < lines.length; i++) {
|
|
3571
3919
|
const line = lines[i];
|
|
3572
3920
|
if (line === void 0) continue;
|
|
3573
|
-
const trimmed = line.trim();
|
|
3921
|
+
const trimmed = (lineComments ? line.slice(0, (blanked[i] ?? "").length) : line).trim();
|
|
3574
3922
|
if (trimmed.length === 0 && (rawLines[i]?.trim().length ?? 0) > 0) {
|
|
3575
3923
|
continue;
|
|
3576
3924
|
}
|
|
3577
|
-
const strippedLine =
|
|
3578
|
-
const
|
|
3579
|
-
if (
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
kind: "selector",
|
|
3585
|
-
lineStart: p.line,
|
|
3586
|
-
lineEnd: p.line,
|
|
3587
|
-
body: p.body,
|
|
3588
|
-
docstring: "",
|
|
3589
|
-
parent: ""
|
|
3590
|
-
});
|
|
3591
|
-
}
|
|
3592
|
-
pending = [];
|
|
3593
|
-
const captureRange = selectorLineMatch.indices?.[1];
|
|
3594
|
-
const rawCapture = captureRange ? line.slice(captureRange[0], captureRange[1]) : selectorLineMatch[1];
|
|
3595
|
-
const strippedCapture = captureRange ? strippedLine.slice(captureRange[0], captureRange[1]) : selectorLineMatch[1];
|
|
3925
|
+
const strippedLine = blanked[i] ?? "";
|
|
3926
|
+
const selectorSpan = matchCssSelectorSpan(strippedLine);
|
|
3927
|
+
if (selectorSpan !== null) {
|
|
3928
|
+
const endIdx = ruleEndIdx(blanked, i, selectorSpan.end);
|
|
3929
|
+
flushPending(endIdx);
|
|
3930
|
+
const rawCapture = line.slice(selectorSpan.start, selectorSpan.end);
|
|
3931
|
+
const strippedCapture = strippedLine.slice(selectorSpan.start, selectorSpan.end);
|
|
3596
3932
|
for (const part of splitTopLevelSelectors(rawCapture, strippedCapture)) {
|
|
3597
3933
|
const name = part.trim();
|
|
3598
3934
|
if (name) {
|
|
3599
|
-
out.push(
|
|
3600
|
-
filePath,
|
|
3601
|
-
name,
|
|
3602
|
-
kind: "selector",
|
|
3603
|
-
lineStart: i + 1,
|
|
3604
|
-
lineEnd: i + 1,
|
|
3605
|
-
body: line.trim(),
|
|
3606
|
-
docstring: "",
|
|
3607
|
-
parent: ""
|
|
3608
|
-
});
|
|
3935
|
+
out.push(spanEntry(name, i, endIdx));
|
|
3609
3936
|
}
|
|
3610
3937
|
}
|
|
3611
3938
|
continue;
|
|
3612
3939
|
}
|
|
3613
3940
|
if (trimmed === "{") {
|
|
3614
|
-
|
|
3615
|
-
out.push({
|
|
3616
|
-
filePath,
|
|
3617
|
-
name: p.name,
|
|
3618
|
-
kind: "selector",
|
|
3619
|
-
lineStart: p.line,
|
|
3620
|
-
lineEnd: p.line,
|
|
3621
|
-
body: p.body,
|
|
3622
|
-
docstring: "",
|
|
3623
|
-
parent: ""
|
|
3624
|
-
});
|
|
3625
|
-
}
|
|
3626
|
-
pending = [];
|
|
3941
|
+
flushPending(ruleEndIdx(blanked, i, blanked[i]?.indexOf("{") ?? 0));
|
|
3627
3942
|
continue;
|
|
3628
3943
|
}
|
|
3629
3944
|
if (trimmed.length > 0 && !trimmed.startsWith("@") && !trimmed.includes("{") && !trimmed.includes("}") && !trimmed.includes(";")) {
|
|
@@ -3638,23 +3953,52 @@ function extractCssSymbols(content, filePath) {
|
|
|
3638
3953
|
}
|
|
3639
3954
|
return out;
|
|
3640
3955
|
}
|
|
3956
|
+
var DOCKERFILE_HEREDOC_RE = /(?<!<)<<(-?)(["']?)([A-Za-z_][\w.-]*)\2/g;
|
|
3957
|
+
function endsWithContinuation(line) {
|
|
3958
|
+
return line.trimEnd().endsWith("\\");
|
|
3959
|
+
}
|
|
3960
|
+
function dockerfileInstructionEnd(lines, i) {
|
|
3961
|
+
let end = i;
|
|
3962
|
+
let continued = endsWithContinuation(lines[i] ?? "");
|
|
3963
|
+
while (continued && end + 1 < lines.length) {
|
|
3964
|
+
end++;
|
|
3965
|
+
const next = lines[end] ?? "";
|
|
3966
|
+
if (!next.trim().startsWith("#")) continued = endsWithContinuation(next);
|
|
3967
|
+
}
|
|
3968
|
+
return end;
|
|
3969
|
+
}
|
|
3970
|
+
function dockerfileHeredocEnd(lines, i, end) {
|
|
3971
|
+
const text = lines.slice(i, end + 1).join("\n");
|
|
3972
|
+
let cursor = end;
|
|
3973
|
+
for (const m of text.matchAll(DOCKERFILE_HEREDOC_RE)) {
|
|
3974
|
+
const stripTabs = m[1] === "-";
|
|
3975
|
+
let close = -1;
|
|
3976
|
+
for (let j = cursor + 1; j < lines.length; j++) {
|
|
3977
|
+
const candidate = stripTabs ? (lines[j] ?? "").replace(/^\t+/, "") : lines[j] ?? "";
|
|
3978
|
+
if (candidate.trimEnd() === m[3]) {
|
|
3979
|
+
close = j;
|
|
3980
|
+
break;
|
|
3981
|
+
}
|
|
3982
|
+
}
|
|
3983
|
+
if (close === -1) return end;
|
|
3984
|
+
cursor = close;
|
|
3985
|
+
}
|
|
3986
|
+
return cursor;
|
|
3987
|
+
}
|
|
3641
3988
|
function extractDockerfileSymbols(content, filePath) {
|
|
3642
3989
|
const out = [];
|
|
3643
3990
|
const lines = content.split(/\r?\n/);
|
|
3644
|
-
let continuing = false;
|
|
3645
3991
|
for (let i = 0; i < lines.length; i++) {
|
|
3646
3992
|
const line = lines[i];
|
|
3647
3993
|
if (line === void 0) continue;
|
|
3648
|
-
if (
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
}
|
|
3652
|
-
const isComment = line.trim().startsWith("#");
|
|
3653
|
-
const match = isComment ? null : /^\s*(FROM|RUN|COPY|ADD|EXPOSE|ENV|WORKDIR|CMD|ENTRYPOINT|ARG|LABEL|VOLUME|USER|HEALTHCHECK|ONBUILD|SHELL|STOPSIGNAL|MAINTAINER)\s+(.+)/i.exec(
|
|
3994
|
+
if (line.trim().startsWith("#")) continue;
|
|
3995
|
+
let end = dockerfileInstructionEnd(lines, i);
|
|
3996
|
+
const match = /^\s*(FROM|RUN|COPY|ADD|EXPOSE|ENV|WORKDIR|CMD|ENTRYPOINT|ARG|LABEL|VOLUME|USER|HEALTHCHECK|ONBUILD|SHELL|STOPSIGNAL|MAINTAINER)\s+(.+)/i.exec(
|
|
3654
3997
|
line
|
|
3655
3998
|
);
|
|
3656
3999
|
if (match !== null && match[1] !== void 0) {
|
|
3657
4000
|
const cmd = match[1];
|
|
4001
|
+
if (/^(?:RUN|COPY|ADD)$/i.test(cmd)) end = dockerfileHeredocEnd(lines, i, end);
|
|
3658
4002
|
const arg = (match[2] ?? "").substring(0, 40);
|
|
3659
4003
|
const name = `${cmd} ${arg}`.trim();
|
|
3660
4004
|
out.push({
|
|
@@ -3662,13 +4006,13 @@ function extractDockerfileSymbols(content, filePath) {
|
|
|
3662
4006
|
name,
|
|
3663
4007
|
kind: "directive",
|
|
3664
4008
|
lineStart: i + 1,
|
|
3665
|
-
lineEnd:
|
|
3666
|
-
body:
|
|
4009
|
+
lineEnd: end + 1,
|
|
4010
|
+
body: lines.slice(i, end + 1).join("\n").trim(),
|
|
3667
4011
|
docstring: "",
|
|
3668
4012
|
parent: ""
|
|
3669
4013
|
});
|
|
3670
4014
|
}
|
|
3671
|
-
|
|
4015
|
+
i = end;
|
|
3672
4016
|
}
|
|
3673
4017
|
return out;
|
|
3674
4018
|
}
|
|
@@ -3738,7 +4082,8 @@ var REF_LANGUAGES = /* @__PURE__ */ new Set([
|
|
|
3738
4082
|
"java",
|
|
3739
4083
|
"c",
|
|
3740
4084
|
"cpp",
|
|
3741
|
-
"ruby"
|
|
4085
|
+
"ruby",
|
|
4086
|
+
"csharp"
|
|
3742
4087
|
]);
|
|
3743
4088
|
var _parserCtor;
|
|
3744
4089
|
var _parserCtorError = null;
|
|
@@ -3789,8 +4134,8 @@ function boundRefContext(context, col) {
|
|
|
3789
4134
|
return (start > 0 ? REF_CONTEXT_ELISION : "") + context.slice(start, end) + (end < context.length ? REF_CONTEXT_ELISION : "");
|
|
3790
4135
|
}
|
|
3791
4136
|
function loadGrammar(lang, filePath, content) {
|
|
3792
|
-
const useTsx = lang === "typescript" && filePath !== void 0 &&
|
|
3793
|
-
const useCppHeader = lang === "c" && filePath !== void 0 &&
|
|
4137
|
+
const useTsx = lang === "typescript" && filePath !== void 0 && path9.extname(filePath).toLowerCase() === ".tsx";
|
|
4138
|
+
const useCppHeader = lang === "c" && filePath !== void 0 && path9.extname(filePath).toLowerCase() === ".h" && content !== void 0 && CPP_HEADER_SNIFF_RE.test(content);
|
|
3794
4139
|
const cacheKey = useTsx ? "typescript:tsx" : useCppHeader ? "c:cpp-header" : lang;
|
|
3795
4140
|
const cached = _grammarCache.get(cacheKey);
|
|
3796
4141
|
if (cached !== void 0) return cached;
|
|
@@ -3926,7 +4271,7 @@ function parseSourceSymbolsTreeSitterOnly(content, filePath, language) {
|
|
|
3926
4271
|
var REGEX_ADAPTERS_SLOT = /* @__PURE__ */ Symbol.for("token-goat.regex-adapters");
|
|
3927
4272
|
var adapterSlot = globalThis;
|
|
3928
4273
|
async function loadRegexExtractors() {
|
|
3929
|
-
adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-
|
|
4274
|
+
adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-A3CK6N5Z.mjs");
|
|
3930
4275
|
}
|
|
3931
4276
|
function regexAdapters() {
|
|
3932
4277
|
const loaded = adapterSlot[REGEX_ADAPTERS_SLOT];
|
|
@@ -3965,6 +4310,17 @@ function extractNoTreeSitter(content, filePath, language) {
|
|
|
3965
4310
|
const r = adapters.extractNatural(content, filePath);
|
|
3966
4311
|
return { symbols: r.symbols, refs: r.refs };
|
|
3967
4312
|
}
|
|
4313
|
+
if (language === "csharp") {
|
|
4314
|
+
const r = adapters.extractCsharp(content, filePath);
|
|
4315
|
+
const symbols = adapters.assignBraceBlockSpans(r.symbols, content, {
|
|
4316
|
+
lineComment: "//",
|
|
4317
|
+
stringEscapes: "csharp",
|
|
4318
|
+
rawStringQuotes: true,
|
|
4319
|
+
expressionBodies: true,
|
|
4320
|
+
interpolation: "csharp"
|
|
4321
|
+
});
|
|
4322
|
+
return { symbols, refs: r.refs };
|
|
4323
|
+
}
|
|
3968
4324
|
const parsed = {
|
|
3969
4325
|
symbols: extractSymbolsNoTreeSitter(content, filePath, language),
|
|
3970
4326
|
refs: []
|
|
@@ -3981,16 +4337,19 @@ function deleteFileRows(db, filePath) {
|
|
|
3981
4337
|
db.prepare(`DELETE FROM refs WHERE ${pathEqClause("file_path")}`).run(folded);
|
|
3982
4338
|
db.prepare(`DELETE FROM files WHERE ${pathEqClause("path")}`).run(folded);
|
|
3983
4339
|
}
|
|
3984
|
-
function
|
|
4340
|
+
function isMinifiedBundlePath(filePath) {
|
|
4341
|
+
return /(?:\.min\.(?:js|mjs|cjs|css)|-min\.js)$/i.test(path9.basename(filePath));
|
|
4342
|
+
}
|
|
4343
|
+
function isUnderSkipDir(filePath, skipDirs, projectRoot) {
|
|
3985
4344
|
if (skipDirs.length === 0) return false;
|
|
3986
|
-
|
|
3987
|
-
return segments.slice(0, -1).some((seg) => skipDirs.includes(seg));
|
|
4345
|
+
return hasSkipSegmentBelowRoot(filePath, projectRoot, (seg) => skipDirs.includes(seg));
|
|
3988
4346
|
}
|
|
3989
|
-
function isParseSkipEligible(filePath, cfg) {
|
|
3990
|
-
if (isUnderSkipDir(filePath, cfg.skip_dirs)) return true;
|
|
3991
|
-
if (cfg.skip_files.includes(
|
|
4347
|
+
function isParseSkipEligible(filePath, cfg, projectRoot) {
|
|
4348
|
+
if (isUnderSkipDir(filePath, cfg.skip_dirs, projectRoot)) return true;
|
|
4349
|
+
if (cfg.skip_files.includes(path9.basename(filePath))) return true;
|
|
4350
|
+
if (cfg.skip_minified !== false && isMinifiedBundlePath(filePath)) return true;
|
|
3992
4351
|
try {
|
|
3993
|
-
const stat =
|
|
4352
|
+
const stat = fs4.statSync(filePath);
|
|
3994
4353
|
if (stat.size > cfg.large_file_skip_kb * 1024) return true;
|
|
3995
4354
|
} catch {
|
|
3996
4355
|
}
|
|
@@ -4061,7 +4420,7 @@ function indexFileSync(rawPath, dbPath = globalDbPath(), preReadBytes) {
|
|
|
4061
4420
|
raw = preReadBytes;
|
|
4062
4421
|
} else {
|
|
4063
4422
|
try {
|
|
4064
|
-
raw =
|
|
4423
|
+
raw = fs4.readFileSync(filePath);
|
|
4065
4424
|
} catch (err) {
|
|
4066
4425
|
if (err.code === "ENOENT") return;
|
|
4067
4426
|
throw err;
|
|
@@ -4113,28 +4472,28 @@ function canonicalizeIndexPath(rawPath) {
|
|
|
4113
4472
|
if (!isCaseInsensitiveFs()) return absPath;
|
|
4114
4473
|
let real;
|
|
4115
4474
|
try {
|
|
4116
|
-
real =
|
|
4475
|
+
real = fs4.realpathSync.native(absPath);
|
|
4117
4476
|
} catch {
|
|
4118
4477
|
return absPath;
|
|
4119
4478
|
}
|
|
4120
4479
|
const cut = Math.max(absPath.lastIndexOf("/"), absPath.lastIndexOf("\\"));
|
|
4121
4480
|
const base = absPath.slice(cut + 1);
|
|
4122
4481
|
const realNorm = normalizePath(real);
|
|
4123
|
-
const realBase =
|
|
4482
|
+
const realBase = path9.basename(realNorm);
|
|
4124
4483
|
if (base === realBase) return absPath;
|
|
4125
4484
|
if (foldPath(base) !== foldPath(realBase)) return absPath;
|
|
4126
|
-
const callerDir = cut < 0 ? normalizePath(
|
|
4127
|
-
if (foldPath(callerDir) !== foldPath(
|
|
4485
|
+
const callerDir = cut < 0 ? normalizePath(path9.resolve(".")) : normalizePath(absPath.slice(0, cut));
|
|
4486
|
+
if (foldPath(callerDir) !== foldPath(path9.dirname(realNorm))) return absPath;
|
|
4128
4487
|
return absPath.slice(0, cut + 1) + realBase;
|
|
4129
4488
|
}
|
|
4130
4489
|
function indexedPathSpellingIsStale(storedPath, absPath) {
|
|
4131
4490
|
if (!isCaseInsensitiveFs()) return false;
|
|
4132
4491
|
const stored = normalizePath(storedPath);
|
|
4133
|
-
const candidate = normalizePath(
|
|
4492
|
+
const candidate = normalizePath(path9.resolve(absPath));
|
|
4134
4493
|
if (foldPath(stored) !== foldPath(candidate)) return false;
|
|
4135
4494
|
let real;
|
|
4136
4495
|
try {
|
|
4137
|
-
real = normalizePath(
|
|
4496
|
+
real = normalizePath(fs4.realpathSync.native(absPath));
|
|
4138
4497
|
} catch {
|
|
4139
4498
|
return false;
|
|
4140
4499
|
}
|
|
@@ -4214,7 +4573,8 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
4214
4573
|
}
|
|
4215
4574
|
try {
|
|
4216
4575
|
const db = getDb(dbPath);
|
|
4217
|
-
const outcome = await indexFile(db, filePath, extracted, []);
|
|
4576
|
+
const outcome = await indexFile(db, filePath, extracted, [], sha);
|
|
4577
|
+
if (outcome === "stale") return;
|
|
4218
4578
|
if (outcome !== "unavailable" && enforceStoredChunkCap(db, filePath, sha, ixCfg.max_chunks_per_file)) return;
|
|
4219
4579
|
stampEmbedSha(db, filePath, sha, (s) => outcome === "unavailable" ? unavailableEmbedSha(s) : s);
|
|
4220
4580
|
} catch (err) {
|
|
@@ -4224,7 +4584,7 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
4224
4584
|
}
|
|
4225
4585
|
let content;
|
|
4226
4586
|
try {
|
|
4227
|
-
content = decodeSource(await
|
|
4587
|
+
content = decodeSource(await fs4.promises.readFile(filePath));
|
|
4228
4588
|
} catch {
|
|
4229
4589
|
return;
|
|
4230
4590
|
}
|
|
@@ -4261,7 +4621,8 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
4261
4621
|
}
|
|
4262
4622
|
try {
|
|
4263
4623
|
const db = getDb(dbPath);
|
|
4264
|
-
const outcome = await indexFile(db, filePath, content, boundaries);
|
|
4624
|
+
const outcome = await indexFile(db, filePath, content, boundaries, sha);
|
|
4625
|
+
if (outcome === "stale") return;
|
|
4265
4626
|
if (outcome !== "unavailable" && enforceStoredChunkCap(db, filePath, sha, ixCfg.max_chunks_per_file)) return;
|
|
4266
4627
|
stampEmbedSha(db, filePath, sha, (s) => outcome === "unavailable" ? unavailableEmbedSha(s) : s);
|
|
4267
4628
|
} catch (err) {
|
|
@@ -4278,7 +4639,7 @@ function stampEmbedSha(db, filePath, sha, makeValue) {
|
|
|
4278
4639
|
}
|
|
4279
4640
|
function safeMtime(filePath) {
|
|
4280
4641
|
try {
|
|
4281
|
-
return
|
|
4642
|
+
return fs4.statSync(filePath).mtimeMs / 1e3;
|
|
4282
4643
|
} catch {
|
|
4283
4644
|
return 0;
|
|
4284
4645
|
}
|
|
@@ -4286,7 +4647,7 @@ function safeMtime(filePath) {
|
|
|
4286
4647
|
|
|
4287
4648
|
// src/section_reader.ts
|
|
4288
4649
|
init_define_import_meta_env();
|
|
4289
|
-
import { readFileSync as
|
|
4650
|
+
import { readFileSync as readFileSync4 } from "node:fs";
|
|
4290
4651
|
function parseHeadingSpec(spec, headers) {
|
|
4291
4652
|
const m = /^([^#\r\n]+)#(\d+)$/.exec(spec);
|
|
4292
4653
|
if (m !== null && m[1] !== void 0 && m[2] !== void 0) {
|
|
@@ -4312,38 +4673,33 @@ function normalizeHeadingStrip(s) {
|
|
|
4312
4673
|
return n.replace(/\s+/g, " ").trim();
|
|
4313
4674
|
}
|
|
4314
4675
|
var MIN_WIDEN_WORD_LEN = 3;
|
|
4315
|
-
var MARKDOWN_HEADER_RE = /^(#{1,6})\s+([^\r\n]+?)(?:\s+#+)?\s*$/;
|
|
4316
|
-
var TABLE_HEADER_RE = /^\s*\[+\s*([^\]]+?)\s*\]+\s*(?:[#;].*)?$/;
|
|
4317
4676
|
var PYTHON_HEADER_RE = /^(\s*)(?:async\s+)?(?:def|class)\s+([A-Za-z_]\w*)/;
|
|
4318
4677
|
var KEYVALUE_HEADER_RE = /^([A-Za-z_][\w.-]*)\s*(?:=|:(?!\/\/))/;
|
|
4319
4678
|
var ENV_KEYVALUE_HEADER_RE = /^(?:export\s+)?([A-Za-z_][\w.-]*)\s*(?:=|:(?!\/\/))/;
|
|
4320
4679
|
function findMarkdownHeaders(lines) {
|
|
4321
4680
|
const headers = [];
|
|
4322
|
-
const
|
|
4681
|
+
const fmEnd = frontMatterEndIndex(lines);
|
|
4682
|
+
const unfenced = Array.from(eachUnfencedLine(lines)).filter(([i]) => i >= fmEnd);
|
|
4683
|
+
let paraStart = -1;
|
|
4323
4684
|
for (let u = 0; u < unfenced.length; u++) {
|
|
4324
4685
|
const [i, line] = unfenced[u];
|
|
4325
|
-
const
|
|
4326
|
-
if (
|
|
4327
|
-
headers.push({ heading:
|
|
4686
|
+
const atx = matchAtxHeading(line, ATX_CLASS_BREAK_RE, true);
|
|
4687
|
+
if (atx !== null) {
|
|
4688
|
+
headers.push({ heading: atx.name, level: atx.level, index: i });
|
|
4689
|
+
paraStart = -1;
|
|
4328
4690
|
continue;
|
|
4329
4691
|
}
|
|
4692
|
+
if (paraStart !== -1 && i !== unfenced[u - 1][0] + 1) paraStart = -1;
|
|
4330
4693
|
const trimmed = line.trim();
|
|
4331
|
-
if (
|
|
4332
|
-
const [
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
4336
|
-
if (eqMatch !== null) {
|
|
4337
|
-
headers.push({ heading: trimmed, level: 1, index: i });
|
|
4338
|
-
u++;
|
|
4339
|
-
continue;
|
|
4340
|
-
} else if (dashMatch !== null) {
|
|
4341
|
-
headers.push({ heading: trimmed, level: 2, index: i });
|
|
4342
|
-
u++;
|
|
4343
|
-
continue;
|
|
4344
|
-
}
|
|
4345
|
-
}
|
|
4694
|
+
if (paraStart !== -1 && /^(=+|-+)$/.test(trimmed)) {
|
|
4695
|
+
const heading = unfenced.slice(paraStart, u).map(([, text]) => text.trim()).join(" ");
|
|
4696
|
+
headers.push({ heading, level: trimmed.startsWith("=") ? 1 : 2, index: unfenced[paraStart][0] });
|
|
4697
|
+
paraStart = -1;
|
|
4698
|
+
continue;
|
|
4346
4699
|
}
|
|
4700
|
+
const isText = trimmed !== "" && !trimmed.startsWith("#") && !trimmed.startsWith("|") && !trimmed.startsWith("```") && !trimmed.startsWith("~~~") && !/^([-*+]|\d+\.)\s/.test(trimmed) && !/^(=+|-+)$/.test(trimmed);
|
|
4701
|
+
if (!isText) paraStart = -1;
|
|
4702
|
+
else if (paraStart === -1) paraStart = u;
|
|
4347
4703
|
}
|
|
4348
4704
|
return headers;
|
|
4349
4705
|
}
|
|
@@ -4358,8 +4714,8 @@ function findTableHeaders(lines, isToml) {
|
|
|
4358
4714
|
const closeIdx = line.indexOf(openDelim);
|
|
4359
4715
|
if (closeIdx === -1) continue;
|
|
4360
4716
|
const restStart = closeIdx + openDelim.length;
|
|
4361
|
-
const
|
|
4362
|
-
if (
|
|
4717
|
+
const name2 = matchTableHeaderName(line.slice(restStart));
|
|
4718
|
+
if (name2 !== null) headers.push({ heading: name2.trim(), level: 1, index: i });
|
|
4363
4719
|
openDelim = lineOpenDelimiterAfter(stripTomlComment(line.slice(restStart)), 0);
|
|
4364
4720
|
continue;
|
|
4365
4721
|
}
|
|
@@ -4367,8 +4723,8 @@ function findTableHeaders(lines, isToml) {
|
|
|
4367
4723
|
arrayDepth = Math.max(0, arrayDepth + tomlBracketDelta(stripTomlComment(line)));
|
|
4368
4724
|
continue;
|
|
4369
4725
|
}
|
|
4370
|
-
const
|
|
4371
|
-
if (
|
|
4726
|
+
const name = matchTableHeaderName(line);
|
|
4727
|
+
if (name !== null) headers.push({ heading: name.trim(), level: 1, index: i });
|
|
4372
4728
|
if (isToml) {
|
|
4373
4729
|
const code = stripTomlComment(line);
|
|
4374
4730
|
openDelim = lineOpenDelimiterAfter(code, 0);
|
|
@@ -4390,9 +4746,9 @@ function extractShellBannerHeading(line) {
|
|
|
4390
4746
|
return { heading: text, level: mdMatch[1].length };
|
|
4391
4747
|
}
|
|
4392
4748
|
}
|
|
4393
|
-
const
|
|
4394
|
-
if (
|
|
4395
|
-
const text =
|
|
4749
|
+
const ruleTitle = matchRuleBannerTitle(trimmed);
|
|
4750
|
+
if (ruleTitle) {
|
|
4751
|
+
const text = ruleTitle.trim();
|
|
4396
4752
|
if (text && !/^[-=]+$/.test(text)) {
|
|
4397
4753
|
return { heading: text, level: 1 };
|
|
4398
4754
|
}
|
|
@@ -4750,7 +5106,7 @@ function readTextForSections(filePath, readFn) {
|
|
|
4750
5106
|
text = read;
|
|
4751
5107
|
} else {
|
|
4752
5108
|
try {
|
|
4753
|
-
text = decodeSource(
|
|
5109
|
+
text = decodeSource(readFileSync4(filePath));
|
|
4754
5110
|
} catch {
|
|
4755
5111
|
return null;
|
|
4756
5112
|
}
|
|
@@ -4820,19 +5176,32 @@ export {
|
|
|
4820
5176
|
extractChangelogVersionHint,
|
|
4821
5177
|
isImagePath,
|
|
4822
5178
|
isNonTextAsset,
|
|
5179
|
+
ORT_WEB_VERSION,
|
|
5180
|
+
ORT_WEB_WASM,
|
|
5181
|
+
NATIVE_RUNTIME_INSTALL,
|
|
5182
|
+
RUNTIME_UNAVAILABLE_ADVICE,
|
|
5183
|
+
activeRuntime,
|
|
5184
|
+
nativeRuntimeLoadError,
|
|
5185
|
+
runtimeVersion,
|
|
5186
|
+
wasmBinaryPresent,
|
|
5187
|
+
ensureWasmBinary,
|
|
5188
|
+
DEFAULT_MODEL,
|
|
5189
|
+
modelDir,
|
|
4823
5190
|
modelFilesPresent,
|
|
4824
5191
|
ensureModelFiles,
|
|
4825
5192
|
checkEmbeddingPreflight,
|
|
5193
|
+
DEFAULT_DISTANCE_THRESHOLD,
|
|
4826
5194
|
OVER_FETCH_FACTOR,
|
|
4827
5195
|
MAX_OVER_FETCH,
|
|
4828
5196
|
isAvailable,
|
|
4829
5197
|
embeddingBackendLoadError,
|
|
4830
5198
|
embedTexts,
|
|
4831
5199
|
searchSemantic,
|
|
4832
|
-
mergeNearbyHits,
|
|
4833
5200
|
embeddingsDepsAvailable,
|
|
4834
5201
|
deleteFileEmbeddings,
|
|
4835
5202
|
ensureEmbeddingProvenance,
|
|
5203
|
+
withDeclaredSkipScopeRoot,
|
|
5204
|
+
hasSkipSegmentBelowRoot,
|
|
4836
5205
|
parserFingerprintForLanguage,
|
|
4837
5206
|
treeSitterCoreAvailable,
|
|
4838
5207
|
treeSitterCoreLoadError,
|