token-goat 2.9.28 → 2.9.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -40
- package/SECURITY.md +28 -10
- package/THIRD_PARTY_NOTICES.md +40 -3
- package/dist/native/linux-arm64/tg-hook +0 -0
- package/dist/native/linux-x64/tg-hook +0 -0
- package/dist/ort-wasm-simd-threaded.mjs +59 -0
- package/dist/token-goat-chunk-3CHUNJ2O.mjs +53 -0
- package/dist/token-goat-chunk-3Z5S4KGM.mjs +918 -0
- package/dist/token-goat-chunk-4FVIAHSI.mjs +211 -0
- package/dist/{token-goat-chunk-66LODAUT.mjs → token-goat-chunk-4JJS54TR.mjs} +4 -3
- package/dist/token-goat-chunk-4LCCNRTG.mjs +313 -0
- package/dist/token-goat-chunk-54K3EUV3.mjs +35 -0
- package/dist/token-goat-chunk-56EIMS3E.mjs +159 -0
- package/dist/token-goat-chunk-5BCIJXMO.mjs +609 -0
- package/dist/token-goat-chunk-5KDMCMF2.mjs +362 -0
- package/dist/token-goat-chunk-5RG5SBL6.mjs +221 -0
- package/dist/{token-goat-chunk-RRNZMM3A.mjs → token-goat-chunk-5Y6ZYL6U.mjs} +21 -2
- package/dist/{token-goat-chunk-NILVK7XM.mjs → token-goat-chunk-64FBPDTK.mjs} +12 -6
- package/dist/{token-goat-chunk-LEG3JNJ6.mjs → token-goat-chunk-664RTTUV.mjs} +31 -15
- package/dist/token-goat-chunk-67TQJ76B.mjs +57 -0
- package/dist/token-goat-chunk-6QMAS7AB.mjs +42 -0
- package/dist/token-goat-chunk-7BXSNBY5.mjs +253 -0
- package/dist/{token-goat-chunk-F7QZRIWM.mjs → token-goat-chunk-7F2UBPD3.mjs} +42 -35
- package/dist/{token-goat-chunk-CC2PYJM7.mjs → token-goat-chunk-7IN2MKDS.mjs} +718 -595
- package/dist/token-goat-chunk-7NADEDKJ.mjs +124 -0
- package/dist/{token-goat-chunk-6SGLIPQB.mjs → token-goat-chunk-A3CK6N5Z.mjs} +621 -72
- package/dist/{token-goat-chunk-HXNAC4ZF.mjs → token-goat-chunk-AFLYLHDH.mjs} +48 -33
- package/dist/token-goat-chunk-AM5V6GSX.mjs +367 -0
- package/dist/{token-goat-chunk-I7CB3X4K.mjs → token-goat-chunk-APGZXSPS.mjs} +44 -148
- package/dist/{token-goat-chunk-BJBRGPBX.mjs → token-goat-chunk-ARDNEC2U.mjs} +11 -7
- package/dist/token-goat-chunk-ATCPY7EQ.mjs +134 -0
- package/dist/token-goat-chunk-B2RZZCSS.mjs +354 -0
- package/dist/{token-goat-chunk-QM76S6YS.mjs → token-goat-chunk-B45VMIIH.mjs} +6 -5
- package/dist/token-goat-chunk-BPGCUMR7.mjs +30 -0
- package/dist/{token-goat-chunk-GGLEE35R.mjs → token-goat-chunk-C63JQV5K.mjs} +49 -10
- package/dist/{token-goat-chunk-RJH64KQH.mjs → token-goat-chunk-COBQGL5S.mjs} +2661 -2440
- package/dist/token-goat-chunk-COTXZHTS.mjs +39 -0
- package/dist/token-goat-chunk-CYCEYAEO.mjs +265 -0
- package/dist/token-goat-chunk-CZ3LUXYZ.mjs +4115 -0
- package/dist/token-goat-chunk-DZX2YDPT.mjs +15 -0
- package/dist/token-goat-chunk-E3OQX2XT.mjs +824 -0
- package/dist/token-goat-chunk-EFHMMIGP.mjs +68 -0
- package/dist/token-goat-chunk-ETBHVDB6.mjs +328 -0
- package/dist/token-goat-chunk-EVMZJ27Z.mjs +147 -0
- package/dist/token-goat-chunk-EXKGNHEG.mjs +2096 -0
- package/dist/{token-goat-chunk-6Z5ULNE5.mjs → token-goat-chunk-FBNQNSJK.mjs} +2490 -4957
- package/dist/{token-goat-chunk-NMTKNYGF.mjs → token-goat-chunk-FNFBKLRD.mjs} +74 -9
- package/dist/{token-goat-chunk-XI2VFAV6.mjs → token-goat-chunk-FZ3NEW5O.mjs} +33 -19
- package/dist/token-goat-chunk-G5PJ7AGM.mjs +36 -0
- package/dist/{token-goat-chunk-VE27HPIF.mjs → token-goat-chunk-GFBSN7BD.mjs} +30 -398
- package/dist/{token-goat-chunk-L45ZQRT7.mjs → token-goat-chunk-GOLEOXZP.mjs} +780 -786
- package/dist/token-goat-chunk-H2K3QP6H.mjs +24 -0
- package/dist/token-goat-chunk-H2PHHGM5.mjs +188 -0
- package/dist/token-goat-chunk-HAVAOB3S.mjs +87 -0
- package/dist/token-goat-chunk-HG45AF2B.mjs +2228 -0
- package/dist/{token-goat-chunk-CBMY2FGI.mjs → token-goat-chunk-HOJLCGDP.mjs} +17113 -15554
- package/dist/{token-goat-chunk-HUDC5GS5.mjs → token-goat-chunk-HYBFA5X3.mjs} +120 -19
- package/dist/token-goat-chunk-IVALLT3S.mjs +48 -0
- package/dist/token-goat-chunk-IVE6U7KV.mjs +53 -0
- package/dist/{token-goat-chunk-UBUKHVYW.mjs → token-goat-chunk-IVKIZWAB.mjs} +3 -3
- package/dist/{token-goat-chunk-4JMWPWCS.mjs → token-goat-chunk-JB6QN6VF.mjs} +6 -4
- package/dist/{token-goat-chunk-Q4S3UQGO.mjs → token-goat-chunk-JDHE26RP.mjs} +10 -3
- package/dist/{token-goat-chunk-7R2XYKC5.mjs → token-goat-chunk-JDU3EXSU.mjs} +14 -10
- package/dist/token-goat-chunk-JDWCRD6M.mjs +253 -0
- package/dist/{token-goat-chunk-XBESNWEX.mjs → token-goat-chunk-JN4XMLSD.mjs} +7 -4
- package/dist/token-goat-chunk-JTQQGXZH.mjs +733 -0
- package/dist/token-goat-chunk-KKUFXAPD.mjs +25 -0
- package/dist/token-goat-chunk-KQNZDCBD.mjs +47 -0
- package/dist/{token-goat-chunk-4RN64ZND.mjs → token-goat-chunk-KWPNJBDH.mjs} +28 -3
- package/dist/token-goat-chunk-LOBGHXPB.mjs +242 -0
- package/dist/token-goat-chunk-MKPF4CL5.mjs +37 -0
- package/dist/token-goat-chunk-MNWKPRAL.mjs +13 -0
- package/dist/{token-goat-chunk-YWSP6IU7.mjs → token-goat-chunk-N2NMER56.mjs} +25 -15
- package/dist/token-goat-chunk-NYKZLXMK.mjs +218 -0
- package/dist/token-goat-chunk-OJBNYTJP.mjs +3087 -0
- package/dist/token-goat-chunk-P4QMM4WK.mjs +295 -0
- package/dist/token-goat-chunk-PD5VKH46.mjs +184 -0
- package/dist/token-goat-chunk-PEPTHRFF.mjs +129 -0
- package/dist/{token-goat-chunk-VWZKZPSR.mjs → token-goat-chunk-PU7V3FD6.mjs} +8752 -11098
- package/dist/token-goat-chunk-PY533T7F.mjs +46 -0
- package/dist/token-goat-chunk-Q5VTGRH7.mjs +761 -0
- package/dist/{token-goat-chunk-IHKZ3QNA.mjs → token-goat-chunk-QCLDAQTS.mjs} +33 -19
- package/dist/{token-goat-chunk-LKSXAMJB.mjs → token-goat-chunk-QSBRTGM2.mjs} +9 -2
- package/dist/{token-goat-chunk-4STDIGYC.mjs → token-goat-chunk-RC35RBUL.mjs} +7 -5
- package/dist/{token-goat-chunk-D5ICEPSB.mjs → token-goat-chunk-RHXKCZX3.mjs} +887 -507
- package/dist/{token-goat-chunk-RSS3TLXU.mjs → token-goat-chunk-RZRTDST7.mjs} +17 -7
- package/dist/{token-goat-chunk-MAMQPC43.mjs → token-goat-chunk-SJHWVNDQ.mjs} +15 -21
- package/dist/token-goat-chunk-SMVRN2HU.mjs +188 -0
- package/dist/{token-goat-chunk-EL7M26WP.mjs → token-goat-chunk-SQSSNQPE.mjs} +2100 -1442
- package/dist/{token-goat-chunk-OZRSREKR.mjs → token-goat-chunk-T4FK7VOV.mjs} +3 -3
- package/dist/token-goat-chunk-U4N3BN2W.mjs +92 -0
- package/dist/token-goat-chunk-UA5V6P3P.mjs +122 -0
- package/dist/{token-goat-chunk-XKY4VKJ4.mjs → token-goat-chunk-UQ5UK2T7.mjs} +15 -4
- package/dist/token-goat-chunk-USILZ3T3.mjs +53 -0
- package/dist/token-goat-chunk-VES7EL7O.mjs +205 -0
- package/dist/{token-goat-chunk-M56BWPBN.mjs → token-goat-chunk-VL3YIFHZ.mjs} +550 -49
- package/dist/{token-goat-chunk-IZXKVT4A.mjs → token-goat-chunk-VPBBTSJR.mjs} +362 -128
- package/dist/{token-goat-chunk-F2ARFHLJ.mjs → token-goat-chunk-W3QCJS6G.mjs} +10 -2
- package/dist/{token-goat-chunk-FSTWAOW6.mjs → token-goat-chunk-WY3L4DVT.mjs} +11 -5
- package/dist/{token-goat-chunk-OSUFN2FV.mjs → token-goat-chunk-XB5KKOCG.mjs} +5 -25
- package/dist/token-goat-chunk-XKI4SEMX.mjs +66 -0
- package/dist/token-goat-chunk-XU4P6Z24.mjs +133 -0
- package/dist/{token-goat-chunk-6CCQDAYY.mjs → token-goat-chunk-Y776AENC.mjs} +138 -33
- package/dist/token-goat-chunk-YARZFCZS.mjs +359 -0
- package/dist/{token-goat-chunk-EEIDFMEM.mjs → token-goat-chunk-YZAAZU2S.mjs} +12 -3
- package/dist/{token-goat-chunk-E3K3BTEQ.mjs → token-goat-chunk-Z44YO3QZ.mjs} +12 -6
- package/dist/token-goat-chunk-Z57YODA2.mjs +35 -0
- package/dist/{token-goat-chunk-DA43OE7L.mjs → token-goat-chunk-ZCDTCVIS.mjs} +49 -30
- package/dist/token-goat-chunk-ZEJVMUQH.mjs +48 -0
- package/dist/{token-goat-chunk-BGHSSJUA.mjs → token-goat-chunk-ZFOBKUWH.mjs} +1 -1
- package/dist/token-goat-chunk-ZMBL6UVV.mjs +110 -0
- package/dist/{token-goat-chunk-SJEKA7Y2.mjs → token-goat-chunk-ZQSJMO2R.mjs} +381 -201
- package/dist/token-goat-hook-client.cjs +539 -0
- package/dist/token-goat-hook-client.mjs +26 -0
- package/dist/token-goat-hook.mjs +48 -28
- package/dist/token-goat.core.mjs +10 -45
- package/docs/C4_RUNTIME_ARCHITECTURE.md +3 -3
- package/docs/cli.md +82 -47
- package/docs/install.md +58 -23
- package/docs/security.md +7 -7
- package/package.json +17 -7
- package/scripts/verify-native-dist.mjs +386 -0
- package/dist/token-goat-chunk-4JJBFT7V.mjs +0 -20
- package/dist/token-goat-chunk-5NNYIUGV.mjs +0 -143
- package/dist/token-goat-chunk-6DQNNFJ6.mjs +0 -294
- package/dist/token-goat-chunk-7MAFDRVR.mjs +0 -47
- package/dist/token-goat-chunk-BCN2YQC6.mjs +0 -32
- package/dist/token-goat-chunk-D7GHQKQD.mjs +0 -30
- package/dist/token-goat-chunk-GMOUBOX4.mjs +0 -386
- package/dist/token-goat-chunk-IFORDBGC.mjs +0 -192
- package/dist/token-goat-chunk-JXK2HQXS.mjs +0 -318
- package/dist/token-goat-chunk-L7SDVGBT.mjs +0 -430
- package/dist/token-goat-chunk-MMLMALWK.mjs +0 -1381
- package/dist/token-goat-chunk-MVSWBNMD.mjs +0 -1985
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
import { createRequire as __cjsRequire } from 'node:module';
|
|
2
2
|
const require = __cjsRequire(import.meta.url);
|
|
3
3
|
import {
|
|
4
|
+
ATX_CLASS_BREAK_RE,
|
|
5
|
+
ATX_DOT_BREAK_RE,
|
|
6
|
+
matchAtxHeading,
|
|
7
|
+
matchCssSelectorSpan,
|
|
8
|
+
matchRuleBannerTitle,
|
|
9
|
+
matchTableHeaderName,
|
|
10
|
+
matchTomlSectionName
|
|
11
|
+
} from "./token-goat-chunk-EVMZJ27Z.mjs";
|
|
12
|
+
import {
|
|
13
|
+
ownProjectScope,
|
|
4
14
|
pathEqClause,
|
|
5
|
-
projectScopeClause,
|
|
6
15
|
querySymbols
|
|
7
|
-
} from "./token-goat-chunk-
|
|
16
|
+
} from "./token-goat-chunk-HYBFA5X3.mjs";
|
|
8
17
|
import {
|
|
9
18
|
TREE_SITTER_LANGUAGES,
|
|
10
19
|
_detectOpenQuote,
|
|
@@ -22,33 +31,38 @@ import {
|
|
|
22
31
|
stripJsComments,
|
|
23
32
|
stripStringLiterals,
|
|
24
33
|
stripXmlComments
|
|
25
|
-
} from "./token-goat-chunk-
|
|
34
|
+
} from "./token-goat-chunk-VL3YIFHZ.mjs";
|
|
26
35
|
import {
|
|
36
|
+
findProject,
|
|
27
37
|
fingerprintContent,
|
|
28
38
|
loadConfig
|
|
29
|
-
} from "./token-goat-chunk-
|
|
39
|
+
} from "./token-goat-chunk-7IN2MKDS.mjs";
|
|
30
40
|
import {
|
|
31
41
|
registerReset
|
|
32
|
-
} from "./token-goat-chunk-
|
|
42
|
+
} from "./token-goat-chunk-YZAAZU2S.mjs";
|
|
43
|
+
import {
|
|
44
|
+
DocumentRefusedError,
|
|
45
|
+
MAX_DOCUMENT_WORK_MILLIS
|
|
46
|
+
} from "./token-goat-chunk-Y4AFKTHK.mjs";
|
|
33
47
|
import {
|
|
34
|
-
SYMBOL_BODY_CHAR_CAP,
|
|
35
48
|
countContentLines,
|
|
36
|
-
dataDir,
|
|
37
49
|
decodeSource,
|
|
38
|
-
|
|
39
|
-
|
|
50
|
+
ensureDirSync
|
|
51
|
+
} from "./token-goat-chunk-GOLEOXZP.mjs";
|
|
52
|
+
import {
|
|
40
53
|
foldPath,
|
|
41
|
-
globalDbPath,
|
|
42
54
|
isCaseInsensitiveFs
|
|
43
|
-
} from "./token-goat-chunk-
|
|
55
|
+
} from "./token-goat-chunk-AM5V6GSX.mjs";
|
|
44
56
|
import {
|
|
45
57
|
displaySafeText,
|
|
46
58
|
normalizePath
|
|
47
|
-
} from "./token-goat-chunk-
|
|
59
|
+
} from "./token-goat-chunk-FNFBKLRD.mjs";
|
|
48
60
|
import {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
61
|
+
SYMBOL_BODY_CHAR_CAP,
|
|
62
|
+
dataDir,
|
|
63
|
+
ensureDataDirPrivate,
|
|
64
|
+
globalDbPath
|
|
65
|
+
} from "./token-goat-chunk-VES7EL7O.mjs";
|
|
52
66
|
import {
|
|
53
67
|
init_define_import_meta_env
|
|
54
68
|
} from "./token-goat-chunk-A37V4PBF.mjs";
|
|
@@ -70,28 +84,28 @@ function isTransientDocumentRefusal(err) {
|
|
|
70
84
|
async function extractEmbeddableDocumentText(filePath) {
|
|
71
85
|
switch (path.extname(filePath).toLowerCase()) {
|
|
72
86
|
case ".pdf": {
|
|
73
|
-
const { extractPdfText, readPdfFileWithinBounds } = await import("./token-goat-chunk-
|
|
87
|
+
const { extractPdfText, readPdfFileWithinBounds } = await import("./token-goat-chunk-W3QCJS6G.mjs");
|
|
74
88
|
const { text } = await extractPdfText(await readPdfFileWithinBounds(filePath));
|
|
75
89
|
return text;
|
|
76
90
|
}
|
|
77
91
|
case ".docx":
|
|
78
|
-
return await (await import("./token-goat-chunk-
|
|
92
|
+
return await (await import("./token-goat-chunk-B45VMIIH.mjs")).docxText(filePath);
|
|
79
93
|
case ".pptx":
|
|
80
|
-
return await (await import("./token-goat-chunk-
|
|
94
|
+
return await (await import("./token-goat-chunk-WY3L4DVT.mjs")).pptxAllSlidesText(filePath, true);
|
|
81
95
|
case ".xlsx":
|
|
82
|
-
return await (await import("./token-goat-chunk-
|
|
96
|
+
return await (await import("./token-goat-chunk-64FBPDTK.mjs")).allSheetsHeadText(filePath, XLSX_SHEET_ROW_CAP);
|
|
83
97
|
default:
|
|
84
98
|
return null;
|
|
85
99
|
}
|
|
86
100
|
}
|
|
87
101
|
|
|
102
|
+
// src/embeddings.ts
|
|
103
|
+
init_define_import_meta_env();
|
|
104
|
+
|
|
88
105
|
// src/embed_model.ts
|
|
89
106
|
init_define_import_meta_env();
|
|
90
|
-
import
|
|
91
|
-
import * as
|
|
92
|
-
import { createRequire } from "node:module";
|
|
93
|
-
import * as path2 from "node:path";
|
|
94
|
-
import { pipeline } from "node:stream/promises";
|
|
107
|
+
import * as fs3 from "node:fs";
|
|
108
|
+
import * as path4 from "node:path";
|
|
95
109
|
|
|
96
110
|
// src/embed_tokenizer.ts
|
|
97
111
|
init_define_import_meta_env();
|
|
@@ -105,20 +119,20 @@ var UnsupportedTokenizerError = class extends Error {
|
|
|
105
119
|
function fail(message) {
|
|
106
120
|
throw new UnsupportedTokenizerError(message);
|
|
107
121
|
}
|
|
108
|
-
function asRecord(value,
|
|
109
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${
|
|
122
|
+
function asRecord(value, path10) {
|
|
123
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${path10} is not an object`);
|
|
110
124
|
return value;
|
|
111
125
|
}
|
|
112
|
-
function asString(value,
|
|
113
|
-
if (typeof value !== "string") fail(`${
|
|
126
|
+
function asString(value, path10) {
|
|
127
|
+
if (typeof value !== "string") fail(`${path10} is not a string`);
|
|
114
128
|
return value;
|
|
115
129
|
}
|
|
116
|
-
function asArray(value,
|
|
117
|
-
if (!Array.isArray(value)) fail(`${
|
|
130
|
+
function asArray(value, path10) {
|
|
131
|
+
if (!Array.isArray(value)) fail(`${path10} is not an array`);
|
|
118
132
|
return value;
|
|
119
133
|
}
|
|
120
|
-
function expectValue(actual, wanted,
|
|
121
|
-
if (actual !== wanted) fail(`${
|
|
134
|
+
function expectValue(actual, wanted, path10) {
|
|
135
|
+
if (actual !== wanted) fail(`${path10} is ${JSON.stringify(actual)}, expected ${JSON.stringify(wanted)}`);
|
|
122
136
|
}
|
|
123
137
|
function isChinese(cp) {
|
|
124
138
|
return cp >= 19968 && cp <= 40959 || cp >= 13312 && cp <= 19903 || cp >= 131072 && cp <= 173791 || cp >= 173824 && cp <= 177983 || cp >= 177984 && cp <= 178207 || cp >= 178208 && cp <= 183983 || cp >= 63744 && cp <= 64255 || cp >= 194560 && cp <= 195103;
|
|
@@ -306,68 +320,25 @@ var BertWordPiece = class _BertWordPiece {
|
|
|
306
320
|
}
|
|
307
321
|
};
|
|
308
322
|
|
|
309
|
-
// src/
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
name: "onnx/model_quantized.onnx",
|
|
323
|
-
sha256: "6c9c6101a956d62dfb5e7190c538226c0c5bb9cb27b651234b6df063ee7dbfe4",
|
|
324
|
-
bytes: 34014426
|
|
325
|
-
}
|
|
326
|
-
];
|
|
327
|
-
function modelDir() {
|
|
328
|
-
return path2.join(dataDir(), "models", ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
329
|
-
}
|
|
330
|
-
function modelFilesPresent() {
|
|
331
|
-
const roots = [modelDir(), sharedModelCacheDir()].filter((d) => d !== null);
|
|
332
|
-
return roots.some(
|
|
333
|
-
(root) => MODEL_FILES.every((file) => {
|
|
334
|
-
try {
|
|
335
|
-
return fs.statSync(path2.join(root, file.name)).size === file.bytes;
|
|
336
|
-
} catch {
|
|
337
|
-
return false;
|
|
338
|
-
}
|
|
339
|
-
})
|
|
340
|
-
);
|
|
341
|
-
}
|
|
342
|
-
function downloadUrl(file) {
|
|
343
|
-
return `https://huggingface.co/${DEFAULT_MODEL}/resolve/${PINNED_MODEL_REVISION}/${file.name}`;
|
|
344
|
-
}
|
|
345
|
-
function sharedModelCacheDir() {
|
|
346
|
-
const raw = process.env["TOKEN_GOAT_MODEL_CACHE_DIR"]?.trim();
|
|
347
|
-
if (!raw) return null;
|
|
348
|
-
return path2.join(raw, ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
323
|
+
// src/embed_runtime.ts
|
|
324
|
+
init_define_import_meta_env();
|
|
325
|
+
import * as fs2 from "node:fs";
|
|
326
|
+
import { createRequire } from "node:module";
|
|
327
|
+
import * as path3 from "node:path";
|
|
328
|
+
import { fileURLToPath } from "node:url";
|
|
329
|
+
|
|
330
|
+
// src/pinned_file.ts
|
|
331
|
+
init_define_import_meta_env();
|
|
332
|
+
import * as fs from "node:fs";
|
|
333
|
+
import * as path2 from "node:path";
|
|
334
|
+
async function sha256Of(filePath) {
|
|
335
|
+
return (await import("./token-goat-chunk-H2K3QP6H.mjs")).sha256Of(filePath);
|
|
349
336
|
}
|
|
350
337
|
async function copyFromSharedCache(shared, file, target) {
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
if (!info?.isFile() || info.size !== file.bytes) return false;
|
|
356
|
-
fs.copyFileSync(source, temp);
|
|
357
|
-
if (await sha256Of(temp) !== file.sha256) {
|
|
358
|
-
fs.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
359
|
-
fs.rmSync(source, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
360
|
-
return false;
|
|
361
|
-
}
|
|
362
|
-
fs.renameSync(temp, target);
|
|
363
|
-
return true;
|
|
364
|
-
} catch {
|
|
365
|
-
try {
|
|
366
|
-
fs.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
367
|
-
} catch {
|
|
368
|
-
}
|
|
369
|
-
return false;
|
|
370
|
-
}
|
|
338
|
+
return (await import("./token-goat-chunk-H2K3QP6H.mjs")).copyFromSharedCache(shared, file, target);
|
|
339
|
+
}
|
|
340
|
+
async function downloadPinned(url, file, target) {
|
|
341
|
+
return (await import("./token-goat-chunk-H2K3QP6H.mjs")).downloadPinned(url, file, target);
|
|
371
342
|
}
|
|
372
343
|
function publishToSharedCache(shared, file, target) {
|
|
373
344
|
const destination = path2.join(shared, file.name);
|
|
@@ -387,38 +358,104 @@ function publishToSharedCache(shared, file, target) {
|
|
|
387
358
|
}
|
|
388
359
|
}
|
|
389
360
|
}
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
var
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
361
|
+
|
|
362
|
+
// src/embed_runtime.ts
|
|
363
|
+
var _require = createRequire(import.meta.url);
|
|
364
|
+
var ORT_WEB_VERSION = "1.30.0";
|
|
365
|
+
var ORT_WEB_WASM = {
|
|
366
|
+
name: "ort-wasm-simd-threaded.wasm",
|
|
367
|
+
sha256: "3398c10d07d229bd91b364548e130e0e51a8e5704b88c7c083ebbeb78842dee2",
|
|
368
|
+
bytes: 14239897
|
|
369
|
+
};
|
|
370
|
+
var WEB_RETRY_AFTER_MS = 10 * 60 * 1e3;
|
|
371
|
+
var START_FAILURE_HOLD_MS = 24 * 60 * 60 * 1e3;
|
|
372
|
+
var NATIVE_RUNTIME_INSTALL = "install the native runtime with: npm install -g onnxruntime-node (drop -g if token-goat is a project dependency)";
|
|
373
|
+
var RUNTIME_UNAVAILABLE_ADVICE = `The WebAssembly runtime is downloaded once from registry.npmjs.org (onnxruntime-web ${ORT_WEB_VERSION}), so check network access and network.offline; or ${NATIVE_RUNTIME_INSTALL}`;
|
|
374
|
+
var _chosen = null;
|
|
375
|
+
var _nodeOrt = null;
|
|
376
|
+
var _nodeError = null;
|
|
377
|
+
var _webFailure = null;
|
|
378
|
+
var _recordChecked = false;
|
|
379
|
+
function asError(e) {
|
|
380
|
+
return e instanceof Error ? e : new Error(String(e));
|
|
381
|
+
}
|
|
382
|
+
function chooseRuntime() {
|
|
383
|
+
if (_chosen !== null) return _chosen;
|
|
396
384
|
try {
|
|
397
|
-
|
|
385
|
+
_nodeOrt = _require("onnxruntime-node");
|
|
386
|
+
_chosen = "onnxruntime-node";
|
|
398
387
|
} catch (e) {
|
|
399
|
-
|
|
388
|
+
_nodeError = asError(e);
|
|
389
|
+
_chosen = "onnxruntime-web";
|
|
400
390
|
}
|
|
391
|
+
return _chosen;
|
|
392
|
+
}
|
|
393
|
+
function recentWebFailure() {
|
|
394
|
+
if (_webFailure === null && !_recordChecked) {
|
|
395
|
+
_recordChecked = true;
|
|
396
|
+
_webFailure = recordedStartFailure();
|
|
397
|
+
}
|
|
398
|
+
if (_webFailure === null) return null;
|
|
399
|
+
if (Date.now() - _webFailure.at >= (_webFailure.started ? START_FAILURE_HOLD_MS : WEB_RETRY_AFTER_MS) || _webFailure.started && !fs2.existsSync(startFailurePath())) {
|
|
400
|
+
_webFailure = null;
|
|
401
|
+
return null;
|
|
402
|
+
}
|
|
403
|
+
return _webFailure.error;
|
|
404
|
+
}
|
|
405
|
+
function startFailurePath() {
|
|
406
|
+
return path3.join(wasmDir(), "start-failure.json");
|
|
407
|
+
}
|
|
408
|
+
var NODE_OPTIONS_AT_START = process.env["NODE_OPTIONS"] ?? "";
|
|
409
|
+
function startFailureKey() {
|
|
410
|
+
let build = "";
|
|
411
|
+
try {
|
|
412
|
+
const self = fs2.statSync(fileURLToPath(import.meta.url));
|
|
413
|
+
build = `${self.size}:${self.mtimeMs}`;
|
|
414
|
+
} catch {
|
|
415
|
+
}
|
|
416
|
+
return JSON.stringify([ORT_WEB_VERSION, ORT_WEB_WASM.sha256, build, process.execPath, process.version, process.platform, process.arch, process.execArgv, NODE_OPTIONS_AT_START]);
|
|
417
|
+
}
|
|
418
|
+
function recordedStartFailure() {
|
|
419
|
+
const file = startFailurePath();
|
|
420
|
+
try {
|
|
421
|
+
const parsed = JSON.parse(fs2.readFileSync(file, "utf8"));
|
|
422
|
+
if (parsed.key !== startFailureKey() || typeof parsed.message !== "string" || typeof parsed.at !== "number") return null;
|
|
423
|
+
const when = new Date(parsed.at).toISOString();
|
|
424
|
+
return {
|
|
425
|
+
error: new Error(`${parsed.message} (in an earlier run, at ${when}; it is tried again once Node, its flags or token-goat change, or a day after, and deleting ${file} tries it at once)`),
|
|
426
|
+
at: parsed.at,
|
|
427
|
+
started: true
|
|
428
|
+
};
|
|
429
|
+
} catch {
|
|
430
|
+
return null;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
function activeRuntime() {
|
|
434
|
+
return chooseRuntime();
|
|
435
|
+
}
|
|
436
|
+
function nativeRuntimeLoadError() {
|
|
437
|
+
chooseRuntime();
|
|
438
|
+
return _nodeError;
|
|
401
439
|
}
|
|
402
440
|
function isRuntimeAvailable() {
|
|
403
|
-
|
|
404
|
-
return
|
|
441
|
+
if (chooseRuntime() === "onnxruntime-node") return true;
|
|
442
|
+
return recentWebFailure() === null;
|
|
405
443
|
}
|
|
406
444
|
function runtimeLoadError() {
|
|
407
|
-
|
|
408
|
-
return
|
|
445
|
+
if (chooseRuntime() === "onnxruntime-node") return null;
|
|
446
|
+
return recentWebFailure();
|
|
409
447
|
}
|
|
410
448
|
function runtimeVersion() {
|
|
411
|
-
|
|
412
|
-
if (_ort === null) return "unknown";
|
|
449
|
+
if (chooseRuntime() === "onnxruntime-web") return ORT_WEB_VERSION;
|
|
413
450
|
try {
|
|
414
|
-
let dir =
|
|
451
|
+
let dir = path3.dirname(_require.resolve("onnxruntime-node"));
|
|
415
452
|
for (let depth = 0; depth < 6; depth++) {
|
|
416
|
-
const manifest =
|
|
417
|
-
if (
|
|
418
|
-
const parsed = JSON.parse(
|
|
453
|
+
const manifest = path3.join(dir, "package.json");
|
|
454
|
+
if (fs2.existsSync(manifest)) {
|
|
455
|
+
const parsed = JSON.parse(fs2.readFileSync(manifest, "utf8"));
|
|
419
456
|
if (parsed.name === "onnxruntime-node" && typeof parsed.version === "string") return parsed.version;
|
|
420
457
|
}
|
|
421
|
-
const up =
|
|
458
|
+
const up = path3.dirname(dir);
|
|
422
459
|
if (up === dir) break;
|
|
423
460
|
dir = up;
|
|
424
461
|
}
|
|
@@ -426,52 +463,101 @@ function runtimeVersion() {
|
|
|
426
463
|
}
|
|
427
464
|
return "unknown";
|
|
428
465
|
}
|
|
429
|
-
function
|
|
430
|
-
return
|
|
431
|
-
const hash = createHash("sha256");
|
|
432
|
-
const stream = fs.createReadStream(filePath);
|
|
433
|
-
stream.on("error", reject);
|
|
434
|
-
stream.on("data", (chunk) => hash.update(chunk));
|
|
435
|
-
stream.on("end", () => resolve2(hash.digest("hex")));
|
|
436
|
-
});
|
|
466
|
+
function wasmDir() {
|
|
467
|
+
return path3.join(dataDir(), "models", "onnxruntime-web", ORT_WEB_VERSION);
|
|
437
468
|
}
|
|
438
|
-
|
|
439
|
-
const
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
const
|
|
445
|
-
|
|
446
|
-
const out = fs.createWriteStream(temp);
|
|
447
|
-
try {
|
|
448
|
-
await pipeline(async function* () {
|
|
449
|
-
for await (const chunk of response.body) {
|
|
450
|
-
written += chunk.byteLength;
|
|
451
|
-
if (written > file.bytes) throw new Error(`${file.name} is longer than the pinned ${file.bytes} bytes`);
|
|
452
|
-
hash.update(chunk);
|
|
453
|
-
yield chunk;
|
|
454
|
-
}
|
|
455
|
-
}, out);
|
|
456
|
-
if (written !== file.bytes) {
|
|
457
|
-
throw new Error(`${file.name} is ${written} bytes, expected the pinned ${file.bytes}`);
|
|
458
|
-
}
|
|
459
|
-
const digest = hash.digest("hex");
|
|
460
|
-
if (digest !== file.sha256) {
|
|
461
|
-
throw new Error(`${file.name} has sha256 ${digest}, expected the pinned ${file.sha256}`);
|
|
462
|
-
}
|
|
463
|
-
fs.renameSync(temp, target);
|
|
464
|
-
} catch (e) {
|
|
465
|
-
await new Promise((resolve2) => {
|
|
466
|
-
if (out.closed) resolve2();
|
|
467
|
-
else out.once("close", () => resolve2());
|
|
468
|
-
});
|
|
469
|
+
function sharedWasmCacheDir() {
|
|
470
|
+
const raw = process.env["TOKEN_GOAT_MODEL_CACHE_DIR"]?.trim();
|
|
471
|
+
if (!raw) return null;
|
|
472
|
+
return path3.join(raw, "onnxruntime-web", ORT_WEB_VERSION);
|
|
473
|
+
}
|
|
474
|
+
function wasmBinaryPresent() {
|
|
475
|
+
const roots = [wasmDir(), sharedWasmCacheDir()].filter((d) => d !== null);
|
|
476
|
+
return roots.some((root) => {
|
|
469
477
|
try {
|
|
470
|
-
|
|
478
|
+
return fs2.statSync(path3.join(root, ORT_WEB_WASM.name)).size === ORT_WEB_WASM.bytes;
|
|
471
479
|
} catch {
|
|
480
|
+
return false;
|
|
472
481
|
}
|
|
473
|
-
|
|
482
|
+
});
|
|
483
|
+
}
|
|
484
|
+
async function ensureWasmBinary() {
|
|
485
|
+
return (await import("./token-goat-chunk-4LCCNRTG.mjs")).ensureWasmBinary(webHost());
|
|
486
|
+
}
|
|
487
|
+
function webHost() {
|
|
488
|
+
return {
|
|
489
|
+
version: ORT_WEB_VERSION,
|
|
490
|
+
wasm: ORT_WEB_WASM,
|
|
491
|
+
dir: wasmDir(),
|
|
492
|
+
sharedDir: sharedWasmCacheDir(),
|
|
493
|
+
publish: publishToSharedCache,
|
|
494
|
+
startFailure: { path: startFailurePath(), key: startFailureKey() },
|
|
495
|
+
failed(e, started) {
|
|
496
|
+
const error = asError(e);
|
|
497
|
+
_webFailure = { error, at: Date.now(), started };
|
|
498
|
+
return error;
|
|
499
|
+
}
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
async function createInferenceSession(modelPath, threads) {
|
|
503
|
+
if (chooseRuntime() === "onnxruntime-node") {
|
|
504
|
+
const ort = _nodeOrt;
|
|
505
|
+
return { session: await ort.InferenceSession.create(modelPath, { intraOpNumThreads: threads, interOpNumThreads: 1 }), Tensor: ort.Tensor };
|
|
474
506
|
}
|
|
507
|
+
const remembered = recentWebFailure();
|
|
508
|
+
if (remembered) throw remembered;
|
|
509
|
+
return (await import("./token-goat-chunk-4LCCNRTG.mjs")).createWebSession(modelPath, threads, webHost());
|
|
510
|
+
}
|
|
511
|
+
registerReset(() => {
|
|
512
|
+
_chosen = null;
|
|
513
|
+
_nodeOrt = null;
|
|
514
|
+
_nodeError = null;
|
|
515
|
+
_webFailure = null;
|
|
516
|
+
_recordChecked = false;
|
|
517
|
+
});
|
|
518
|
+
|
|
519
|
+
// src/embed_model.ts
|
|
520
|
+
var DEFAULT_MODEL = "Xenova/bge-small-en-v1.5";
|
|
521
|
+
var DEFAULT_DIM = 384;
|
|
522
|
+
var DEFAULT_EMBED_THREADS = 4;
|
|
523
|
+
var PINNED_MODEL_REVISION = "ea104dacec62c0de699686887e3f920caeb4f3e3";
|
|
524
|
+
var MODEL_FILES = [
|
|
525
|
+
{
|
|
526
|
+
name: "tokenizer.json",
|
|
527
|
+
sha256: "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66",
|
|
528
|
+
bytes: 711396
|
|
529
|
+
},
|
|
530
|
+
{
|
|
531
|
+
name: "onnx/model_quantized.onnx",
|
|
532
|
+
sha256: "6c9c6101a956d62dfb5e7190c538226c0c5bb9cb27b651234b6df063ee7dbfe4",
|
|
533
|
+
bytes: 34014426
|
|
534
|
+
}
|
|
535
|
+
];
|
|
536
|
+
function modelDir() {
|
|
537
|
+
return path4.join(dataDir(), "models", ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
538
|
+
}
|
|
539
|
+
function modelFilesPresent() {
|
|
540
|
+
const roots = [modelDir(), sharedModelCacheDir()].filter((d) => d !== null);
|
|
541
|
+
return roots.some(
|
|
542
|
+
(root) => MODEL_FILES.every((file) => {
|
|
543
|
+
try {
|
|
544
|
+
return fs3.statSync(path4.join(root, file.name)).size === file.bytes;
|
|
545
|
+
} catch {
|
|
546
|
+
return false;
|
|
547
|
+
}
|
|
548
|
+
})
|
|
549
|
+
);
|
|
550
|
+
}
|
|
551
|
+
function downloadUrl(file) {
|
|
552
|
+
return `https://huggingface.co/${DEFAULT_MODEL}/resolve/${PINNED_MODEL_REVISION}/${file.name}`;
|
|
553
|
+
}
|
|
554
|
+
function sharedModelCacheDir() {
|
|
555
|
+
const raw = process.env["TOKEN_GOAT_MODEL_CACHE_DIR"]?.trim();
|
|
556
|
+
if (!raw) return null;
|
|
557
|
+
return path4.join(raw, ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
558
|
+
}
|
|
559
|
+
function download(file, target) {
|
|
560
|
+
return downloadPinned(downloadUrl(file), file, target);
|
|
475
561
|
}
|
|
476
562
|
var _inFlightModelDownload = null;
|
|
477
563
|
function ensureModelFiles(modelName = DEFAULT_MODEL) {
|
|
@@ -500,14 +586,14 @@ async function _ensureModelFilesInner(_modelName) {
|
|
|
500
586
|
const offline = loadConfig().network.offline;
|
|
501
587
|
const shared = sharedModelCacheDir();
|
|
502
588
|
for (const file of MODEL_FILES) {
|
|
503
|
-
const target =
|
|
504
|
-
if (
|
|
589
|
+
const target = path4.join(dir, file.name);
|
|
590
|
+
if (fs3.existsSync(target)) {
|
|
505
591
|
const digest = await sha256Of(target);
|
|
506
592
|
if (digest === file.sha256) continue;
|
|
507
|
-
|
|
593
|
+
fs3.rmSync(target, { force: true });
|
|
508
594
|
}
|
|
509
595
|
if (shared) {
|
|
510
|
-
ensureDirSync(
|
|
596
|
+
ensureDirSync(path4.dirname(target));
|
|
511
597
|
if (await copyFromSharedCache(shared, file, target)) continue;
|
|
512
598
|
}
|
|
513
599
|
if (offline) {
|
|
@@ -515,7 +601,7 @@ async function _ensureModelFilesInner(_modelName) {
|
|
|
515
601
|
`Offline mode is on (network.offline): refusing to download ${file.name} for the embedding model. Copy the pinned files into ${dir} on a connected machine to use semantic search here.`
|
|
516
602
|
);
|
|
517
603
|
}
|
|
518
|
-
ensureDirSync(
|
|
604
|
+
ensureDirSync(path4.dirname(target));
|
|
519
605
|
console.warn(
|
|
520
606
|
`Downloading the embedding model, once (${file.name}, ${Math.round(file.bytes / 1024 / 1024)} MB) into ${dir}`
|
|
521
607
|
);
|
|
@@ -553,17 +639,13 @@ var EmbeddingModel = class _EmbeddingModel {
|
|
|
553
639
|
tensorFactory;
|
|
554
640
|
static async load(modelName = DEFAULT_MODEL) {
|
|
555
641
|
if (!isRuntimeAvailable()) {
|
|
556
|
-
throw new Error(`
|
|
642
|
+
throw new Error(`the inference runtime is not available: ${runtimeLoadError()?.message ?? "unknown error"}`);
|
|
557
643
|
}
|
|
558
644
|
const dir = await ensureModelFiles(modelName);
|
|
559
|
-
const tokenizer = BertWordPiece.fromJson(
|
|
560
|
-
const ort = _ort;
|
|
645
|
+
const tokenizer = BertWordPiece.fromJson(fs3.readFileSync(path4.join(dir, "tokenizer.json"), "utf8"));
|
|
561
646
|
const threads = loadConfig().worker.embed_threads ?? DEFAULT_EMBED_THREADS;
|
|
562
|
-
const session = await
|
|
563
|
-
|
|
564
|
-
interOpNumThreads: 1
|
|
565
|
-
});
|
|
566
|
-
return new _EmbeddingModel(tokenizer, session, ort.Tensor);
|
|
647
|
+
const { session, Tensor } = await createInferenceSession(path4.join(dir, "onnx", "model_quantized.onnx"), threads);
|
|
648
|
+
return new _EmbeddingModel(tokenizer, session, Tensor);
|
|
567
649
|
}
|
|
568
650
|
/** Wordpieces `text` becomes under this model's own tokenizer, markers excluded. */
|
|
569
651
|
countTokens(text) {
|
|
@@ -599,8 +681,10 @@ async function checkEmbeddingPreflight(options) {
|
|
|
599
681
|
const cfg = loadConfig();
|
|
600
682
|
const enabled = cfg.indexing?.embeddings_enabled ?? true;
|
|
601
683
|
const mDir = modelDir();
|
|
684
|
+
const rt = activeRuntime();
|
|
602
685
|
const rtVer = runtimeVersion();
|
|
603
686
|
const rtAvail = isRuntimeAvailable();
|
|
687
|
+
const binaryPresent = rt === "onnxruntime-web" ? wasmBinaryPresent() : null;
|
|
604
688
|
const filesPresent = modelFilesPresent();
|
|
605
689
|
let warmed = false;
|
|
606
690
|
const buildResult = (params) => {
|
|
@@ -613,8 +697,10 @@ async function checkEmbeddingPreflight(options) {
|
|
|
613
697
|
message: params.message,
|
|
614
698
|
summary: params.message,
|
|
615
699
|
modelName,
|
|
700
|
+
runtime: rt,
|
|
616
701
|
runtimeVersion: rtVer,
|
|
617
702
|
runtimeAvailable: rtAvail,
|
|
703
|
+
runtimeBinaryPresent: binaryPresent,
|
|
618
704
|
configEnabled: enabled,
|
|
619
705
|
modelFilesPresent: filesPresent,
|
|
620
706
|
modelWarmed: warmed,
|
|
@@ -640,11 +726,19 @@ async function checkEmbeddingPreflight(options) {
|
|
|
640
726
|
return buildResult({
|
|
641
727
|
status: "missing_runtime",
|
|
642
728
|
available: false,
|
|
643
|
-
message: `Inference runtime is not available: ${err?.message ?? "
|
|
644
|
-
suggestion:
|
|
729
|
+
message: `Inference runtime is not available: ${err?.message ?? "unknown error"}`,
|
|
730
|
+
suggestion: RUNTIME_UNAVAILABLE_ADVICE,
|
|
645
731
|
...err?.message ? { error: err.message } : {}
|
|
646
732
|
});
|
|
647
733
|
}
|
|
734
|
+
if (binaryPresent === false && cfg?.network?.offline) {
|
|
735
|
+
return buildResult({
|
|
736
|
+
status: "missing_runtime",
|
|
737
|
+
available: false,
|
|
738
|
+
message: `The WebAssembly inference runtime's ${ORT_WEB_WASM.name} is not downloaded yet and offline mode (network.offline) prevents fetching it`,
|
|
739
|
+
suggestion: `Copy ${ORT_WEB_WASM.name} from the onnxruntime-web package's dist/ into ${wasmDir()}, or ${NATIVE_RUNTIME_INSTALL}`
|
|
740
|
+
});
|
|
741
|
+
}
|
|
648
742
|
if (!filesPresent && cfg?.network?.offline) {
|
|
649
743
|
return buildResult({
|
|
650
744
|
status: "missing_model_files",
|
|
@@ -687,33 +781,27 @@ async function checkEmbeddingPreflight(options) {
|
|
|
687
781
|
});
|
|
688
782
|
}
|
|
689
783
|
registerReset(() => {
|
|
690
|
-
_ort = null;
|
|
691
|
-
_ortError = null;
|
|
692
|
-
_ortLoadAttempted = false;
|
|
693
784
|
_inFlightModelDownload = null;
|
|
694
785
|
});
|
|
695
786
|
|
|
696
|
-
// src/embeddings.ts
|
|
697
|
-
init_define_import_meta_env();
|
|
698
|
-
|
|
699
787
|
// src/embed_fingerprint.ts
|
|
700
788
|
init_define_import_meta_env();
|
|
701
|
-
var EMBED_FINGERPRINT = "
|
|
789
|
+
var EMBED_FINGERPRINT = "98753e3639c3c9e1";
|
|
702
790
|
var EMBED_KIND_FINGERPRINTS = /* @__PURE__ */ new Map([
|
|
703
|
-
["docx", "
|
|
704
|
-
["markdown", "
|
|
705
|
-
["pdf", "
|
|
706
|
-
["pptx", "
|
|
707
|
-
["xlsx", "
|
|
791
|
+
["docx", "ea76a9e656cab4a7"],
|
|
792
|
+
["markdown", "7c9b8d7beb37ac60"],
|
|
793
|
+
["pdf", "ef7a41b66bc04bc1"],
|
|
794
|
+
["pptx", "8b18b531b42106e7"],
|
|
795
|
+
["xlsx", "67a12e733136381e"]
|
|
708
796
|
]);
|
|
709
797
|
var PRE_KIND_EMBED_FINGERPRINT = "b7b2ff71de288d13";
|
|
710
798
|
var SPLIT_EMBED_FINGERPRINT = "2310db32d9bc8a2b";
|
|
711
799
|
|
|
712
800
|
// src/embed_stamp.ts
|
|
713
801
|
init_define_import_meta_env();
|
|
714
|
-
import * as
|
|
802
|
+
import * as path5 from "node:path";
|
|
715
803
|
function embedKindForPath(filePath) {
|
|
716
|
-
if (isEmbeddableDocument(filePath)) return
|
|
804
|
+
if (isEmbeddableDocument(filePath)) return path5.extname(filePath).toLowerCase().slice(1);
|
|
717
805
|
if (detectLanguage(filePath) === "markdown") return "markdown";
|
|
718
806
|
return null;
|
|
719
807
|
}
|
|
@@ -745,6 +833,8 @@ async function buildExtractorWithRetry(pipelineFn, modelName) {
|
|
|
745
833
|
try {
|
|
746
834
|
return await pipelineFn("feature-extraction", modelName);
|
|
747
835
|
} catch (e) {
|
|
836
|
+
const { failedAtOf } = await import("./token-goat-chunk-COTXZHTS.mjs");
|
|
837
|
+
if (failedAtOf(e) !== null) throw e;
|
|
748
838
|
lastError = e;
|
|
749
839
|
if (attempt < PIPELINE_RETRY_ATTEMPTS) await sleep(PIPELINE_RETRY_DELAY_MS * attempt);
|
|
750
840
|
}
|
|
@@ -1069,7 +1159,7 @@ function chunkFile(filePath, content, chunkSize = MAX_CHUNK_CHARS, overlap = 200
|
|
|
1069
1159
|
function insertChunkVector(stmt, rowid, embedding) {
|
|
1070
1160
|
stmt.run(BigInt(rowid), packVec(embedding));
|
|
1071
1161
|
}
|
|
1072
|
-
async function upsertChunks(db, chunks) {
|
|
1162
|
+
async function upsertChunks(db, chunks, expectedSha) {
|
|
1073
1163
|
if (chunks.length === 0) {
|
|
1074
1164
|
return "embedded";
|
|
1075
1165
|
}
|
|
@@ -1095,6 +1185,10 @@ async function upsertChunks(db, chunks) {
|
|
|
1095
1185
|
VALUES (?, ?)
|
|
1096
1186
|
`);
|
|
1097
1187
|
const tx = db.transaction(() => {
|
|
1188
|
+
if (expectedSha !== void 0) {
|
|
1189
|
+
const current = db.prepare(`SELECT sha FROM files WHERE ${pathEqClause("path")}`).pluck().get(foldPath(filePath));
|
|
1190
|
+
if (current !== expectedSha) return "stale";
|
|
1191
|
+
}
|
|
1098
1192
|
deleteFileEmbeddings(db, filePath);
|
|
1099
1193
|
for (let i = 0; i < chunks.length; i++) {
|
|
1100
1194
|
const chunk = chunks[i];
|
|
@@ -1111,9 +1205,9 @@ async function upsertChunks(db, chunks) {
|
|
|
1111
1205
|
);
|
|
1112
1206
|
insertChunkVector(vectorInsertStmt, chunkResult.lastInsertRowid, embedding);
|
|
1113
1207
|
}
|
|
1208
|
+
return "embedded";
|
|
1114
1209
|
});
|
|
1115
|
-
tx.immediate();
|
|
1116
|
-
return "embedded";
|
|
1210
|
+
return tx.immediate();
|
|
1117
1211
|
}
|
|
1118
1212
|
var BACKFILL_MULTIPLIER = 3;
|
|
1119
1213
|
function countStoredVectors(db) {
|
|
@@ -1131,10 +1225,10 @@ function fetchScopedHits(db, queryVec, k, maxDistance, rootDir) {
|
|
|
1131
1225
|
if (!rows || rows.length === 0) {
|
|
1132
1226
|
return { hits: [], candidateCount: 0 };
|
|
1133
1227
|
}
|
|
1134
|
-
const scope = rootDir !== void 0 ?
|
|
1228
|
+
const scope = rootDir !== void 0 ? ownProjectScope("file_path", rootDir) : void 0;
|
|
1135
1229
|
const chunkSql = scope !== void 0 ? `SELECT file_path, start_line, end_line, text, kind FROM chunks WHERE id = ? AND ${scope.clause}` : `SELECT file_path, start_line, end_line, text, kind FROM chunks WHERE id = ?`;
|
|
1136
1230
|
const chunkStmt = db.prepare(chunkSql);
|
|
1137
|
-
const scopeParams = scope
|
|
1231
|
+
const scopeParams = scope?.params;
|
|
1138
1232
|
const hits = [];
|
|
1139
1233
|
for (const row of rows) {
|
|
1140
1234
|
if (!row) {
|
|
@@ -1157,7 +1251,7 @@ function fetchScopedHits(db, queryVec, k, maxDistance, rootDir) {
|
|
|
1157
1251
|
return { hits, candidateCount: rows.length };
|
|
1158
1252
|
}
|
|
1159
1253
|
function fetchScopedExactHits(db, queryVec, limit, maxDistance, rootDir) {
|
|
1160
|
-
const scope =
|
|
1254
|
+
const scope = ownProjectScope("c.file_path", rootDir);
|
|
1161
1255
|
const rows = db.prepare(
|
|
1162
1256
|
`SELECT file_path, start_line, end_line, text, kind, distance FROM (
|
|
1163
1257
|
SELECT c.id AS id, c.file_path AS file_path, c.start_line AS start_line, c.end_line AS end_line, c.text AS text, c.kind AS kind, vec_distance_L2(v.embedding, ?) AS distance
|
|
@@ -1167,7 +1261,7 @@ function fetchScopedExactHits(db, queryVec, limit, maxDistance, rootDir) {
|
|
|
1167
1261
|
WHERE distance IS NOT NULL AND distance <= ?
|
|
1168
1262
|
ORDER BY distance ASC, file_path ASC, start_line ASC, id ASC
|
|
1169
1263
|
LIMIT ?`
|
|
1170
|
-
).all(packVec(queryVec), ...scope.params
|
|
1264
|
+
).all(packVec(queryVec), ...scope.params, maxDistance, limit);
|
|
1171
1265
|
const hits = [];
|
|
1172
1266
|
for (const row of rows) {
|
|
1173
1267
|
if (!row || typeof row.distance !== "number" || !Number.isFinite(row.distance) || row.distance > maxDistance) {
|
|
@@ -1244,82 +1338,14 @@ function rerankHits(hits, query, topK) {
|
|
|
1244
1338
|
scored.sort((a, b) => a.adjusted - b.adjusted || a.index - b.index);
|
|
1245
1339
|
return scored.slice(0, topK).map((entry) => ({ ...entry.hit, adjustedDistance: entry.adjusted }));
|
|
1246
1340
|
}
|
|
1247
|
-
function
|
|
1248
|
-
if (hits.length <= 1) {
|
|
1249
|
-
return hits;
|
|
1250
|
-
}
|
|
1251
|
-
const byFile = /* @__PURE__ */ new Map();
|
|
1252
|
-
for (const hit of hits) {
|
|
1253
|
-
const fileHits = byFile.get(hit.filePath);
|
|
1254
|
-
if (fileHits) {
|
|
1255
|
-
fileHits.push(hit);
|
|
1256
|
-
} else {
|
|
1257
|
-
byFile.set(hit.filePath, [hit]);
|
|
1258
|
-
}
|
|
1259
|
-
}
|
|
1260
|
-
const merged = [];
|
|
1261
|
-
for (const fileHits of byFile.values()) {
|
|
1262
|
-
fileHits.sort((a, b) => a.startLine - b.startLine);
|
|
1263
|
-
let current = fileHits[0];
|
|
1264
|
-
if (!current) {
|
|
1265
|
-
continue;
|
|
1266
|
-
}
|
|
1267
|
-
let curStart = current.startLine;
|
|
1268
|
-
let curEnd = current.endLine;
|
|
1269
|
-
let curDist = current.distance;
|
|
1270
|
-
let curAdjusted = current.adjustedDistance ?? current.distance;
|
|
1271
|
-
const mergedTexts = [current.text];
|
|
1272
|
-
for (let i = 1; i < fileHits.length; i++) {
|
|
1273
|
-
const hit = fileHits[i];
|
|
1274
|
-
if (!hit) {
|
|
1275
|
-
continue;
|
|
1276
|
-
}
|
|
1277
|
-
const gap = hit.startLine - curEnd - 1;
|
|
1278
|
-
if (gap <= proximity) {
|
|
1279
|
-
curEnd = Math.max(curEnd, hit.endLine);
|
|
1280
|
-
curDist = Math.min(curDist, hit.distance);
|
|
1281
|
-
curAdjusted = Math.min(curAdjusted, hit.adjustedDistance ?? hit.distance);
|
|
1282
|
-
mergedTexts.push(hit.text);
|
|
1283
|
-
} else {
|
|
1284
|
-
merged.push({
|
|
1285
|
-
filePath: current.filePath,
|
|
1286
|
-
startLine: curStart,
|
|
1287
|
-
endLine: curEnd,
|
|
1288
|
-
kind: current.kind,
|
|
1289
|
-
distance: curDist,
|
|
1290
|
-
adjustedDistance: curAdjusted,
|
|
1291
|
-
text: mergedTexts.join("\n---\n")
|
|
1292
|
-
});
|
|
1293
|
-
current = hit;
|
|
1294
|
-
curStart = hit.startLine;
|
|
1295
|
-
curEnd = hit.endLine;
|
|
1296
|
-
curDist = hit.distance;
|
|
1297
|
-
curAdjusted = hit.adjustedDistance ?? hit.distance;
|
|
1298
|
-
mergedTexts.length = 0;
|
|
1299
|
-
mergedTexts.push(hit.text);
|
|
1300
|
-
}
|
|
1301
|
-
}
|
|
1302
|
-
merged.push({
|
|
1303
|
-
filePath: current.filePath,
|
|
1304
|
-
startLine: curStart,
|
|
1305
|
-
endLine: curEnd,
|
|
1306
|
-
kind: current.kind,
|
|
1307
|
-
distance: curDist,
|
|
1308
|
-
adjustedDistance: curAdjusted,
|
|
1309
|
-
text: mergedTexts.join("\n---\n")
|
|
1310
|
-
});
|
|
1311
|
-
}
|
|
1312
|
-
merged.sort((a, b) => (a.adjustedDistance ?? a.distance) - (b.adjustedDistance ?? b.distance));
|
|
1313
|
-
return merged;
|
|
1314
|
-
}
|
|
1315
|
-
async function indexFile(db, filePath, content, boundaries = []) {
|
|
1341
|
+
async function indexFile(db, filePath, content, boundaries = [], expectedSha) {
|
|
1316
1342
|
let chunks = chunkFile(filePath, content, void 0, void 0, boundaries);
|
|
1317
1343
|
if (chunks.length > 0) {
|
|
1318
1344
|
if (embeddingsDepsAvailable(db)) {
|
|
1319
1345
|
const { countTokens } = await getExtractor(DEFAULT_MODEL);
|
|
1320
1346
|
if (countTokens !== void 0) chunks = chunkFile(filePath, content, void 0, void 0, boundaries, countTokens);
|
|
1321
1347
|
}
|
|
1322
|
-
return upsertChunks(db, chunks);
|
|
1348
|
+
return upsertChunks(db, chunks, expectedSha);
|
|
1323
1349
|
}
|
|
1324
1350
|
deleteFileEmbeddings(db, filePath);
|
|
1325
1351
|
return "embedded";
|
|
@@ -1402,7 +1428,7 @@ function resetEmbeddingsForKinds(db, kinds) {
|
|
|
1402
1428
|
return paths.length;
|
|
1403
1429
|
}
|
|
1404
1430
|
function backendId() {
|
|
1405
|
-
return
|
|
1431
|
+
return `${activeRuntime()}@${majorMinor(runtimeVersion())}`;
|
|
1406
1432
|
}
|
|
1407
1433
|
function majorMinor(version) {
|
|
1408
1434
|
const parts = version.split(".");
|
|
@@ -1415,15 +1441,21 @@ function vectorSpaceOf(provenance) {
|
|
|
1415
1441
|
}
|
|
1416
1442
|
function ensureEmbeddingProvenance(db, modelName = DEFAULT_MODEL) {
|
|
1417
1443
|
if (_provenanceChecked.has(db)) return;
|
|
1418
|
-
_provenanceChecked.add(db);
|
|
1419
1444
|
const current = embeddingProvenance(modelName);
|
|
1420
1445
|
const stored = db.prepare("SELECT provenance FROM embedding_provenance WHERE id = 1").pluck().get();
|
|
1421
|
-
if (stored === current)
|
|
1446
|
+
if (stored === current) {
|
|
1447
|
+
_provenanceChecked.add(db);
|
|
1448
|
+
return;
|
|
1449
|
+
}
|
|
1422
1450
|
const keepVectors = stored !== void 0 && vectorSpaceOf(stored) === vectorSpaceOf(current);
|
|
1423
|
-
const cleared =
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1451
|
+
const cleared = db.transaction(() => {
|
|
1452
|
+
const count = stored !== void 0 && keepVectors ? resetStaleChunking(db, stored, current) : resetAllEmbeddings(db, false);
|
|
1453
|
+
db.prepare(
|
|
1454
|
+
"INSERT INTO embedding_provenance (id, provenance) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET provenance = excluded.provenance"
|
|
1455
|
+
).run(current);
|
|
1456
|
+
return count;
|
|
1457
|
+
}).immediate();
|
|
1458
|
+
_provenanceChecked.add(db);
|
|
1427
1459
|
if (cleared > 0 && !keepVectors) {
|
|
1428
1460
|
console.warn(
|
|
1429
1461
|
`Embedding stack changed (${stored ?? "unrecorded"} -> ${current}); discarded ${cleared} file${cleared === 1 ? "" : "s"} worth of vectors because they no longer reliably describe this file's current chunks. Run \`token-goat index\` to rebuild them.`
|
|
@@ -1465,65 +1497,110 @@ function _pathPriorityPenalty(filePath) {
|
|
|
1465
1497
|
return 0;
|
|
1466
1498
|
}
|
|
1467
1499
|
|
|
1500
|
+
// src/skip_scope.ts
|
|
1501
|
+
init_define_import_meta_env();
|
|
1502
|
+
import * as os from "node:os";
|
|
1503
|
+
import * as path6 from "node:path";
|
|
1504
|
+
var declaredRoot;
|
|
1505
|
+
function withDeclaredSkipScopeRoot(root, fn) {
|
|
1506
|
+
const previous = declaredRoot;
|
|
1507
|
+
declaredRoot = root;
|
|
1508
|
+
try {
|
|
1509
|
+
return fn();
|
|
1510
|
+
} finally {
|
|
1511
|
+
declaredRoot = previous;
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
function pathSegments(p) {
|
|
1515
|
+
return p.split(/[/\\]/).filter((s) => s.length > 0);
|
|
1516
|
+
}
|
|
1517
|
+
function outermostProjectRoot(filePath) {
|
|
1518
|
+
const first = findProject(path6.dirname(filePath));
|
|
1519
|
+
if (first === null) return void 0;
|
|
1520
|
+
const home = foldPath(os.homedir()).replace(/\\/g, "/");
|
|
1521
|
+
let root = first.root;
|
|
1522
|
+
for (; ; ) {
|
|
1523
|
+
const parent = path6.dirname(root);
|
|
1524
|
+
if (parent === root) return root;
|
|
1525
|
+
const outer = findProject(parent);
|
|
1526
|
+
if (outer === null || outer.root === root) return root;
|
|
1527
|
+
const outerFolded = foldPath(outer.root).replace(/\\/g, "/").replace(/\/+$/, "");
|
|
1528
|
+
if (home === outerFolded || home.startsWith(`${outerFolded}/`)) return root;
|
|
1529
|
+
root = outer.root;
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
function hasSkipSegmentBelowRoot(filePath, projectRoot, isSkipSegment) {
|
|
1533
|
+
const dirSegments = pathSegments(filePath).slice(0, -1);
|
|
1534
|
+
if (!dirSegments.some(isSkipSegment)) return false;
|
|
1535
|
+
const root = projectRoot ?? declaredRoot ?? outermostProjectRoot(filePath);
|
|
1536
|
+
if (root === void 0) return true;
|
|
1537
|
+
const rootSegments = pathSegments(root);
|
|
1538
|
+
if (rootSegments.length > dirSegments.length) return true;
|
|
1539
|
+
for (let i = 0; i < rootSegments.length; i++) {
|
|
1540
|
+
if (foldPath(rootSegments[i] ?? "") !== foldPath(dirSegments[i] ?? "")) return true;
|
|
1541
|
+
}
|
|
1542
|
+
return dirSegments.slice(rootSegments.length).some(isSkipSegment);
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1468
1545
|
// src/parser_stamp.ts
|
|
1469
1546
|
init_define_import_meta_env();
|
|
1470
1547
|
|
|
1471
1548
|
// src/parser_fingerprint.ts
|
|
1472
1549
|
init_define_import_meta_env();
|
|
1473
|
-
var PARSER_FINGERPRINT = "
|
|
1550
|
+
var PARSER_FINGERPRINT = "c8a3aa41909be8c0";
|
|
1474
1551
|
var LANGUAGE_PARSER_FINGERPRINTS = /* @__PURE__ */ new Map([
|
|
1475
|
-
["abap", "
|
|
1476
|
-
["abl", "
|
|
1477
|
-
["apache", "
|
|
1478
|
-
["apex", "
|
|
1479
|
-
["asm", "
|
|
1480
|
-
["bash", "
|
|
1481
|
-
["batch", "
|
|
1482
|
-
["caddy", "
|
|
1483
|
-
["clojure", "
|
|
1484
|
-
["cmake", "
|
|
1485
|
-
["common_lisp", "
|
|
1486
|
-
["csharp", "
|
|
1487
|
-
["dart", "
|
|
1488
|
-
["elixir", "
|
|
1489
|
-
["emacs_lisp", "
|
|
1490
|
-
["erlang", "
|
|
1491
|
-
["fortran", "
|
|
1492
|
-
["fsharp", "
|
|
1493
|
-
["graphql", "
|
|
1494
|
-
["groovy", "
|
|
1495
|
-
["haskell", "
|
|
1496
|
-
["jcl", "
|
|
1497
|
-
["kotlin", "
|
|
1498
|
-
["liquid", "
|
|
1499
|
-
["lua", "
|
|
1500
|
-
["makefile", "
|
|
1501
|
-
["matlab", "
|
|
1502
|
-
["nginx", "
|
|
1503
|
-
["nix", "
|
|
1504
|
-
["objc", "
|
|
1505
|
-
["ocaml", "
|
|
1506
|
-
["pascal", "
|
|
1507
|
-
["perl", "
|
|
1508
|
-
["php", "
|
|
1509
|
-
["pli", "
|
|
1510
|
-
["powershell", "
|
|
1511
|
-
["proto", "
|
|
1512
|
-
["r", "
|
|
1513
|
-
["racket", "
|
|
1514
|
-
["rpg", "
|
|
1515
|
-
["salesforce_metadata", "
|
|
1516
|
-
["sas", "
|
|
1517
|
-
["scala", "
|
|
1518
|
-
["scheme", "
|
|
1519
|
-
["solidity", "
|
|
1520
|
-
["sql", "
|
|
1521
|
-
["swift", "
|
|
1522
|
-
["terraform", "
|
|
1523
|
-
["thrift", "
|
|
1524
|
-
["vb", "
|
|
1525
|
-
["vhdl", "
|
|
1526
|
-
["zig", "
|
|
1552
|
+
["abap", "accec34183d6fc45"],
|
|
1553
|
+
["abl", "b721e43e5720a56d"],
|
|
1554
|
+
["apache", "6e840e4ac38d88c1"],
|
|
1555
|
+
["apex", "6e621f41fce0d396"],
|
|
1556
|
+
["asm", "845db656efdc0fca"],
|
|
1557
|
+
["bash", "656e307df22168bb"],
|
|
1558
|
+
["batch", "d20823bbe2a8eec4"],
|
|
1559
|
+
["caddy", "b0f8fdf4ea685095"],
|
|
1560
|
+
["clojure", "85b583da30a1a44e"],
|
|
1561
|
+
["cmake", "1e0b257e06742425"],
|
|
1562
|
+
["common_lisp", "b6c3bc62b3a255a5"],
|
|
1563
|
+
["csharp", "68d9c4bb7fb18e1c"],
|
|
1564
|
+
["dart", "40a3921e9143f0f4"],
|
|
1565
|
+
["elixir", "719739d859fb5a25"],
|
|
1566
|
+
["emacs_lisp", "463153969879ead0"],
|
|
1567
|
+
["erlang", "3aaebe53622499d3"],
|
|
1568
|
+
["fortran", "7ecf0ab7c3f52b6f"],
|
|
1569
|
+
["fsharp", "b9c219bd5fbdc8b5"],
|
|
1570
|
+
["graphql", "34c5923656df5406"],
|
|
1571
|
+
["groovy", "780274c5b5c7d79a"],
|
|
1572
|
+
["haskell", "942d08d94feba789"],
|
|
1573
|
+
["jcl", "d0868f4420b941ea"],
|
|
1574
|
+
["kotlin", "df0ce105cba0fc0b"],
|
|
1575
|
+
["liquid", "436055b0a6b64ceb"],
|
|
1576
|
+
["lua", "ded20290e6d00d06"],
|
|
1577
|
+
["makefile", "dbbd6b32d5cc62eb"],
|
|
1578
|
+
["matlab", "d8a2f8c83f070614"],
|
|
1579
|
+
["nginx", "63248e67cc521def"],
|
|
1580
|
+
["nix", "056a50108db861a6"],
|
|
1581
|
+
["objc", "c21c0d7c13775c0f"],
|
|
1582
|
+
["ocaml", "22bf5a0eaa2b5f80"],
|
|
1583
|
+
["pascal", "24579492a16ad7e4"],
|
|
1584
|
+
["perl", "bd8ac6c679b29abe"],
|
|
1585
|
+
["php", "b823c3df1138d2b7"],
|
|
1586
|
+
["pli", "38f7027519e34e39"],
|
|
1587
|
+
["powershell", "70dd31bfbcf69a68"],
|
|
1588
|
+
["proto", "9c42c66e3fee093f"],
|
|
1589
|
+
["r", "5db6f150582b5550"],
|
|
1590
|
+
["racket", "e900071a3d70d0b3"],
|
|
1591
|
+
["rpg", "5839e0f9c099c11b"],
|
|
1592
|
+
["salesforce_metadata", "a4408c65eb712933"],
|
|
1593
|
+
["sas", "7e8a6772363c76a1"],
|
|
1594
|
+
["scala", "72c64544a81c0c98"],
|
|
1595
|
+
["scheme", "592b9b37b5494f0e"],
|
|
1596
|
+
["solidity", "3f0947056f925fad"],
|
|
1597
|
+
["sql", "5d423d87e7a0c2b3"],
|
|
1598
|
+
["swift", "3c7306bc8c96279d"],
|
|
1599
|
+
["terraform", "ead711fa99e96a85"],
|
|
1600
|
+
["thrift", "a773492ef26655c7"],
|
|
1601
|
+
["vb", "97ade1ad9cd06689"],
|
|
1602
|
+
["vhdl", "cd0e1ee6c5da9bd1"],
|
|
1603
|
+
["zig", "9034a79cede0b52f"]
|
|
1527
1604
|
]);
|
|
1528
1605
|
|
|
1529
1606
|
// src/parser_stamp.ts
|
|
@@ -1533,13 +1610,13 @@ function parserFingerprintForLanguage(language) {
|
|
|
1533
1610
|
|
|
1534
1611
|
// src/parser.ts
|
|
1535
1612
|
init_define_import_meta_env();
|
|
1536
|
-
import * as
|
|
1613
|
+
import * as fs4 from "node:fs";
|
|
1537
1614
|
import { createRequire as createRequire2 } from "node:module";
|
|
1538
|
-
import * as
|
|
1615
|
+
import * as path9 from "node:path";
|
|
1539
1616
|
|
|
1540
1617
|
// src/asset_extensions.ts
|
|
1541
1618
|
init_define_import_meta_env();
|
|
1542
|
-
import * as
|
|
1619
|
+
import * as path7 from "node:path";
|
|
1543
1620
|
var IMAGE_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
1544
1621
|
".png",
|
|
1545
1622
|
".jpg",
|
|
@@ -1554,7 +1631,7 @@ var IMAGE_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
|
1554
1631
|
".heif"
|
|
1555
1632
|
]);
|
|
1556
1633
|
function isImagePath(p) {
|
|
1557
|
-
return IMAGE_EXTENSIONS.has(
|
|
1634
|
+
return IMAGE_EXTENSIONS.has(path7.extname(p).toLowerCase());
|
|
1558
1635
|
}
|
|
1559
1636
|
var NON_TEXT_ASSET_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
1560
1637
|
...IMAGE_EXTENSIONS,
|
|
@@ -1602,7 +1679,7 @@ var NON_TEXT_ASSET_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
|
1602
1679
|
".sqlite3"
|
|
1603
1680
|
]);
|
|
1604
1681
|
function isNonTextAsset(p) {
|
|
1605
|
-
return NON_TEXT_ASSET_EXTENSIONS.has(
|
|
1682
|
+
return NON_TEXT_ASSET_EXTENSIONS.has(path7.extname(p).toLowerCase());
|
|
1606
1683
|
}
|
|
1607
1684
|
var NON_TEXT_ASSET_SET_ID = (() => {
|
|
1608
1685
|
const joined = [...NON_TEXT_ASSET_EXTENSIONS].sort().join(",");
|
|
@@ -1624,9 +1701,14 @@ init_define_import_meta_env();
|
|
|
1624
1701
|
init_define_import_meta_env();
|
|
1625
1702
|
function* eachUnfencedLine(lines) {
|
|
1626
1703
|
let fence = null;
|
|
1704
|
+
let inComment = false;
|
|
1627
1705
|
for (let i = 0; i < lines.length; i++) {
|
|
1628
1706
|
const line = lines[i];
|
|
1629
1707
|
if (line === void 0) continue;
|
|
1708
|
+
if (inComment) {
|
|
1709
|
+
if (line.includes("-->")) inComment = false;
|
|
1710
|
+
continue;
|
|
1711
|
+
}
|
|
1630
1712
|
const fm = /^\s*(`{3,}|~{3,})([^\n]*)$/.exec(line);
|
|
1631
1713
|
if (fm !== null && fm[1] !== void 0) {
|
|
1632
1714
|
const run = fm[1];
|
|
@@ -1640,10 +1722,28 @@ function* eachUnfencedLine(lines) {
|
|
|
1640
1722
|
continue;
|
|
1641
1723
|
}
|
|
1642
1724
|
if (fence !== null) continue;
|
|
1725
|
+
if (/^ {0,3}<!--/.test(line)) {
|
|
1726
|
+
inComment = !line.includes("-->");
|
|
1727
|
+
continue;
|
|
1728
|
+
}
|
|
1643
1729
|
yield [i, line];
|
|
1644
1730
|
}
|
|
1645
1731
|
}
|
|
1646
1732
|
|
|
1733
|
+
// src/markdown_frontmatter.ts
|
|
1734
|
+
init_define_import_meta_env();
|
|
1735
|
+
function frontMatterEndIndex(lines) {
|
|
1736
|
+
const first = lines[0];
|
|
1737
|
+
if (first === void 0) return 0;
|
|
1738
|
+
const open = first.replace(/^\uFEFF/, "").trimEnd();
|
|
1739
|
+
if (open !== "---" && open !== "+++") return 0;
|
|
1740
|
+
for (let i = 1; i < lines.length; i++) {
|
|
1741
|
+
const t = (lines[i] ?? "").trimEnd();
|
|
1742
|
+
if (t === open || open === "---" && t === "...") return i + 1;
|
|
1743
|
+
}
|
|
1744
|
+
return 0;
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1647
1747
|
// src/hints/markdown_hints.ts
|
|
1648
1748
|
var MARKDOWN_SIZE_THRESHOLD = 4e3;
|
|
1649
1749
|
var MAX_HEADINGS = 40;
|
|
@@ -1651,47 +1751,44 @@ var MAX_OUTPUT_LINES = 60;
|
|
|
1651
1751
|
function extractMarkdownHeadings(content, limit = MAX_HEADINGS) {
|
|
1652
1752
|
const headings = [];
|
|
1653
1753
|
const lines = content.split("\n");
|
|
1654
|
-
const
|
|
1754
|
+
const fmEnd = frontMatterEndIndex(lines);
|
|
1755
|
+
const unfenced = Array.from(eachUnfencedLine(lines)).filter(([i]) => i >= fmEnd);
|
|
1756
|
+
let paraStart = -1;
|
|
1655
1757
|
for (let u = 0; u < unfenced.length; u++) {
|
|
1656
1758
|
const [i, line] = unfenced[u];
|
|
1657
|
-
if (!line)
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1759
|
+
if (!line) {
|
|
1760
|
+
paraStart = -1;
|
|
1761
|
+
continue;
|
|
1762
|
+
}
|
|
1763
|
+
const atx = matchAtxHeading(line, ATX_CLASS_BREAK_RE, true);
|
|
1764
|
+
if (atx !== null) {
|
|
1663
1765
|
const maxLevel = limit === Infinity ? 6 : 3;
|
|
1664
|
-
if (level <= maxLevel) {
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
});
|
|
1672
|
-
if (headings.length >= limit) break;
|
|
1673
|
-
}
|
|
1766
|
+
if (atx.level <= maxLevel && atx.name) {
|
|
1767
|
+
headings.push({
|
|
1768
|
+
level: atx.level,
|
|
1769
|
+
text: atx.name,
|
|
1770
|
+
lineNumber: i + 1
|
|
1771
|
+
});
|
|
1772
|
+
if (headings.length >= limit) break;
|
|
1674
1773
|
}
|
|
1774
|
+
paraStart = -1;
|
|
1675
1775
|
continue;
|
|
1676
1776
|
}
|
|
1777
|
+
if (paraStart !== -1 && i !== unfenced[u - 1][0] + 1) paraStart = -1;
|
|
1677
1778
|
const trimmed = line.trim();
|
|
1678
|
-
if (
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
text: trimmed,
|
|
1688
|
-
lineNumber: i + 1
|
|
1689
|
-
});
|
|
1690
|
-
u++;
|
|
1691
|
-
if (headings.length >= limit) break;
|
|
1692
|
-
}
|
|
1693
|
-
}
|
|
1779
|
+
if (paraStart !== -1 && /^(=+|-+)$/.test(trimmed)) {
|
|
1780
|
+
headings.push({
|
|
1781
|
+
level: trimmed.startsWith("=") ? 1 : 2,
|
|
1782
|
+
text: unfenced.slice(paraStart, u).map(([, text]) => text.trim()).join(" "),
|
|
1783
|
+
lineNumber: unfenced[paraStart][0] + 1
|
|
1784
|
+
});
|
|
1785
|
+
paraStart = -1;
|
|
1786
|
+
if (headings.length >= limit) break;
|
|
1787
|
+
continue;
|
|
1694
1788
|
}
|
|
1789
|
+
const isText = trimmed !== "" && !trimmed.startsWith("#") && !trimmed.startsWith("|") && !trimmed.startsWith("```") && !trimmed.startsWith("~~~") && !/^([-*+]|\d+\.)\s/.test(trimmed) && !/^(=+|-+)$/.test(trimmed);
|
|
1790
|
+
if (!isText) paraStart = -1;
|
|
1791
|
+
else if (paraStart === -1) paraStart = u;
|
|
1695
1792
|
}
|
|
1696
1793
|
return headings;
|
|
1697
1794
|
}
|
|
@@ -1779,11 +1876,11 @@ function buildEmbeddingBoundaries(filePath, content, dbPath) {
|
|
|
1779
1876
|
|
|
1780
1877
|
// src/languages/salesforce_frontend.ts
|
|
1781
1878
|
init_define_import_meta_env();
|
|
1782
|
-
import * as
|
|
1879
|
+
import * as path8 from "node:path";
|
|
1783
1880
|
function bundleName(filePath) {
|
|
1784
1881
|
const normalized = filePath.replaceAll("\\", "/");
|
|
1785
|
-
const parent =
|
|
1786
|
-
const base =
|
|
1882
|
+
const parent = path8.posix.basename(path8.posix.dirname(normalized));
|
|
1883
|
+
const base = path8.posix.basename(normalized).replace(/\.[^.]+$/, "");
|
|
1787
1884
|
return parent === "lwc" || parent === "aura" ? base : parent;
|
|
1788
1885
|
}
|
|
1789
1886
|
function lwcTagAlias(name) {
|
|
@@ -1881,7 +1978,7 @@ var MARKUP_KIND = {
|
|
|
1881
1978
|
};
|
|
1882
1979
|
function markupArtifactName(filePath, extension) {
|
|
1883
1980
|
if (extension === ".page" || extension === ".component" || extension === ".email") {
|
|
1884
|
-
return
|
|
1981
|
+
return path8.posix.basename(filePath.replaceAll("\\", "/")).replace(new RegExp(`${extension.replace(".", "\\.")}$`, "i"), "");
|
|
1885
1982
|
}
|
|
1886
1983
|
return bundleName(filePath);
|
|
1887
1984
|
}
|
|
@@ -1901,7 +1998,7 @@ function attributeRefs(refs, content, lineIndex, filePath, attribute, split = fa
|
|
|
1901
1998
|
}
|
|
1902
1999
|
function extractSalesforceMarkup(content, filePath) {
|
|
1903
2000
|
const normalized = filePath.replaceAll("\\", "/");
|
|
1904
|
-
const extension =
|
|
2001
|
+
const extension = path8.posix.extname(normalized).toLowerCase();
|
|
1905
2002
|
const kind = MARKUP_KIND[extension] ?? "salesforce_markup";
|
|
1906
2003
|
const symbols = [
|
|
1907
2004
|
symbol(filePath, markupArtifactName(normalized, extension), kind, 1, countContentLines(content))
|
|
@@ -2153,6 +2250,39 @@ function isModuleWrapperCallback(node) {
|
|
|
2153
2250
|
}
|
|
2154
2251
|
return false;
|
|
2155
2252
|
}
|
|
2253
|
+
var TSJS_FN_VALUE_TYPES = /* @__PURE__ */ new Set(["arrow_function", "function_expression", "function"]);
|
|
2254
|
+
function isModuleExports(node) {
|
|
2255
|
+
if (node.type !== "member_expression") return false;
|
|
2256
|
+
const obj = node.childForFieldName("object");
|
|
2257
|
+
const prop = node.childForFieldName("property");
|
|
2258
|
+
return obj !== null && prop !== null && obj.type === "identifier" && obj.text === "module" && prop.text === "exports";
|
|
2259
|
+
}
|
|
2260
|
+
function commonJsAssignmentSymbol(stmt, filePath, lines) {
|
|
2261
|
+
const assign = stmt.namedChildren[0];
|
|
2262
|
+
if (assign === void 0 || assign.type !== "assignment_expression") return null;
|
|
2263
|
+
const left = assign.childForFieldName("left");
|
|
2264
|
+
const right = assign.childForFieldName("right");
|
|
2265
|
+
if (left === null || right === null || left.type !== "member_expression") return null;
|
|
2266
|
+
const isFn = TSJS_FN_VALUE_TYPES.has(right.type);
|
|
2267
|
+
if (isModuleExports(left)) {
|
|
2268
|
+
const own = right.type === "function_expression" || right.type === "class" ? right.childForFieldName("name") : null;
|
|
2269
|
+
if (own === null || own.text === "") return null;
|
|
2270
|
+
return makeSymbol(filePath, own.text, right.type === "class" ? "class" : "function", stmt, lines, "c");
|
|
2271
|
+
}
|
|
2272
|
+
if (!isFn) return null;
|
|
2273
|
+
const obj = left.childForFieldName("object");
|
|
2274
|
+
const prop = left.childForFieldName("property");
|
|
2275
|
+
if (obj === null || prop === null || prop.type !== "property_identifier") return null;
|
|
2276
|
+
if (obj.type === "identifier" && obj.text === "exports") return makeSymbol(filePath, prop.text, "function", stmt, lines, "c");
|
|
2277
|
+
if (isModuleExports(obj)) return makeSymbol(filePath, prop.text, "function", stmt, lines, "c");
|
|
2278
|
+
if (obj.type === "member_expression" && obj.childForFieldName("property")?.text === "prototype") {
|
|
2279
|
+
const owner = obj.childForFieldName("object");
|
|
2280
|
+
if (owner === null) return null;
|
|
2281
|
+
const sym = makeSymbol(filePath, prop.text, "method", stmt, lines, "c");
|
|
2282
|
+
return { ...sym, parent: owner.text };
|
|
2283
|
+
}
|
|
2284
|
+
return null;
|
|
2285
|
+
}
|
|
2156
2286
|
function extractTsJsSymbols(root, filePath, lines) {
|
|
2157
2287
|
const out = [];
|
|
2158
2288
|
const visit = (node, insideFunction) => {
|
|
@@ -2209,6 +2339,10 @@ function extractTsJsSymbols(root, filePath, lines) {
|
|
|
2209
2339
|
}
|
|
2210
2340
|
}
|
|
2211
2341
|
}
|
|
2342
|
+
if (!insideFunction && node.type === "expression_statement") {
|
|
2343
|
+
const assigned = commonJsAssignmentSymbol(node, filePath, lines);
|
|
2344
|
+
if (assigned !== null) out.push(assigned);
|
|
2345
|
+
}
|
|
2212
2346
|
if (node.type === "public_field_definition" || node.type === "field_definition") {
|
|
2213
2347
|
const fieldName = node.childForFieldName("name") ?? node.childForFieldName("property");
|
|
2214
2348
|
const value = node.childForFieldName("value");
|
|
@@ -2338,6 +2472,12 @@ var GO_LOCAL_KINDS = /* @__PURE__ */ new Set([
|
|
|
2338
2472
|
"method_elem"
|
|
2339
2473
|
]);
|
|
2340
2474
|
var GO_MULTI_NAME_SPECS = /* @__PURE__ */ new Set(["var_spec", "const_spec"]);
|
|
2475
|
+
function goReceiverType(node) {
|
|
2476
|
+
const receiver = node.childForFieldName("receiver");
|
|
2477
|
+
const param = receiver?.namedChildren.find((c) => c.type === "parameter_declaration");
|
|
2478
|
+
const typeText = param?.childForFieldName("type")?.text ?? "";
|
|
2479
|
+
return typeText.replace(/\[[^\]]*\]/g, "").replace(/[*()\s]/g, "");
|
|
2480
|
+
}
|
|
2341
2481
|
function extractGoSymbols(root, filePath, lines) {
|
|
2342
2482
|
const out = [];
|
|
2343
2483
|
const visit = (node, insideFunction) => {
|
|
@@ -2355,7 +2495,9 @@ function extractGoSymbols(root, filePath, lines) {
|
|
|
2355
2495
|
} else {
|
|
2356
2496
|
const name = nodeName(node);
|
|
2357
2497
|
if (name !== null && name !== "") {
|
|
2358
|
-
|
|
2498
|
+
const sym = makeSymbol(filePath, name, kind, node, lines, "c");
|
|
2499
|
+
const receiverType = node.type === "method_declaration" ? goReceiverType(node) : "";
|
|
2500
|
+
out.push(receiverType !== "" ? { ...sym, parent: receiverType } : sym);
|
|
2359
2501
|
}
|
|
2360
2502
|
}
|
|
2361
2503
|
}
|
|
@@ -2446,12 +2588,18 @@ function leadingRustAttributes(node) {
|
|
|
2446
2588
|
}
|
|
2447
2589
|
return attrs;
|
|
2448
2590
|
}
|
|
2591
|
+
function rustImplTypeName(type) {
|
|
2592
|
+
if (type === null) return null;
|
|
2593
|
+
const inner = type.type === "generic_type" || type.type === "reference_type" ? type.childForFieldName("type") : null;
|
|
2594
|
+
if (inner !== null) return rustImplTypeName(inner);
|
|
2595
|
+
return type.type === "scoped_type_identifier" ? type.childForFieldName("name")?.text ?? type.text : type.text;
|
|
2596
|
+
}
|
|
2449
2597
|
function extractRustSymbols(root, filePath, lines) {
|
|
2450
2598
|
const out = [];
|
|
2451
2599
|
const visit = (node, insideFunction) => {
|
|
2452
2600
|
const kind = RUST_KIND_BY_TYPE.get(node.type);
|
|
2453
2601
|
if (kind !== void 0 && !(insideFunction && RUST_LOCAL_KINDS.has(node.type))) {
|
|
2454
|
-
const name = node.type === "impl_item" ? node.childForFieldName("type")
|
|
2602
|
+
const name = node.type === "impl_item" ? rustImplTypeName(node.childForFieldName("type")) : node.type === "foreign_mod_item" ? node.namedChildren.find((c) => c.type === "extern_modifier")?.text ?? "extern" : nodeName(node);
|
|
2455
2603
|
if (name !== null && name !== "") {
|
|
2456
2604
|
const attrs = leadingRustAttributes(node);
|
|
2457
2605
|
if (attrs.length === 0) {
|
|
@@ -2485,14 +2633,16 @@ var RUBY_KIND_BY_TYPE = /* @__PURE__ */ new Map([
|
|
|
2485
2633
|
["class", "class"],
|
|
2486
2634
|
["module", "module"]
|
|
2487
2635
|
]);
|
|
2488
|
-
function extractSimpleSymbols(root, filePath, kindByType, lines, style, nameFor = nodeName) {
|
|
2636
|
+
function extractSimpleSymbols(root, filePath, kindByType, lines, style, nameFor = nodeName, parentFor) {
|
|
2489
2637
|
const out = [];
|
|
2490
2638
|
const visit = (node) => {
|
|
2491
2639
|
const kind = kindByType.get(node.type);
|
|
2492
2640
|
if (kind !== void 0) {
|
|
2493
2641
|
const name = nameFor(node);
|
|
2494
2642
|
if (name !== null && name !== "") {
|
|
2495
|
-
|
|
2643
|
+
const sym = makeSymbol(filePath, name, kind, node, lines, style);
|
|
2644
|
+
const parent = parentFor === void 0 ? "" : parentFor(node);
|
|
2645
|
+
out.push(parent !== "" ? { ...sym, parent } : sym);
|
|
2496
2646
|
}
|
|
2497
2647
|
}
|
|
2498
2648
|
for (const child of node.namedChildren) {
|
|
@@ -2582,21 +2732,46 @@ function extractCppSymbols(root, filePath, lines) {
|
|
|
2582
2732
|
CPP_KIND_BY_TYPE,
|
|
2583
2733
|
lines,
|
|
2584
2734
|
"c",
|
|
2585
|
-
(node) => node.type === "function_definition" ? cFunctionName(node) : node.type === "type_definition" ? cTypedefAliasName(node) : node.type === "declaration" ? cFunctionPrototypeName(node) : nodeName(node)
|
|
2735
|
+
(node) => node.type === "function_definition" ? cFunctionName(node) : node.type === "type_definition" ? cTypedefAliasName(node) : node.type === "declaration" ? cFunctionPrototypeName(node) : C_TAG_SPECIFIER_TYPES.has(node.type) && node.childForFieldName("body") === null ? null : nodeName(node),
|
|
2736
|
+
cMethodParent
|
|
2586
2737
|
);
|
|
2587
2738
|
}
|
|
2739
|
+
var C_TAG_SPECIFIER_TYPES = /* @__PURE__ */ new Set(["struct_specifier", "class_specifier", "union_specifier", "enum_specifier"]);
|
|
2740
|
+
function cMethodParent(node) {
|
|
2741
|
+
if (node.type !== "function_definition" && node.type !== "declaration") return "";
|
|
2742
|
+
const body = node.parent;
|
|
2743
|
+
const owner = body?.parent ?? null;
|
|
2744
|
+
if (body !== null && body.type === "field_declaration_list" && owner !== null && (owner.type === "class_specifier" || owner.type === "struct_specifier" || owner.type === "union_specifier")) {
|
|
2745
|
+
return owner.childForFieldName("name")?.text ?? "";
|
|
2746
|
+
}
|
|
2747
|
+
let cur = node.childForFieldName("declarator");
|
|
2748
|
+
for (let i = 0; cur !== null && i < 16; i++) {
|
|
2749
|
+
if (cur.type === "qualified_identifier") {
|
|
2750
|
+
let inner = cur;
|
|
2751
|
+
for (let j = 0; j < 16; j++) {
|
|
2752
|
+
const next = inner.childForFieldName("name");
|
|
2753
|
+
if (next === null || next.type !== "qualified_identifier") break;
|
|
2754
|
+
inner = next;
|
|
2755
|
+
}
|
|
2756
|
+
const scope = inner.childForFieldName("scope");
|
|
2757
|
+
return scope === null ? "" : scope.text.replace(/<.*$/s, "").trim();
|
|
2758
|
+
}
|
|
2759
|
+
cur = innerDeclarator(cur);
|
|
2760
|
+
}
|
|
2761
|
+
return "";
|
|
2762
|
+
}
|
|
2588
2763
|
function cFunctionPrototypeName(node) {
|
|
2589
2764
|
let cur = node.childForFieldName("declarator");
|
|
2590
2765
|
for (let i = 0; cur !== null && i < 16; i++) {
|
|
2591
2766
|
if (cur.type === "function_declarator") {
|
|
2592
2767
|
const inner = cur.childForFieldName("declarator");
|
|
2593
2768
|
if (inner === null) return null;
|
|
2594
|
-
if (inner.type === "identifier" || inner.type === "field_identifier") return inner.text;
|
|
2769
|
+
if (inner.type === "identifier" || inner.type === "field_identifier" || inner.type === "destructor_name" || inner.type === "operator_name") return inner.text;
|
|
2595
2770
|
if (inner.type === "qualified_identifier") return lastSegment(inner.text);
|
|
2596
2771
|
return null;
|
|
2597
2772
|
}
|
|
2598
2773
|
if (cur.type === "pointer_declarator" || cur.type === "reference_declarator") {
|
|
2599
|
-
cur = cur
|
|
2774
|
+
cur = innerDeclarator(cur);
|
|
2600
2775
|
continue;
|
|
2601
2776
|
}
|
|
2602
2777
|
return null;
|
|
@@ -2616,12 +2791,18 @@ function lastSegment(text) {
|
|
|
2616
2791
|
const parts = text.split(/::|\./);
|
|
2617
2792
|
return parts[parts.length - 1] ?? text;
|
|
2618
2793
|
}
|
|
2794
|
+
function innerDeclarator(node) {
|
|
2795
|
+
const named = node.childForFieldName("declarator");
|
|
2796
|
+
if (named !== null) return named;
|
|
2797
|
+
return node.type === "reference_declarator" ? node.namedChildren[0] ?? null : null;
|
|
2798
|
+
}
|
|
2619
2799
|
function cFunctionName(node) {
|
|
2620
2800
|
let cur = node.childForFieldName("declarator");
|
|
2621
2801
|
for (let i = 0; cur !== null && i < 16; i++) {
|
|
2622
2802
|
if (cur.type === "identifier" || cur.type === "field_identifier") return cur.text;
|
|
2803
|
+
if (cur.type === "destructor_name" || cur.type === "operator_name") return cur.text;
|
|
2623
2804
|
if (cur.type === "qualified_identifier") return lastSegment(cur.text);
|
|
2624
|
-
cur = cur
|
|
2805
|
+
cur = innerDeclarator(cur);
|
|
2625
2806
|
}
|
|
2626
2807
|
return null;
|
|
2627
2808
|
}
|
|
@@ -2660,8 +2841,8 @@ var SCOPE_TYPES_BY_LANG = /* @__PURE__ */ new Map([
|
|
|
2660
2841
|
["ruby", /* @__PURE__ */ new Set(["method", "singleton_method", "class", "module"])]
|
|
2661
2842
|
]);
|
|
2662
2843
|
var CALL_TYPES_BY_LANG = /* @__PURE__ */ new Map([
|
|
2663
|
-
["typescript", /* @__PURE__ */ new Set(["call_expression", "new_expression"])],
|
|
2664
|
-
["javascript", /* @__PURE__ */ new Set(["call_expression", "new_expression"])],
|
|
2844
|
+
["typescript", /* @__PURE__ */ new Set(["call_expression", "new_expression", "jsx_self_closing_element", "jsx_opening_element"])],
|
|
2845
|
+
["javascript", /* @__PURE__ */ new Set(["call_expression", "new_expression", "jsx_self_closing_element", "jsx_opening_element"])],
|
|
2665
2846
|
["python", /* @__PURE__ */ new Set(["call"])],
|
|
2666
2847
|
["go", /* @__PURE__ */ new Set(["call_expression"])],
|
|
2667
2848
|
["rust", /* @__PURE__ */ new Set(["call_expression", "macro_invocation"])],
|
|
@@ -2823,7 +3004,13 @@ var REF_NOISE_BY_LANG = /* @__PURE__ */ new Map([
|
|
|
2823
3004
|
"proc",
|
|
2824
3005
|
"sleep",
|
|
2825
3006
|
"exit",
|
|
2826
|
-
"freeze"
|
|
3007
|
+
"freeze",
|
|
3008
|
+
"private",
|
|
3009
|
+
"protected",
|
|
3010
|
+
"public",
|
|
3011
|
+
"module_function",
|
|
3012
|
+
"block_given?",
|
|
3013
|
+
"binding"
|
|
2827
3014
|
])
|
|
2828
3015
|
]
|
|
2829
3016
|
]);
|
|
@@ -2833,6 +3020,19 @@ function lastSegment2(text) {
|
|
|
2833
3020
|
const parts = text.split(/::|\./);
|
|
2834
3021
|
return parts[parts.length - 1] ?? text;
|
|
2835
3022
|
}
|
|
3023
|
+
var TSJS_FUNCTION_VALUE_TYPES = /* @__PURE__ */ new Set(["arrow_function", "function_expression", "function"]);
|
|
3024
|
+
function tsJsFunctionValuedMemberName(node) {
|
|
3025
|
+
const isField = node.type === "public_field_definition" || node.type === "field_definition";
|
|
3026
|
+
const isPair = node.type === "pair" && node.parent?.type === "object";
|
|
3027
|
+
if (!isField && !isPair) return void 0;
|
|
3028
|
+
const value = node.childForFieldName("value");
|
|
3029
|
+
if (value === null || !TSJS_FUNCTION_VALUE_TYPES.has(value.type)) return void 0;
|
|
3030
|
+
if (isField) return (node.childForFieldName("name") ?? node.childForFieldName("property"))?.text;
|
|
3031
|
+
const key = node.childForFieldName("key");
|
|
3032
|
+
if (key === null || key.type !== "identifier" && key.type !== "property_identifier" && key.type !== "string") return void 0;
|
|
3033
|
+
const name = key.text.replace(/^['"]|['"]$/g, "");
|
|
3034
|
+
return name === "" ? void 0 : name;
|
|
3035
|
+
}
|
|
2836
3036
|
function scopeName(node, language) {
|
|
2837
3037
|
if ((language === "typescript" || language === "javascript") && node.type === "variable_declarator") {
|
|
2838
3038
|
const value = node.childForFieldName("value");
|
|
@@ -2841,6 +3041,10 @@ function scopeName(node, language) {
|
|
|
2841
3041
|
}
|
|
2842
3042
|
return null;
|
|
2843
3043
|
}
|
|
3044
|
+
if (language === "typescript" || language === "javascript") {
|
|
3045
|
+
const member = tsJsFunctionValuedMemberName(node);
|
|
3046
|
+
if (member !== void 0) return member;
|
|
3047
|
+
}
|
|
2844
3048
|
const scopeTypes = SCOPE_TYPES_BY_LANG.get(language);
|
|
2845
3049
|
if (scopeTypes !== void 0 && scopeTypes.has(node.type)) {
|
|
2846
3050
|
if ((language === "c" || language === "cpp") && node.type === "function_definition") {
|
|
@@ -2895,6 +3099,14 @@ function calleeName(call, language) {
|
|
|
2895
3099
|
switch (language) {
|
|
2896
3100
|
case "typescript":
|
|
2897
3101
|
case "javascript": {
|
|
3102
|
+
if (call.type === "jsx_self_closing_element" || call.type === "jsx_opening_element") {
|
|
3103
|
+
const tag = call.childForFieldName("name");
|
|
3104
|
+
if (tag === null) return null;
|
|
3105
|
+
if (tag.type === "identifier") return /^[A-Z]/.test(tag.text) ? tag.text : null;
|
|
3106
|
+
if (tag.type === "member_expression") return tag.childForFieldName("property")?.text ?? null;
|
|
3107
|
+
if (tag.type === "nested_identifier") return lastSegment2(tag.text);
|
|
3108
|
+
return null;
|
|
3109
|
+
}
|
|
2898
3110
|
if (call.type === "new_expression") {
|
|
2899
3111
|
const c = call.childForFieldName("constructor");
|
|
2900
3112
|
if (c === null) return null;
|
|
@@ -2938,7 +3150,9 @@ function calleeName(call, language) {
|
|
|
2938
3150
|
}
|
|
2939
3151
|
case "java": {
|
|
2940
3152
|
const n = call.childForFieldName("name") ?? call.childForFieldName("type");
|
|
2941
|
-
|
|
3153
|
+
if (n === null) return null;
|
|
3154
|
+
const named = n.type === "generic_type" ? n.namedChildren[0] ?? n : n;
|
|
3155
|
+
return lastSegment2(named.text);
|
|
2942
3156
|
}
|
|
2943
3157
|
case "c":
|
|
2944
3158
|
case "cpp": {
|
|
@@ -3088,12 +3302,43 @@ function valueRefIdentifiers(node, language) {
|
|
|
3088
3302
|
}
|
|
3089
3303
|
return result;
|
|
3090
3304
|
}
|
|
3305
|
+
var RUBY_FRESH_SCOPES = /* @__PURE__ */ new Set(["program", "method", "singleton_method", "class", "singleton_class", "module"]);
|
|
3306
|
+
var RUBY_BLOCK_SCOPES = /* @__PURE__ */ new Set(["block", "do_block", "lambda"]);
|
|
3307
|
+
var RUBY_IMPLICIT_PARAM = /^(?:it|_[1-9])$/;
|
|
3308
|
+
function sameNode2(a, b) {
|
|
3309
|
+
return a !== null && a.type === b.type && a.startPosition.row === b.startPosition.row && a.startPosition.column === b.startPosition.column && a.endPosition.column === b.endPosition.column;
|
|
3310
|
+
}
|
|
3311
|
+
function rubyIsLocal(scopes, name) {
|
|
3312
|
+
for (let i = scopes.length - 1; i >= 0; i--) {
|
|
3313
|
+
const scope = scopes[i];
|
|
3314
|
+
if (scope.vars.has(name)) return true;
|
|
3315
|
+
if (!scope.inherits) return false;
|
|
3316
|
+
}
|
|
3317
|
+
return false;
|
|
3318
|
+
}
|
|
3319
|
+
function rubyIdentifierRole(node) {
|
|
3320
|
+
const parent = node.parent;
|
|
3321
|
+
if (parent === null) return "use";
|
|
3322
|
+
const t = parent.type;
|
|
3323
|
+
if (t === "method" || t === "singleton_method") return sameNode2(parent.childForFieldName("name"), node) ? "skip" : "use";
|
|
3324
|
+
if (t === "call") return sameNode2(parent.childForFieldName("method"), node) ? "skip" : "use";
|
|
3325
|
+
if (t === "alias" || t === "undef" || t === "scope_resolution") return "skip";
|
|
3326
|
+
if (t.endsWith("_parameters") || t === "destructured_parameter") return "declare";
|
|
3327
|
+
if (t.endsWith("_parameter")) return sameNode2(parent.childForFieldName("name"), node) ? "declare" : "use";
|
|
3328
|
+
if (t === "assignment" || t === "operator_assignment") return sameNode2(parent.childForFieldName("left"), node) ? "declare" : "use";
|
|
3329
|
+
if (t === "left_assignment_list" || t === "rest_assignment" || t === "destructured_left_assignment" || t === "exception_variable") return "declare";
|
|
3330
|
+
if (t === "for") return sameNode2(parent.childForFieldName("pattern"), node) ? "declare" : "use";
|
|
3331
|
+
if (t === "in_clause") return sameNode2(parent.childForFieldName("pattern"), node) ? "declare" : "use";
|
|
3332
|
+
if (t.endsWith("_pattern")) return "declare";
|
|
3333
|
+
return "use";
|
|
3334
|
+
}
|
|
3091
3335
|
function extractRefs(root, filePath, language) {
|
|
3092
3336
|
const out = [];
|
|
3093
3337
|
const seen = /* @__PURE__ */ new Set();
|
|
3094
3338
|
const callTypes = CALL_TYPES_BY_LANG.get(language) ?? EMPTY_STRING_SET;
|
|
3095
3339
|
const noise = language === "javascript" ? JS_NOISE : REF_NOISE_BY_LANG.get(language) ?? EMPTY_STRING_SET;
|
|
3096
3340
|
const stack = [];
|
|
3341
|
+
const rubyScopes = [];
|
|
3097
3342
|
const record = (name, node) => {
|
|
3098
3343
|
if (name.length <= 1 || noise.has(name)) return;
|
|
3099
3344
|
const line = node.startPosition.row + 1;
|
|
@@ -3111,6 +3356,12 @@ function extractRefs(root, filePath, language) {
|
|
|
3111
3356
|
const visit = (node) => {
|
|
3112
3357
|
const enclosing = scopeName(node, language);
|
|
3113
3358
|
if (enclosing !== null && enclosing !== "") stack.push(enclosing);
|
|
3359
|
+
let rubyScoped = false;
|
|
3360
|
+
if (language === "ruby") {
|
|
3361
|
+
rubyScoped = RUBY_FRESH_SCOPES.has(node.type) || RUBY_BLOCK_SCOPES.has(node.type);
|
|
3362
|
+
if (rubyScoped) rubyScopes.push({ vars: /* @__PURE__ */ new Set(), inherits: RUBY_BLOCK_SCOPES.has(node.type) });
|
|
3363
|
+
visitRubyBinding(node);
|
|
3364
|
+
}
|
|
3114
3365
|
if (callTypes.has(node.type)) {
|
|
3115
3366
|
const callee = calleeName(node, language);
|
|
3116
3367
|
if (callee !== null) record(callee, node);
|
|
@@ -3119,8 +3370,22 @@ function extractRefs(root, filePath, language) {
|
|
|
3119
3370
|
record(idNode.text, idNode);
|
|
3120
3371
|
}
|
|
3121
3372
|
for (const child of node.namedChildren) visit(child);
|
|
3373
|
+
if (rubyScoped) rubyScopes.pop();
|
|
3122
3374
|
if (enclosing !== null && enclosing !== "") stack.pop();
|
|
3123
3375
|
};
|
|
3376
|
+
const visitRubyBinding = (node) => {
|
|
3377
|
+
const scope = rubyScopes[rubyScopes.length - 1];
|
|
3378
|
+
if (scope === void 0) return;
|
|
3379
|
+
if (node.type === "keyword_pattern" && node.childForFieldName("value") === null) {
|
|
3380
|
+
const key = node.childForFieldName("key");
|
|
3381
|
+
if (key !== null) scope.vars.add(key.text);
|
|
3382
|
+
return;
|
|
3383
|
+
}
|
|
3384
|
+
if (node.type !== "identifier") return;
|
|
3385
|
+
const role = rubyIdentifierRole(node);
|
|
3386
|
+
if (role === "declare") scope.vars.add(node.text);
|
|
3387
|
+
else if (role === "use" && !RUBY_IMPLICIT_PARAM.test(node.text) && !rubyIsLocal(rubyScopes, node.text)) record(node.text, node);
|
|
3388
|
+
};
|
|
3124
3389
|
visit(root);
|
|
3125
3390
|
return out;
|
|
3126
3391
|
}
|
|
@@ -3191,13 +3456,18 @@ function countNewlines(s) {
|
|
|
3191
3456
|
function extractMarkdownSymbols(content, filePath) {
|
|
3192
3457
|
const out = [];
|
|
3193
3458
|
const lines = content.split(/\r?\n/);
|
|
3194
|
-
const
|
|
3459
|
+
const fmEnd = frontMatterEndIndex(lines);
|
|
3460
|
+
const unfenced = Array.from(eachUnfencedLine(lines)).filter(([i]) => i >= fmEnd);
|
|
3461
|
+
const bounds = [];
|
|
3462
|
+
let paraStart = -1;
|
|
3195
3463
|
for (let u = 0; u < unfenced.length; u++) {
|
|
3196
3464
|
const [i, line] = unfenced[u];
|
|
3197
|
-
const
|
|
3198
|
-
if (
|
|
3199
|
-
const name =
|
|
3465
|
+
const atx = matchAtxHeading(line, ATX_DOT_BREAK_RE, false);
|
|
3466
|
+
if (atx !== null) {
|
|
3467
|
+
const name = atx.name;
|
|
3468
|
+
let at = -1;
|
|
3200
3469
|
if (name !== "") {
|
|
3470
|
+
at = out.length;
|
|
3201
3471
|
out.push({
|
|
3202
3472
|
filePath,
|
|
3203
3473
|
name,
|
|
@@ -3209,29 +3479,47 @@ function extractMarkdownSymbols(content, filePath) {
|
|
|
3209
3479
|
parent: ""
|
|
3210
3480
|
});
|
|
3211
3481
|
}
|
|
3482
|
+
bounds.push({ line: i + 1, level: atx.level, at });
|
|
3483
|
+
paraStart = -1;
|
|
3212
3484
|
continue;
|
|
3213
3485
|
}
|
|
3486
|
+
if (paraStart !== -1 && i !== unfenced[u - 1][0] + 1) paraStart = -1;
|
|
3214
3487
|
const trimmed = line.trim();
|
|
3215
|
-
if (
|
|
3216
|
-
const
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3488
|
+
if (paraStart !== -1 && /^(=+|-+)$/.test(trimmed)) {
|
|
3489
|
+
const para = unfenced.slice(paraStart, u).map(([, text]) => text.trim());
|
|
3490
|
+
const at = out.length;
|
|
3491
|
+
out.push({
|
|
3492
|
+
filePath,
|
|
3493
|
+
name: para.join(" "),
|
|
3494
|
+
kind: "heading",
|
|
3495
|
+
lineStart: unfenced[paraStart][0] + 1,
|
|
3496
|
+
lineEnd: i + 1,
|
|
3497
|
+
body: `${para.join("\n")}
|
|
3498
|
+
${trimmed}`,
|
|
3499
|
+
docstring: "",
|
|
3500
|
+
parent: ""
|
|
3501
|
+
});
|
|
3502
|
+
bounds.push({ line: unfenced[paraStart][0] + 1, level: trimmed.startsWith("=") ? 1 : 2, at });
|
|
3503
|
+
paraStart = -1;
|
|
3504
|
+
continue;
|
|
3505
|
+
}
|
|
3506
|
+
const isText = trimmed !== "" && !trimmed.startsWith("#") && !trimmed.startsWith("|") && !trimmed.startsWith("```") && !trimmed.startsWith("~~~") && !/^([-*+]|\d+\.)\s/.test(trimmed) && !/^(=+|-+)$/.test(trimmed);
|
|
3507
|
+
if (!isText) paraStart = -1;
|
|
3508
|
+
else if (paraStart === -1) paraStart = u;
|
|
3509
|
+
}
|
|
3510
|
+
for (let k = 0; k < bounds.length; k++) {
|
|
3511
|
+
const cur = bounds[k];
|
|
3512
|
+
const sym = out[cur.at];
|
|
3513
|
+
if (sym === void 0) continue;
|
|
3514
|
+
let end = lines.length;
|
|
3515
|
+
for (let j = k + 1; j < bounds.length; j++) {
|
|
3516
|
+
if (bounds[j].level <= cur.level) {
|
|
3517
|
+
end = bounds[j].line - 1;
|
|
3518
|
+
break;
|
|
3233
3519
|
}
|
|
3234
3520
|
}
|
|
3521
|
+
while (end > sym.lineEnd && (lines[end - 1] ?? "").trim() === "") end--;
|
|
3522
|
+
out[cur.at] = { ...sym, lineEnd: Math.max(sym.lineEnd, end) };
|
|
3235
3523
|
}
|
|
3236
3524
|
return out;
|
|
3237
3525
|
}
|
|
@@ -3397,10 +3685,13 @@ function yamlLineClosesQuote(line, quote) {
|
|
|
3397
3685
|
}
|
|
3398
3686
|
return false;
|
|
3399
3687
|
}
|
|
3688
|
+
var YAML_TOP_KEY_RE = /^(?:([a-zA-Z_0-9][\w.-]*)|"((?:[^"\\]|\\.)*)"|'((?:[^']|'')*)')\s*:(?!\/\/)/;
|
|
3400
3689
|
function extractYamlSymbols(content, filePath) {
|
|
3401
3690
|
const out = [];
|
|
3402
3691
|
const lines = content.split(/\r?\n/);
|
|
3403
3692
|
let openQuote = null;
|
|
3693
|
+
const boundaries = [];
|
|
3694
|
+
const keyAt = /* @__PURE__ */ new Map();
|
|
3404
3695
|
for (let i = 0; i < lines.length; i++) {
|
|
3405
3696
|
const line = lines[i];
|
|
3406
3697
|
if (line === void 0) continue;
|
|
@@ -3408,11 +3699,18 @@ function extractYamlSymbols(content, filePath) {
|
|
|
3408
3699
|
if (yamlLineClosesQuote(line, openQuote)) openQuote = null;
|
|
3409
3700
|
continue;
|
|
3410
3701
|
}
|
|
3411
|
-
|
|
3412
|
-
|
|
3702
|
+
if (/^(?:---|\.\.\.)(?:\s|$)/.test(line)) {
|
|
3703
|
+
boundaries.push(i);
|
|
3704
|
+
continue;
|
|
3705
|
+
}
|
|
3706
|
+
const match = YAML_TOP_KEY_RE.exec(line);
|
|
3707
|
+
if (match !== null) {
|
|
3708
|
+
const name = match[2] !== void 0 ? match[2].replace(/\\(["\\/])/g, "$1") : match[3] !== void 0 ? match[3].replace(/''/g, "'") : match[1] ?? "";
|
|
3709
|
+
boundaries.push(i);
|
|
3710
|
+
keyAt.set(i, out.length);
|
|
3413
3711
|
out.push({
|
|
3414
3712
|
filePath,
|
|
3415
|
-
name
|
|
3713
|
+
name,
|
|
3416
3714
|
kind: "key",
|
|
3417
3715
|
lineStart: i + 1,
|
|
3418
3716
|
lineEnd: i + 1,
|
|
@@ -3423,6 +3721,12 @@ function extractYamlSymbols(content, filePath) {
|
|
|
3423
3721
|
openQuote = yamlOpenQuoteAfter(line, match[0].length);
|
|
3424
3722
|
}
|
|
3425
3723
|
}
|
|
3724
|
+
for (const [start, at] of keyAt) {
|
|
3725
|
+
const sym = out[at];
|
|
3726
|
+
let end = (boundaries.find((b) => b > start) ?? lines.length) - 1;
|
|
3727
|
+
while (end > start && (lines[end].trim() === "" || lines[end].startsWith("#"))) end--;
|
|
3728
|
+
out[at] = { ...sym, lineEnd: end + 1, body: lines.slice(start, end + 1).join("\n").trim() };
|
|
3729
|
+
}
|
|
3426
3730
|
return out;
|
|
3427
3731
|
}
|
|
3428
3732
|
function lineOpenDelimiterAfter(line, startIdx) {
|
|
@@ -3483,11 +3787,11 @@ function extractTomlSymbols(content, filePath) {
|
|
|
3483
3787
|
const out = [];
|
|
3484
3788
|
const lines = content.split(/\r?\n/);
|
|
3485
3789
|
function matchLine(line, lineNum) {
|
|
3486
|
-
const
|
|
3487
|
-
if (
|
|
3790
|
+
const sectionName = matchTomlSectionName(line);
|
|
3791
|
+
if (sectionName !== null) {
|
|
3488
3792
|
out.push({
|
|
3489
3793
|
filePath,
|
|
3490
|
-
name:
|
|
3794
|
+
name: sectionName.trim(),
|
|
3491
3795
|
kind: "section",
|
|
3492
3796
|
lineStart: lineNum + 1,
|
|
3493
3797
|
lineEnd: lineNum + 1,
|
|
@@ -3532,6 +3836,14 @@ function extractTomlSymbols(content, filePath) {
|
|
|
3532
3836
|
openDelim = lineOpenDelimiterAfter(code, 0);
|
|
3533
3837
|
if (openDelim === null) arrayDepth = Math.max(0, tomlBracketDelta(code));
|
|
3534
3838
|
}
|
|
3839
|
+
const headers = out.filter((sym) => sym.kind === "section");
|
|
3840
|
+
headers.forEach((sym, h) => {
|
|
3841
|
+
const next = headers[h + 1];
|
|
3842
|
+
let end = (next === void 0 ? lines.length : next.lineStart - 1) - 1;
|
|
3843
|
+
while (end > sym.lineStart - 1 && (lines[end].trim() === "" || lines[end].trim().startsWith("#"))) end--;
|
|
3844
|
+
const at = out.indexOf(sym);
|
|
3845
|
+
out[at] = { ...sym, lineEnd: end + 1, body: lines.slice(sym.lineStart - 1, end + 1).join("\n").trim() };
|
|
3846
|
+
});
|
|
3535
3847
|
return out;
|
|
3536
3848
|
}
|
|
3537
3849
|
function splitTopLevelSelectors(rawCapture, strippedCapture) {
|
|
@@ -3558,68 +3870,75 @@ function nextContentLineOpensBrace(lines, i) {
|
|
|
3558
3870
|
}
|
|
3559
3871
|
return false;
|
|
3560
3872
|
}
|
|
3873
|
+
function ruleEndIdx(blanked, startIdx, braceCol) {
|
|
3874
|
+
let depth = 0;
|
|
3875
|
+
for (let i = startIdx; i < blanked.length; i++) {
|
|
3876
|
+
const text = blanked[i] ?? "";
|
|
3877
|
+
for (let c = i === startIdx ? braceCol : 0; c < text.length; c++) {
|
|
3878
|
+
if (text[c] === "{") depth++;
|
|
3879
|
+
else if (text[c] === "}" && --depth === 0) return i;
|
|
3880
|
+
}
|
|
3881
|
+
}
|
|
3882
|
+
return startIdx;
|
|
3883
|
+
}
|
|
3884
|
+
function stripCssLineComment(line) {
|
|
3885
|
+
let inUrl = false;
|
|
3886
|
+
for (let c = 0; c < line.length; c++) {
|
|
3887
|
+
if (inUrl) {
|
|
3888
|
+
if (line[c] === ")") inUrl = false;
|
|
3889
|
+
} else if (line[c] === "/" && line[c + 1] === "/") {
|
|
3890
|
+
return line.slice(0, c);
|
|
3891
|
+
} else if (line[c] === "(" && /url$/i.test(line.slice(Math.max(0, c - 3), c))) {
|
|
3892
|
+
inUrl = true;
|
|
3893
|
+
}
|
|
3894
|
+
}
|
|
3895
|
+
return line;
|
|
3896
|
+
}
|
|
3561
3897
|
function extractCssSymbols(content, filePath) {
|
|
3562
3898
|
const out = [];
|
|
3563
3899
|
const lines = stripCstyleComments(content).split(/\r?\n/);
|
|
3564
3900
|
const rawLines = content.split(/\r?\n/);
|
|
3565
3901
|
let pending = [];
|
|
3902
|
+
const lineComments = /\.(?:scss|sass|less)$/i.test(filePath);
|
|
3903
|
+
const blanked = lines.map((l) => lineComments ? stripCssLineComment(stripStringLiterals(l)) : stripStringLiterals(l));
|
|
3904
|
+
const spanEntry = (name, startIdx, endIdx) => ({
|
|
3905
|
+
filePath,
|
|
3906
|
+
name,
|
|
3907
|
+
kind: "selector",
|
|
3908
|
+
lineStart: startIdx + 1,
|
|
3909
|
+
lineEnd: endIdx + 1,
|
|
3910
|
+
body: endIdx > startIdx ? rawLines.slice(startIdx, endIdx + 1).join("\n") : (lines[startIdx] ?? "").trim(),
|
|
3911
|
+
docstring: "",
|
|
3912
|
+
parent: ""
|
|
3913
|
+
});
|
|
3914
|
+
const flushPending = (endIdx) => {
|
|
3915
|
+
for (const p of pending) out.push(spanEntry(p.name, p.line - 1, endIdx));
|
|
3916
|
+
pending = [];
|
|
3917
|
+
};
|
|
3566
3918
|
for (let i = 0; i < lines.length; i++) {
|
|
3567
3919
|
const line = lines[i];
|
|
3568
3920
|
if (line === void 0) continue;
|
|
3569
|
-
const trimmed = line.trim();
|
|
3921
|
+
const trimmed = (lineComments ? line.slice(0, (blanked[i] ?? "").length) : line).trim();
|
|
3570
3922
|
if (trimmed.length === 0 && (rawLines[i]?.trim().length ?? 0) > 0) {
|
|
3571
3923
|
continue;
|
|
3572
3924
|
}
|
|
3573
|
-
const strippedLine =
|
|
3574
|
-
const
|
|
3575
|
-
if (
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
kind: "selector",
|
|
3581
|
-
lineStart: p.line,
|
|
3582
|
-
lineEnd: p.line,
|
|
3583
|
-
body: p.body,
|
|
3584
|
-
docstring: "",
|
|
3585
|
-
parent: ""
|
|
3586
|
-
});
|
|
3587
|
-
}
|
|
3588
|
-
pending = [];
|
|
3589
|
-
const captureRange = selectorLineMatch.indices?.[1];
|
|
3590
|
-
const rawCapture = captureRange ? line.slice(captureRange[0], captureRange[1]) : selectorLineMatch[1];
|
|
3591
|
-
const strippedCapture = captureRange ? strippedLine.slice(captureRange[0], captureRange[1]) : selectorLineMatch[1];
|
|
3925
|
+
const strippedLine = blanked[i] ?? "";
|
|
3926
|
+
const selectorSpan = matchCssSelectorSpan(strippedLine);
|
|
3927
|
+
if (selectorSpan !== null) {
|
|
3928
|
+
const endIdx = ruleEndIdx(blanked, i, selectorSpan.end);
|
|
3929
|
+
flushPending(endIdx);
|
|
3930
|
+
const rawCapture = line.slice(selectorSpan.start, selectorSpan.end);
|
|
3931
|
+
const strippedCapture = strippedLine.slice(selectorSpan.start, selectorSpan.end);
|
|
3592
3932
|
for (const part of splitTopLevelSelectors(rawCapture, strippedCapture)) {
|
|
3593
3933
|
const name = part.trim();
|
|
3594
3934
|
if (name) {
|
|
3595
|
-
out.push(
|
|
3596
|
-
filePath,
|
|
3597
|
-
name,
|
|
3598
|
-
kind: "selector",
|
|
3599
|
-
lineStart: i + 1,
|
|
3600
|
-
lineEnd: i + 1,
|
|
3601
|
-
body: line.trim(),
|
|
3602
|
-
docstring: "",
|
|
3603
|
-
parent: ""
|
|
3604
|
-
});
|
|
3935
|
+
out.push(spanEntry(name, i, endIdx));
|
|
3605
3936
|
}
|
|
3606
3937
|
}
|
|
3607
3938
|
continue;
|
|
3608
3939
|
}
|
|
3609
3940
|
if (trimmed === "{") {
|
|
3610
|
-
|
|
3611
|
-
out.push({
|
|
3612
|
-
filePath,
|
|
3613
|
-
name: p.name,
|
|
3614
|
-
kind: "selector",
|
|
3615
|
-
lineStart: p.line,
|
|
3616
|
-
lineEnd: p.line,
|
|
3617
|
-
body: p.body,
|
|
3618
|
-
docstring: "",
|
|
3619
|
-
parent: ""
|
|
3620
|
-
});
|
|
3621
|
-
}
|
|
3622
|
-
pending = [];
|
|
3941
|
+
flushPending(ruleEndIdx(blanked, i, blanked[i]?.indexOf("{") ?? 0));
|
|
3623
3942
|
continue;
|
|
3624
3943
|
}
|
|
3625
3944
|
if (trimmed.length > 0 && !trimmed.startsWith("@") && !trimmed.includes("{") && !trimmed.includes("}") && !trimmed.includes(";")) {
|
|
@@ -3634,23 +3953,52 @@ function extractCssSymbols(content, filePath) {
|
|
|
3634
3953
|
}
|
|
3635
3954
|
return out;
|
|
3636
3955
|
}
|
|
3956
|
+
var DOCKERFILE_HEREDOC_RE = /(?<!<)<<(-?)(["']?)([A-Za-z_][\w.-]*)\2/g;
|
|
3957
|
+
function endsWithContinuation(line) {
|
|
3958
|
+
return line.trimEnd().endsWith("\\");
|
|
3959
|
+
}
|
|
3960
|
+
function dockerfileInstructionEnd(lines, i) {
|
|
3961
|
+
let end = i;
|
|
3962
|
+
let continued = endsWithContinuation(lines[i] ?? "");
|
|
3963
|
+
while (continued && end + 1 < lines.length) {
|
|
3964
|
+
end++;
|
|
3965
|
+
const next = lines[end] ?? "";
|
|
3966
|
+
if (!next.trim().startsWith("#")) continued = endsWithContinuation(next);
|
|
3967
|
+
}
|
|
3968
|
+
return end;
|
|
3969
|
+
}
|
|
3970
|
+
function dockerfileHeredocEnd(lines, i, end) {
|
|
3971
|
+
const text = lines.slice(i, end + 1).join("\n");
|
|
3972
|
+
let cursor = end;
|
|
3973
|
+
for (const m of text.matchAll(DOCKERFILE_HEREDOC_RE)) {
|
|
3974
|
+
const stripTabs = m[1] === "-";
|
|
3975
|
+
let close = -1;
|
|
3976
|
+
for (let j = cursor + 1; j < lines.length; j++) {
|
|
3977
|
+
const candidate = stripTabs ? (lines[j] ?? "").replace(/^\t+/, "") : lines[j] ?? "";
|
|
3978
|
+
if (candidate.trimEnd() === m[3]) {
|
|
3979
|
+
close = j;
|
|
3980
|
+
break;
|
|
3981
|
+
}
|
|
3982
|
+
}
|
|
3983
|
+
if (close === -1) return end;
|
|
3984
|
+
cursor = close;
|
|
3985
|
+
}
|
|
3986
|
+
return cursor;
|
|
3987
|
+
}
|
|
3637
3988
|
function extractDockerfileSymbols(content, filePath) {
|
|
3638
3989
|
const out = [];
|
|
3639
3990
|
const lines = content.split(/\r?\n/);
|
|
3640
|
-
let continuing = false;
|
|
3641
3991
|
for (let i = 0; i < lines.length; i++) {
|
|
3642
3992
|
const line = lines[i];
|
|
3643
3993
|
if (line === void 0) continue;
|
|
3644
|
-
if (
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
}
|
|
3648
|
-
const isComment = line.trim().startsWith("#");
|
|
3649
|
-
const match = isComment ? null : /^\s*(FROM|RUN|COPY|ADD|EXPOSE|ENV|WORKDIR|CMD|ENTRYPOINT|ARG|LABEL|VOLUME|USER|HEALTHCHECK|ONBUILD|SHELL|STOPSIGNAL|MAINTAINER)\s+(.+)/i.exec(
|
|
3994
|
+
if (line.trim().startsWith("#")) continue;
|
|
3995
|
+
let end = dockerfileInstructionEnd(lines, i);
|
|
3996
|
+
const match = /^\s*(FROM|RUN|COPY|ADD|EXPOSE|ENV|WORKDIR|CMD|ENTRYPOINT|ARG|LABEL|VOLUME|USER|HEALTHCHECK|ONBUILD|SHELL|STOPSIGNAL|MAINTAINER)\s+(.+)/i.exec(
|
|
3650
3997
|
line
|
|
3651
3998
|
);
|
|
3652
3999
|
if (match !== null && match[1] !== void 0) {
|
|
3653
4000
|
const cmd = match[1];
|
|
4001
|
+
if (/^(?:RUN|COPY|ADD)$/i.test(cmd)) end = dockerfileHeredocEnd(lines, i, end);
|
|
3654
4002
|
const arg = (match[2] ?? "").substring(0, 40);
|
|
3655
4003
|
const name = `${cmd} ${arg}`.trim();
|
|
3656
4004
|
out.push({
|
|
@@ -3658,13 +4006,13 @@ function extractDockerfileSymbols(content, filePath) {
|
|
|
3658
4006
|
name,
|
|
3659
4007
|
kind: "directive",
|
|
3660
4008
|
lineStart: i + 1,
|
|
3661
|
-
lineEnd:
|
|
3662
|
-
body:
|
|
4009
|
+
lineEnd: end + 1,
|
|
4010
|
+
body: lines.slice(i, end + 1).join("\n").trim(),
|
|
3663
4011
|
docstring: "",
|
|
3664
4012
|
parent: ""
|
|
3665
4013
|
});
|
|
3666
4014
|
}
|
|
3667
|
-
|
|
4015
|
+
i = end;
|
|
3668
4016
|
}
|
|
3669
4017
|
return out;
|
|
3670
4018
|
}
|
|
@@ -3734,7 +4082,8 @@ var REF_LANGUAGES = /* @__PURE__ */ new Set([
|
|
|
3734
4082
|
"java",
|
|
3735
4083
|
"c",
|
|
3736
4084
|
"cpp",
|
|
3737
|
-
"ruby"
|
|
4085
|
+
"ruby",
|
|
4086
|
+
"csharp"
|
|
3738
4087
|
]);
|
|
3739
4088
|
var _parserCtor;
|
|
3740
4089
|
var _parserCtorError = null;
|
|
@@ -3785,8 +4134,8 @@ function boundRefContext(context, col) {
|
|
|
3785
4134
|
return (start > 0 ? REF_CONTEXT_ELISION : "") + context.slice(start, end) + (end < context.length ? REF_CONTEXT_ELISION : "");
|
|
3786
4135
|
}
|
|
3787
4136
|
function loadGrammar(lang, filePath, content) {
|
|
3788
|
-
const useTsx = lang === "typescript" && filePath !== void 0 &&
|
|
3789
|
-
const useCppHeader = lang === "c" && filePath !== void 0 &&
|
|
4137
|
+
const useTsx = lang === "typescript" && filePath !== void 0 && path9.extname(filePath).toLowerCase() === ".tsx";
|
|
4138
|
+
const useCppHeader = lang === "c" && filePath !== void 0 && path9.extname(filePath).toLowerCase() === ".h" && content !== void 0 && CPP_HEADER_SNIFF_RE.test(content);
|
|
3790
4139
|
const cacheKey = useTsx ? "typescript:tsx" : useCppHeader ? "c:cpp-header" : lang;
|
|
3791
4140
|
const cached = _grammarCache.get(cacheKey);
|
|
3792
4141
|
if (cached !== void 0) return cached;
|
|
@@ -3922,7 +4271,7 @@ function parseSourceSymbolsTreeSitterOnly(content, filePath, language) {
|
|
|
3922
4271
|
var REGEX_ADAPTERS_SLOT = /* @__PURE__ */ Symbol.for("token-goat.regex-adapters");
|
|
3923
4272
|
var adapterSlot = globalThis;
|
|
3924
4273
|
async function loadRegexExtractors() {
|
|
3925
|
-
adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-
|
|
4274
|
+
adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-A3CK6N5Z.mjs");
|
|
3926
4275
|
}
|
|
3927
4276
|
function regexAdapters() {
|
|
3928
4277
|
const loaded = adapterSlot[REGEX_ADAPTERS_SLOT];
|
|
@@ -3961,6 +4310,17 @@ function extractNoTreeSitter(content, filePath, language) {
|
|
|
3961
4310
|
const r = adapters.extractNatural(content, filePath);
|
|
3962
4311
|
return { symbols: r.symbols, refs: r.refs };
|
|
3963
4312
|
}
|
|
4313
|
+
if (language === "csharp") {
|
|
4314
|
+
const r = adapters.extractCsharp(content, filePath);
|
|
4315
|
+
const symbols = adapters.assignBraceBlockSpans(r.symbols, content, {
|
|
4316
|
+
lineComment: "//",
|
|
4317
|
+
stringEscapes: "csharp",
|
|
4318
|
+
rawStringQuotes: true,
|
|
4319
|
+
expressionBodies: true,
|
|
4320
|
+
interpolation: "csharp"
|
|
4321
|
+
});
|
|
4322
|
+
return { symbols, refs: r.refs };
|
|
4323
|
+
}
|
|
3964
4324
|
const parsed = {
|
|
3965
4325
|
symbols: extractSymbolsNoTreeSitter(content, filePath, language),
|
|
3966
4326
|
refs: []
|
|
@@ -3977,16 +4337,19 @@ function deleteFileRows(db, filePath) {
|
|
|
3977
4337
|
db.prepare(`DELETE FROM refs WHERE ${pathEqClause("file_path")}`).run(folded);
|
|
3978
4338
|
db.prepare(`DELETE FROM files WHERE ${pathEqClause("path")}`).run(folded);
|
|
3979
4339
|
}
|
|
3980
|
-
function
|
|
4340
|
+
function isMinifiedBundlePath(filePath) {
|
|
4341
|
+
return /(?:\.min\.(?:js|mjs|cjs|css)|-min\.js)$/i.test(path9.basename(filePath));
|
|
4342
|
+
}
|
|
4343
|
+
function isUnderSkipDir(filePath, skipDirs, projectRoot) {
|
|
3981
4344
|
if (skipDirs.length === 0) return false;
|
|
3982
|
-
|
|
3983
|
-
return segments.slice(0, -1).some((seg) => skipDirs.includes(seg));
|
|
4345
|
+
return hasSkipSegmentBelowRoot(filePath, projectRoot, (seg) => skipDirs.includes(seg));
|
|
3984
4346
|
}
|
|
3985
|
-
function isParseSkipEligible(filePath, cfg) {
|
|
3986
|
-
if (isUnderSkipDir(filePath, cfg.skip_dirs)) return true;
|
|
3987
|
-
if (cfg.skip_files.includes(
|
|
4347
|
+
function isParseSkipEligible(filePath, cfg, projectRoot) {
|
|
4348
|
+
if (isUnderSkipDir(filePath, cfg.skip_dirs, projectRoot)) return true;
|
|
4349
|
+
if (cfg.skip_files.includes(path9.basename(filePath))) return true;
|
|
4350
|
+
if (cfg.skip_minified !== false && isMinifiedBundlePath(filePath)) return true;
|
|
3988
4351
|
try {
|
|
3989
|
-
const stat =
|
|
4352
|
+
const stat = fs4.statSync(filePath);
|
|
3990
4353
|
if (stat.size > cfg.large_file_skip_kb * 1024) return true;
|
|
3991
4354
|
} catch {
|
|
3992
4355
|
}
|
|
@@ -4057,7 +4420,7 @@ function indexFileSync(rawPath, dbPath = globalDbPath(), preReadBytes) {
|
|
|
4057
4420
|
raw = preReadBytes;
|
|
4058
4421
|
} else {
|
|
4059
4422
|
try {
|
|
4060
|
-
raw =
|
|
4423
|
+
raw = fs4.readFileSync(filePath);
|
|
4061
4424
|
} catch (err) {
|
|
4062
4425
|
if (err.code === "ENOENT") return;
|
|
4063
4426
|
throw err;
|
|
@@ -4072,6 +4435,10 @@ var DISABLED_EMBED_SHA_PREFIX = "disabled:";
|
|
|
4072
4435
|
function disabledEmbedSha(sha) {
|
|
4073
4436
|
return DISABLED_EMBED_SHA_PREFIX + sha;
|
|
4074
4437
|
}
|
|
4438
|
+
var disabledStamps = 0;
|
|
4439
|
+
function disabledStampCount() {
|
|
4440
|
+
return disabledStamps;
|
|
4441
|
+
}
|
|
4075
4442
|
var UNAVAILABLE_EMBED_SHA_PREFIX = "unavailable:";
|
|
4076
4443
|
function unavailableEmbedSha(sha) {
|
|
4077
4444
|
return UNAVAILABLE_EMBED_SHA_PREFIX + sha;
|
|
@@ -4105,28 +4472,28 @@ function canonicalizeIndexPath(rawPath) {
|
|
|
4105
4472
|
if (!isCaseInsensitiveFs()) return absPath;
|
|
4106
4473
|
let real;
|
|
4107
4474
|
try {
|
|
4108
|
-
real =
|
|
4475
|
+
real = fs4.realpathSync.native(absPath);
|
|
4109
4476
|
} catch {
|
|
4110
4477
|
return absPath;
|
|
4111
4478
|
}
|
|
4112
4479
|
const cut = Math.max(absPath.lastIndexOf("/"), absPath.lastIndexOf("\\"));
|
|
4113
4480
|
const base = absPath.slice(cut + 1);
|
|
4114
4481
|
const realNorm = normalizePath(real);
|
|
4115
|
-
const realBase =
|
|
4482
|
+
const realBase = path9.basename(realNorm);
|
|
4116
4483
|
if (base === realBase) return absPath;
|
|
4117
4484
|
if (foldPath(base) !== foldPath(realBase)) return absPath;
|
|
4118
|
-
const callerDir = cut < 0 ? normalizePath(
|
|
4119
|
-
if (foldPath(callerDir) !== foldPath(
|
|
4485
|
+
const callerDir = cut < 0 ? normalizePath(path9.resolve(".")) : normalizePath(absPath.slice(0, cut));
|
|
4486
|
+
if (foldPath(callerDir) !== foldPath(path9.dirname(realNorm))) return absPath;
|
|
4120
4487
|
return absPath.slice(0, cut + 1) + realBase;
|
|
4121
4488
|
}
|
|
4122
4489
|
function indexedPathSpellingIsStale(storedPath, absPath) {
|
|
4123
4490
|
if (!isCaseInsensitiveFs()) return false;
|
|
4124
4491
|
const stored = normalizePath(storedPath);
|
|
4125
|
-
const candidate = normalizePath(
|
|
4492
|
+
const candidate = normalizePath(path9.resolve(absPath));
|
|
4126
4493
|
if (foldPath(stored) !== foldPath(candidate)) return false;
|
|
4127
4494
|
let real;
|
|
4128
4495
|
try {
|
|
4129
|
-
real = normalizePath(
|
|
4496
|
+
real = normalizePath(fs4.realpathSync.native(absPath));
|
|
4130
4497
|
} catch {
|
|
4131
4498
|
return false;
|
|
4132
4499
|
}
|
|
@@ -4164,6 +4531,7 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
4164
4531
|
const ixCfg = loadConfig().indexing;
|
|
4165
4532
|
if (!ixCfg.embeddings_enabled) {
|
|
4166
4533
|
stampEmbedSha(getDb(dbPath), filePath, sha, disabledEmbedSha);
|
|
4534
|
+
disabledStamps += 1;
|
|
4167
4535
|
return;
|
|
4168
4536
|
}
|
|
4169
4537
|
if (filePath.toLowerCase().endsWith(".profile-meta.xml")) {
|
|
@@ -4205,7 +4573,8 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
4205
4573
|
}
|
|
4206
4574
|
try {
|
|
4207
4575
|
const db = getDb(dbPath);
|
|
4208
|
-
const outcome = await indexFile(db, filePath, extracted, []);
|
|
4576
|
+
const outcome = await indexFile(db, filePath, extracted, [], sha);
|
|
4577
|
+
if (outcome === "stale") return;
|
|
4209
4578
|
if (outcome !== "unavailable" && enforceStoredChunkCap(db, filePath, sha, ixCfg.max_chunks_per_file)) return;
|
|
4210
4579
|
stampEmbedSha(db, filePath, sha, (s) => outcome === "unavailable" ? unavailableEmbedSha(s) : s);
|
|
4211
4580
|
} catch (err) {
|
|
@@ -4215,7 +4584,7 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
4215
4584
|
}
|
|
4216
4585
|
let content;
|
|
4217
4586
|
try {
|
|
4218
|
-
content = decodeSource(await
|
|
4587
|
+
content = decodeSource(await fs4.promises.readFile(filePath));
|
|
4219
4588
|
} catch {
|
|
4220
4589
|
return;
|
|
4221
4590
|
}
|
|
@@ -4252,7 +4621,8 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
4252
4621
|
}
|
|
4253
4622
|
try {
|
|
4254
4623
|
const db = getDb(dbPath);
|
|
4255
|
-
const outcome = await indexFile(db, filePath, content, boundaries);
|
|
4624
|
+
const outcome = await indexFile(db, filePath, content, boundaries, sha);
|
|
4625
|
+
if (outcome === "stale") return;
|
|
4256
4626
|
if (outcome !== "unavailable" && enforceStoredChunkCap(db, filePath, sha, ixCfg.max_chunks_per_file)) return;
|
|
4257
4627
|
stampEmbedSha(db, filePath, sha, (s) => outcome === "unavailable" ? unavailableEmbedSha(s) : s);
|
|
4258
4628
|
} catch (err) {
|
|
@@ -4269,7 +4639,7 @@ function stampEmbedSha(db, filePath, sha, makeValue) {
|
|
|
4269
4639
|
}
|
|
4270
4640
|
function safeMtime(filePath) {
|
|
4271
4641
|
try {
|
|
4272
|
-
return
|
|
4642
|
+
return fs4.statSync(filePath).mtimeMs / 1e3;
|
|
4273
4643
|
} catch {
|
|
4274
4644
|
return 0;
|
|
4275
4645
|
}
|
|
@@ -4277,7 +4647,7 @@ function safeMtime(filePath) {
|
|
|
4277
4647
|
|
|
4278
4648
|
// src/section_reader.ts
|
|
4279
4649
|
init_define_import_meta_env();
|
|
4280
|
-
import { readFileSync as
|
|
4650
|
+
import { readFileSync as readFileSync4 } from "node:fs";
|
|
4281
4651
|
function parseHeadingSpec(spec, headers) {
|
|
4282
4652
|
const m = /^([^#\r\n]+)#(\d+)$/.exec(spec);
|
|
4283
4653
|
if (m !== null && m[1] !== void 0 && m[2] !== void 0) {
|
|
@@ -4303,38 +4673,33 @@ function normalizeHeadingStrip(s) {
|
|
|
4303
4673
|
return n.replace(/\s+/g, " ").trim();
|
|
4304
4674
|
}
|
|
4305
4675
|
var MIN_WIDEN_WORD_LEN = 3;
|
|
4306
|
-
var MARKDOWN_HEADER_RE = /^(#{1,6})\s+([^\r\n]+?)(?:\s+#+)?\s*$/;
|
|
4307
|
-
var TABLE_HEADER_RE = /^\s*\[+\s*([^\]]+?)\s*\]+\s*(?:[#;].*)?$/;
|
|
4308
4676
|
var PYTHON_HEADER_RE = /^(\s*)(?:async\s+)?(?:def|class)\s+([A-Za-z_]\w*)/;
|
|
4309
4677
|
var KEYVALUE_HEADER_RE = /^([A-Za-z_][\w.-]*)\s*(?:=|:(?!\/\/))/;
|
|
4310
4678
|
var ENV_KEYVALUE_HEADER_RE = /^(?:export\s+)?([A-Za-z_][\w.-]*)\s*(?:=|:(?!\/\/))/;
|
|
4311
4679
|
function findMarkdownHeaders(lines) {
|
|
4312
4680
|
const headers = [];
|
|
4313
|
-
const
|
|
4681
|
+
const fmEnd = frontMatterEndIndex(lines);
|
|
4682
|
+
const unfenced = Array.from(eachUnfencedLine(lines)).filter(([i]) => i >= fmEnd);
|
|
4683
|
+
let paraStart = -1;
|
|
4314
4684
|
for (let u = 0; u < unfenced.length; u++) {
|
|
4315
4685
|
const [i, line] = unfenced[u];
|
|
4316
|
-
const
|
|
4317
|
-
if (
|
|
4318
|
-
headers.push({ heading:
|
|
4686
|
+
const atx = matchAtxHeading(line, ATX_CLASS_BREAK_RE, true);
|
|
4687
|
+
if (atx !== null) {
|
|
4688
|
+
headers.push({ heading: atx.name, level: atx.level, index: i });
|
|
4689
|
+
paraStart = -1;
|
|
4319
4690
|
continue;
|
|
4320
4691
|
}
|
|
4692
|
+
if (paraStart !== -1 && i !== unfenced[u - 1][0] + 1) paraStart = -1;
|
|
4321
4693
|
const trimmed = line.trim();
|
|
4322
|
-
if (
|
|
4323
|
-
const [
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
if (eqMatch !== null) {
|
|
4328
|
-
headers.push({ heading: trimmed, level: 1, index: i });
|
|
4329
|
-
u++;
|
|
4330
|
-
continue;
|
|
4331
|
-
} else if (dashMatch !== null) {
|
|
4332
|
-
headers.push({ heading: trimmed, level: 2, index: i });
|
|
4333
|
-
u++;
|
|
4334
|
-
continue;
|
|
4335
|
-
}
|
|
4336
|
-
}
|
|
4694
|
+
if (paraStart !== -1 && /^(=+|-+)$/.test(trimmed)) {
|
|
4695
|
+
const heading = unfenced.slice(paraStart, u).map(([, text]) => text.trim()).join(" ");
|
|
4696
|
+
headers.push({ heading, level: trimmed.startsWith("=") ? 1 : 2, index: unfenced[paraStart][0] });
|
|
4697
|
+
paraStart = -1;
|
|
4698
|
+
continue;
|
|
4337
4699
|
}
|
|
4700
|
+
const isText = trimmed !== "" && !trimmed.startsWith("#") && !trimmed.startsWith("|") && !trimmed.startsWith("```") && !trimmed.startsWith("~~~") && !/^([-*+]|\d+\.)\s/.test(trimmed) && !/^(=+|-+)$/.test(trimmed);
|
|
4701
|
+
if (!isText) paraStart = -1;
|
|
4702
|
+
else if (paraStart === -1) paraStart = u;
|
|
4338
4703
|
}
|
|
4339
4704
|
return headers;
|
|
4340
4705
|
}
|
|
@@ -4349,8 +4714,8 @@ function findTableHeaders(lines, isToml) {
|
|
|
4349
4714
|
const closeIdx = line.indexOf(openDelim);
|
|
4350
4715
|
if (closeIdx === -1) continue;
|
|
4351
4716
|
const restStart = closeIdx + openDelim.length;
|
|
4352
|
-
const
|
|
4353
|
-
if (
|
|
4717
|
+
const name2 = matchTableHeaderName(line.slice(restStart));
|
|
4718
|
+
if (name2 !== null) headers.push({ heading: name2.trim(), level: 1, index: i });
|
|
4354
4719
|
openDelim = lineOpenDelimiterAfter(stripTomlComment(line.slice(restStart)), 0);
|
|
4355
4720
|
continue;
|
|
4356
4721
|
}
|
|
@@ -4358,8 +4723,8 @@ function findTableHeaders(lines, isToml) {
|
|
|
4358
4723
|
arrayDepth = Math.max(0, arrayDepth + tomlBracketDelta(stripTomlComment(line)));
|
|
4359
4724
|
continue;
|
|
4360
4725
|
}
|
|
4361
|
-
const
|
|
4362
|
-
if (
|
|
4726
|
+
const name = matchTableHeaderName(line);
|
|
4727
|
+
if (name !== null) headers.push({ heading: name.trim(), level: 1, index: i });
|
|
4363
4728
|
if (isToml) {
|
|
4364
4729
|
const code = stripTomlComment(line);
|
|
4365
4730
|
openDelim = lineOpenDelimiterAfter(code, 0);
|
|
@@ -4381,9 +4746,9 @@ function extractShellBannerHeading(line) {
|
|
|
4381
4746
|
return { heading: text, level: mdMatch[1].length };
|
|
4382
4747
|
}
|
|
4383
4748
|
}
|
|
4384
|
-
const
|
|
4385
|
-
if (
|
|
4386
|
-
const text =
|
|
4749
|
+
const ruleTitle = matchRuleBannerTitle(trimmed);
|
|
4750
|
+
if (ruleTitle) {
|
|
4751
|
+
const text = ruleTitle.trim();
|
|
4387
4752
|
if (text && !/^[-=]+$/.test(text)) {
|
|
4388
4753
|
return { heading: text, level: 1 };
|
|
4389
4754
|
}
|
|
@@ -4741,7 +5106,7 @@ function readTextForSections(filePath, readFn) {
|
|
|
4741
5106
|
text = read;
|
|
4742
5107
|
} else {
|
|
4743
5108
|
try {
|
|
4744
|
-
text = decodeSource(
|
|
5109
|
+
text = decodeSource(readFileSync4(filePath));
|
|
4745
5110
|
} catch {
|
|
4746
5111
|
return null;
|
|
4747
5112
|
}
|
|
@@ -4811,19 +5176,32 @@ export {
|
|
|
4811
5176
|
extractChangelogVersionHint,
|
|
4812
5177
|
isImagePath,
|
|
4813
5178
|
isNonTextAsset,
|
|
5179
|
+
ORT_WEB_VERSION,
|
|
5180
|
+
ORT_WEB_WASM,
|
|
5181
|
+
NATIVE_RUNTIME_INSTALL,
|
|
5182
|
+
RUNTIME_UNAVAILABLE_ADVICE,
|
|
5183
|
+
activeRuntime,
|
|
5184
|
+
nativeRuntimeLoadError,
|
|
5185
|
+
runtimeVersion,
|
|
5186
|
+
wasmBinaryPresent,
|
|
5187
|
+
ensureWasmBinary,
|
|
5188
|
+
DEFAULT_MODEL,
|
|
5189
|
+
modelDir,
|
|
4814
5190
|
modelFilesPresent,
|
|
4815
5191
|
ensureModelFiles,
|
|
4816
5192
|
checkEmbeddingPreflight,
|
|
5193
|
+
DEFAULT_DISTANCE_THRESHOLD,
|
|
4817
5194
|
OVER_FETCH_FACTOR,
|
|
4818
5195
|
MAX_OVER_FETCH,
|
|
4819
5196
|
isAvailable,
|
|
4820
5197
|
embeddingBackendLoadError,
|
|
4821
5198
|
embedTexts,
|
|
4822
5199
|
searchSemantic,
|
|
4823
|
-
mergeNearbyHits,
|
|
4824
5200
|
embeddingsDepsAvailable,
|
|
4825
5201
|
deleteFileEmbeddings,
|
|
4826
5202
|
ensureEmbeddingProvenance,
|
|
5203
|
+
withDeclaredSkipScopeRoot,
|
|
5204
|
+
hasSkipSegmentBelowRoot,
|
|
4827
5205
|
parserFingerprintForLanguage,
|
|
4828
5206
|
treeSitterCoreAvailable,
|
|
4829
5207
|
treeSitterCoreLoadError,
|
|
@@ -4834,6 +5212,8 @@ export {
|
|
|
4834
5212
|
deleteFileRows,
|
|
4835
5213
|
isParseSkipEligible,
|
|
4836
5214
|
indexFileSync,
|
|
5215
|
+
disabledStampCount,
|
|
5216
|
+
oversizeEmbedSha,
|
|
4837
5217
|
assetEmbedSha,
|
|
4838
5218
|
maxChunksEmbedSha,
|
|
4839
5219
|
indexedPathSpellingIsStale,
|