token-goat 2.9.12 → 2.9.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/{token-goat-chunk-6NIGPPN6.mjs → token-goat-chunk-2WC4ZUXN.mjs} +1 -1
- package/dist/{token-goat-chunk-GCXX67HM.mjs → token-goat-chunk-3ESRORNM.mjs} +188 -97
- package/dist/token-goat-chunk-4NXUKV7D.mjs +56 -0
- package/dist/{token-goat-chunk-OTW7LC4Y.mjs → token-goat-chunk-6B44WLIF.mjs} +5 -5
- package/dist/{token-goat-chunk-6F5TLJC7.mjs → token-goat-chunk-B3CTCQTH.mjs} +1 -1
- package/dist/{token-goat-chunk-A6QTLAWO.mjs → token-goat-chunk-FQCNJV4V.mjs} +20 -6
- package/dist/{token-goat-chunk-SBYRP3X4.mjs → token-goat-chunk-FZU7GMUS.mjs} +49 -24
- package/dist/{token-goat-chunk-TOGYS5A7.mjs → token-goat-chunk-JOXLE672.mjs} +344 -5699
- package/dist/{token-goat-chunk-HKFOH6JH.mjs → token-goat-chunk-JVNPCQB7.mjs} +10 -7
- package/dist/{token-goat-chunk-LHLQFGWQ.mjs → token-goat-chunk-OMNQUUIT.mjs} +9 -27
- package/dist/{token-goat-chunk-VRWX6QYW.mjs → token-goat-chunk-P2PU4CR5.mjs} +8 -6
- package/dist/{token-goat-chunk-LOCOX2ML.mjs → token-goat-chunk-QKXBGBQR.mjs} +110 -21
- package/dist/{token-goat-chunk-THLHC6QJ.mjs → token-goat-chunk-QWSUZWFP.mjs} +44 -33
- package/dist/{token-goat-chunk-SF2CKCFQ.mjs → token-goat-chunk-RDITECDL.mjs} +7 -5
- package/dist/{token-goat-chunk-4CK445AW.mjs → token-goat-chunk-U7X6LQD2.mjs} +271 -69
- package/dist/token-goat-chunk-UM47DRD3.mjs +242 -0
- package/dist/token-goat-chunk-UMXJN7DI.mjs +5521 -0
- package/dist/token-goat-chunk-YOA4N6WA.mjs +308 -0
- package/dist/{token-goat-chunk-GM6QZWCF.mjs → token-goat-chunk-ZZI3IDQZ.mjs} +963 -966
- package/dist/token-goat-hook.mjs +8 -6
- package/dist/token-goat.core.mjs +10 -7
- package/docs/cli.md +11 -8
- package/docs/security.md +1 -1
- package/package.json +1 -1
|
@@ -2,46 +2,57 @@ import { createRequire as __cjsRequire } from 'node:module';
|
|
|
2
2
|
const require = __cjsRequire(import.meta.url);
|
|
3
3
|
import {
|
|
4
4
|
OUTPUT_FIRST_TOOL_RESPONSE_KEYS,
|
|
5
|
-
SOURCE_HINT,
|
|
6
|
-
TREE_SITTER_LANGUAGES,
|
|
7
5
|
VSCODE_TOOL_NAME_KEY,
|
|
8
6
|
contextOutput,
|
|
9
7
|
denyOutput,
|
|
10
|
-
detectHarness,
|
|
11
|
-
detectLanguage,
|
|
12
|
-
detectLanguageOfFile,
|
|
13
8
|
emitRewrite,
|
|
14
9
|
estimateTokens,
|
|
15
10
|
extractToolResponseField,
|
|
11
|
+
getCwd,
|
|
12
|
+
getFilePath,
|
|
13
|
+
hasUnquotedOperator,
|
|
14
|
+
isRewriteWorthwhile,
|
|
15
|
+
materializeShrunkImageFile,
|
|
16
|
+
passOutput,
|
|
17
|
+
registerHook,
|
|
18
|
+
resolveMinNetSavingsBytes,
|
|
19
|
+
sessionStateKey
|
|
20
|
+
} from "./token-goat-chunk-JOXLE672.mjs";
|
|
21
|
+
import {
|
|
22
|
+
DEFAULT_MAX_AGE_MS,
|
|
23
|
+
DEFAULT_MAX_COUNT,
|
|
24
|
+
blobPath,
|
|
25
|
+
isBlobStale,
|
|
26
|
+
loadBlob,
|
|
27
|
+
storeBlob,
|
|
28
|
+
sweepCacheRoots
|
|
29
|
+
} from "./token-goat-chunk-UM47DRD3.mjs";
|
|
30
|
+
import {
|
|
31
|
+
SOURCE_HINT,
|
|
32
|
+
TREE_SITTER_LANGUAGES,
|
|
33
|
+
detectHarness,
|
|
34
|
+
detectLanguage,
|
|
35
|
+
detectLanguageOfFile,
|
|
16
36
|
fileIsAbsent,
|
|
17
37
|
findProject,
|
|
18
38
|
fingerprintContent,
|
|
19
39
|
fingerprintFile,
|
|
20
|
-
getCwd,
|
|
21
40
|
getDb,
|
|
22
|
-
getFilePath,
|
|
23
41
|
getHarnessName,
|
|
24
|
-
hasUnquotedOperator,
|
|
25
42
|
isAutoTriggerMultiplierExplicit,
|
|
26
|
-
isRewriteWorthwhile,
|
|
27
43
|
isUnderSystemTemp,
|
|
28
44
|
languageHasFlag,
|
|
29
45
|
loadConfig,
|
|
30
46
|
makeProjectAt,
|
|
31
|
-
materializeShrunkImageFile,
|
|
32
|
-
passOutput,
|
|
33
47
|
recordStat,
|
|
34
48
|
redactIfDotenv,
|
|
35
49
|
redactSecrets,
|
|
36
50
|
refineLanguageByContent,
|
|
37
|
-
registerHook,
|
|
38
|
-
resolveMinNetSavingsBytes,
|
|
39
51
|
resolveProjectRoot,
|
|
40
52
|
savedTokensFromBytes,
|
|
41
|
-
sessionStateKey,
|
|
42
53
|
shortFingerprint,
|
|
43
54
|
summarize
|
|
44
|
-
} from "./token-goat-chunk-
|
|
55
|
+
} from "./token-goat-chunk-UMXJN7DI.mjs";
|
|
45
56
|
import {
|
|
46
57
|
registerReset
|
|
47
58
|
} from "./token-goat-chunk-EEIDFMEM.mjs";
|
|
@@ -71,18 +82,20 @@ import {
|
|
|
71
82
|
extractErrorMessage,
|
|
72
83
|
fenceUntrustedContent,
|
|
73
84
|
fenceUntrustedFileContent,
|
|
74
|
-
fenceUntrustedOcrText,
|
|
75
85
|
filtersFilteredToEmptyNotice,
|
|
76
86
|
findHtmlHeadingMatches,
|
|
77
87
|
foldPath,
|
|
78
88
|
foldPathForContainment,
|
|
89
|
+
formatSymbolLocation,
|
|
79
90
|
globalDbPath,
|
|
91
|
+
ipynbToVirtualSource,
|
|
80
92
|
isCaseInsensitiveFs,
|
|
81
93
|
isCodeFenceDelimiter,
|
|
82
94
|
isInsideRoot,
|
|
83
95
|
isTestFile,
|
|
84
96
|
isUncOrDevicePath,
|
|
85
97
|
isUnderBlockedRoot,
|
|
98
|
+
isVirtualIndexedPath,
|
|
86
99
|
isWithinQuietHours,
|
|
87
100
|
normalizePath,
|
|
88
101
|
normalizePathForwardSlash,
|
|
@@ -103,7 +116,7 @@ import {
|
|
|
103
116
|
toKB,
|
|
104
117
|
tokenGoatHome,
|
|
105
118
|
withFileLock
|
|
106
|
-
} from "./token-goat-chunk-
|
|
119
|
+
} from "./token-goat-chunk-QKXBGBQR.mjs";
|
|
107
120
|
import {
|
|
108
121
|
__commonJS,
|
|
109
122
|
__toESM,
|
|
@@ -2258,9 +2271,9 @@ var require_decoder = __commonJS({
|
|
|
2258
2271
|
return a < 0 ? 0 : a > 255 ? 255 : a;
|
|
2259
2272
|
}
|
|
2260
2273
|
constructor.prototype = {
|
|
2261
|
-
load: function load3(
|
|
2274
|
+
load: function load3(path22) {
|
|
2262
2275
|
var xhr = new XMLHttpRequest();
|
|
2263
|
-
xhr.open("GET",
|
|
2276
|
+
xhr.open("GET", path22, true);
|
|
2264
2277
|
xhr.responseType = "arraybuffer";
|
|
2265
2278
|
xhr.onload = (function() {
|
|
2266
2279
|
var data = new Uint8Array(xhr.response || xhr.mozResponseArrayBuffer);
|
|
@@ -2836,10 +2849,13 @@ import * as fs from "node:fs";
|
|
|
2836
2849
|
|
|
2837
2850
|
// src/document_refusal.ts
|
|
2838
2851
|
init_define_import_meta_env();
|
|
2852
|
+
var MAX_DOCUMENT_WORK_MILLIS = 6e4;
|
|
2839
2853
|
var DocumentRefusedError = class extends Error {
|
|
2840
|
-
|
|
2854
|
+
transient;
|
|
2855
|
+
constructor(message, name, transient = false) {
|
|
2841
2856
|
super(message);
|
|
2842
2857
|
this.name = name;
|
|
2858
|
+
this.transient = transient;
|
|
2843
2859
|
}
|
|
2844
2860
|
};
|
|
2845
2861
|
|
|
@@ -2863,6 +2879,11 @@ function createLazyModuleLoader(load3, errorLabel) {
|
|
|
2863
2879
|
// src/xml_parser.ts
|
|
2864
2880
|
init_define_import_meta_env();
|
|
2865
2881
|
var MAX_XML_DEPTH = 512;
|
|
2882
|
+
var XmlTooDeepError = class extends DocumentRefusedError {
|
|
2883
|
+
constructor(message) {
|
|
2884
|
+
super(message, "XmlTooDeepError");
|
|
2885
|
+
}
|
|
2886
|
+
};
|
|
2866
2887
|
var NAMED_ENTITIES = {
|
|
2867
2888
|
lt: "<",
|
|
2868
2889
|
gt: ">",
|
|
@@ -3060,7 +3081,7 @@ function parseXml(xml) {
|
|
|
3060
3081
|
continue;
|
|
3061
3082
|
}
|
|
3062
3083
|
if (stack.length > MAX_XML_DEPTH) {
|
|
3063
|
-
throw new
|
|
3084
|
+
throw new XmlTooDeepError(`XML nesting deeper than ${MAX_XML_DEPTH} elements; refusing to parse (this file is not something any office application produces)`);
|
|
3064
3085
|
}
|
|
3065
3086
|
stack.push(frame);
|
|
3066
3087
|
}
|
|
@@ -3145,10 +3166,16 @@ function unzipBounded(mod, data, opts) {
|
|
|
3145
3166
|
}
|
|
3146
3167
|
|
|
3147
3168
|
// src/ooxml_extract.ts
|
|
3148
|
-
var MAX_OOXML_WORK_MILLIS =
|
|
3169
|
+
var MAX_OOXML_WORK_MILLIS = MAX_DOCUMENT_WORK_MILLIS;
|
|
3149
3170
|
var OoxmlTookTooLongError = class extends DocumentRefusedError {
|
|
3150
3171
|
constructor(message) {
|
|
3151
|
-
super(message, "OoxmlTookTooLongError");
|
|
3172
|
+
super(message, "OoxmlTookTooLongError", true);
|
|
3173
|
+
}
|
|
3174
|
+
};
|
|
3175
|
+
var NotAnOfficeDocumentError = class extends DocumentRefusedError {
|
|
3176
|
+
constructor(message, cause) {
|
|
3177
|
+
super(message, "NotAnOfficeDocumentError");
|
|
3178
|
+
if (cause !== void 0) this.cause = cause;
|
|
3152
3179
|
}
|
|
3153
3180
|
};
|
|
3154
3181
|
function ooxmlWorkDeadline() {
|
|
@@ -3177,7 +3204,7 @@ async function readOoxmlZip(filePath, kind) {
|
|
|
3177
3204
|
} catch (err) {
|
|
3178
3205
|
throw new Error(accessFailureMessage(err, filePath), { cause: err });
|
|
3179
3206
|
}
|
|
3180
|
-
if (!stat2.isFile()) throw new
|
|
3207
|
+
if (!stat2.isFile()) throw new NotAnOfficeDocumentError(`not a valid ${kind} file: ${filePath}`);
|
|
3181
3208
|
if (stat2.size > MAX_ZIP_INPUT_BYTES) throw new ZipInputTooLargeError(filePath, stat2.size, MAX_ZIP_INPUT_BYTES);
|
|
3182
3209
|
let data;
|
|
3183
3210
|
try {
|
|
@@ -3189,12 +3216,30 @@ async function readOoxmlZip(filePath, kind) {
|
|
|
3189
3216
|
return unzipBounded(fflate, new Uint8Array(data), { limitBytes: MAX_ZIP_OUTPUT_BYTES, shouldExtract: () => true });
|
|
3190
3217
|
} catch (err) {
|
|
3191
3218
|
if (err instanceof ZipOutputTooLargeError) throw err;
|
|
3192
|
-
throw new
|
|
3219
|
+
throw new NotAnOfficeDocumentError(`not a valid ${kind} file: ${filePath}`, err);
|
|
3220
|
+
}
|
|
3221
|
+
}
|
|
3222
|
+
var MAX_OOXML_PART_BYTES = 32 * 1024 * 1024;
|
|
3223
|
+
var OoxmlPartTooLargeError = class extends DocumentRefusedError {
|
|
3224
|
+
constructor(entryPath, bytes) {
|
|
3225
|
+
super(`${entryPath} is ${bytes} bytes, past the ${MAX_OOXML_PART_BYTES}-byte limit for one part of an office file. Split the document, or extract from a smaller copy.`, "OoxmlPartTooLargeError");
|
|
3226
|
+
}
|
|
3227
|
+
};
|
|
3228
|
+
var MAX_OOXML_DOCUMENT_PART_BYTES = 64 * 1024 * 1024;
|
|
3229
|
+
var OoxmlDocumentTooLargeError = class extends DocumentRefusedError {
|
|
3230
|
+
constructor(entryPath, spentBytes, partBytes) {
|
|
3231
|
+
super(`decoding ${entryPath} (${partBytes} bytes, after ${spentBytes} already decoded) would take this office file past the ${MAX_OOXML_DOCUMENT_PART_BYTES}-byte limit on the XML one document may have decoded at once. Split the document, or extract from a smaller copy.`, "OoxmlDocumentTooLargeError");
|
|
3193
3232
|
}
|
|
3233
|
+
};
|
|
3234
|
+
function ooxmlPartBudget() {
|
|
3235
|
+
return { spent: 0 };
|
|
3194
3236
|
}
|
|
3195
|
-
function decodeZipEntry(entries, entryPath) {
|
|
3237
|
+
function decodeZipEntry(entries, entryPath, budget) {
|
|
3196
3238
|
const bytes = entries[entryPath];
|
|
3197
3239
|
if (bytes === void 0) return null;
|
|
3240
|
+
if (bytes.length > MAX_OOXML_PART_BYTES) throw new OoxmlPartTooLargeError(entryPath, bytes.length);
|
|
3241
|
+
if (budget.spent + bytes.length > MAX_OOXML_DOCUMENT_PART_BYTES) throw new OoxmlDocumentTooLargeError(entryPath, budget.spent, bytes.length);
|
|
3242
|
+
budget.spent += bytes.length;
|
|
3198
3243
|
return new TextDecoder("utf-8").decode(bytes);
|
|
3199
3244
|
}
|
|
3200
3245
|
async function parseOoxmlPart(xmlText) {
|
|
@@ -3273,8 +3318,8 @@ function headingLevel(styleVal) {
|
|
|
3273
3318
|
}
|
|
3274
3319
|
async function loadDocumentBody(filePath) {
|
|
3275
3320
|
const entries = await readOoxmlZip(filePath, ".docx");
|
|
3276
|
-
const xml = decodeZipEntry(entries, "word/document.xml");
|
|
3277
|
-
if (xml === null) throw new
|
|
3321
|
+
const xml = decodeZipEntry(entries, "word/document.xml", ooxmlPartBudget());
|
|
3322
|
+
if (xml === null) throw new NotAnOfficeDocumentError(`no word/document.xml found in ${filePath} (not a valid .docx?)`);
|
|
3278
3323
|
return parseOoxmlPart(xml);
|
|
3279
3324
|
}
|
|
3280
3325
|
async function docxOutline(filePath) {
|
|
@@ -3295,6 +3340,68 @@ async function docxText(filePath) {
|
|
|
3295
3340
|
const lines2 = paragraphs.map((p) => collectTextRuns(p, "w:t").join("")).filter((t) => t.trim().length > 0);
|
|
3296
3341
|
return lines2.join("\n\n");
|
|
3297
3342
|
}
|
|
3343
|
+
async function docxTables(filePath) {
|
|
3344
|
+
const parsed = await loadDocumentBody(filePath);
|
|
3345
|
+
const tbls = collectElements(parsed, "w:tbl");
|
|
3346
|
+
const out = [];
|
|
3347
|
+
for (let i = 0; i < tbls.length; i++) {
|
|
3348
|
+
const tbl = tbls[i];
|
|
3349
|
+
const trElements = collectElements(tbl, "w:tr");
|
|
3350
|
+
const rows = [];
|
|
3351
|
+
let maxCols = 0;
|
|
3352
|
+
for (const tr of trElements) {
|
|
3353
|
+
const tcElements = collectElements(tr, "w:tc");
|
|
3354
|
+
const rowCells = [];
|
|
3355
|
+
for (const tc of tcElements) {
|
|
3356
|
+
const cellText2 = collectTextRuns(tc, "w:t").join("").replace(/\r?\n/g, " ").trim();
|
|
3357
|
+
rowCells.push(cellText2);
|
|
3358
|
+
}
|
|
3359
|
+
if (rowCells.length > maxCols) maxCols = rowCells.length;
|
|
3360
|
+
rows.push(rowCells);
|
|
3361
|
+
}
|
|
3362
|
+
if (rows.length > 0) {
|
|
3363
|
+
out.push({
|
|
3364
|
+
tableIndex: i + 1,
|
|
3365
|
+
rowCount: rows.length,
|
|
3366
|
+
colCount: maxCols,
|
|
3367
|
+
rows
|
|
3368
|
+
});
|
|
3369
|
+
}
|
|
3370
|
+
}
|
|
3371
|
+
return out;
|
|
3372
|
+
}
|
|
3373
|
+
function formatDocxTables(tables, opts) {
|
|
3374
|
+
if (tables.length === 0) return "no tables found in document";
|
|
3375
|
+
const selected = opts?.tableIndex !== void 0 ? tables.filter((t) => t.tableIndex === opts.tableIndex) : tables;
|
|
3376
|
+
if (selected.length === 0) {
|
|
3377
|
+
return `table ${opts?.tableIndex} not found (document has ${tables.length} table${tables.length === 1 ? "" : "s"})`;
|
|
3378
|
+
}
|
|
3379
|
+
const sections = [];
|
|
3380
|
+
for (const table of selected) {
|
|
3381
|
+
const lines2 = [
|
|
3382
|
+
`Table ${table.tableIndex} (${table.rowCount} row${table.rowCount === 1 ? "" : "s"} x ${table.colCount} col${table.colCount === 1 ? "" : "s"}):`
|
|
3383
|
+
];
|
|
3384
|
+
if (table.rows.length === 0) {
|
|
3385
|
+
lines2.push(" (empty)");
|
|
3386
|
+
sections.push(lines2.join("\n"));
|
|
3387
|
+
continue;
|
|
3388
|
+
}
|
|
3389
|
+
const header = table.rows[0];
|
|
3390
|
+
const padCols = Math.max(table.colCount, 1);
|
|
3391
|
+
const paddedHeader = [...header];
|
|
3392
|
+
while (paddedHeader.length < padCols) paddedHeader.push("");
|
|
3393
|
+
lines2.push("| " + paddedHeader.map((c) => displaySafeText(c.replace(/\|/g, "\\|"))).join(" | ") + " |");
|
|
3394
|
+
lines2.push("| " + Array.from({ length: padCols }, () => "---").join(" | ") + " |");
|
|
3395
|
+
for (let r = 1; r < table.rows.length; r++) {
|
|
3396
|
+
const row = table.rows[r];
|
|
3397
|
+
const paddedRow = [...row];
|
|
3398
|
+
while (paddedRow.length < padCols) paddedRow.push("");
|
|
3399
|
+
lines2.push("| " + paddedRow.map((c) => displaySafeText(c.replace(/\|/g, "\\|"))).join(" | ") + " |");
|
|
3400
|
+
}
|
|
3401
|
+
sections.push(lines2.join("\n"));
|
|
3402
|
+
}
|
|
3403
|
+
return sections.join("\n\n");
|
|
3404
|
+
}
|
|
3298
3405
|
|
|
3299
3406
|
// src/pptx_extract.ts
|
|
3300
3407
|
init_define_import_meta_env();
|
|
@@ -3319,9 +3426,9 @@ function tableRowBlocks(parsedSlide) {
|
|
|
3319
3426
|
}
|
|
3320
3427
|
return blocks;
|
|
3321
3428
|
}
|
|
3322
|
-
async function slidePathsInPresentationOrder(entries) {
|
|
3323
|
-
const presXml = decodeZipEntry(entries, "ppt/presentation.xml");
|
|
3324
|
-
const relsXml = decodeZipEntry(entries, "ppt/_rels/presentation.xml.rels");
|
|
3429
|
+
async function slidePathsInPresentationOrder(entries, budget) {
|
|
3430
|
+
const presXml = decodeZipEntry(entries, "ppt/presentation.xml", budget);
|
|
3431
|
+
const relsXml = decodeZipEntry(entries, "ppt/_rels/presentation.xml.rels", budget);
|
|
3325
3432
|
if (presXml === null || relsXml === null) return null;
|
|
3326
3433
|
const presParsed = await parseOoxmlPart(presXml);
|
|
3327
3434
|
const relsParsed = await parseOoxmlPart(relsXml);
|
|
@@ -3332,24 +3439,28 @@ async function slidePathsInPresentationOrder(entries) {
|
|
|
3332
3439
|
}
|
|
3333
3440
|
}
|
|
3334
3441
|
const ordered = [];
|
|
3442
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3335
3443
|
for (const sldId of collectElements(presParsed, "p:sldId")) {
|
|
3336
3444
|
const rid = sldId["@_r:id"];
|
|
3337
3445
|
if (rid === void 0) continue;
|
|
3338
3446
|
const target = ridToTarget.get(rid);
|
|
3339
3447
|
if (target === void 0) continue;
|
|
3340
3448
|
const normalized = target.startsWith("slides/") ? `ppt/${target}` : `ppt/slides/${target}`;
|
|
3341
|
-
if (entries[normalized]
|
|
3449
|
+
if (entries[normalized] === void 0 || seen.has(normalized)) continue;
|
|
3450
|
+
seen.add(normalized);
|
|
3451
|
+
ordered.push(normalized);
|
|
3342
3452
|
}
|
|
3343
3453
|
return ordered.length > 0 ? ordered : null;
|
|
3344
3454
|
}
|
|
3345
3455
|
async function listSlideParts(filePath) {
|
|
3346
3456
|
const entries = await readOoxmlZip(filePath, ".pptx");
|
|
3347
|
-
const
|
|
3457
|
+
const budget = ooxmlPartBudget();
|
|
3458
|
+
const slidePaths = await slidePathsInPresentationOrder(entries, budget) ?? sortNumberedParts(
|
|
3348
3459
|
Object.keys(entries).filter((p) => /^ppt\/slides\/slide\d+\.xml$/.test(p)),
|
|
3349
3460
|
/slide(\d+)\.xml$/
|
|
3350
3461
|
);
|
|
3351
|
-
if (slidePaths.length === 0) throw new
|
|
3352
|
-
return { entries, slidePaths };
|
|
3462
|
+
if (slidePaths.length === 0) throw new NotAnOfficeDocumentError(`no slides found in ${filePath} (not a valid .pptx?)`);
|
|
3463
|
+
return { entries, slidePaths, budget };
|
|
3353
3464
|
}
|
|
3354
3465
|
function relsPathFor(slidePath) {
|
|
3355
3466
|
const idx = slidePath.lastIndexOf("/");
|
|
@@ -3366,8 +3477,8 @@ function resolveRelativeTarget(basePath, target) {
|
|
|
3366
3477
|
}
|
|
3367
3478
|
return resolved.join("/");
|
|
3368
3479
|
}
|
|
3369
|
-
async function notesPathFor(entries, slidePath) {
|
|
3370
|
-
const relsXml = decodeZipEntry(entries, relsPathFor(slidePath));
|
|
3480
|
+
async function notesPathFor(entries, slidePath, budget) {
|
|
3481
|
+
const relsXml = decodeZipEntry(entries, relsPathFor(slidePath), budget);
|
|
3371
3482
|
if (relsXml === null) return null;
|
|
3372
3483
|
const relsParsed = await parseOoxmlPart(relsXml);
|
|
3373
3484
|
for (const rel of collectElements(relsParsed, "Relationship")) {
|
|
@@ -3377,26 +3488,27 @@ async function notesPathFor(entries, slidePath) {
|
|
|
3377
3488
|
}
|
|
3378
3489
|
return null;
|
|
3379
3490
|
}
|
|
3380
|
-
async function parseSlide(entries,
|
|
3381
|
-
const xml = decodeZipEntry(entries,
|
|
3382
|
-
if (xml === null) throw new Error(`missing part: ${
|
|
3491
|
+
async function parseSlide(entries, path22, budget) {
|
|
3492
|
+
const xml = decodeZipEntry(entries, path22, budget);
|
|
3493
|
+
if (xml === null) throw new Error(`missing part: ${path22}`);
|
|
3383
3494
|
return parseOoxmlPart(xml);
|
|
3384
3495
|
}
|
|
3385
|
-
async function notesTextFor(entries, notesPath) {
|
|
3496
|
+
async function notesTextFor(entries, notesPath, budget) {
|
|
3386
3497
|
if (notesPath === null) return "";
|
|
3387
|
-
const xml = decodeZipEntry(entries, notesPath);
|
|
3498
|
+
const xml = decodeZipEntry(entries, notesPath, budget);
|
|
3388
3499
|
if (xml === null) return "";
|
|
3389
3500
|
const parsed = await parseOoxmlPart(xml);
|
|
3390
3501
|
const shapes = collectElements(parsed, "p:sp");
|
|
3391
3502
|
const bodyShape = shapes.find((s) => shapePlaceholderType(s) === "body");
|
|
3392
3503
|
return bodyShape !== void 0 ? shapeText(bodyShape) : collectTextRuns(parsed, "a:t").join(" ").trim();
|
|
3393
3504
|
}
|
|
3394
|
-
async function pptxOutline(filePath) {
|
|
3395
|
-
const { entries, slidePaths } = await listSlideParts(filePath);
|
|
3505
|
+
async function pptxOutline(filePath, deadline = ooxmlWorkDeadline()) {
|
|
3506
|
+
const { entries, slidePaths, budget } = await listSlideParts(filePath);
|
|
3396
3507
|
const out = [];
|
|
3397
3508
|
for (let i = 0; i < slidePaths.length; i++) {
|
|
3398
|
-
|
|
3399
|
-
const
|
|
3509
|
+
assertOoxmlWithinDeadline(deadline, "Narrow the read to specific slides with pptx-slide, or use a smaller deck.");
|
|
3510
|
+
const path22 = slidePaths[i];
|
|
3511
|
+
const parsed = await parseSlide(entries, path22, budget);
|
|
3400
3512
|
const shapes = slideShapes(parsed);
|
|
3401
3513
|
const titleShape = shapes.find((s) => {
|
|
3402
3514
|
const t = shapePlaceholderType(s);
|
|
@@ -3405,61 +3517,63 @@ async function pptxOutline(filePath) {
|
|
|
3405
3517
|
const title = titleShape !== void 0 ? shapeText(titleShape) : "";
|
|
3406
3518
|
const allText = collectTextRuns(parsed, "a:t").join(" ");
|
|
3407
3519
|
const bodyChars = Math.max(0, allText.length - title.length);
|
|
3408
|
-
const hasNotes = (await notesTextFor(entries, await notesPathFor(entries,
|
|
3520
|
+
const hasNotes = (await notesTextFor(entries, await notesPathFor(entries, path22, budget), budget)).length > 0;
|
|
3409
3521
|
out.push({ slide: i + 1, title, bodyChars, hasNotes });
|
|
3410
3522
|
}
|
|
3411
3523
|
return out;
|
|
3412
3524
|
}
|
|
3413
|
-
async function slideTextFromParts(entries, slidePaths, slideNumber, includeNotes) {
|
|
3525
|
+
async function slideTextFromParts(entries, slidePaths, slideNumber, includeNotes, budget) {
|
|
3414
3526
|
if (slideNumber < 1 || slideNumber > slidePaths.length) {
|
|
3415
3527
|
throw new Error(`slide ${slideNumber} out of range (this deck has ${slidePaths.length} slides)`);
|
|
3416
3528
|
}
|
|
3417
|
-
const
|
|
3418
|
-
const parsed = await parseSlide(entries,
|
|
3529
|
+
const path22 = slidePaths[slideNumber - 1];
|
|
3530
|
+
const parsed = await parseSlide(entries, path22, budget);
|
|
3419
3531
|
const shapes = slideShapes(parsed);
|
|
3420
3532
|
const blocks = [...shapes.map(shapeText).filter((t) => t.length > 0), ...tableRowBlocks(parsed)];
|
|
3421
3533
|
const lines2 = [`# Slide ${slideNumber}`, ...blocks];
|
|
3422
3534
|
if (includeNotes) {
|
|
3423
|
-
const notes = await notesTextFor(entries, await notesPathFor(entries,
|
|
3535
|
+
const notes = await notesTextFor(entries, await notesPathFor(entries, path22, budget), budget);
|
|
3424
3536
|
if (notes.length > 0) lines2.push("", "## Speaker notes", notes);
|
|
3425
3537
|
}
|
|
3426
3538
|
return lines2.join("\n\n");
|
|
3427
3539
|
}
|
|
3428
3540
|
async function pptxSlideText(filePath, slideNumber, includeNotes) {
|
|
3429
|
-
const { entries, slidePaths } = await listSlideParts(filePath);
|
|
3430
|
-
return slideTextFromParts(entries, slidePaths, slideNumber, includeNotes);
|
|
3541
|
+
const { entries, slidePaths, budget } = await listSlideParts(filePath);
|
|
3542
|
+
return slideTextFromParts(entries, slidePaths, slideNumber, includeNotes, budget);
|
|
3431
3543
|
}
|
|
3432
3544
|
async function pptxAllSlidesText(filePath, includeNotes, deadline = ooxmlWorkDeadline()) {
|
|
3433
|
-
const { entries, slidePaths } = await listSlideParts(filePath);
|
|
3545
|
+
const { entries, slidePaths, budget } = await listSlideParts(filePath);
|
|
3434
3546
|
const slideTexts = [];
|
|
3435
3547
|
for (let i = 1; i <= slidePaths.length; i++) {
|
|
3436
3548
|
assertOoxmlWithinDeadline(deadline, "Narrow the read to specific slides with pptx-slide, or use a smaller deck.");
|
|
3437
|
-
slideTexts.push(await slideTextFromParts(entries, slidePaths, i, includeNotes));
|
|
3549
|
+
slideTexts.push(await slideTextFromParts(entries, slidePaths, i, includeNotes, budget));
|
|
3438
3550
|
}
|
|
3439
3551
|
return slideTexts.join("\n\n");
|
|
3440
3552
|
}
|
|
3441
|
-
async function pptxNotesText(filePath, slideNumber) {
|
|
3442
|
-
const { entries, slidePaths } = await listSlideParts(filePath);
|
|
3553
|
+
async function pptxNotesText(filePath, slideNumber, deadline = ooxmlWorkDeadline()) {
|
|
3554
|
+
const { entries, slidePaths, budget } = await listSlideParts(filePath);
|
|
3443
3555
|
const targets = slideNumber !== void 0 ? [slideNumber] : slidePaths.map((_, i) => i + 1);
|
|
3444
3556
|
const sections = [];
|
|
3445
3557
|
for (const n of targets) {
|
|
3558
|
+
assertOoxmlWithinDeadline(deadline, "Narrow the read to one slide with the slide-number argument, or use a smaller deck.");
|
|
3446
3559
|
if (n < 1 || n > slidePaths.length) throw new Error(`slide ${n} out of range (this deck has ${slidePaths.length} slides)`);
|
|
3447
|
-
const notesPath = await notesPathFor(entries, slidePaths[n - 1]);
|
|
3448
|
-
const text = await notesTextFor(entries, notesPath);
|
|
3560
|
+
const notesPath = await notesPathFor(entries, slidePaths[n - 1], budget);
|
|
3561
|
+
const text = await notesTextFor(entries, notesPath, budget);
|
|
3449
3562
|
if (text.length > 0) sections.push(`# Slide ${n} notes
|
|
3450
3563
|
|
|
3451
3564
|
${text}`);
|
|
3452
3565
|
}
|
|
3453
3566
|
return sections.join("\n\n");
|
|
3454
3567
|
}
|
|
3455
|
-
async function pptxTextGrep(filePath, pattern) {
|
|
3456
|
-
const { entries, slidePaths } = await listSlideParts(filePath);
|
|
3568
|
+
async function pptxTextGrep(filePath, pattern, deadline = ooxmlWorkDeadline()) {
|
|
3569
|
+
const { entries, slidePaths, budget } = await listSlideParts(filePath);
|
|
3457
3570
|
const guarded = compileGuardedRegex(pattern, "i");
|
|
3458
3571
|
if (!guarded.ok) throw new Error(`invalid --grep pattern: ${pattern} -- the pattern ${guarded.reason}`);
|
|
3459
3572
|
const re = guarded.re;
|
|
3460
3573
|
const out = [];
|
|
3461
3574
|
for (let i = 0; i < slidePaths.length; i++) {
|
|
3462
|
-
|
|
3575
|
+
assertOoxmlWithinDeadline(deadline, "Narrow the read to specific slides with pptx-slide, or use a smaller deck.");
|
|
3576
|
+
const parsed = await parseSlide(entries, slidePaths[i], budget);
|
|
3463
3577
|
const text = collectTextRuns(parsed, "a:t").join(" ");
|
|
3464
3578
|
if (re.test(text)) {
|
|
3465
3579
|
const idx = text.search(re);
|
|
@@ -5198,13 +5312,30 @@ var parse = function(data, opts = {}) {
|
|
|
5198
5312
|
};
|
|
5199
5313
|
|
|
5200
5314
|
// src/csv_query.ts
|
|
5315
|
+
var DELIMITER_CANDIDATES = [",", " ", ";", "|"];
|
|
5316
|
+
var DELIMITER_SAMPLE_ROWS = 5;
|
|
5317
|
+
function detectDelimiter(content) {
|
|
5318
|
+
const slice = content.slice(0, 1e4);
|
|
5319
|
+
if (slice.trim() === "") return ",";
|
|
5320
|
+
for (const cand of DELIMITER_CANDIDATES) {
|
|
5321
|
+
let rows;
|
|
5322
|
+
try {
|
|
5323
|
+
rows = parse(slice, { columns: false, skip_empty_lines: true, trim: true, delimiter: cand, bom: true, relax_quotes: true, to: DELIMITER_SAMPLE_ROWS });
|
|
5324
|
+
} catch {
|
|
5325
|
+
continue;
|
|
5326
|
+
}
|
|
5327
|
+
if ((rows[0]?.length ?? 0) < 2) continue;
|
|
5328
|
+
return cand;
|
|
5329
|
+
}
|
|
5330
|
+
return ",";
|
|
5331
|
+
}
|
|
5201
5332
|
function parseRecords(content, opts) {
|
|
5202
|
-
const delimiter = opts.delimiter ??
|
|
5333
|
+
const delimiter = opts.delimiter ?? detectDelimiter(content);
|
|
5203
5334
|
if (opts.noHeader === true) {
|
|
5204
|
-
const rows = parse(content, { columns: false, skip_empty_lines: true, trim: true, delimiter, bom: true, relax_column_count: true });
|
|
5335
|
+
const rows = parse(content, { columns: false, skip_empty_lines: true, trim: true, delimiter, bom: true, relax_column_count: true, relax_quotes: true });
|
|
5205
5336
|
return rows.map((row) => Object.fromEntries(row.map((cell, i) => [`col${i + 1}`, cell])));
|
|
5206
5337
|
}
|
|
5207
|
-
const header = csvHeader(content, opts);
|
|
5338
|
+
const header = csvHeader(content, { ...opts, delimiter });
|
|
5208
5339
|
const dupes = header.filter((name, i) => name !== "" && header.indexOf(name) !== i);
|
|
5209
5340
|
if (dupes.length > 0) {
|
|
5210
5341
|
const unique = [...new Set(dupes)];
|
|
@@ -5212,13 +5343,13 @@ function parseRecords(content, opts) {
|
|
|
5212
5343
|
`duplicate column ${unique.length === 1 ? "name" : "names"} in header: ${unique.join(", ")} \u2014 rename the duplicates or pass --no-header to address columns positionally as col1, col2, \u2026`
|
|
5213
5344
|
);
|
|
5214
5345
|
}
|
|
5215
|
-
return parse(content, { columns: true, skip_empty_lines: true, trim: true, delimiter, bom: true, relax_column_count: true });
|
|
5346
|
+
return parse(content, { columns: true, skip_empty_lines: true, trim: true, delimiter, bom: true, relax_column_count: true, relax_quotes: true });
|
|
5216
5347
|
}
|
|
5217
5348
|
function csvHeader(content, opts) {
|
|
5218
5349
|
if (opts.noHeader === true) return [];
|
|
5219
|
-
const delimiter = opts.delimiter ??
|
|
5350
|
+
const delimiter = opts.delimiter ?? detectDelimiter(content);
|
|
5220
5351
|
try {
|
|
5221
|
-
const rows = parse(content, { columns: false, skip_empty_lines: true, trim: true, delimiter, bom: true, to: 1 });
|
|
5352
|
+
const rows = parse(content, { columns: false, skip_empty_lines: true, trim: true, delimiter, bom: true, to: 1, relax_quotes: true });
|
|
5222
5353
|
return rows[0] ?? [];
|
|
5223
5354
|
} catch {
|
|
5224
5355
|
return [];
|
|
@@ -5602,30 +5733,39 @@ function makeWorksheet(name, data) {
|
|
|
5602
5733
|
}
|
|
5603
5734
|
};
|
|
5604
5735
|
}
|
|
5605
|
-
async function readXlsxWorkbook(filePath) {
|
|
5736
|
+
async function readXlsxWorkbook(filePath, deadline = ooxmlWorkDeadline()) {
|
|
5606
5737
|
const entries = await readOoxmlZip(filePath, ".xlsx");
|
|
5607
|
-
const
|
|
5608
|
-
|
|
5738
|
+
const budget = ooxmlPartBudget();
|
|
5739
|
+
const workbookXml = decodeZipEntry(entries, "xl/workbook.xml", budget);
|
|
5740
|
+
if (workbookXml === null) throw new NotAnOfficeDocumentError(`not a valid .xlsx file: ${filePath}`);
|
|
5609
5741
|
const workbookRoot = await parseOoxmlPart(workbookXml);
|
|
5610
5742
|
const wbNode = workbookRoot?.["workbook"];
|
|
5611
5743
|
if (wbNode === void 0 || wbNode === null || typeof wbNode !== "object") {
|
|
5612
|
-
throw new
|
|
5744
|
+
throw new NotAnOfficeDocumentError(`not a valid .xlsx file: ${filePath}`);
|
|
5613
5745
|
}
|
|
5614
5746
|
const wb = wbNode;
|
|
5615
5747
|
const date1904 = isTruthyAttr(attr(wb["workbookPr"], "date1904"));
|
|
5616
|
-
const relsXml = decodeZipEntry(entries, "xl/_rels/workbook.xml.rels");
|
|
5748
|
+
const relsXml = decodeZipEntry(entries, "xl/_rels/workbook.xml.rels", budget);
|
|
5617
5749
|
const rels = parseWorkbookRels(relsXml === null ? null : await parseOoxmlPart(relsXml));
|
|
5618
|
-
const sharedXml = decodeZipEntry(entries, "xl/sharedStrings.xml");
|
|
5750
|
+
const sharedXml = decodeZipEntry(entries, "xl/sharedStrings.xml", budget);
|
|
5619
5751
|
const shared = sharedXml === null ? [] : parseSharedStrings(await parseOoxmlPart(sharedXml));
|
|
5620
|
-
const stylesXml = decodeZipEntry(entries, "xl/styles.xml");
|
|
5752
|
+
const stylesXml = decodeZipEntry(entries, "xl/styles.xml", budget);
|
|
5621
5753
|
const styles = parseStyles(stylesXml, stylesXml === null ? null : await parseOoxmlPart(stylesXml));
|
|
5622
5754
|
const worksheets = [];
|
|
5755
|
+
const parsedByPart = /* @__PURE__ */ new Map();
|
|
5623
5756
|
for (const sheet of asArray(wb["sheets"]?.["sheet"])) {
|
|
5624
5757
|
const name = attr(sheet, "name") ?? "";
|
|
5625
5758
|
const rid = attr(sheet, "r:id") ?? attr(sheet, "relationshipId");
|
|
5626
5759
|
const partPath = rid === void 0 ? void 0 : rels.get(rid);
|
|
5627
|
-
const
|
|
5760
|
+
const cached = partPath === void 0 ? void 0 : parsedByPart.get(partPath);
|
|
5761
|
+
if (cached !== void 0) {
|
|
5762
|
+
worksheets.push(makeWorksheet(name, cached));
|
|
5763
|
+
continue;
|
|
5764
|
+
}
|
|
5765
|
+
assertOoxmlWithinDeadline(deadline, "Narrow the read to specific sheets with xlsx-head, or use a smaller workbook.");
|
|
5766
|
+
const sheetXml = partPath === void 0 ? null : decodeZipEntry(entries, partPath, budget);
|
|
5628
5767
|
const data = sheetXml === null ? { cells: /* @__PURE__ */ new Map(), rowCount: 0, columnCount: 0, populatedRows: 0 } : parseSheet(await parseOoxmlPart(sheetXml), shared, styles, date1904);
|
|
5768
|
+
if (partPath !== void 0) parsedByPart.set(partPath, data);
|
|
5629
5769
|
worksheets.push(makeWorksheet(name, data));
|
|
5630
5770
|
}
|
|
5631
5771
|
return {
|
|
@@ -5637,11 +5777,18 @@ async function readXlsxWorkbook(filePath) {
|
|
|
5637
5777
|
// src/xlsx_extract.ts
|
|
5638
5778
|
var loadWorkbook = readXlsxWorkbook;
|
|
5639
5779
|
function requireSheet(wb, sheetName) {
|
|
5640
|
-
|
|
5641
|
-
|
|
5642
|
-
|
|
5780
|
+
if (sheetName !== void 0 && sheetName.trim() !== "") {
|
|
5781
|
+
const ws = wb.getWorksheet(sheetName);
|
|
5782
|
+
if (ws === void 0) {
|
|
5783
|
+
throw new Error(`unknown sheet: ${sheetName} (available: ${wb.worksheets.map((s) => s.name).join(", ")})`);
|
|
5784
|
+
}
|
|
5785
|
+
return ws;
|
|
5643
5786
|
}
|
|
5644
|
-
|
|
5787
|
+
const first = wb.worksheets[0];
|
|
5788
|
+
if (first === void 0) {
|
|
5789
|
+
throw new Error("workbook contains no worksheets");
|
|
5790
|
+
}
|
|
5791
|
+
return first;
|
|
5645
5792
|
}
|
|
5646
5793
|
var MAX_XLSX_SCAN_CELLS = 2e7;
|
|
5647
5794
|
var XlsxScanTooLargeError = class extends DocumentRefusedError {
|
|
@@ -5741,49 +5888,68 @@ function usedRange(ws) {
|
|
|
5741
5888
|
}
|
|
5742
5889
|
return { ref: `A1:${indexToColLetters(cols)}${rows}`, rows, cols };
|
|
5743
5890
|
}
|
|
5744
|
-
async function listSheets(filePath) {
|
|
5745
|
-
const wb = await loadWorkbook(filePath);
|
|
5891
|
+
async function listSheets(filePath, deadline = ooxmlWorkDeadline()) {
|
|
5892
|
+
const wb = await loadWorkbook(filePath, deadline);
|
|
5746
5893
|
return wb.worksheets.map((ws) => {
|
|
5894
|
+
assertOoxmlWithinDeadline(deadline, "Narrow the read to specific sheets with xlsx-head, or use a smaller workbook.");
|
|
5747
5895
|
const { ref: ref2, rows, cols } = usedRange(ws);
|
|
5748
5896
|
return { name: ws.name, ref: ref2, rows, cols };
|
|
5749
5897
|
});
|
|
5750
5898
|
}
|
|
5751
|
-
function headSheetFromWorksheet(ws, rows) {
|
|
5899
|
+
function headSheetFromWorksheet(ws, rows, columns) {
|
|
5752
5900
|
assertScannableExtent(ws);
|
|
5753
5901
|
const rowCount = ws.rowCount || 0;
|
|
5902
|
+
const sheetCols = ws.columnCount || 0;
|
|
5903
|
+
const scanRows = Math.min(rowCount, rows + 1);
|
|
5754
5904
|
const aoa = [];
|
|
5755
|
-
let
|
|
5756
|
-
for (let r = 1; r <= rowCount; r++) {
|
|
5757
|
-
const row = ws.getRow(r);
|
|
5758
|
-
let maxCol = 0;
|
|
5759
|
-
row.eachCell({ includeEmpty: false }, (_c, colNumber) => {
|
|
5760
|
-
if (colNumber > maxCol) maxCol = colNumber;
|
|
5761
|
-
});
|
|
5762
|
-
if (maxCol > sheetCols) sheetCols = maxCol;
|
|
5905
|
+
for (let r = 1; r <= scanRows; r++) {
|
|
5763
5906
|
const rowVals = [];
|
|
5764
|
-
for (let c = 1; c <=
|
|
5907
|
+
for (let c = 1; c <= sheetCols; c++) {
|
|
5765
5908
|
rowVals.push(cellText(ws.getCell(encodeCell({ r, c }))));
|
|
5766
5909
|
}
|
|
5767
5910
|
aoa.push(rowVals);
|
|
5768
5911
|
}
|
|
5769
|
-
|
|
5770
|
-
|
|
5912
|
+
let header = Array.from({ length: sheetCols }, (_, i) => String(aoa[0]?.[i] ?? ""));
|
|
5913
|
+
let dataRows = aoa.slice(1, 1 + rows).map(
|
|
5771
5914
|
(r) => Array.from({ length: sheetCols }, (_, i) => String(r[i] ?? ""))
|
|
5772
5915
|
);
|
|
5916
|
+
if (columns !== void 0 && columns.length > 0) {
|
|
5917
|
+
const normCols = columns.map((c) => c.trim()).filter(Boolean);
|
|
5918
|
+
const selectedIndices = [];
|
|
5919
|
+
for (const req of normCols) {
|
|
5920
|
+
let idx = header.findIndex((h) => h === req);
|
|
5921
|
+
if (idx === -1) {
|
|
5922
|
+
idx = header.findIndex((h) => h.toLowerCase() === req.toLowerCase());
|
|
5923
|
+
}
|
|
5924
|
+
if (idx === -1 && /^[A-Za-z]+$/.test(req)) {
|
|
5925
|
+
const colNum = colLettersToIndex(req);
|
|
5926
|
+
if (colNum >= 1 && colNum <= sheetCols) {
|
|
5927
|
+
idx = colNum - 1;
|
|
5928
|
+
}
|
|
5929
|
+
}
|
|
5930
|
+
if (idx !== -1 && !selectedIndices.includes(idx)) {
|
|
5931
|
+
selectedIndices.push(idx);
|
|
5932
|
+
} else if (idx === -1) {
|
|
5933
|
+
throw new Error(`unknown column: ${req} (available: ${header.filter(Boolean).join(", ")})`);
|
|
5934
|
+
}
|
|
5935
|
+
}
|
|
5936
|
+
header = selectedIndices.map((i) => header[i] ?? "");
|
|
5937
|
+
dataRows = dataRows.map((r) => selectedIndices.map((i) => r[i] ?? ""));
|
|
5938
|
+
}
|
|
5773
5939
|
const lines2 = [header.map(quoteCsvCell).join(",")];
|
|
5774
5940
|
for (const r of dataRows) lines2.push(r.map(quoteCsvCell).join(","));
|
|
5775
|
-
if (
|
|
5776
|
-
lines2.push(`...(${
|
|
5941
|
+
if (rowCount - 1 > dataRows.length) {
|
|
5942
|
+
lines2.push(`...(${rowCount - 1 - dataRows.length} more rows elided; use --rows to see more, or xlsx-query for filtering)`);
|
|
5777
5943
|
}
|
|
5778
5944
|
return lines2.join("\n");
|
|
5779
5945
|
}
|
|
5780
|
-
async function headSheet(filePath, sheetName, rows) {
|
|
5946
|
+
async function headSheet(filePath, sheetName, rows = 20, columns) {
|
|
5781
5947
|
const wb = await loadWorkbook(filePath);
|
|
5782
5948
|
const ws = requireSheet(wb, sheetName);
|
|
5783
|
-
return headSheetFromWorksheet(ws, rows);
|
|
5949
|
+
return headSheetFromWorksheet(ws, rows, columns);
|
|
5784
5950
|
}
|
|
5785
5951
|
async function allSheetsHeadText(filePath, rows, deadline = ooxmlWorkDeadline()) {
|
|
5786
|
-
const wb = await loadWorkbook(filePath);
|
|
5952
|
+
const wb = await loadWorkbook(filePath, deadline);
|
|
5787
5953
|
const sheetTexts = [];
|
|
5788
5954
|
for (const ws of wb.worksheets) {
|
|
5789
5955
|
assertOoxmlWithinDeadline(deadline, "Narrow the read to specific sheets with xlsx-head, or use a smaller workbook.");
|
|
@@ -5840,6 +6006,60 @@ async function querySheet(filePath, sheetName, opts) {
|
|
|
5840
6006
|
const csvText = await sheetToCsv(ws);
|
|
5841
6007
|
return queryCsv(csvText, opts);
|
|
5842
6008
|
}
|
|
6009
|
+
async function xlsxColumns(filePath, sheetName, sampleLimit = 100) {
|
|
6010
|
+
const wb = await loadWorkbook(filePath);
|
|
6011
|
+
const ws = requireSheet(wb, sheetName);
|
|
6012
|
+
assertScannableExtent(ws);
|
|
6013
|
+
const rowCount = ws.rowCount || 0;
|
|
6014
|
+
const { cols: sheetCols } = usedRange(ws);
|
|
6015
|
+
if (rowCount === 0 || sheetCols === 0) {
|
|
6016
|
+
return { sheetName: ws.name, totalSheetRows: 0, sampleRows: 0, columns: [] };
|
|
6017
|
+
}
|
|
6018
|
+
const sampleRows = Math.min(Math.max(1, sampleLimit), Math.max(1, rowCount - 1));
|
|
6019
|
+
const headers = [];
|
|
6020
|
+
for (let c = 1; c <= sheetCols; c++) {
|
|
6021
|
+
const val = cellText(ws.getCell(encodeCell({ r: 1, c })));
|
|
6022
|
+
headers.push(val || `(col ${indexToColLetters(c)})`);
|
|
6023
|
+
}
|
|
6024
|
+
const colSummaries = headers.map((name, i) => ({
|
|
6025
|
+
letter: indexToColLetters(i + 1),
|
|
6026
|
+
name,
|
|
6027
|
+
index: i + 1,
|
|
6028
|
+
nonEmptyRows: 0,
|
|
6029
|
+
sampleRows,
|
|
6030
|
+
sampleValues: []
|
|
6031
|
+
}));
|
|
6032
|
+
for (let r = 2; r <= 1 + sampleRows; r++) {
|
|
6033
|
+
for (let c = 1; c <= sheetCols; c++) {
|
|
6034
|
+
const txt = cellText(ws.getCell(encodeCell({ r, c })));
|
|
6035
|
+
if (txt !== "") {
|
|
6036
|
+
const summary = colSummaries[c - 1];
|
|
6037
|
+
summary.nonEmptyRows++;
|
|
6038
|
+
if (summary.sampleValues.length < 3 && !summary.sampleValues.includes(txt)) {
|
|
6039
|
+
summary.sampleValues.push(txt);
|
|
6040
|
+
}
|
|
6041
|
+
}
|
|
6042
|
+
}
|
|
6043
|
+
}
|
|
6044
|
+
return {
|
|
6045
|
+
sheetName: ws.name,
|
|
6046
|
+
totalSheetRows: Math.max(0, rowCount - 1),
|
|
6047
|
+
sampleRows,
|
|
6048
|
+
columns: colSummaries
|
|
6049
|
+
};
|
|
6050
|
+
}
|
|
6051
|
+
function formatXlsxColumns(result) {
|
|
6052
|
+
if (result.columns.length === 0) return `Sheet "${displaySafeText(result.sheetName)}" is empty`;
|
|
6053
|
+
const lines2 = [
|
|
6054
|
+
`Sheet: ${displaySafeText(result.sheetName)} (${result.totalSheetRows} data rows, ${result.columns.length} columns; sampled first ${result.sampleRows} rows)`
|
|
6055
|
+
];
|
|
6056
|
+
for (const c of result.columns) {
|
|
6057
|
+
const pct = result.sampleRows > 0 ? Math.round(c.nonEmptyRows / c.sampleRows * 100) : 0;
|
|
6058
|
+
const samples = c.sampleValues.length > 0 ? ` (e.g. ${c.sampleValues.map((v) => JSON.stringify(v.length > 30 ? v.slice(0, 27) + "..." : v)).join(", ")})` : " (all empty)";
|
|
6059
|
+
lines2.push(` ${c.letter.padEnd(4)} ${displaySafeText(c.name).padEnd(25)} ${c.nonEmptyRows}/${c.sampleRows} (${pct}%)${samples}`);
|
|
6060
|
+
}
|
|
6061
|
+
return lines2.join("\n");
|
|
6062
|
+
}
|
|
5843
6063
|
|
|
5844
6064
|
// src/doc_embed_extract.ts
|
|
5845
6065
|
init_define_import_meta_env();
|
|
@@ -5865,7 +6085,7 @@ var MAX_OUTLINE_ENTRIES = 2e4;
|
|
|
5865
6085
|
var MAX_OUTLINE_TITLE_CHARS = 500;
|
|
5866
6086
|
var MAX_LOCATE_MATCHES = 1e3;
|
|
5867
6087
|
var MAX_LOCATE_CONTEXT_CHARS = 4e3;
|
|
5868
|
-
var MAX_PDF_WORK_MILLIS =
|
|
6088
|
+
var MAX_PDF_WORK_MILLIS = MAX_DOCUMENT_WORK_MILLIS;
|
|
5869
6089
|
var PdfRefusedError = class extends DocumentRefusedError {
|
|
5870
6090
|
};
|
|
5871
6091
|
var PdfTooLargeError = class extends PdfRefusedError {
|
|
@@ -5875,7 +6095,7 @@ var PdfTooLargeError = class extends PdfRefusedError {
|
|
|
5875
6095
|
};
|
|
5876
6096
|
var PdfTookTooLongError = class extends PdfRefusedError {
|
|
5877
6097
|
constructor(message) {
|
|
5878
|
-
super(message, "PdfTookTooLongError");
|
|
6098
|
+
super(message, "PdfTookTooLongError", true);
|
|
5879
6099
|
}
|
|
5880
6100
|
};
|
|
5881
6101
|
function pdfWorkDeadline() {
|
|
@@ -5988,6 +6208,10 @@ function reconstructLayout(items, deadline) {
|
|
|
5988
6208
|
for (const item of items) {
|
|
5989
6209
|
if ((checked++ & 1023) === 0 && Date.now() > deadline) throw pdfWorkTookTooLong();
|
|
5990
6210
|
const y = item.transform[5];
|
|
6211
|
+
if (!Number.isFinite(y)) {
|
|
6212
|
+
rows.push([item]);
|
|
6213
|
+
continue;
|
|
6214
|
+
}
|
|
5991
6215
|
const home = bucketOf(y);
|
|
5992
6216
|
let found = -1;
|
|
5993
6217
|
for (const bucket of [home - 1, home, home + 1]) {
|
|
@@ -6011,7 +6235,16 @@ function reconstructLayout(items, deadline) {
|
|
|
6011
6235
|
place(home, found);
|
|
6012
6236
|
}
|
|
6013
6237
|
}
|
|
6014
|
-
rows.sort((a, b) =>
|
|
6238
|
+
rows.sort((a, b) => {
|
|
6239
|
+
const ya = a[0].transform[5];
|
|
6240
|
+
const yb = b[0].transform[5];
|
|
6241
|
+
const finiteA = Number.isFinite(ya);
|
|
6242
|
+
const finiteB = Number.isFinite(yb);
|
|
6243
|
+
if (finiteA && finiteB) return yb - ya;
|
|
6244
|
+
if (finiteA) return -1;
|
|
6245
|
+
if (finiteB) return 1;
|
|
6246
|
+
return 0;
|
|
6247
|
+
});
|
|
6015
6248
|
const lines2 = [];
|
|
6016
6249
|
for (const row of rows) {
|
|
6017
6250
|
row.sort((a, b) => a.transform[4] - b.transform[4]);
|
|
@@ -6141,16 +6374,17 @@ function locateSnippet(text, index, matchLen, context) {
|
|
|
6141
6374
|
const snippet = text.slice(from, to).replace(/\s+/g, " ").trim();
|
|
6142
6375
|
return matchLen > kept ? `${snippet}...` : snippet;
|
|
6143
6376
|
}
|
|
6144
|
-
async function resolveDestPage(doc, dest) {
|
|
6377
|
+
async function resolveDestPage(doc, dest, deadline) {
|
|
6145
6378
|
let explicitDest = dest;
|
|
6146
6379
|
if (typeof explicitDest === "string") {
|
|
6147
|
-
explicitDest = await doc.getDestination(explicitDest);
|
|
6380
|
+
explicitDest = await raceDeadline(doc.getDestination(explicitDest), deadline);
|
|
6148
6381
|
}
|
|
6149
6382
|
if (!Array.isArray(explicitDest) || explicitDest.length === 0) return null;
|
|
6150
6383
|
try {
|
|
6151
|
-
const pageIndex = await doc.getPageIndex(explicitDest[0]);
|
|
6384
|
+
const pageIndex = await raceDeadline(doc.getPageIndex(explicitDest[0]), deadline);
|
|
6152
6385
|
return pageIndex + 1;
|
|
6153
|
-
} catch {
|
|
6386
|
+
} catch (err) {
|
|
6387
|
+
if (err instanceof PdfTookTooLongError) throw err;
|
|
6154
6388
|
return null;
|
|
6155
6389
|
}
|
|
6156
6390
|
}
|
|
@@ -6167,7 +6401,7 @@ async function extractPdfOutline(data) {
|
|
|
6167
6401
|
for (const item of items) {
|
|
6168
6402
|
if (entries.length >= MAX_OUTLINE_ENTRIES) return;
|
|
6169
6403
|
if (Date.now() > deadline) throw pdfWorkTookTooLong();
|
|
6170
|
-
const page = await resolveDestPage(doc, item.dest);
|
|
6404
|
+
const page = await resolveDestPage(doc, item.dest, deadline);
|
|
6171
6405
|
const title = item.title.trim();
|
|
6172
6406
|
entries.push({ level, title: title.length > MAX_OUTLINE_TITLE_CHARS ? `${title.slice(0, MAX_OUTLINE_TITLE_CHARS)}...` : title, page });
|
|
6173
6407
|
if (item.items.length > 0) await walk(item.items, level + 1);
|
|
@@ -6220,6 +6454,9 @@ function isEmbeddableDocument(filePath) {
|
|
|
6220
6454
|
function isDocumentRefusal(err) {
|
|
6221
6455
|
return err instanceof DocumentRefusedError;
|
|
6222
6456
|
}
|
|
6457
|
+
function isTransientDocumentRefusal(err) {
|
|
6458
|
+
return err instanceof DocumentRefusedError && err.transient;
|
|
6459
|
+
}
|
|
6223
6460
|
async function extractEmbeddableDocumentText(filePath) {
|
|
6224
6461
|
switch (path.extname(filePath).toLowerCase()) {
|
|
6225
6462
|
case ".pdf": {
|
|
@@ -7175,20 +7412,26 @@ function queryRefsByContext(context, filePath, dbPath = globalDbPath()) {
|
|
|
7175
7412
|
const rows = db.prepare(sql).all(context, foldPath(filePath));
|
|
7176
7413
|
return rows.map(toRefEntry);
|
|
7177
7414
|
}
|
|
7415
|
+
var REF_COUNT_BATCH = 900;
|
|
7178
7416
|
function queryRefCounts(names, dbPath = globalDbPath(), rootDir) {
|
|
7179
7417
|
const counts = /* @__PURE__ */ new Map();
|
|
7180
7418
|
if (names.length === 0) return counts;
|
|
7181
7419
|
const db = getDb(dbPath);
|
|
7182
|
-
const placeholders = names.map(() => "?").join(", ");
|
|
7183
|
-
const params = [...names];
|
|
7184
7420
|
const scopeWhere = [];
|
|
7185
|
-
|
|
7421
|
+
const scopeParams = [];
|
|
7422
|
+
applyRootDirScope(rootDir, "file_path", scopeWhere, scopeParams);
|
|
7186
7423
|
const scopeSql = scopeWhere.length > 0 ? ` AND ${scopeWhere.join(" AND ")}` : "";
|
|
7187
|
-
const
|
|
7188
|
-
const
|
|
7189
|
-
|
|
7190
|
-
|
|
7191
|
-
|
|
7424
|
+
const sqlFor = (width) => `SELECT name, COUNT(*) as c FROM refs WHERE name IN (${Array.from({ length: width }, () => "?").join(", ")})${scopeSql} GROUP BY name`;
|
|
7425
|
+
const full = names.length >= REF_COUNT_BATCH ? db.prepare(sqlFor(REF_COUNT_BATCH)) : void 0;
|
|
7426
|
+
db.transaction(() => {
|
|
7427
|
+
for (let start = 0; start < names.length; start += REF_COUNT_BATCH) {
|
|
7428
|
+
const batch = names.slice(start, start + REF_COUNT_BATCH);
|
|
7429
|
+
const stmt = batch.length === REF_COUNT_BATCH && full !== void 0 ? full : db.prepare(sqlFor(batch.length));
|
|
7430
|
+
for (const row of stmt.all(...batch, ...scopeParams)) {
|
|
7431
|
+
counts.set(row.name, row.c);
|
|
7432
|
+
}
|
|
7433
|
+
}
|
|
7434
|
+
})();
|
|
7192
7435
|
return counts;
|
|
7193
7436
|
}
|
|
7194
7437
|
function getProjectFileEntries(rootDir, dbPath = globalDbPath()) {
|
|
@@ -7225,9 +7468,9 @@ function getFileEntry(filePath, dbPath = globalDbPath()) {
|
|
|
7225
7468
|
parserSha: row.parser_sha ?? ""
|
|
7226
7469
|
};
|
|
7227
7470
|
}
|
|
7228
|
-
function sanitizeFtsQuery(query,
|
|
7471
|
+
function sanitizeFtsQuery(query, join16 = "AND") {
|
|
7229
7472
|
const terms = query.split(/\s+/).filter((t) => t.length > 0).map((t) => '"' + t.replace(/"/g, '""') + '"');
|
|
7230
|
-
return terms.join(
|
|
7473
|
+
return terms.join(join16 === "OR" ? " OR " : " ");
|
|
7231
7474
|
}
|
|
7232
7475
|
function runFtsQuery(db, match, limit, scope, rootDir) {
|
|
7233
7476
|
const sql = `SELECT s.file_path, s.name, s.kind, s.line_start, s.line_end, s.body, s.docstring, s.parent FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid WHERE symbols_fts MATCH ?${scope !== void 0 ? ` AND ${scope.clause}` : ""} ORDER BY bm25(symbols_fts) LIMIT ?`;
|
|
@@ -7257,230 +7500,14 @@ function searchSymbolsFts(query, limit = 50, dbPath = globalDbPath(), rootDir) {
|
|
|
7257
7500
|
|
|
7258
7501
|
// src/recall_index.ts
|
|
7259
7502
|
init_define_import_meta_env();
|
|
7260
|
-
import * as fs7 from "node:fs";
|
|
7261
|
-
|
|
7262
|
-
// src/disk_cache.ts
|
|
7263
|
-
init_define_import_meta_env();
|
|
7264
7503
|
import * as fs6 from "node:fs";
|
|
7265
|
-
import * as path3 from "node:path";
|
|
7266
|
-
var DEFAULT_MAX_COUNT = 200;
|
|
7267
|
-
var DEFAULT_MAX_AGE_MS = 24 * 3600 * 1e3;
|
|
7268
|
-
function sanitizeId(id) {
|
|
7269
|
-
return sanitizeIdForFilename(id, 64);
|
|
7270
|
-
}
|
|
7271
|
-
function blobDir(subdir) {
|
|
7272
|
-
return path3.join(tokenGoatHome(), subdir);
|
|
7273
|
-
}
|
|
7274
|
-
function blobPath(subdir, id) {
|
|
7275
|
-
const safe = sanitizeId(id);
|
|
7276
|
-
if (!safe) return null;
|
|
7277
|
-
const dir = blobDir(subdir);
|
|
7278
|
-
const candidate = path3.join(dir, `${safe}.json`);
|
|
7279
|
-
try {
|
|
7280
|
-
const rel = path3.relative(dir, candidate);
|
|
7281
|
-
if (rel.startsWith("..")) return null;
|
|
7282
|
-
} catch {
|
|
7283
|
-
return null;
|
|
7284
|
-
}
|
|
7285
|
-
return candidate;
|
|
7286
|
-
}
|
|
7287
|
-
function isBlobStale(subdir, id) {
|
|
7288
|
-
const p = blobPath(subdir, id);
|
|
7289
|
-
if (p === null) return false;
|
|
7290
|
-
try {
|
|
7291
|
-
const stat2 = fs6.statSync(p);
|
|
7292
|
-
return Date.now() - stat2.mtimeMs > DEFAULT_MAX_AGE_MS;
|
|
7293
|
-
} catch {
|
|
7294
|
-
return false;
|
|
7295
|
-
}
|
|
7296
|
-
}
|
|
7297
|
-
function subdirCacheDefaults(subdir) {
|
|
7298
|
-
try {
|
|
7299
|
-
if (subdir === "bash_outputs") {
|
|
7300
|
-
const bc = loadConfig().bash_compress;
|
|
7301
|
-
return { maxCount: bc.cache_max_file_count, maxBytes: bc.cache_max_bytes, maxBytesPerItem: bc.cache_max_bytes_per_output };
|
|
7302
|
-
}
|
|
7303
|
-
if (subdir === "web_outputs") {
|
|
7304
|
-
const wf = loadConfig().webfetch;
|
|
7305
|
-
return { maxCount: wf.max_file_count, maxBytes: wf.max_bytes, maxBytesPerItem: Number.POSITIVE_INFINITY };
|
|
7306
|
-
}
|
|
7307
|
-
} catch {
|
|
7308
|
-
}
|
|
7309
|
-
return { maxCount: DEFAULT_MAX_COUNT, maxBytes: Number.POSITIVE_INFINITY, maxBytesPerItem: Number.POSITIVE_INFINITY };
|
|
7310
|
-
}
|
|
7311
|
-
function storeBlob(subdir, id, value, opts = {}) {
|
|
7312
|
-
const p = blobPath(subdir, id);
|
|
7313
|
-
if (!p) return false;
|
|
7314
|
-
const defaults = subdirCacheDefaults(subdir);
|
|
7315
|
-
const maxBytesPerItem = opts.maxBytesPerItem ?? defaults.maxBytesPerItem;
|
|
7316
|
-
let rawJson;
|
|
7317
|
-
try {
|
|
7318
|
-
rawJson = JSON.stringify(value);
|
|
7319
|
-
} catch {
|
|
7320
|
-
return false;
|
|
7321
|
-
}
|
|
7322
|
-
let json;
|
|
7323
|
-
try {
|
|
7324
|
-
const result = redactSecrets(rawJson);
|
|
7325
|
-
json = result.text;
|
|
7326
|
-
if (result.count > 0) recordStat("secret_redacted", 0, result.count, void 0, subdir);
|
|
7327
|
-
} catch {
|
|
7328
|
-
return false;
|
|
7329
|
-
}
|
|
7330
|
-
if (Number.isFinite(maxBytesPerItem) && Buffer.byteLength(json, "utf-8") > maxBytesPerItem) return false;
|
|
7331
|
-
try {
|
|
7332
|
-
const dir = path3.dirname(p);
|
|
7333
|
-
if (!fs6.existsSync(dir)) ensureDirSync(dir);
|
|
7334
|
-
atomicWriteText(p, json);
|
|
7335
|
-
} catch {
|
|
7336
|
-
return false;
|
|
7337
|
-
}
|
|
7338
|
-
pruneBlobs(
|
|
7339
|
-
subdir,
|
|
7340
|
-
opts.maxCount ?? defaults.maxCount,
|
|
7341
|
-
opts.maxAgeMs ?? DEFAULT_MAX_AGE_MS,
|
|
7342
|
-
opts.maxBytes ?? defaults.maxBytes,
|
|
7343
|
-
p
|
|
7344
|
-
);
|
|
7345
|
-
return true;
|
|
7346
|
-
}
|
|
7347
|
-
function loadBlob(subdir, id) {
|
|
7348
|
-
const p = blobPath(subdir, id);
|
|
7349
|
-
if (!p) return null;
|
|
7350
|
-
try {
|
|
7351
|
-
if (!fs6.existsSync(p)) return null;
|
|
7352
|
-
return JSON.parse(fs6.readFileSync(p, "utf8"));
|
|
7353
|
-
} catch {
|
|
7354
|
-
return null;
|
|
7355
|
-
}
|
|
7356
|
-
}
|
|
7357
|
-
function listBlobs(subdir) {
|
|
7358
|
-
const dir = blobDir(subdir);
|
|
7359
|
-
const out = [];
|
|
7360
|
-
try {
|
|
7361
|
-
if (!fs6.existsSync(dir)) return out;
|
|
7362
|
-
for (const file of fs6.readdirSync(dir)) {
|
|
7363
|
-
if (!file.endsWith(".json")) continue;
|
|
7364
|
-
const id = file.slice(0, -5);
|
|
7365
|
-
let mtime = 0;
|
|
7366
|
-
try {
|
|
7367
|
-
mtime = fs6.statSync(path3.join(dir, file)).mtimeMs;
|
|
7368
|
-
} catch {
|
|
7369
|
-
}
|
|
7370
|
-
const value = loadBlob(subdir, id);
|
|
7371
|
-
if (value !== null) out.push({ id, mtime, value });
|
|
7372
|
-
}
|
|
7373
|
-
} catch {
|
|
7374
|
-
return out;
|
|
7375
|
-
}
|
|
7376
|
-
return out;
|
|
7377
|
-
}
|
|
7378
|
-
function pruneBlobs(subdir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_AGE_MS, maxBytes = Number.POSITIVE_INFINITY, protectedPath) {
|
|
7379
|
-
return pruneBlobDir(blobDir(subdir), maxCount, maxAgeMs, maxBytes, protectedPath);
|
|
7380
|
-
}
|
|
7381
|
-
function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_AGE_MS, maxBytes = Number.POSITIVE_INFINITY, protectedPath) {
|
|
7382
|
-
let removed = 0;
|
|
7383
|
-
try {
|
|
7384
|
-
if (!fs6.existsSync(dir)) return 0;
|
|
7385
|
-
const cutoff = Date.now() - maxAgeMs;
|
|
7386
|
-
let kept = [];
|
|
7387
|
-
let protectedEntry;
|
|
7388
|
-
for (const file of fs6.readdirSync(dir)) {
|
|
7389
|
-
const full = path3.join(dir, file);
|
|
7390
|
-
let stat2;
|
|
7391
|
-
try {
|
|
7392
|
-
stat2 = fs6.statSync(full);
|
|
7393
|
-
} catch {
|
|
7394
|
-
continue;
|
|
7395
|
-
}
|
|
7396
|
-
if (!stat2.isFile()) continue;
|
|
7397
|
-
if (protectedPath !== void 0 && full === protectedPath) {
|
|
7398
|
-
protectedEntry = [full, stat2.mtimeMs, stat2.size];
|
|
7399
|
-
continue;
|
|
7400
|
-
}
|
|
7401
|
-
if (stat2.mtimeMs < cutoff) {
|
|
7402
|
-
try {
|
|
7403
|
-
fs6.unlinkSync(full);
|
|
7404
|
-
removed++;
|
|
7405
|
-
} catch {
|
|
7406
|
-
continue;
|
|
7407
|
-
}
|
|
7408
|
-
} else if (file.endsWith(".json")) {
|
|
7409
|
-
kept.push([full, stat2.mtimeMs, stat2.size]);
|
|
7410
|
-
}
|
|
7411
|
-
}
|
|
7412
|
-
const countBudget = protectedEntry ? Math.max(0, maxCount - 1) : maxCount;
|
|
7413
|
-
if (kept.length > countBudget) {
|
|
7414
|
-
kept.sort((a, b) => a[1] - b[1]);
|
|
7415
|
-
const excess = kept.slice(0, kept.length - countBudget);
|
|
7416
|
-
kept = kept.slice(kept.length - countBudget);
|
|
7417
|
-
for (const [full] of excess) {
|
|
7418
|
-
try {
|
|
7419
|
-
fs6.unlinkSync(full);
|
|
7420
|
-
removed++;
|
|
7421
|
-
} catch {
|
|
7422
|
-
continue;
|
|
7423
|
-
}
|
|
7424
|
-
}
|
|
7425
|
-
}
|
|
7426
|
-
if (Number.isFinite(maxBytes)) {
|
|
7427
|
-
kept.sort((a, b) => a[1] - b[1]);
|
|
7428
|
-
let total = kept.reduce((sum, [, , size]) => sum + size, 0) + (protectedEntry ? protectedEntry[2] : 0);
|
|
7429
|
-
while (total > maxBytes && kept.length > 0) {
|
|
7430
|
-
const oldest = kept.shift();
|
|
7431
|
-
if (!oldest) break;
|
|
7432
|
-
const [full, , size] = oldest;
|
|
7433
|
-
try {
|
|
7434
|
-
fs6.unlinkSync(full);
|
|
7435
|
-
removed++;
|
|
7436
|
-
total -= size;
|
|
7437
|
-
} catch {
|
|
7438
|
-
break;
|
|
7439
|
-
}
|
|
7440
|
-
}
|
|
7441
|
-
}
|
|
7442
|
-
} catch {
|
|
7443
|
-
return removed;
|
|
7444
|
-
}
|
|
7445
|
-
return removed;
|
|
7446
|
-
}
|
|
7447
|
-
var SWEEPABLE_CACHE_SUBDIRS = [
|
|
7448
|
-
{ subdir: "bash_outputs", countCapped: true },
|
|
7449
|
-
{ subdir: "web_outputs", countCapped: true },
|
|
7450
|
-
{ subdir: "mcp_outputs", countCapped: true },
|
|
7451
|
-
{ subdir: "sessions", countCapped: false }
|
|
7452
|
-
];
|
|
7453
|
-
function sweepCacheRoots(extraRoots = []) {
|
|
7454
|
-
let removed = 0;
|
|
7455
|
-
const roots = [tokenGoatHome(), ...extraRoots];
|
|
7456
|
-
const seen = /* @__PURE__ */ new Set();
|
|
7457
|
-
for (const root of roots) {
|
|
7458
|
-
if (!root || seen.has(root)) continue;
|
|
7459
|
-
seen.add(root);
|
|
7460
|
-
for (const { subdir, countCapped } of SWEEPABLE_CACHE_SUBDIRS) {
|
|
7461
|
-
try {
|
|
7462
|
-
const defaults = subdirCacheDefaults(subdir);
|
|
7463
|
-
removed += pruneBlobDir(
|
|
7464
|
-
path3.join(root, subdir),
|
|
7465
|
-
countCapped ? defaults.maxCount : Number.POSITIVE_INFINITY,
|
|
7466
|
-
DEFAULT_MAX_AGE_MS,
|
|
7467
|
-
countCapped ? defaults.maxBytes : Number.POSITIVE_INFINITY
|
|
7468
|
-
);
|
|
7469
|
-
} catch {
|
|
7470
|
-
}
|
|
7471
|
-
}
|
|
7472
|
-
}
|
|
7473
|
-
return removed;
|
|
7474
|
-
}
|
|
7475
|
-
|
|
7476
|
-
// src/recall_index.ts
|
|
7477
7504
|
function subdirForCacheType(cacheType) {
|
|
7478
7505
|
return cacheType === "web" ? "web_outputs" : "bash_outputs";
|
|
7479
7506
|
}
|
|
7480
7507
|
function blobStillExists(cacheType, id) {
|
|
7481
7508
|
try {
|
|
7482
7509
|
const p = blobPath(subdirForCacheType(cacheType), id);
|
|
7483
|
-
return p !== null &&
|
|
7510
|
+
return p !== null && fs6.existsSync(p);
|
|
7484
7511
|
} catch {
|
|
7485
7512
|
return false;
|
|
7486
7513
|
}
|
|
@@ -7631,7 +7658,7 @@ function searchRecall(query, opts = {}) {
|
|
|
7631
7658
|
|
|
7632
7659
|
// src/bash_output_cache.ts
|
|
7633
7660
|
init_define_import_meta_env();
|
|
7634
|
-
import { readdirSync
|
|
7661
|
+
import { readdirSync, readFileSync as readFileSync2, statSync as statSync3 } from "fs";
|
|
7635
7662
|
import { resolve } from "path";
|
|
7636
7663
|
|
|
7637
7664
|
// src/hints/lang_patterns.ts
|
|
@@ -7977,11 +8004,11 @@ function gitStateFingerprintSync(cwd) {
|
|
|
7977
8004
|
}
|
|
7978
8005
|
}
|
|
7979
8006
|
var DIR_FINGERPRINT_LISTING_CAP_ENTRIES = 1e4;
|
|
7980
|
-
function dirStateFingerprintSync(
|
|
8007
|
+
function dirStateFingerprintSync(path22) {
|
|
7981
8008
|
try {
|
|
7982
|
-
const stat2 =
|
|
8009
|
+
const stat2 = statSync3(path22);
|
|
7983
8010
|
if (!stat2.isDirectory()) return null;
|
|
7984
|
-
const entries =
|
|
8011
|
+
const entries = readdirSync(path22);
|
|
7985
8012
|
if (entries.length <= DIR_FINGERPRINT_LISTING_CAP_ENTRIES) {
|
|
7986
8013
|
return shortFingerprint(entries.slice().sort().join("\0"));
|
|
7987
8014
|
}
|
|
@@ -7991,12 +8018,12 @@ function dirStateFingerprintSync(path24) {
|
|
|
7991
8018
|
}
|
|
7992
8019
|
}
|
|
7993
8020
|
var FILE_FINGERPRINT_CONTENT_CAP_BYTES = 2 * 1024 * 1024;
|
|
7994
|
-
function fileStateFingerprintSync(
|
|
8021
|
+
function fileStateFingerprintSync(path22) {
|
|
7995
8022
|
try {
|
|
7996
|
-
const stat2 =
|
|
8023
|
+
const stat2 = statSync3(path22);
|
|
7997
8024
|
if (!stat2.isFile()) return null;
|
|
7998
8025
|
if (stat2.size <= FILE_FINGERPRINT_CONTENT_CAP_BYTES) {
|
|
7999
|
-
return shortFingerprint(
|
|
8026
|
+
return shortFingerprint(readFileSync2(path22));
|
|
8000
8027
|
}
|
|
8001
8028
|
return shortFingerprint(`${stat2.mtimeMs}\0${stat2.size}`);
|
|
8002
8029
|
} catch {
|
|
@@ -8012,7 +8039,7 @@ function depLockfileFingerprintSync(cmd, cwd) {
|
|
|
8012
8039
|
if (!candidates) return null;
|
|
8013
8040
|
for (const lockfile of candidates) {
|
|
8014
8041
|
try {
|
|
8015
|
-
const content =
|
|
8042
|
+
const content = readFileSync2(resolve(cwd, lockfile));
|
|
8016
8043
|
return shortFingerprint(content);
|
|
8017
8044
|
} catch {
|
|
8018
8045
|
continue;
|
|
@@ -8265,8 +8292,8 @@ registerReset(() => {
|
|
|
8265
8292
|
// src/session_store.ts
|
|
8266
8293
|
init_define_import_meta_env();
|
|
8267
8294
|
import { createHash } from "node:crypto";
|
|
8268
|
-
import * as
|
|
8269
|
-
import * as
|
|
8295
|
+
import * as fs7 from "node:fs";
|
|
8296
|
+
import * as path3 from "node:path";
|
|
8270
8297
|
var MAX_FILES = 500;
|
|
8271
8298
|
var SESSIONS_SUBDIR = "sessions";
|
|
8272
8299
|
var AGENT_SALT_MARKER = sanitizeIdForFilename(":agent:");
|
|
@@ -8291,10 +8318,10 @@ function sessionSidecarPath(sessionId, suffix) {
|
|
|
8291
8318
|
const safe = sessionFileStem(sessionId);
|
|
8292
8319
|
if (!safe) return null;
|
|
8293
8320
|
if (suffix.includes("/") || suffix.includes("\\") || suffix.includes("..")) return null;
|
|
8294
|
-
const dir =
|
|
8295
|
-
const candidate =
|
|
8321
|
+
const dir = path3.join(tokenGoatHome(), SESSIONS_SUBDIR);
|
|
8322
|
+
const candidate = path3.join(dir, `${safe}${suffix}`);
|
|
8296
8323
|
try {
|
|
8297
|
-
const rel =
|
|
8324
|
+
const rel = path3.relative(dir, candidate);
|
|
8298
8325
|
if (rel.startsWith("..")) return null;
|
|
8299
8326
|
} catch {
|
|
8300
8327
|
return null;
|
|
@@ -8568,8 +8595,8 @@ function capFiles(s, max) {
|
|
|
8568
8595
|
}
|
|
8569
8596
|
function readDiskState(p) {
|
|
8570
8597
|
try {
|
|
8571
|
-
if (!
|
|
8572
|
-
return coerce(JSON.parse(
|
|
8598
|
+
if (!fs7.existsSync(p)) return null;
|
|
8599
|
+
return coerce(JSON.parse(fs7.readFileSync(p, "utf8")));
|
|
8573
8600
|
} catch {
|
|
8574
8601
|
return null;
|
|
8575
8602
|
}
|
|
@@ -8583,13 +8610,13 @@ function listSiblingSessionStates(sessionId) {
|
|
|
8583
8610
|
if (!sessionId) return [];
|
|
8584
8611
|
const prefix = saltedStemPrefix(sessionId);
|
|
8585
8612
|
if (prefix === AGENT_SALT_MARKER) return [];
|
|
8586
|
-
const dir =
|
|
8613
|
+
const dir = path3.join(tokenGoatHome(), SESSIONS_SUBDIR);
|
|
8587
8614
|
const out = [];
|
|
8588
8615
|
try {
|
|
8589
|
-
if (!
|
|
8590
|
-
for (const file of
|
|
8616
|
+
if (!fs7.existsSync(dir)) return out;
|
|
8617
|
+
for (const file of fs7.readdirSync(dir)) {
|
|
8591
8618
|
if (!file.endsWith(".json") || !file.startsWith(prefix)) continue;
|
|
8592
|
-
const state = readDiskState(
|
|
8619
|
+
const state = readDiskState(path3.join(dir, file));
|
|
8593
8620
|
if (state !== null) out.push(state);
|
|
8594
8621
|
}
|
|
8595
8622
|
} catch {
|
|
@@ -8608,8 +8635,8 @@ function saveSessionState(sessionId) {
|
|
|
8608
8635
|
const p = sessionPath(sessionId);
|
|
8609
8636
|
if (!p) return;
|
|
8610
8637
|
try {
|
|
8611
|
-
const dir =
|
|
8612
|
-
if (!
|
|
8638
|
+
const dir = path3.dirname(p);
|
|
8639
|
+
if (!fs7.existsSync(dir)) ensureDirSync(dir);
|
|
8613
8640
|
const mem = exportSessionState();
|
|
8614
8641
|
const writeMerged = () => {
|
|
8615
8642
|
const disk = readDiskState(p);
|
|
@@ -8631,10 +8658,10 @@ function saveSessionState(sessionId) {
|
|
|
8631
8658
|
|
|
8632
8659
|
// src/skill_cache.ts
|
|
8633
8660
|
init_define_import_meta_env();
|
|
8634
|
-
import * as
|
|
8661
|
+
import * as fs8 from "fs/promises";
|
|
8635
8662
|
import { resolve as resolve2 } from "path";
|
|
8636
8663
|
import { homedir } from "os";
|
|
8637
|
-
import { readdirSync as
|
|
8664
|
+
import { readdirSync as readdirSync3, readFileSync as readFileSync4, existsSync as existsSync5, statSync as statSync4, unlinkSync } from "node:fs";
|
|
8638
8665
|
var COMPACT_END_MARKER = "<!-- COMPACT_END -->";
|
|
8639
8666
|
var SKILLS_OUTPUT_SUBDIR = "skills";
|
|
8640
8667
|
var _skillOutputsDirOverride = null;
|
|
@@ -8799,14 +8826,14 @@ function extractChecklistSection(body) {
|
|
|
8799
8826
|
async function listOutputs() {
|
|
8800
8827
|
try {
|
|
8801
8828
|
const dir = skillOutputsDir();
|
|
8802
|
-
const entries = await
|
|
8829
|
+
const entries = await fs8.readdir(dir, { withFileTypes: true });
|
|
8803
8830
|
const metas = [];
|
|
8804
8831
|
for (const entry of entries) {
|
|
8805
8832
|
if (!entry.isFile() || !entry.name.endsWith(".meta")) {
|
|
8806
8833
|
continue;
|
|
8807
8834
|
}
|
|
8808
8835
|
try {
|
|
8809
|
-
const content = await
|
|
8836
|
+
const content = await fs8.readFile(resolve2(dir, entry.name), "utf-8");
|
|
8810
8837
|
const meta = JSON.parse(content);
|
|
8811
8838
|
metas.push(meta);
|
|
8812
8839
|
} catch {
|
|
@@ -8857,7 +8884,7 @@ async function findCrossSessionEntry(skillName, contentSha) {
|
|
|
8857
8884
|
const dir = skillOutputsDir();
|
|
8858
8885
|
const bodyPath = resolve2(dir, `${meta.outputId}.txt`);
|
|
8859
8886
|
try {
|
|
8860
|
-
const bodyExists = await
|
|
8887
|
+
const bodyExists = await fs8.access(bodyPath).then(() => true).catch(() => false);
|
|
8861
8888
|
if (bodyExists) {
|
|
8862
8889
|
return meta;
|
|
8863
8890
|
}
|
|
@@ -8960,11 +8987,11 @@ function getCompactAnySessionSync(skillName) {
|
|
|
8960
8987
|
const suffix = compactSessionSuffix(skillName);
|
|
8961
8988
|
if (!suffix) return null;
|
|
8962
8989
|
const dir = skillOutputsDir();
|
|
8963
|
-
const entries =
|
|
8990
|
+
const entries = readdirSync3(dir, { withFileTypes: true });
|
|
8964
8991
|
for (const entry of entries) {
|
|
8965
8992
|
if (!matchesCompactSuffix(entry.name, entry.isFile(), suffix)) continue;
|
|
8966
8993
|
try {
|
|
8967
|
-
const text =
|
|
8994
|
+
const text = readFileSync4(resolve2(dir, entry.name), "utf-8");
|
|
8968
8995
|
if (text.trim()) return text;
|
|
8969
8996
|
} catch {
|
|
8970
8997
|
continue;
|
|
@@ -8993,7 +9020,7 @@ async function readSkillHits(skillName) {
|
|
|
8993
9020
|
if (!name) return { count: 0, lastTs: 0 };
|
|
8994
9021
|
const dir = skillOutputsDir();
|
|
8995
9022
|
const hitsFile = resolve2(dir, `${sanitizeSkillId(name)}.hits`);
|
|
8996
|
-
const content = await
|
|
9023
|
+
const content = await fs8.readFile(hitsFile, "utf-8").catch(() => null);
|
|
8997
9024
|
if (content) {
|
|
8998
9025
|
const parsed = JSON.parse(content);
|
|
8999
9026
|
return parsed;
|
|
@@ -9021,14 +9048,14 @@ function runExclusiveInProcess(key, fn) {
|
|
|
9021
9048
|
async function acquireSkillHitLock(lockPath) {
|
|
9022
9049
|
for (let attempt = 0; attempt < SKILL_HIT_LOCK_MAX_ATTEMPTS; attempt++) {
|
|
9023
9050
|
try {
|
|
9024
|
-
await
|
|
9051
|
+
await fs8.mkdir(lockPath);
|
|
9025
9052
|
return true;
|
|
9026
9053
|
} catch (err) {
|
|
9027
9054
|
if (err.code !== "EEXIST") return false;
|
|
9028
9055
|
try {
|
|
9029
|
-
const st = await
|
|
9056
|
+
const st = await fs8.stat(lockPath);
|
|
9030
9057
|
if (Date.now() - st.mtimeMs > SKILL_HIT_LOCK_STALE_MS) {
|
|
9031
|
-
await
|
|
9058
|
+
await fs8.rmdir(lockPath).catch(() => {
|
|
9032
9059
|
});
|
|
9033
9060
|
continue;
|
|
9034
9061
|
}
|
|
@@ -9057,7 +9084,7 @@ async function incrementSkillHit(skillName) {
|
|
|
9057
9084
|
hits.lastTs = Date.now();
|
|
9058
9085
|
await atomicWriteText(hitsFile, JSON.stringify(hits, null, 2));
|
|
9059
9086
|
} finally {
|
|
9060
|
-
await
|
|
9087
|
+
await fs8.rmdir(lockPath).catch(() => {
|
|
9061
9088
|
});
|
|
9062
9089
|
}
|
|
9063
9090
|
});
|
|
@@ -9093,14 +9120,14 @@ async function listSkills(sessionId) {
|
|
|
9093
9120
|
let compactLen = 0;
|
|
9094
9121
|
let compactText = "";
|
|
9095
9122
|
try {
|
|
9096
|
-
const stat2 = await
|
|
9123
|
+
const stat2 = await fs8.stat(resolve2(dir, compactFileId));
|
|
9097
9124
|
compactLen = stat2.size;
|
|
9098
|
-
compactText = await
|
|
9125
|
+
compactText = await fs8.readFile(resolve2(dir, compactFileId), "utf-8").catch(() => "");
|
|
9099
9126
|
} catch {
|
|
9100
9127
|
compactLen = 0;
|
|
9101
9128
|
}
|
|
9102
9129
|
const hasMarker = extractCompactFromMarker(
|
|
9103
|
-
await
|
|
9130
|
+
await fs8.readFile(resolve2(dir, `${meta.outputId}.txt`), "utf-8").catch(() => "")
|
|
9104
9131
|
) !== null;
|
|
9105
9132
|
const compactStale = isCompactStale(compactText, meta.skillName, meta.contentSha);
|
|
9106
9133
|
const { count: hitCount } = await readSkillHits(meta.skillName);
|
|
@@ -9139,7 +9166,7 @@ async function resolvePluginSkillPath(pluginName, skillSlug) {
|
|
|
9139
9166
|
const manifestPath = _pluginsManifestPathOverride ?? resolve2(homedir(), ".claude", "plugins", "installed_plugins.json");
|
|
9140
9167
|
let raw;
|
|
9141
9168
|
try {
|
|
9142
|
-
raw = await
|
|
9169
|
+
raw = await fs8.readFile(manifestPath, "utf8");
|
|
9143
9170
|
} catch {
|
|
9144
9171
|
return null;
|
|
9145
9172
|
}
|
|
@@ -9161,7 +9188,7 @@ async function resolvePluginSkillPath(pluginName, skillSlug) {
|
|
|
9161
9188
|
if (typeof installPath !== "string" || installPath === "") continue;
|
|
9162
9189
|
const diskPath = resolve2(installPath, "skills", skillSlug, "SKILL.md");
|
|
9163
9190
|
try {
|
|
9164
|
-
await
|
|
9191
|
+
await fs8.access(diskPath);
|
|
9165
9192
|
return diskPath;
|
|
9166
9193
|
} catch {
|
|
9167
9194
|
continue;
|
|
@@ -9180,7 +9207,7 @@ async function installedSkillPath(skillName) {
|
|
|
9180
9207
|
}
|
|
9181
9208
|
const diskPath = resolve2(skillsSourceDir(), name, "SKILL.md");
|
|
9182
9209
|
try {
|
|
9183
|
-
await
|
|
9210
|
+
await fs8.access(diskPath);
|
|
9184
9211
|
return diskPath;
|
|
9185
9212
|
} catch {
|
|
9186
9213
|
return null;
|
|
@@ -9190,19 +9217,19 @@ function pruneSkillOutputs(maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
|
|
|
9190
9217
|
const dir = skillOutputsDir();
|
|
9191
9218
|
let removed = 0;
|
|
9192
9219
|
try {
|
|
9193
|
-
if (!
|
|
9220
|
+
if (!existsSync5(dir)) return 0;
|
|
9194
9221
|
const cutoff = Date.now() - maxAgeMs;
|
|
9195
9222
|
const entries = [];
|
|
9196
|
-
for (const file of
|
|
9223
|
+
for (const file of readdirSync3(dir)) {
|
|
9197
9224
|
if (!file.endsWith(".meta")) continue;
|
|
9198
9225
|
const outputId = file.slice(0, -".meta".length);
|
|
9199
9226
|
let ts;
|
|
9200
9227
|
try {
|
|
9201
|
-
const parsed = JSON.parse(
|
|
9228
|
+
const parsed = JSON.parse(readFileSync4(resolve2(dir, file), "utf-8"));
|
|
9202
9229
|
ts = parsed.ts;
|
|
9203
9230
|
} catch {
|
|
9204
9231
|
try {
|
|
9205
|
-
ts =
|
|
9232
|
+
ts = statSync4(resolve2(dir, file)).mtimeMs;
|
|
9206
9233
|
} catch {
|
|
9207
9234
|
continue;
|
|
9208
9235
|
}
|
|
@@ -9212,7 +9239,7 @@ function pruneSkillOutputs(maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
|
|
|
9212
9239
|
const removeEntry = (outputId) => {
|
|
9213
9240
|
for (const ext of [".meta", ".txt"]) {
|
|
9214
9241
|
try {
|
|
9215
|
-
|
|
9242
|
+
unlinkSync(resolve2(dir, `${outputId}${ext}`));
|
|
9216
9243
|
} catch {
|
|
9217
9244
|
}
|
|
9218
9245
|
}
|
|
@@ -9241,9 +9268,9 @@ function pruneSkillOutputs(maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
|
|
|
9241
9268
|
// src/image_shrink.ts
|
|
9242
9269
|
init_define_import_meta_env();
|
|
9243
9270
|
var import_omggif2 = __toESM(require_omggif(), 1);
|
|
9244
|
-
import { createHash as
|
|
9245
|
-
import * as
|
|
9246
|
-
import * as
|
|
9271
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
9272
|
+
import * as fs9 from "node:fs";
|
|
9273
|
+
import * as path5 from "node:path";
|
|
9247
9274
|
|
|
9248
9275
|
// src/image_engine.ts
|
|
9249
9276
|
init_define_import_meta_env();
|
|
@@ -9803,10 +9830,10 @@ function calculateFitInside(width, height, maxDimension) {
|
|
|
9803
9830
|
|
|
9804
9831
|
// src/vscode_path_gate.ts
|
|
9805
9832
|
init_define_import_meta_env();
|
|
9806
|
-
import * as
|
|
9833
|
+
import * as path4 from "node:path";
|
|
9807
9834
|
function resolvesToUncPath(target, cwd) {
|
|
9808
9835
|
try {
|
|
9809
|
-
return isUncOrDevicePath(
|
|
9836
|
+
return isUncOrDevicePath(path4.resolve(cwd ?? process.cwd(), target));
|
|
9810
9837
|
} catch {
|
|
9811
9838
|
return false;
|
|
9812
9839
|
}
|
|
@@ -9814,8 +9841,8 @@ function resolvesToUncPath(target, cwd) {
|
|
|
9814
9841
|
function vscodePathAllowed(filePath, workspace) {
|
|
9815
9842
|
if (isUncOrDevicePath(filePath)) return false;
|
|
9816
9843
|
if (workspace === void 0 || isUncOrDevicePath(workspace)) return false;
|
|
9817
|
-
const resolvedRoot =
|
|
9818
|
-
const resolvedTarget =
|
|
9844
|
+
const resolvedRoot = path4.resolve(workspace);
|
|
9845
|
+
const resolvedTarget = path4.resolve(workspace, filePath);
|
|
9819
9846
|
const root = foldPathForContainment(resolvedRoot.replace(/\\/g, "/"));
|
|
9820
9847
|
const target = foldPathForContainment(resolvedTarget.replace(/\\/g, "/"));
|
|
9821
9848
|
if (target !== root && !target.startsWith(root.endsWith("/") ? root : root + "/")) return false;
|
|
@@ -9832,163 +9859,6 @@ function preToolPathDeclined(event, target) {
|
|
|
9832
9859
|
return escapesOntoNetworkThroughLinks(target, cwd);
|
|
9833
9860
|
}
|
|
9834
9861
|
|
|
9835
|
-
// src/image_ocr.ts
|
|
9836
|
-
init_define_import_meta_env();
|
|
9837
|
-
import { spawn } from "node:child_process";
|
|
9838
|
-
import { createHash as createHash2 } from "node:crypto";
|
|
9839
|
-
import * as fs10 from "node:fs";
|
|
9840
|
-
import { createRequire } from "node:module";
|
|
9841
|
-
import * as path6 from "node:path";
|
|
9842
|
-
var _ocrTimeoutMs = 12e3;
|
|
9843
|
-
var OCR_LANG_FILE = "eng.traineddata";
|
|
9844
|
-
var OCR_LANG_PATH = "https://cdn.jsdelivr.net/npm/@tesseract.js-data/eng@1.0.0/4.0.0_best_int";
|
|
9845
|
-
var OCR_LANG_SHA256 = "5dc5d8d640a212c9d6184921ba103b186f50e0fed9ee716c53e6b312b400d747";
|
|
9846
|
-
function verifyOcrLangCache() {
|
|
9847
|
-
const file = path6.join(ocrCacheDir(), OCR_LANG_FILE);
|
|
9848
|
-
let bytes;
|
|
9849
|
-
try {
|
|
9850
|
-
if (!fs10.existsSync(file)) return "absent";
|
|
9851
|
-
bytes = fs10.readFileSync(file);
|
|
9852
|
-
} catch {
|
|
9853
|
-
return "unreadable";
|
|
9854
|
-
}
|
|
9855
|
-
return createHash2("sha256").update(bytes).digest("hex") === OCR_LANG_SHA256 ? "ok" : "mismatch";
|
|
9856
|
-
}
|
|
9857
|
-
function quarantineOcrLangCache() {
|
|
9858
|
-
try {
|
|
9859
|
-
fs10.rmSync(path6.join(ocrCacheDir(), OCR_LANG_FILE), { force: true });
|
|
9860
|
-
} catch {
|
|
9861
|
-
}
|
|
9862
|
-
}
|
|
9863
|
-
var _ocrIntegrityFailed = false;
|
|
9864
|
-
function ocrIntegrityFailed() {
|
|
9865
|
-
return _ocrIntegrityFailed;
|
|
9866
|
-
}
|
|
9867
|
-
function ocrBlockedOffline() {
|
|
9868
|
-
if (!loadConfig().network.offline) return false;
|
|
9869
|
-
return !fs10.existsSync(path6.join(ocrCacheDir(), OCR_LANG_FILE));
|
|
9870
|
-
}
|
|
9871
|
-
function ocrCacheDir() {
|
|
9872
|
-
return path6.join(tokenGoatHome(), "ocr-cache");
|
|
9873
|
-
}
|
|
9874
|
-
function ensureOcrCacheDir() {
|
|
9875
|
-
try {
|
|
9876
|
-
ensureDirSync(ocrCacheDir());
|
|
9877
|
-
} catch {
|
|
9878
|
-
}
|
|
9879
|
-
}
|
|
9880
|
-
var _require = createRequire(import.meta.url);
|
|
9881
|
-
var _tesseractEntryPath;
|
|
9882
|
-
function resolveTesseractEntry() {
|
|
9883
|
-
if (_tesseractEntryPath !== void 0) return _tesseractEntryPath;
|
|
9884
|
-
try {
|
|
9885
|
-
_tesseractEntryPath = _require.resolve("tesseract.js");
|
|
9886
|
-
} catch {
|
|
9887
|
-
_tesseractEntryPath = null;
|
|
9888
|
-
}
|
|
9889
|
-
return _tesseractEntryPath;
|
|
9890
|
-
}
|
|
9891
|
-
function isOcrEngineAvailable() {
|
|
9892
|
-
return !_ocrUnavailableThisProcess && resolveTesseractEntry() !== null;
|
|
9893
|
-
}
|
|
9894
|
-
var _ocrUnavailableThisProcess = false;
|
|
9895
|
-
function buildChildScript(entryPath, cacheDir) {
|
|
9896
|
-
return [
|
|
9897
|
-
`const { createWorker } = require(${JSON.stringify(entryPath)});`,
|
|
9898
|
-
"const chunks = [];",
|
|
9899
|
-
"process.stdin.on('data', (c) => chunks.push(c));",
|
|
9900
|
-
"process.stdin.on('end', async () => {",
|
|
9901
|
-
" try {",
|
|
9902
|
-
" const buf = Buffer.concat(chunks);",
|
|
9903
|
-
` const worker = await createWorker('eng', 1, { cachePath: ${JSON.stringify(cacheDir)}, langPath: ${JSON.stringify(OCR_LANG_PATH)}, errorHandler: () => {} });`,
|
|
9904
|
-
" const { data } = await worker.recognize(buf);",
|
|
9905
|
-
" process.stdout.write(JSON.stringify({ text: data.text || '', confidence: data.confidence || 0 }));",
|
|
9906
|
-
" await worker.terminate();",
|
|
9907
|
-
" process.exit(0);",
|
|
9908
|
-
" } catch (e) {",
|
|
9909
|
-
" process.exit(1);",
|
|
9910
|
-
" }",
|
|
9911
|
-
"});"
|
|
9912
|
-
].join("\n");
|
|
9913
|
-
}
|
|
9914
|
-
async function ocrImage(input) {
|
|
9915
|
-
if (_ocrUnavailableThisProcess) return null;
|
|
9916
|
-
const entryPath = resolveTesseractEntry();
|
|
9917
|
-
if (entryPath === null) return null;
|
|
9918
|
-
if (ocrBlockedOffline()) return null;
|
|
9919
|
-
if (_ocrIntegrityFailed) return null;
|
|
9920
|
-
if (verifyOcrLangCache() === "mismatch") {
|
|
9921
|
-
_ocrIntegrityFailed = true;
|
|
9922
|
-
quarantineOcrLangCache();
|
|
9923
|
-
return null;
|
|
9924
|
-
}
|
|
9925
|
-
return new Promise((resolve8) => {
|
|
9926
|
-
let settled = false;
|
|
9927
|
-
let child;
|
|
9928
|
-
try {
|
|
9929
|
-
ensureOcrCacheDir();
|
|
9930
|
-
child = spawn(process.execPath, ["-e", buildChildScript(entryPath, ocrCacheDir())], {
|
|
9931
|
-
stdio: ["pipe", "pipe", "ignore"]
|
|
9932
|
-
});
|
|
9933
|
-
} catch {
|
|
9934
|
-
_ocrUnavailableThisProcess = true;
|
|
9935
|
-
resolve8(null);
|
|
9936
|
-
return;
|
|
9937
|
-
}
|
|
9938
|
-
const chunks = [];
|
|
9939
|
-
const finish = (result, subprocessBroken) => {
|
|
9940
|
-
if (settled) return;
|
|
9941
|
-
settled = true;
|
|
9942
|
-
clearTimeout(timer);
|
|
9943
|
-
if (subprocessBroken) _ocrUnavailableThisProcess = true;
|
|
9944
|
-
try {
|
|
9945
|
-
child.kill();
|
|
9946
|
-
} catch {
|
|
9947
|
-
}
|
|
9948
|
-
resolve8(result);
|
|
9949
|
-
};
|
|
9950
|
-
const timer = setTimeout(() => finish(null, true), _ocrTimeoutMs);
|
|
9951
|
-
child.stdout?.on("data", (c) => chunks.push(c));
|
|
9952
|
-
child.on("error", () => finish(null, true));
|
|
9953
|
-
child.on("close", (code) => {
|
|
9954
|
-
if (code !== 0) {
|
|
9955
|
-
finish(null, false);
|
|
9956
|
-
return;
|
|
9957
|
-
}
|
|
9958
|
-
if (verifyOcrLangCache() === "mismatch") {
|
|
9959
|
-
_ocrIntegrityFailed = true;
|
|
9960
|
-
quarantineOcrLangCache();
|
|
9961
|
-
finish(null, false);
|
|
9962
|
-
return;
|
|
9963
|
-
}
|
|
9964
|
-
try {
|
|
9965
|
-
const raw = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
9966
|
-
const parsed = raw;
|
|
9967
|
-
const text = typeof parsed.text === "string" ? parsed.text.trim() : "";
|
|
9968
|
-
const confidence = typeof parsed.confidence === "number" ? parsed.confidence : 0;
|
|
9969
|
-
finish({ text, confidence }, false);
|
|
9970
|
-
} catch {
|
|
9971
|
-
finish(null, false);
|
|
9972
|
-
}
|
|
9973
|
-
});
|
|
9974
|
-
try {
|
|
9975
|
-
child.stdin?.end(input);
|
|
9976
|
-
} catch {
|
|
9977
|
-
finish(null, false);
|
|
9978
|
-
}
|
|
9979
|
-
});
|
|
9980
|
-
}
|
|
9981
|
-
function isTextHeavy(result, minConfidence, minChars = 40) {
|
|
9982
|
-
return result.confidence >= minConfidence && result.text.length >= minChars;
|
|
9983
|
-
}
|
|
9984
|
-
function formatOcrSummary(result, subject, originalBytes) {
|
|
9985
|
-
const kb = Math.round(originalBytes / 1024);
|
|
9986
|
-
const summary = `token-goat OCR'd ${subject} instead of shrinking it: text-heavy image detected (${Math.round(result.confidence)}% confidence), extracted ${result.text.length} chars of text from ${kb}kb of pixels.`;
|
|
9987
|
-
return `${summary}
|
|
9988
|
-
|
|
9989
|
-
${fenceUntrustedOcrText(result.text)}`;
|
|
9990
|
-
}
|
|
9991
|
-
|
|
9992
9862
|
// src/image_shrink.ts
|
|
9993
9863
|
var IMAGE_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
9994
9864
|
".png",
|
|
@@ -10060,7 +9930,7 @@ function formatShrinkSummary(result, subject) {
|
|
|
10060
9930
|
return { summary, dataUrl };
|
|
10061
9931
|
}
|
|
10062
9932
|
function isImagePath(p) {
|
|
10063
|
-
return IMAGE_EXTENSIONS.has(
|
|
9933
|
+
return IMAGE_EXTENSIONS.has(path5.extname(p).toLowerCase());
|
|
10064
9934
|
}
|
|
10065
9935
|
var ImageDecodeError = class extends Error {
|
|
10066
9936
|
};
|
|
@@ -10175,17 +10045,17 @@ async function shrinkImage(input, opts) {
|
|
|
10175
10045
|
}
|
|
10176
10046
|
function statInfo(absPath) {
|
|
10177
10047
|
try {
|
|
10178
|
-
const st =
|
|
10048
|
+
const st = fs9.statSync(absPath);
|
|
10179
10049
|
return st.isFile() ? { size: st.size, mtimeMs: st.mtimeMs } : null;
|
|
10180
10050
|
} catch {
|
|
10181
10051
|
return null;
|
|
10182
10052
|
}
|
|
10183
10053
|
}
|
|
10184
10054
|
function imageShrinkCacheDir() {
|
|
10185
|
-
return
|
|
10055
|
+
return path5.join(tokenGoatHome(), "image_shrink_cache");
|
|
10186
10056
|
}
|
|
10187
10057
|
function shrinkCacheKey(originalPath, size, mtimeMs, quality) {
|
|
10188
|
-
return
|
|
10058
|
+
return createHash2("sha256").update(`${originalPath}:${size}:${mtimeMs}:${quality}`).digest("hex").slice(0, 16);
|
|
10189
10059
|
}
|
|
10190
10060
|
var SHRINK_CACHE_EXTENSION = /* @__PURE__ */ new Map([
|
|
10191
10061
|
["jpeg", ".jpg"],
|
|
@@ -10203,7 +10073,7 @@ function findCachedShrink(originalPath, size, mtimeMs, quality) {
|
|
|
10203
10073
|
const dir = imageShrinkCacheDir();
|
|
10204
10074
|
let entries;
|
|
10205
10075
|
try {
|
|
10206
|
-
entries =
|
|
10076
|
+
entries = fs9.readdirSync(dir);
|
|
10207
10077
|
} catch {
|
|
10208
10078
|
return null;
|
|
10209
10079
|
}
|
|
@@ -10214,7 +10084,7 @@ function findCachedShrink(originalPath, size, mtimeMs, quality) {
|
|
|
10214
10084
|
const format = SHRINK_CACHE_FORMAT.get(m[3] ?? "");
|
|
10215
10085
|
if (format === void 0) continue;
|
|
10216
10086
|
return {
|
|
10217
|
-
filePath:
|
|
10087
|
+
filePath: path5.join(dir, file),
|
|
10218
10088
|
format,
|
|
10219
10089
|
originalWidth: Number(m[1]),
|
|
10220
10090
|
originalHeight: Number(m[2])
|
|
@@ -10229,7 +10099,7 @@ function writeCachedShrink(originalPath, result, mtimeMs, quality) {
|
|
|
10229
10099
|
const key = shrinkCacheKey(originalPath, result.originalBytes, mtimeMs, quality);
|
|
10230
10100
|
const ext = SHRINK_CACHE_EXTENSION.get(result.format);
|
|
10231
10101
|
if (ext === void 0) return;
|
|
10232
|
-
atomicWriteBytes(
|
|
10102
|
+
atomicWriteBytes(path5.join(dir, `token-goat-shrink-${key}-${result.originalWidth}x${result.originalHeight}${ext}`), result.data);
|
|
10233
10103
|
} catch {
|
|
10234
10104
|
}
|
|
10235
10105
|
}
|
|
@@ -10241,14 +10111,14 @@ function pruneShrinkCache() {
|
|
|
10241
10111
|
lastShrinkCachePruneAtMs = now;
|
|
10242
10112
|
try {
|
|
10243
10113
|
const dir = imageShrinkCacheDir();
|
|
10244
|
-
if (!
|
|
10114
|
+
if (!fs9.existsSync(dir)) return;
|
|
10245
10115
|
const cutoff = now - DEFAULT_MAX_AGE_MS;
|
|
10246
|
-
for (const file of
|
|
10116
|
+
for (const file of fs9.readdirSync(dir)) {
|
|
10247
10117
|
if (!file.startsWith("token-goat-shrink-")) continue;
|
|
10248
|
-
const full =
|
|
10118
|
+
const full = path5.join(dir, file);
|
|
10249
10119
|
try {
|
|
10250
|
-
const st =
|
|
10251
|
-
if (st.mtimeMs < cutoff)
|
|
10120
|
+
const st = fs9.statSync(full);
|
|
10121
|
+
if (st.mtimeMs < cutoff) fs9.unlinkSync(full);
|
|
10252
10122
|
} catch {
|
|
10253
10123
|
}
|
|
10254
10124
|
}
|
|
@@ -10257,7 +10127,7 @@ function pruneShrinkCache() {
|
|
|
10257
10127
|
}
|
|
10258
10128
|
var HOST_MATERIALIZED_HARNESSES = /* @__PURE__ */ new Set(["copilot_cli", "pi", "opencode", "openclaw"]);
|
|
10259
10129
|
async function finalizeShrinkResult(result, filePath, event) {
|
|
10260
|
-
const basename10 = displaySafePath(
|
|
10130
|
+
const basename10 = displaySafePath(path5.basename(filePath));
|
|
10261
10131
|
const shrinkSaved = result.originalBytes - result.shrunkBytes;
|
|
10262
10132
|
const tier = loadConfig().image_shrink.vision_tier;
|
|
10263
10133
|
const shrinkTokens = visionTokensSaved(result.originalWidth, result.originalHeight, result.width, result.height, tier);
|
|
@@ -10275,6 +10145,7 @@ ${dataUrl2}`);
|
|
|
10275
10145
|
}
|
|
10276
10146
|
recordStat("image_shrink", shrinkSaved, shrinkTokens, void 0, basename10);
|
|
10277
10147
|
if (loadConfig().image_shrink.ocr_enabled) {
|
|
10148
|
+
const { ocrImage, isTextHeavy, formatOcrSummary } = await import("./token-goat-chunk-4NXUKV7D.mjs");
|
|
10278
10149
|
const ocr = await ocrImage(result.data);
|
|
10279
10150
|
if (ocr !== null && isTextHeavy(ocr, loadConfig().image_shrink.ocr_min_confidence)) {
|
|
10280
10151
|
const emitted = formatOcrSummary(ocr, basename10, result.originalBytes);
|
|
@@ -10305,7 +10176,7 @@ async function preReadImageHandler(event) {
|
|
|
10305
10176
|
if (cached !== null) {
|
|
10306
10177
|
let cachedData;
|
|
10307
10178
|
try {
|
|
10308
|
-
cachedData =
|
|
10179
|
+
cachedData = fs9.readFileSync(cached.filePath);
|
|
10309
10180
|
} catch {
|
|
10310
10181
|
cachedData = null;
|
|
10311
10182
|
}
|
|
@@ -10331,13 +10202,13 @@ async function preReadImageHandler(event) {
|
|
|
10331
10202
|
return finalizeShrinkResult(result2, filePath, event);
|
|
10332
10203
|
}
|
|
10333
10204
|
try {
|
|
10334
|
-
|
|
10205
|
+
fs9.unlinkSync(cached.filePath);
|
|
10335
10206
|
} catch {
|
|
10336
10207
|
}
|
|
10337
10208
|
}
|
|
10338
10209
|
let input;
|
|
10339
10210
|
try {
|
|
10340
|
-
input =
|
|
10211
|
+
input = fs9.readFileSync(filePath);
|
|
10341
10212
|
} catch {
|
|
10342
10213
|
return passOutput();
|
|
10343
10214
|
}
|
|
@@ -10357,10 +10228,10 @@ init_define_import_meta_env();
|
|
|
10357
10228
|
|
|
10358
10229
|
// src/embed_model.ts
|
|
10359
10230
|
init_define_import_meta_env();
|
|
10360
|
-
import { createHash as
|
|
10361
|
-
import * as
|
|
10362
|
-
import { createRequire
|
|
10363
|
-
import * as
|
|
10231
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
10232
|
+
import * as fs10 from "node:fs";
|
|
10233
|
+
import { createRequire } from "node:module";
|
|
10234
|
+
import * as path6 from "node:path";
|
|
10364
10235
|
import { pipeline } from "node:stream/promises";
|
|
10365
10236
|
|
|
10366
10237
|
// src/embed_tokenizer.ts
|
|
@@ -10375,20 +10246,20 @@ var UnsupportedTokenizerError = class extends Error {
|
|
|
10375
10246
|
function fail(message) {
|
|
10376
10247
|
throw new UnsupportedTokenizerError(message);
|
|
10377
10248
|
}
|
|
10378
|
-
function asRecord(value,
|
|
10379
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${
|
|
10249
|
+
function asRecord(value, path22) {
|
|
10250
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${path22} is not an object`);
|
|
10380
10251
|
return value;
|
|
10381
10252
|
}
|
|
10382
|
-
function asString(value,
|
|
10383
|
-
if (typeof value !== "string") fail(`${
|
|
10253
|
+
function asString(value, path22) {
|
|
10254
|
+
if (typeof value !== "string") fail(`${path22} is not a string`);
|
|
10384
10255
|
return value;
|
|
10385
10256
|
}
|
|
10386
|
-
function asArray2(value,
|
|
10387
|
-
if (!Array.isArray(value)) fail(`${
|
|
10257
|
+
function asArray2(value, path22) {
|
|
10258
|
+
if (!Array.isArray(value)) fail(`${path22} is not an array`);
|
|
10388
10259
|
return value;
|
|
10389
10260
|
}
|
|
10390
|
-
function expectValue(actual, wanted,
|
|
10391
|
-
if (actual !== wanted) fail(`${
|
|
10261
|
+
function expectValue(actual, wanted, path22) {
|
|
10262
|
+
if (actual !== wanted) fail(`${path22} is ${JSON.stringify(actual)}, expected ${JSON.stringify(wanted)}`);
|
|
10392
10263
|
}
|
|
10393
10264
|
function isChinese(cp) {
|
|
10394
10265
|
return cp >= 19968 && cp <= 40959 || cp >= 13312 && cp <= 19903 || cp >= 131072 && cp <= 173791 || cp >= 173824 && cp <= 177983 || cp >= 177984 && cp <= 178207 || cp >= 178208 && cp <= 183983 || cp >= 63744 && cp <= 64255 || cp >= 194560 && cp <= 195103;
|
|
@@ -10571,7 +10442,7 @@ var BertWordPiece = class _BertWordPiece {
|
|
|
10571
10442
|
};
|
|
10572
10443
|
|
|
10573
10444
|
// src/embed_model.ts
|
|
10574
|
-
var
|
|
10445
|
+
var _require = createRequire(import.meta.url);
|
|
10575
10446
|
var DEFAULT_MODEL = "Xenova/bge-small-en-v1.5";
|
|
10576
10447
|
var DEFAULT_DIM = 384;
|
|
10577
10448
|
var DEFAULT_EMBED_THREADS = 2;
|
|
@@ -10589,7 +10460,7 @@ var MODEL_FILES = [
|
|
|
10589
10460
|
}
|
|
10590
10461
|
];
|
|
10591
10462
|
function modelDir() {
|
|
10592
|
-
return
|
|
10463
|
+
return path6.join(dataDir(), "models", ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
10593
10464
|
}
|
|
10594
10465
|
function downloadUrl(file) {
|
|
10595
10466
|
return `https://huggingface.co/${DEFAULT_MODEL}/resolve/${PINNED_MODEL_REVISION}/${file.name}`;
|
|
@@ -10597,44 +10468,44 @@ function downloadUrl(file) {
|
|
|
10597
10468
|
function sharedModelCacheDir() {
|
|
10598
10469
|
const raw = process.env["TOKEN_GOAT_MODEL_CACHE_DIR"]?.trim();
|
|
10599
10470
|
if (!raw) return null;
|
|
10600
|
-
return
|
|
10471
|
+
return path6.join(raw, ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
10601
10472
|
}
|
|
10602
10473
|
async function copyFromSharedCache(shared, file, target) {
|
|
10603
|
-
const source =
|
|
10474
|
+
const source = path6.join(shared, file.name);
|
|
10604
10475
|
const temp = `${target}.${process.pid}.shared`;
|
|
10605
10476
|
try {
|
|
10606
|
-
const info =
|
|
10477
|
+
const info = fs10.lstatSync(source, { throwIfNoEntry: false });
|
|
10607
10478
|
if (!info?.isFile() || info.size !== file.bytes) return false;
|
|
10608
|
-
|
|
10479
|
+
fs10.copyFileSync(source, temp);
|
|
10609
10480
|
if (await sha256Of(temp) !== file.sha256) {
|
|
10610
|
-
|
|
10611
|
-
|
|
10481
|
+
fs10.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
10482
|
+
fs10.rmSync(source, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
10612
10483
|
return false;
|
|
10613
10484
|
}
|
|
10614
|
-
|
|
10485
|
+
fs10.renameSync(temp, target);
|
|
10615
10486
|
return true;
|
|
10616
10487
|
} catch {
|
|
10617
10488
|
try {
|
|
10618
|
-
|
|
10489
|
+
fs10.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
10619
10490
|
} catch {
|
|
10620
10491
|
}
|
|
10621
10492
|
return false;
|
|
10622
10493
|
}
|
|
10623
10494
|
}
|
|
10624
10495
|
function publishToSharedCache(shared, file, target) {
|
|
10625
|
-
const destination =
|
|
10496
|
+
const destination = path6.join(shared, file.name);
|
|
10626
10497
|
const temp = `${destination}.${process.pid}.partial`;
|
|
10627
10498
|
let created = false;
|
|
10628
10499
|
try {
|
|
10629
|
-
if (
|
|
10630
|
-
ensureDirSync(
|
|
10631
|
-
|
|
10500
|
+
if (fs10.existsSync(destination)) return;
|
|
10501
|
+
ensureDirSync(path6.dirname(destination));
|
|
10502
|
+
fs10.copyFileSync(target, temp, fs10.constants.COPYFILE_EXCL);
|
|
10632
10503
|
created = true;
|
|
10633
|
-
|
|
10504
|
+
fs10.renameSync(temp, destination);
|
|
10634
10505
|
} catch {
|
|
10635
10506
|
if (!created) return;
|
|
10636
10507
|
try {
|
|
10637
|
-
|
|
10508
|
+
fs10.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
10638
10509
|
} catch {
|
|
10639
10510
|
}
|
|
10640
10511
|
}
|
|
@@ -10646,7 +10517,7 @@ function ensureRuntimeLoaded() {
|
|
|
10646
10517
|
if (_ortLoadAttempted) return;
|
|
10647
10518
|
_ortLoadAttempted = true;
|
|
10648
10519
|
try {
|
|
10649
|
-
_ort =
|
|
10520
|
+
_ort = _require("onnxruntime-node");
|
|
10650
10521
|
} catch (e) {
|
|
10651
10522
|
_ortError = e instanceof Error ? e : new Error(String(e));
|
|
10652
10523
|
}
|
|
@@ -10663,14 +10534,14 @@ function runtimeVersion() {
|
|
|
10663
10534
|
ensureRuntimeLoaded();
|
|
10664
10535
|
if (_ort === null) return "unknown";
|
|
10665
10536
|
try {
|
|
10666
|
-
let dir =
|
|
10537
|
+
let dir = path6.dirname(_require.resolve("onnxruntime-node"));
|
|
10667
10538
|
for (let depth = 0; depth < 6; depth++) {
|
|
10668
|
-
const manifest =
|
|
10669
|
-
if (
|
|
10670
|
-
const parsed = JSON.parse(
|
|
10539
|
+
const manifest = path6.join(dir, "package.json");
|
|
10540
|
+
if (fs10.existsSync(manifest)) {
|
|
10541
|
+
const parsed = JSON.parse(fs10.readFileSync(manifest, "utf8"));
|
|
10671
10542
|
if (parsed.name === "onnxruntime-node" && typeof parsed.version === "string") return parsed.version;
|
|
10672
10543
|
}
|
|
10673
|
-
const up =
|
|
10544
|
+
const up = path6.dirname(dir);
|
|
10674
10545
|
if (up === dir) break;
|
|
10675
10546
|
dir = up;
|
|
10676
10547
|
}
|
|
@@ -10680,8 +10551,8 @@ function runtimeVersion() {
|
|
|
10680
10551
|
}
|
|
10681
10552
|
function sha256Of(filePath) {
|
|
10682
10553
|
return new Promise((resolve8, reject) => {
|
|
10683
|
-
const hash2 =
|
|
10684
|
-
const stream =
|
|
10554
|
+
const hash2 = createHash3("sha256");
|
|
10555
|
+
const stream = fs10.createReadStream(filePath);
|
|
10685
10556
|
stream.on("error", reject);
|
|
10686
10557
|
stream.on("data", (chunk) => hash2.update(chunk));
|
|
10687
10558
|
stream.on("end", () => resolve8(hash2.digest("hex")));
|
|
@@ -10693,9 +10564,9 @@ async function download(file, target) {
|
|
|
10693
10564
|
if (!response.ok) throw new Error(`GET ${url} returned ${response.status} ${response.statusText}`);
|
|
10694
10565
|
if (!response.body) throw new Error(`GET ${url} returned no body`);
|
|
10695
10566
|
const temp = `${target}.${process.pid}.partial`;
|
|
10696
|
-
const hash2 =
|
|
10567
|
+
const hash2 = createHash3("sha256");
|
|
10697
10568
|
let written = 0;
|
|
10698
|
-
const out =
|
|
10569
|
+
const out = fs10.createWriteStream(temp);
|
|
10699
10570
|
try {
|
|
10700
10571
|
await pipeline(async function* () {
|
|
10701
10572
|
for await (const chunk of response.body) {
|
|
@@ -10712,14 +10583,14 @@ async function download(file, target) {
|
|
|
10712
10583
|
if (digest !== file.sha256) {
|
|
10713
10584
|
throw new Error(`${file.name} has sha256 ${digest}, expected the pinned ${file.sha256}`);
|
|
10714
10585
|
}
|
|
10715
|
-
|
|
10586
|
+
fs10.renameSync(temp, target);
|
|
10716
10587
|
} catch (e) {
|
|
10717
10588
|
await new Promise((resolve8) => {
|
|
10718
10589
|
if (out.closed) resolve8();
|
|
10719
10590
|
else out.once("close", () => resolve8());
|
|
10720
10591
|
});
|
|
10721
10592
|
try {
|
|
10722
|
-
|
|
10593
|
+
fs10.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
10723
10594
|
} catch {
|
|
10724
10595
|
}
|
|
10725
10596
|
throw e;
|
|
@@ -10736,14 +10607,14 @@ async function ensureModelFiles(modelName = DEFAULT_MODEL) {
|
|
|
10736
10607
|
const offline = loadConfig().network.offline;
|
|
10737
10608
|
const shared = sharedModelCacheDir();
|
|
10738
10609
|
for (const file of MODEL_FILES) {
|
|
10739
|
-
const target =
|
|
10740
|
-
if (
|
|
10610
|
+
const target = path6.join(dir, file.name);
|
|
10611
|
+
if (fs10.existsSync(target)) {
|
|
10741
10612
|
const digest = await sha256Of(target);
|
|
10742
10613
|
if (digest === file.sha256) continue;
|
|
10743
|
-
|
|
10614
|
+
fs10.rmSync(target, { force: true });
|
|
10744
10615
|
}
|
|
10745
10616
|
if (shared) {
|
|
10746
|
-
ensureDirSync(
|
|
10617
|
+
ensureDirSync(path6.dirname(target));
|
|
10747
10618
|
if (await copyFromSharedCache(shared, file, target)) continue;
|
|
10748
10619
|
}
|
|
10749
10620
|
if (offline) {
|
|
@@ -10751,7 +10622,7 @@ async function ensureModelFiles(modelName = DEFAULT_MODEL) {
|
|
|
10751
10622
|
`Offline mode is on (network.offline): refusing to download ${file.name} for the embedding model. Copy the pinned files into ${dir} on a connected machine to use semantic search here.`
|
|
10752
10623
|
);
|
|
10753
10624
|
}
|
|
10754
|
-
ensureDirSync(
|
|
10625
|
+
ensureDirSync(path6.dirname(target));
|
|
10755
10626
|
console.warn(
|
|
10756
10627
|
`Downloading the embedding model, once (${file.name}, ${Math.round(file.bytes / 1024 / 1024)} MB) into ${dir}`
|
|
10757
10628
|
);
|
|
@@ -10792,10 +10663,10 @@ var EmbeddingModel = class _EmbeddingModel {
|
|
|
10792
10663
|
throw new Error(`onnxruntime-node is not available: ${_ortError?.message ?? "unknown error"}`);
|
|
10793
10664
|
}
|
|
10794
10665
|
const dir = await ensureModelFiles(modelName);
|
|
10795
|
-
const tokenizer = BertWordPiece.fromJson(
|
|
10666
|
+
const tokenizer = BertWordPiece.fromJson(fs10.readFileSync(path6.join(dir, "tokenizer.json"), "utf8"));
|
|
10796
10667
|
const ort = _ort;
|
|
10797
10668
|
const threads = loadConfig().worker.embed_threads ?? DEFAULT_EMBED_THREADS;
|
|
10798
|
-
const session = await ort.InferenceSession.create(
|
|
10669
|
+
const session = await ort.InferenceSession.create(path6.join(dir, "onnx", "model_quantized.onnx"), {
|
|
10799
10670
|
intraOpNumThreads: threads,
|
|
10800
10671
|
interOpNumThreads: 1
|
|
10801
10672
|
});
|
|
@@ -10908,6 +10779,7 @@ var _TOKEN_RE = /\w+/g;
|
|
|
10908
10779
|
var _MIN_TOKEN_LEN = 3;
|
|
10909
10780
|
var OVER_FETCH_FACTOR = 4;
|
|
10910
10781
|
var MAX_OVER_FETCH = 100;
|
|
10782
|
+
var VEC_MAX_K = 4096;
|
|
10911
10783
|
function isAvailable() {
|
|
10912
10784
|
return isRuntimeAvailable();
|
|
10913
10785
|
}
|
|
@@ -11210,6 +11082,34 @@ function fetchScopedHits(db, queryVec, k, maxDistance, rootDir) {
|
|
|
11210
11082
|
}
|
|
11211
11083
|
return { hits, candidateCount: rows.length };
|
|
11212
11084
|
}
|
|
11085
|
+
function fetchScopedExactHits(db, queryVec, limit, maxDistance, rootDir) {
|
|
11086
|
+
const scope = projectScopeClause("c.file_path");
|
|
11087
|
+
const rows = db.prepare(
|
|
11088
|
+
`SELECT file_path, start_line, end_line, text, kind, distance FROM (
|
|
11089
|
+
SELECT c.id AS id, c.file_path AS file_path, c.start_line AS start_line, c.end_line AS end_line, c.text AS text, c.kind AS kind, vec_distance_L2(v.embedding, ?) AS distance
|
|
11090
|
+
FROM chunks c JOIN chunk_vectors v ON v.rowid = c.id
|
|
11091
|
+
WHERE ${scope.clause}
|
|
11092
|
+
)
|
|
11093
|
+
WHERE distance IS NOT NULL AND distance <= ?
|
|
11094
|
+
ORDER BY distance ASC, file_path ASC, start_line ASC, id ASC
|
|
11095
|
+
LIMIT ?`
|
|
11096
|
+
).all(packVec(queryVec), ...scope.params(rootDir), maxDistance, limit);
|
|
11097
|
+
const hits = [];
|
|
11098
|
+
for (const row of rows) {
|
|
11099
|
+
if (!row || typeof row.distance !== "number" || !Number.isFinite(row.distance) || row.distance > maxDistance) {
|
|
11100
|
+
continue;
|
|
11101
|
+
}
|
|
11102
|
+
hits.push({
|
|
11103
|
+
filePath: row.file_path,
|
|
11104
|
+
startLine: row.start_line,
|
|
11105
|
+
endLine: row.end_line,
|
|
11106
|
+
kind: row.kind,
|
|
11107
|
+
distance: row.distance,
|
|
11108
|
+
text: row.text
|
|
11109
|
+
});
|
|
11110
|
+
}
|
|
11111
|
+
return hits;
|
|
11112
|
+
}
|
|
11213
11113
|
async function searchSemantic(db, query, topK = 8, modelName = DEFAULT_MODEL, maxDistance = DEFAULT_DISTANCE_THRESHOLD, rootDir) {
|
|
11214
11114
|
if (!isAvailable()) {
|
|
11215
11115
|
return [];
|
|
@@ -11233,16 +11133,19 @@ async function searchSemantic(db, query, topK = 8, modelName = DEFAULT_MODEL, ma
|
|
|
11233
11133
|
MAX_OVER_FETCH,
|
|
11234
11134
|
Math.ceil(topK * OVER_FETCH_FACTOR)
|
|
11235
11135
|
);
|
|
11236
|
-
let k = overFetchK;
|
|
11136
|
+
let k = Math.min(overFetchK, VEC_MAX_K);
|
|
11237
11137
|
let pass = fetchScopedHits(db, queryVec, k, maxDistance, rootDir);
|
|
11238
11138
|
let hits = pass.hits;
|
|
11239
11139
|
if (rootDir !== void 0) {
|
|
11240
|
-
const ceiling = countStoredVectors(db);
|
|
11140
|
+
const ceiling = Math.min(countStoredVectors(db), VEC_MAX_K);
|
|
11241
11141
|
while (hits.length < topK && pass.candidateCount === k && k < ceiling) {
|
|
11242
11142
|
k = Math.min(ceiling, k * BACKFILL_MULTIPLIER);
|
|
11243
11143
|
pass = fetchScopedHits(db, queryVec, k, maxDistance, rootDir);
|
|
11244
11144
|
hits = pass.hits;
|
|
11245
11145
|
}
|
|
11146
|
+
if (hits.length < topK && pass.candidateCount === k && k >= ceiling && countStoredVectors(db) > VEC_MAX_K) {
|
|
11147
|
+
hits = fetchScopedExactHits(db, queryVec, VEC_MAX_K, maxDistance, rootDir);
|
|
11148
|
+
}
|
|
11246
11149
|
}
|
|
11247
11150
|
return rerankHits(hits, query, topK);
|
|
11248
11151
|
}
|
|
@@ -11446,13 +11349,13 @@ function _pathPriorityPenalty(filePath) {
|
|
|
11446
11349
|
|
|
11447
11350
|
// src/parser_fingerprint.ts
|
|
11448
11351
|
init_define_import_meta_env();
|
|
11449
|
-
var PARSER_FINGERPRINT = "
|
|
11352
|
+
var PARSER_FINGERPRINT = "9c0e5a5680783f75";
|
|
11450
11353
|
|
|
11451
11354
|
// src/parser.ts
|
|
11452
11355
|
init_define_import_meta_env();
|
|
11453
|
-
import * as
|
|
11454
|
-
import { createRequire as
|
|
11455
|
-
import * as
|
|
11356
|
+
import * as fs11 from "node:fs";
|
|
11357
|
+
import { createRequire as createRequire2 } from "node:module";
|
|
11358
|
+
import * as path8 from "node:path";
|
|
11456
11359
|
|
|
11457
11360
|
// src/markdown_lines.ts
|
|
11458
11361
|
init_define_import_meta_env();
|
|
@@ -11575,14 +11478,14 @@ function extractChangelogVersionHint(content, filePath) {
|
|
|
11575
11478
|
|
|
11576
11479
|
// src/languages/salesforce_frontend.ts
|
|
11577
11480
|
init_define_import_meta_env();
|
|
11578
|
-
import * as
|
|
11481
|
+
import * as path7 from "node:path";
|
|
11579
11482
|
function lines(content) {
|
|
11580
11483
|
return content.split("\n");
|
|
11581
11484
|
}
|
|
11582
11485
|
function bundleName(filePath) {
|
|
11583
11486
|
const normalized = filePath.replaceAll("\\", "/");
|
|
11584
|
-
const parent =
|
|
11585
|
-
const base =
|
|
11487
|
+
const parent = path7.posix.basename(path7.posix.dirname(normalized));
|
|
11488
|
+
const base = path7.posix.basename(normalized).replace(/\.[^.]+$/, "");
|
|
11586
11489
|
return parent === "lwc" || parent === "aura" ? base : parent;
|
|
11587
11490
|
}
|
|
11588
11491
|
function lwcTagAlias(name) {
|
|
@@ -11686,7 +11589,7 @@ var MARKUP_KIND = {
|
|
|
11686
11589
|
};
|
|
11687
11590
|
function markupArtifactName(filePath, extension) {
|
|
11688
11591
|
if (extension === ".page" || extension === ".component" || extension === ".email") {
|
|
11689
|
-
return
|
|
11592
|
+
return path7.posix.basename(filePath.replaceAll("\\", "/")).replace(new RegExp(`${extension.replace(".", "\\.")}$`, "i"), "");
|
|
11690
11593
|
}
|
|
11691
11594
|
return bundleName(filePath);
|
|
11692
11595
|
}
|
|
@@ -11706,7 +11609,7 @@ function attributeRefs(refs, content, filePath, attribute, split = false) {
|
|
|
11706
11609
|
}
|
|
11707
11610
|
function extractSalesforceMarkup(content, filePath) {
|
|
11708
11611
|
const normalized = filePath.replaceAll("\\", "/");
|
|
11709
|
-
const extension =
|
|
11612
|
+
const extension = path7.posix.extname(normalized).toLowerCase();
|
|
11710
11613
|
const kind = MARKUP_KIND[extension] ?? "salesforce_markup";
|
|
11711
11614
|
const symbols = [
|
|
11712
11615
|
symbol(filePath, markupArtifactName(normalized, extension), kind, 1, countContentLines(content))
|
|
@@ -11737,56 +11640,8 @@ function extractSalesforceMarkup(content, filePath) {
|
|
|
11737
11640
|
};
|
|
11738
11641
|
}
|
|
11739
11642
|
|
|
11740
|
-
// src/languages/ipynb_idx.ts
|
|
11741
|
-
init_define_import_meta_env();
|
|
11742
|
-
function normalizeSource(source) {
|
|
11743
|
-
if (typeof source === "string") return source;
|
|
11744
|
-
if (!Array.isArray(source)) return "";
|
|
11745
|
-
const lines2 = source.filter((s) => typeof s === "string");
|
|
11746
|
-
if (lines2.length === 0) return "";
|
|
11747
|
-
const noneEndWithNewline = lines2.every((l) => !l.endsWith("\n"));
|
|
11748
|
-
return noneEndWithNewline ? lines2.join("\n") : lines2.join("");
|
|
11749
|
-
}
|
|
11750
|
-
function resolveKernelLanguage(metadata) {
|
|
11751
|
-
const kernelspecLang = metadata?.kernelspec?.language;
|
|
11752
|
-
if (typeof kernelspecLang === "string" && kernelspecLang !== "") return kernelspecLang.toLowerCase();
|
|
11753
|
-
const infoLang = metadata?.language_info?.name;
|
|
11754
|
-
if (typeof infoLang === "string" && infoLang !== "") return infoLang.toLowerCase();
|
|
11755
|
-
return void 0;
|
|
11756
|
-
}
|
|
11757
|
-
function ipynbToVirtualSource(raw) {
|
|
11758
|
-
let data;
|
|
11759
|
-
try {
|
|
11760
|
-
data = JSON.parse(raw);
|
|
11761
|
-
} catch {
|
|
11762
|
-
return { content: "", cellLanguage: null };
|
|
11763
|
-
}
|
|
11764
|
-
if (typeof data !== "object" || data === null) return { content: "", cellLanguage: null };
|
|
11765
|
-
const nb = data;
|
|
11766
|
-
if (!Array.isArray(nb.cells)) return { content: "", cellLanguage: null };
|
|
11767
|
-
const declared = resolveKernelLanguage(nb.metadata);
|
|
11768
|
-
const isPython = declared === void 0 || declared === "python" || declared === "python3";
|
|
11769
|
-
if (!isPython) return { content: "", cellLanguage: null };
|
|
11770
|
-
const blocks = [];
|
|
11771
|
-
nb.cells.forEach((cell, index) => {
|
|
11772
|
-
if (typeof cell !== "object" || cell === null) return;
|
|
11773
|
-
const c = cell;
|
|
11774
|
-
if (c.cell_type === "code") {
|
|
11775
|
-
const src = normalizeSource(c.source).replace(/\n+$/, "");
|
|
11776
|
-
blocks.push(`# %% cell ${index}
|
|
11777
|
-
${src}`);
|
|
11778
|
-
} else if (c.cell_type === "markdown") {
|
|
11779
|
-
const src = normalizeSource(c.source).replace(/\n+$/, "");
|
|
11780
|
-
const commented = src.split("\n").map((line) => line === "" ? "#" : `# ${line}`).join("\n");
|
|
11781
|
-
blocks.push(`# %% [markdown] cell ${index}
|
|
11782
|
-
${commented}`);
|
|
11783
|
-
}
|
|
11784
|
-
});
|
|
11785
|
-
return { content: blocks.join("\n\n"), cellLanguage: "python" };
|
|
11786
|
-
}
|
|
11787
|
-
|
|
11788
11643
|
// src/parser.ts
|
|
11789
|
-
var
|
|
11644
|
+
var _require2 = createRequire2(import.meta.url);
|
|
11790
11645
|
var COMPILER_ARTIFACT_REF_NAMES = /* @__PURE__ */ new Set(["__name"]);
|
|
11791
11646
|
var _parserCtor;
|
|
11792
11647
|
var _parserCtorError = null;
|
|
@@ -11797,7 +11652,7 @@ function loadParserCtor() {
|
|
|
11797
11652
|
if (_parserCtorOverride !== void 0) return _parserCtorOverride;
|
|
11798
11653
|
if (_parserCtor !== void 0) return _parserCtor;
|
|
11799
11654
|
try {
|
|
11800
|
-
_parserCtor =
|
|
11655
|
+
_parserCtor = _require2("tree-sitter");
|
|
11801
11656
|
} catch (e) {
|
|
11802
11657
|
_parserCtor = null;
|
|
11803
11658
|
_parserCtorError = e instanceof Error ? e : new Error(String(e));
|
|
@@ -11886,32 +11741,32 @@ function countNewlines(s) {
|
|
|
11886
11741
|
return n;
|
|
11887
11742
|
}
|
|
11888
11743
|
function loadGrammar(lang, filePath, content) {
|
|
11889
|
-
const useTsx = lang === "typescript" && filePath !== void 0 &&
|
|
11890
|
-
const useCppHeader = lang === "c" && filePath !== void 0 &&
|
|
11744
|
+
const useTsx = lang === "typescript" && filePath !== void 0 && path8.extname(filePath).toLowerCase() === ".tsx";
|
|
11745
|
+
const useCppHeader = lang === "c" && filePath !== void 0 && path8.extname(filePath).toLowerCase() === ".h" && content !== void 0 && CPP_HEADER_SNIFF_RE.test(content);
|
|
11891
11746
|
const cacheKey = useTsx ? "typescript:tsx" : useCppHeader ? "c:cpp-header" : lang;
|
|
11892
11747
|
const cached = _grammarCache.get(cacheKey);
|
|
11893
11748
|
if (cached !== void 0) return cached;
|
|
11894
11749
|
let grammar = null;
|
|
11895
11750
|
try {
|
|
11896
11751
|
if (lang === "typescript") {
|
|
11897
|
-
const mod =
|
|
11752
|
+
const mod = _require2("tree-sitter-typescript");
|
|
11898
11753
|
grammar = useTsx ? mod.tsx : mod.typescript;
|
|
11899
11754
|
} else if (lang === "javascript") {
|
|
11900
|
-
grammar =
|
|
11755
|
+
grammar = _require2("tree-sitter-javascript");
|
|
11901
11756
|
} else if (lang === "python") {
|
|
11902
|
-
grammar =
|
|
11757
|
+
grammar = _require2("tree-sitter-python");
|
|
11903
11758
|
} else if (lang === "go") {
|
|
11904
|
-
grammar =
|
|
11759
|
+
grammar = _require2("tree-sitter-go");
|
|
11905
11760
|
} else if (lang === "rust") {
|
|
11906
|
-
grammar =
|
|
11761
|
+
grammar = _require2("tree-sitter-rust");
|
|
11907
11762
|
} else if (lang === "ruby") {
|
|
11908
|
-
grammar =
|
|
11763
|
+
grammar = _require2("tree-sitter-ruby");
|
|
11909
11764
|
} else if (lang === "java") {
|
|
11910
|
-
grammar =
|
|
11765
|
+
grammar = _require2("tree-sitter-java");
|
|
11911
11766
|
} else if (lang === "c") {
|
|
11912
|
-
grammar = useCppHeader ?
|
|
11767
|
+
grammar = useCppHeader ? _require2("tree-sitter-cpp") : _require2("tree-sitter-c");
|
|
11913
11768
|
} else if (lang === "cpp") {
|
|
11914
|
-
grammar =
|
|
11769
|
+
grammar = _require2("tree-sitter-cpp");
|
|
11915
11770
|
}
|
|
11916
11771
|
} catch {
|
|
11917
11772
|
grammar = null;
|
|
@@ -11937,7 +11792,7 @@ var TREE_SITTER_GRAMMAR_PACKAGES = [
|
|
|
11937
11792
|
function missingTreeSitterGrammarPackages() {
|
|
11938
11793
|
return TREE_SITTER_GRAMMAR_PACKAGES.filter((pkg) => {
|
|
11939
11794
|
try {
|
|
11940
|
-
|
|
11795
|
+
_require2.resolve(pkg);
|
|
11941
11796
|
return false;
|
|
11942
11797
|
} catch {
|
|
11943
11798
|
return true;
|
|
@@ -12578,7 +12433,7 @@ var CALL_TYPES_BY_LANG = /* @__PURE__ */ new Map([
|
|
|
12578
12433
|
["rust", /* @__PURE__ */ new Set(["call_expression", "macro_invocation"])],
|
|
12579
12434
|
["java", /* @__PURE__ */ new Set(["method_invocation", "object_creation_expression"])],
|
|
12580
12435
|
["c", /* @__PURE__ */ new Set(["call_expression"])],
|
|
12581
|
-
["cpp", /* @__PURE__ */ new Set(["call_expression"])],
|
|
12436
|
+
["cpp", /* @__PURE__ */ new Set(["call_expression", "new_expression"])],
|
|
12582
12437
|
["ruby", /* @__PURE__ */ new Set(["call"])]
|
|
12583
12438
|
]);
|
|
12584
12439
|
var REF_NOISE_BY_LANG = /* @__PURE__ */ new Map([
|
|
@@ -12770,6 +12625,38 @@ function cFunctionName(node) {
|
|
|
12770
12625
|
}
|
|
12771
12626
|
return null;
|
|
12772
12627
|
}
|
|
12628
|
+
function cppCalleeName(node) {
|
|
12629
|
+
switch (node.type) {
|
|
12630
|
+
case "identifier":
|
|
12631
|
+
case "field_identifier":
|
|
12632
|
+
case "type_identifier":
|
|
12633
|
+
return node.text;
|
|
12634
|
+
// An overloaded operator's name is its own node type, and it is the spelling the extractor gives the operator's definition too, so returning the text here is what makes the two match. Measured: `r.template operator()<int>(3)` parses as `dependent_name(template, template_method(operator_name "operator()", template_argument_list "<int>"))`, so without this case the name resolved to null and the field_expression fallback below recorded `template operator()<int>` -- a spelling no symbol row ever holds, leaving the call permanently unresolvable.
|
|
12635
|
+
case "operator_name":
|
|
12636
|
+
return node.text;
|
|
12637
|
+
case "template_function":
|
|
12638
|
+
case "template_method":
|
|
12639
|
+
case "template_type": {
|
|
12640
|
+
const name = node.childForFieldName("name");
|
|
12641
|
+
return name !== null ? cppCalleeName(name) : null;
|
|
12642
|
+
}
|
|
12643
|
+
case "qualified_identifier": {
|
|
12644
|
+
const name = node.childForFieldName("name");
|
|
12645
|
+
return name !== null ? cppCalleeName(name) : lastSegment(node.text);
|
|
12646
|
+
}
|
|
12647
|
+
case "dependent_name": {
|
|
12648
|
+
const last = node.namedChildren[node.namedChildren.length - 1];
|
|
12649
|
+
return last !== void 0 ? cppCalleeName(last) : null;
|
|
12650
|
+
}
|
|
12651
|
+
case "field_expression": {
|
|
12652
|
+
const field = node.childForFieldName("field");
|
|
12653
|
+
if (field === null) return null;
|
|
12654
|
+
return cppCalleeName(field) ?? field.text;
|
|
12655
|
+
}
|
|
12656
|
+
default:
|
|
12657
|
+
return null;
|
|
12658
|
+
}
|
|
12659
|
+
}
|
|
12773
12660
|
function calleeName(call, language) {
|
|
12774
12661
|
switch (language) {
|
|
12775
12662
|
case "typescript":
|
|
@@ -12821,12 +12708,9 @@ function calleeName(call, language) {
|
|
|
12821
12708
|
}
|
|
12822
12709
|
case "c":
|
|
12823
12710
|
case "cpp": {
|
|
12824
|
-
const fn = call.childForFieldName("function");
|
|
12711
|
+
const fn = call.type === "new_expression" ? call.childForFieldName("type") : call.childForFieldName("function");
|
|
12825
12712
|
if (fn === null) return null;
|
|
12826
|
-
|
|
12827
|
-
if (fn.type === "field_expression") return fn.childForFieldName("field")?.text ?? null;
|
|
12828
|
-
if (fn.type === "qualified_identifier") return lastSegment(fn.text);
|
|
12829
|
-
return null;
|
|
12713
|
+
return cppCalleeName(fn);
|
|
12830
12714
|
}
|
|
12831
12715
|
case "ruby": {
|
|
12832
12716
|
const m = call.childForFieldName("method");
|
|
@@ -13594,7 +13478,7 @@ function parseSourceSymbolsTreeSitterOnly(content, filePath, language) {
|
|
|
13594
13478
|
var REGEX_ADAPTERS_SLOT = /* @__PURE__ */ Symbol.for("token-goat.regex-adapters");
|
|
13595
13479
|
var adapterSlot = globalThis;
|
|
13596
13480
|
async function loadRegexExtractors() {
|
|
13597
|
-
adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-
|
|
13481
|
+
adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-6B44WLIF.mjs");
|
|
13598
13482
|
}
|
|
13599
13483
|
function regexAdapters() {
|
|
13600
13484
|
const loaded = adapterSlot[REGEX_ADAPTERS_SLOT];
|
|
@@ -13656,9 +13540,9 @@ function isUnderSkipDir(filePath, skipDirs) {
|
|
|
13656
13540
|
}
|
|
13657
13541
|
function isParseSkipEligible(filePath, cfg) {
|
|
13658
13542
|
if (isUnderSkipDir(filePath, cfg.skip_dirs)) return true;
|
|
13659
|
-
if (cfg.skip_files.includes(
|
|
13543
|
+
if (cfg.skip_files.includes(path8.basename(filePath))) return true;
|
|
13660
13544
|
try {
|
|
13661
|
-
const stat2 =
|
|
13545
|
+
const stat2 = fs11.statSync(filePath);
|
|
13662
13546
|
if (stat2.size > cfg.large_file_skip_kb * 1024) return true;
|
|
13663
13547
|
} catch {
|
|
13664
13548
|
}
|
|
@@ -13717,7 +13601,7 @@ function indexFileSync(rawPath, dbPath = globalDbPath(), preReadBytes) {
|
|
|
13717
13601
|
raw = preReadBytes;
|
|
13718
13602
|
} else {
|
|
13719
13603
|
try {
|
|
13720
|
-
raw =
|
|
13604
|
+
raw = fs11.readFileSync(filePath);
|
|
13721
13605
|
} catch (err) {
|
|
13722
13606
|
if (err.code === "ENOENT") return;
|
|
13723
13607
|
throw err;
|
|
@@ -13753,32 +13637,36 @@ var OVERSIZE_EMBED_SHA_PREFIX = "oversize:";
|
|
|
13753
13637
|
function oversizeEmbedSha(sha, symbolOnlyKb) {
|
|
13754
13638
|
return `${OVERSIZE_EMBED_SHA_PREFIX}${symbolOnlyKb}:${sha}`;
|
|
13755
13639
|
}
|
|
13640
|
+
var TIMEOUT_EMBED_SHA_PREFIX = "timeout:";
|
|
13641
|
+
function timeoutEmbedSha(sha, workMillis) {
|
|
13642
|
+
return `${TIMEOUT_EMBED_SHA_PREFIX}${workMillis}:${sha}`;
|
|
13643
|
+
}
|
|
13756
13644
|
function canonicalizeIndexPath(absPath) {
|
|
13757
13645
|
if (!isCaseInsensitiveFs()) return absPath;
|
|
13758
13646
|
let real;
|
|
13759
13647
|
try {
|
|
13760
|
-
real =
|
|
13648
|
+
real = fs11.realpathSync.native(absPath);
|
|
13761
13649
|
} catch {
|
|
13762
13650
|
return absPath;
|
|
13763
13651
|
}
|
|
13764
13652
|
const cut = Math.max(absPath.lastIndexOf("/"), absPath.lastIndexOf("\\"));
|
|
13765
13653
|
const base = absPath.slice(cut + 1);
|
|
13766
13654
|
const realNorm = normalizePath(real);
|
|
13767
|
-
const realBase =
|
|
13655
|
+
const realBase = path8.basename(realNorm);
|
|
13768
13656
|
if (base === realBase) return absPath;
|
|
13769
13657
|
if (foldPath(base) !== foldPath(realBase)) return absPath;
|
|
13770
|
-
const callerDir = cut < 0 ? normalizePath(
|
|
13771
|
-
if (foldPath(callerDir) !== foldPath(
|
|
13658
|
+
const callerDir = cut < 0 ? normalizePath(path8.resolve(".")) : normalizePath(absPath.slice(0, cut));
|
|
13659
|
+
if (foldPath(callerDir) !== foldPath(path8.dirname(realNorm))) return absPath;
|
|
13772
13660
|
return absPath.slice(0, cut + 1) + realBase;
|
|
13773
13661
|
}
|
|
13774
13662
|
function indexedPathSpellingIsStale(storedPath, absPath) {
|
|
13775
13663
|
if (!isCaseInsensitiveFs()) return false;
|
|
13776
13664
|
const stored = normalizePath(storedPath);
|
|
13777
|
-
const candidate = normalizePath(
|
|
13665
|
+
const candidate = normalizePath(path8.resolve(absPath));
|
|
13778
13666
|
if (foldPath(stored) !== foldPath(candidate)) return false;
|
|
13779
13667
|
let real;
|
|
13780
13668
|
try {
|
|
13781
|
-
real = normalizePath(
|
|
13669
|
+
real = normalizePath(fs11.realpathSync.native(absPath));
|
|
13782
13670
|
} catch {
|
|
13783
13671
|
return false;
|
|
13784
13672
|
}
|
|
@@ -13806,6 +13694,7 @@ function isEmbedFresh(storedEmbedSha, sha, embeddingsEnabled, depsAvailable, sym
|
|
|
13806
13694
|
if (storedEmbedSha === sha) return true;
|
|
13807
13695
|
if (!depsAvailable && storedEmbedSha === unavailableEmbedSha(sha)) return true;
|
|
13808
13696
|
if (storedEmbedSha === oversizeEmbedSha(sha, symbolOnlyKb)) return true;
|
|
13697
|
+
if (storedEmbedSha === timeoutEmbedSha(sha, MAX_DOCUMENT_WORK_MILLIS)) return true;
|
|
13809
13698
|
return false;
|
|
13810
13699
|
}
|
|
13811
13700
|
async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onError) {
|
|
@@ -13823,6 +13712,7 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
13823
13712
|
}
|
|
13824
13713
|
if (isEmbeddableDocument(filePath)) {
|
|
13825
13714
|
let extracted;
|
|
13715
|
+
let refusedOnTheClock = false;
|
|
13826
13716
|
try {
|
|
13827
13717
|
extracted = await extractEmbeddableDocumentText(filePath);
|
|
13828
13718
|
} catch (err) {
|
|
@@ -13830,12 +13720,13 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
13830
13720
|
onError?.(err);
|
|
13831
13721
|
return;
|
|
13832
13722
|
}
|
|
13723
|
+
refusedOnTheClock = isTransientDocumentRefusal(err);
|
|
13833
13724
|
extracted = null;
|
|
13834
13725
|
}
|
|
13835
13726
|
if (extracted === null || extracted.trim().length === 0) {
|
|
13836
13727
|
const db = getDb(dbPath);
|
|
13837
13728
|
deleteFileEmbeddings(db, filePath);
|
|
13838
|
-
stampEmbedSha(db, filePath, sha, (s) => s);
|
|
13729
|
+
stampEmbedSha(db, filePath, sha, (s) => refusedOnTheClock ? timeoutEmbedSha(s, MAX_DOCUMENT_WORK_MILLIS) : s);
|
|
13839
13730
|
return;
|
|
13840
13731
|
}
|
|
13841
13732
|
if (extracted.length > ixCfg.large_file_symbol_only_kb * 1024) {
|
|
@@ -13855,7 +13746,7 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
13855
13746
|
}
|
|
13856
13747
|
let content;
|
|
13857
13748
|
try {
|
|
13858
|
-
content = decodeSource(await
|
|
13749
|
+
content = decodeSource(await fs11.promises.readFile(filePath));
|
|
13859
13750
|
} catch {
|
|
13860
13751
|
return;
|
|
13861
13752
|
}
|
|
@@ -13901,7 +13792,7 @@ function stampEmbedSha(db, filePath, sha, makeValue) {
|
|
|
13901
13792
|
}
|
|
13902
13793
|
function safeMtime(filePath) {
|
|
13903
13794
|
try {
|
|
13904
|
-
return
|
|
13795
|
+
return fs11.statSync(filePath).mtimeMs / 1e3;
|
|
13905
13796
|
} catch {
|
|
13906
13797
|
return 0;
|
|
13907
13798
|
}
|
|
@@ -13909,8 +13800,8 @@ function safeMtime(filePath) {
|
|
|
13909
13800
|
|
|
13910
13801
|
// src/index_prune.ts
|
|
13911
13802
|
init_define_import_meta_env();
|
|
13912
|
-
import * as
|
|
13913
|
-
import * as
|
|
13803
|
+
import * as fs12 from "node:fs";
|
|
13804
|
+
import * as path9 from "node:path";
|
|
13914
13805
|
function removeFileFromIndex(db, filePath) {
|
|
13915
13806
|
const tx = db.transaction(() => {
|
|
13916
13807
|
deleteFileRows(db, filePath);
|
|
@@ -13947,7 +13838,7 @@ function findDeletablePaths(rootPrefix, dbPath) {
|
|
|
13947
13838
|
for (const p of foldedPathsUnderRoot(rootPrefix, dbPath)) {
|
|
13948
13839
|
let stillExists;
|
|
13949
13840
|
try {
|
|
13950
|
-
const st =
|
|
13841
|
+
const st = fs12.statSync(p, { throwIfNoEntry: false });
|
|
13951
13842
|
stillExists = st !== void 0 && st.isFile();
|
|
13952
13843
|
} catch {
|
|
13953
13844
|
continue;
|
|
@@ -13975,7 +13866,7 @@ function removeDeletedFilesBestEffort(db, paths) {
|
|
|
13975
13866
|
for (const p of paths) {
|
|
13976
13867
|
let gone;
|
|
13977
13868
|
try {
|
|
13978
|
-
const st =
|
|
13869
|
+
const st = fs12.statSync(p, { throwIfNoEntry: false });
|
|
13979
13870
|
gone = st === void 0 || !st.isFile();
|
|
13980
13871
|
} catch {
|
|
13981
13872
|
continue;
|
|
@@ -14042,7 +13933,7 @@ function pruneOrphanedChunks(dbPath = globalDbPath()) {
|
|
|
14042
13933
|
return removed;
|
|
14043
13934
|
}
|
|
14044
13935
|
function recordKnownRoot(filePath, dbPath = globalDbPath()) {
|
|
14045
|
-
const project = findProject(
|
|
13936
|
+
const project = findProject(path9.dirname(filePath));
|
|
14046
13937
|
if (project === null || isTooShallowToPrune(project.root)) return;
|
|
14047
13938
|
const db = getDb(dbPath);
|
|
14048
13939
|
db.prepare(
|
|
@@ -14065,7 +13956,7 @@ function sweepKnownRoots(dbPath = globalDbPath(), opts) {
|
|
|
14065
13956
|
if (isTooShallowToPrune(root)) continue;
|
|
14066
13957
|
let reachable;
|
|
14067
13958
|
try {
|
|
14068
|
-
reachable =
|
|
13959
|
+
reachable = fs12.statSync(root, { throwIfNoEntry: false })?.isDirectory() === true;
|
|
14069
13960
|
} catch {
|
|
14070
13961
|
reachable = false;
|
|
14071
13962
|
}
|
|
@@ -14100,19 +13991,19 @@ function sweepKnownRoots(dbPath = globalDbPath(), opts) {
|
|
|
14100
13991
|
}
|
|
14101
13992
|
var KNOWN_ROOT_RECORD_MIN_INTERVAL_MS = 60 * 60 * 1e3;
|
|
14102
13993
|
function knownRootRecordMarkerPath(dir, filePath) {
|
|
14103
|
-
return
|
|
13994
|
+
return path9.join(dir, `known-root-record-${shortFingerprint(path9.dirname(filePath))}.marker`);
|
|
14104
13995
|
}
|
|
14105
13996
|
var KNOWN_ROOT_MARKER_PREFIX = "known-root-record-";
|
|
14106
13997
|
function sweepExpiredKnownRootMarkers(dir = dataDir()) {
|
|
14107
13998
|
let removed = 0;
|
|
14108
13999
|
try {
|
|
14109
14000
|
const cutoff = Date.now() - KNOWN_ROOT_RECORD_MIN_INTERVAL_MS;
|
|
14110
|
-
for (const file of
|
|
14001
|
+
for (const file of fs12.readdirSync(dir)) {
|
|
14111
14002
|
if (!file.startsWith(KNOWN_ROOT_MARKER_PREFIX) || !file.endsWith(".marker")) continue;
|
|
14112
|
-
const full =
|
|
14003
|
+
const full = path9.join(dir, file);
|
|
14113
14004
|
try {
|
|
14114
|
-
if (
|
|
14115
|
-
|
|
14005
|
+
if (fs12.statSync(full).mtimeMs < cutoff) {
|
|
14006
|
+
fs12.unlinkSync(full);
|
|
14116
14007
|
removed += 1;
|
|
14117
14008
|
}
|
|
14118
14009
|
} catch {
|
|
@@ -14125,13 +14016,13 @@ function sweepExpiredKnownRootMarkers(dir = dataDir()) {
|
|
|
14125
14016
|
function recordKnownRootThrottled(filePath, dir = dataDir(), dbPath = globalDbPath()) {
|
|
14126
14017
|
const markerPath = knownRootRecordMarkerPath(dir, filePath);
|
|
14127
14018
|
try {
|
|
14128
|
-
const stat2 =
|
|
14019
|
+
const stat2 = fs12.statSync(markerPath);
|
|
14129
14020
|
if (Date.now() - stat2.mtimeMs < KNOWN_ROOT_RECORD_MIN_INTERVAL_MS) return;
|
|
14130
14021
|
} catch {
|
|
14131
14022
|
}
|
|
14132
14023
|
try {
|
|
14133
14024
|
ensureDirSync(dir);
|
|
14134
|
-
|
|
14025
|
+
fs12.writeFileSync(markerPath, "");
|
|
14135
14026
|
} catch {
|
|
14136
14027
|
}
|
|
14137
14028
|
recordKnownRoot(filePath, dbPath);
|
|
@@ -14161,15 +14052,15 @@ function applyIndexingPriority() {
|
|
|
14161
14052
|
|
|
14162
14053
|
// src/worker.ts
|
|
14163
14054
|
init_define_import_meta_env();
|
|
14164
|
-
import { spawn
|
|
14165
|
-
import * as
|
|
14166
|
-
import * as
|
|
14055
|
+
import { spawn } from "node:child_process";
|
|
14056
|
+
import * as fs14 from "node:fs";
|
|
14057
|
+
import * as path11 from "node:path";
|
|
14167
14058
|
import { fileURLToPath } from "node:url";
|
|
14168
14059
|
|
|
14169
14060
|
// src/snapshots.ts
|
|
14170
14061
|
init_define_import_meta_env();
|
|
14171
|
-
import * as
|
|
14172
|
-
import * as
|
|
14062
|
+
import * as fs13 from "node:fs";
|
|
14063
|
+
import * as path10 from "node:path";
|
|
14173
14064
|
var MAX_SNAPSHOTS_PER_SESSION = 150;
|
|
14174
14065
|
var MAX_SNAPSHOT_BYTES = 256 * 1024;
|
|
14175
14066
|
var SNAPSHOT_TRUNCATE_BYTES = 50 * 1024;
|
|
@@ -14179,10 +14070,10 @@ var VALID_KINDS = /* @__PURE__ */ new Set([KIND_READ, KIND_PREDICTIVE]);
|
|
|
14179
14070
|
function sessionDir(sessionId) {
|
|
14180
14071
|
if (!sessionId) return null;
|
|
14181
14072
|
const safe = sanitizeIdForFilename(sessionId, 64, "anon");
|
|
14182
|
-
const base =
|
|
14183
|
-
const candidate =
|
|
14073
|
+
const base = path10.join(tokenGoatHome(), "session_snapshots");
|
|
14074
|
+
const candidate = path10.join(base, safe);
|
|
14184
14075
|
try {
|
|
14185
|
-
const rel =
|
|
14076
|
+
const rel = path10.relative(base, candidate);
|
|
14186
14077
|
if (rel.startsWith("..")) return null;
|
|
14187
14078
|
} catch {
|
|
14188
14079
|
return null;
|
|
@@ -14195,7 +14086,7 @@ function pathKey(filePath) {
|
|
|
14195
14086
|
function snapshot_path(sessionId, filePath) {
|
|
14196
14087
|
const d = sessionDir(sessionId);
|
|
14197
14088
|
if (!d) return null;
|
|
14198
|
-
return
|
|
14089
|
+
return path10.join(d, `${pathKey(filePath)}.bin`);
|
|
14199
14090
|
}
|
|
14200
14091
|
function kindSidecarPath(snapshotPath) {
|
|
14201
14092
|
return snapshotPath + ".kind";
|
|
@@ -14203,11 +14094,11 @@ function kindSidecarPath(snapshotPath) {
|
|
|
14203
14094
|
function writeSnapshotKind(sidecarPath, kind) {
|
|
14204
14095
|
try {
|
|
14205
14096
|
const safeKind = VALID_KINDS.has(kind) ? kind : KIND_READ;
|
|
14206
|
-
const dir =
|
|
14207
|
-
if (!
|
|
14097
|
+
const dir = path10.dirname(sidecarPath);
|
|
14098
|
+
if (!fs13.existsSync(dir)) {
|
|
14208
14099
|
ensureDirSync(dir);
|
|
14209
14100
|
}
|
|
14210
|
-
|
|
14101
|
+
fs13.writeFileSync(sidecarPath, safeKind, "utf8");
|
|
14211
14102
|
return true;
|
|
14212
14103
|
} catch {
|
|
14213
14104
|
return false;
|
|
@@ -14216,12 +14107,12 @@ function writeSnapshotKind(sidecarPath, kind) {
|
|
|
14216
14107
|
function evictOldest(d, maxCount) {
|
|
14217
14108
|
try {
|
|
14218
14109
|
const entries = [];
|
|
14219
|
-
const files =
|
|
14110
|
+
const files = fs13.readdirSync(d);
|
|
14220
14111
|
for (const file of files) {
|
|
14221
|
-
const fullPath =
|
|
14112
|
+
const fullPath = path10.join(d, file);
|
|
14222
14113
|
if (!file.endsWith(".bin")) continue;
|
|
14223
14114
|
try {
|
|
14224
|
-
const stat2 =
|
|
14115
|
+
const stat2 = fs13.statSync(fullPath);
|
|
14225
14116
|
entries.push([fullPath, stat2.mtimeMs]);
|
|
14226
14117
|
} catch {
|
|
14227
14118
|
continue;
|
|
@@ -14233,10 +14124,10 @@ function evictOldest(d, maxCount) {
|
|
|
14233
14124
|
const over = entries.length - maxCount;
|
|
14234
14125
|
for (const [p] of entries.slice(0, over)) {
|
|
14235
14126
|
try {
|
|
14236
|
-
|
|
14127
|
+
fs13.unlinkSync(p);
|
|
14237
14128
|
removed++;
|
|
14238
14129
|
try {
|
|
14239
|
-
|
|
14130
|
+
fs13.unlinkSync(kindSidecarPath(p));
|
|
14240
14131
|
} catch {
|
|
14241
14132
|
}
|
|
14242
14133
|
} catch {
|
|
@@ -14265,10 +14156,10 @@ function store(sessionId, filePath, content, opts = {}) {
|
|
|
14265
14156
|
if (!p) return null;
|
|
14266
14157
|
const sha = fingerprintContent(stored);
|
|
14267
14158
|
try {
|
|
14268
|
-
const isNewEntry = !
|
|
14159
|
+
const isNewEntry = !fs13.existsSync(p);
|
|
14269
14160
|
if (!isNewEntry) {
|
|
14270
14161
|
try {
|
|
14271
|
-
const existing =
|
|
14162
|
+
const existing = fs13.readFileSync(p);
|
|
14272
14163
|
if (Buffer.from(existing).equals(stored)) {
|
|
14273
14164
|
return {
|
|
14274
14165
|
path: p,
|
|
@@ -14279,8 +14170,8 @@ function store(sessionId, filePath, content, opts = {}) {
|
|
|
14279
14170
|
} catch {
|
|
14280
14171
|
}
|
|
14281
14172
|
}
|
|
14282
|
-
const dir =
|
|
14283
|
-
if (!
|
|
14173
|
+
const dir = path10.dirname(p);
|
|
14174
|
+
if (!fs13.existsSync(dir)) {
|
|
14284
14175
|
ensureDirSync(dir);
|
|
14285
14176
|
}
|
|
14286
14177
|
if (isNewEntry) {
|
|
@@ -14300,9 +14191,9 @@ function store(sessionId, filePath, content, opts = {}) {
|
|
|
14300
14191
|
}
|
|
14301
14192
|
function load(sessionId, filePath, opts = {}) {
|
|
14302
14193
|
const p = snapshot_path(sessionId, filePath);
|
|
14303
|
-
if (!p || !
|
|
14194
|
+
if (!p || !fs13.existsSync(p)) return null;
|
|
14304
14195
|
try {
|
|
14305
|
-
const stat2 =
|
|
14196
|
+
const stat2 = fs13.statSync(p);
|
|
14306
14197
|
if (stat2.size > MAX_SNAPSHOT_BYTES) {
|
|
14307
14198
|
return null;
|
|
14308
14199
|
}
|
|
@@ -14310,7 +14201,7 @@ function load(sessionId, filePath, opts = {}) {
|
|
|
14310
14201
|
return null;
|
|
14311
14202
|
}
|
|
14312
14203
|
try {
|
|
14313
|
-
const data =
|
|
14204
|
+
const data = fs13.readFileSync(p);
|
|
14314
14205
|
if (opts.expected_sha) {
|
|
14315
14206
|
const actualSha = fingerprintContent(data);
|
|
14316
14207
|
if (actualSha.toLowerCase() !== opts.expected_sha.toLowerCase()) {
|
|
@@ -14323,31 +14214,31 @@ function load(sessionId, filePath, opts = {}) {
|
|
|
14323
14214
|
}
|
|
14324
14215
|
}
|
|
14325
14216
|
function removeEligibleSnapshotFile(fullPath, file, cutoff) {
|
|
14326
|
-
const stat2 =
|
|
14217
|
+
const stat2 = fs13.lstatSync(fullPath);
|
|
14327
14218
|
if ((stat2.mode & 61440) === 40960) return false;
|
|
14328
14219
|
if (cutoff !== void 0 && stat2.mtimeMs >= cutoff) return false;
|
|
14329
|
-
|
|
14220
|
+
fs13.unlinkSync(fullPath);
|
|
14330
14221
|
return file.endsWith(".bin");
|
|
14331
14222
|
}
|
|
14332
14223
|
function cleanup_stale(maxAgeHours = 24) {
|
|
14333
|
-
const base =
|
|
14334
|
-
if (!
|
|
14224
|
+
const base = path10.join(tokenGoatHome(), "session_snapshots");
|
|
14225
|
+
if (!fs13.existsSync(base)) return 0;
|
|
14335
14226
|
const cutoff = Date.now() - maxAgeHours * 3600 * 1e3;
|
|
14336
14227
|
let removed = 0;
|
|
14337
14228
|
try {
|
|
14338
|
-
const sessionDirs =
|
|
14229
|
+
const sessionDirs = fs13.readdirSync(base);
|
|
14339
14230
|
for (const sessionDir2 of sessionDirs) {
|
|
14340
|
-
const sessionPath2 =
|
|
14231
|
+
const sessionPath2 = path10.join(base, sessionDir2);
|
|
14341
14232
|
try {
|
|
14342
|
-
const stat2 =
|
|
14233
|
+
const stat2 = fs13.statSync(sessionPath2);
|
|
14343
14234
|
if (!stat2.isDirectory()) continue;
|
|
14344
14235
|
} catch {
|
|
14345
14236
|
continue;
|
|
14346
14237
|
}
|
|
14347
14238
|
try {
|
|
14348
|
-
const files =
|
|
14239
|
+
const files = fs13.readdirSync(sessionPath2);
|
|
14349
14240
|
for (const file of files) {
|
|
14350
|
-
const fullPath =
|
|
14241
|
+
const fullPath = path10.join(sessionPath2, file);
|
|
14351
14242
|
try {
|
|
14352
14243
|
if (removeEligibleSnapshotFile(fullPath, file, cutoff)) removed++;
|
|
14353
14244
|
} catch {
|
|
@@ -14355,7 +14246,7 @@ function cleanup_stale(maxAgeHours = 24) {
|
|
|
14355
14246
|
}
|
|
14356
14247
|
}
|
|
14357
14248
|
try {
|
|
14358
|
-
|
|
14249
|
+
fs13.rmdirSync(sessionPath2);
|
|
14359
14250
|
} catch {
|
|
14360
14251
|
}
|
|
14361
14252
|
} catch {
|
|
@@ -14387,7 +14278,7 @@ var unclearedDrainingSnapshots = /* @__PURE__ */ new Map();
|
|
|
14387
14278
|
function drainingSnapshotStamp(file, content) {
|
|
14388
14279
|
let identity = "unknown";
|
|
14389
14280
|
try {
|
|
14390
|
-
const stat2 =
|
|
14281
|
+
const stat2 = fs14.statSync(file);
|
|
14391
14282
|
identity = `${stat2.mtimeMs}:${stat2.size}:${stat2.ino}`;
|
|
14392
14283
|
} catch {
|
|
14393
14284
|
}
|
|
@@ -14396,15 +14287,15 @@ function drainingSnapshotStamp(file, content) {
|
|
|
14396
14287
|
var DRAINING_READ_ATTEMPTS = 5;
|
|
14397
14288
|
var DRAINING_READ_RETRY_DELAY_MS = 50;
|
|
14398
14289
|
function listDrainingFiles(queuePath) {
|
|
14399
|
-
const dir =
|
|
14400
|
-
const base = `${
|
|
14290
|
+
const dir = path11.dirname(queuePath);
|
|
14291
|
+
const base = `${path11.basename(queuePath)}.draining`;
|
|
14401
14292
|
let entries;
|
|
14402
14293
|
try {
|
|
14403
|
-
entries =
|
|
14294
|
+
entries = fs14.readdirSync(dir);
|
|
14404
14295
|
} catch {
|
|
14405
14296
|
return [];
|
|
14406
14297
|
}
|
|
14407
|
-
return entries.filter((name) => name === base || name.startsWith(`${base}.alt-`) && !name.includes(".corrupt-")).sort().map((name) =>
|
|
14298
|
+
return entries.filter((name) => name === base || name.startsWith(`${base}.alt-`) && !name.includes(".corrupt-")).sort().map((name) => path11.join(dir, name));
|
|
14408
14299
|
}
|
|
14409
14300
|
var MAX_TRANSIENT_RETRIES = 5;
|
|
14410
14301
|
function bumpRetryCount(dbPath, absPath) {
|
|
@@ -14432,17 +14323,17 @@ function clearRetryCount(dbPath, absPath) {
|
|
|
14432
14323
|
}
|
|
14433
14324
|
}
|
|
14434
14325
|
function dirtyQueuePathFor(dir) {
|
|
14435
|
-
return
|
|
14326
|
+
return path11.join(dir, "queue", "dirty.txt");
|
|
14436
14327
|
}
|
|
14437
14328
|
function drainHeartbeatPathFor(dir) {
|
|
14438
|
-
return
|
|
14329
|
+
return path11.join(dir, "queue", "drain-heartbeat");
|
|
14439
14330
|
}
|
|
14440
14331
|
function writeDrainHeartbeat(dir, force = false) {
|
|
14441
14332
|
const now = Date.now();
|
|
14442
14333
|
if (!force && now - (heartbeatWriteTimes.get(dir) ?? 0) < WORKER_HEARTBEAT_REFRESH_MS) return;
|
|
14443
14334
|
try {
|
|
14444
|
-
ensureDirSync(
|
|
14445
|
-
|
|
14335
|
+
ensureDirSync(path11.dirname(drainHeartbeatPathFor(dir)));
|
|
14336
|
+
fs14.writeFileSync(drainHeartbeatPathFor(dir), `${process.pid}
|
|
14446
14337
|
`);
|
|
14447
14338
|
heartbeatWriteTimes.set(dir, now);
|
|
14448
14339
|
} catch {
|
|
@@ -14451,15 +14342,15 @@ function writeDrainHeartbeat(dir, force = false) {
|
|
|
14451
14342
|
function hasFreshWorkerHeartbeat(dir, pid) {
|
|
14452
14343
|
try {
|
|
14453
14344
|
const heartbeatPath = drainHeartbeatPathFor(dir);
|
|
14454
|
-
if (Date.now() -
|
|
14455
|
-
return
|
|
14345
|
+
if (Date.now() - fs14.statSync(heartbeatPath).mtimeMs > WORKER_HEARTBEAT_STALE_MS) return false;
|
|
14346
|
+
return fs14.readFileSync(heartbeatPath, "utf8").trim() === String(pid);
|
|
14456
14347
|
} catch {
|
|
14457
14348
|
return false;
|
|
14458
14349
|
}
|
|
14459
14350
|
}
|
|
14460
14351
|
function pidFileIsWithinStartupGrace(dir) {
|
|
14461
14352
|
try {
|
|
14462
|
-
return Date.now() -
|
|
14353
|
+
return Date.now() - fs14.statSync(workerPidPath(dir)).mtimeMs < WORKER_STARTUP_GRACE_MS;
|
|
14463
14354
|
} catch {
|
|
14464
14355
|
return false;
|
|
14465
14356
|
}
|
|
@@ -14493,28 +14384,28 @@ function parseDirtyQueueLines(raw) {
|
|
|
14493
14384
|
return out;
|
|
14494
14385
|
}
|
|
14495
14386
|
function workerPidPath(dir = dataDir()) {
|
|
14496
|
-
return
|
|
14387
|
+
return path11.join(dir, "worker.pid");
|
|
14497
14388
|
}
|
|
14498
14389
|
function getDirtyPathsFor(dir) {
|
|
14499
14390
|
let raw;
|
|
14500
14391
|
try {
|
|
14501
|
-
raw =
|
|
14392
|
+
raw = fs14.readFileSync(dirtyQueuePathFor(dir), "utf8");
|
|
14502
14393
|
} catch {
|
|
14503
14394
|
return [];
|
|
14504
14395
|
}
|
|
14505
14396
|
return parseDirtyQueueLines(raw);
|
|
14506
14397
|
}
|
|
14507
14398
|
function workerErrorLogPath(dir) {
|
|
14508
|
-
return
|
|
14399
|
+
return path11.join(dir, "worker-errors.log");
|
|
14509
14400
|
}
|
|
14510
14401
|
var WORKER_ERROR_LOG_MAX_BYTES = 5 * 1024 * 1024;
|
|
14511
14402
|
var CORRUPT_QUARANTINE_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1e3;
|
|
14512
14403
|
function cleanupWorkerStateFiles(dir) {
|
|
14513
14404
|
try {
|
|
14514
14405
|
const logPath = workerErrorLogPath(dir);
|
|
14515
|
-
const stat2 =
|
|
14406
|
+
const stat2 = fs14.statSync(logPath);
|
|
14516
14407
|
if (stat2.size > WORKER_ERROR_LOG_MAX_BYTES) {
|
|
14517
|
-
|
|
14408
|
+
fs14.writeFileSync(
|
|
14518
14409
|
logPath,
|
|
14519
14410
|
`${(/* @__PURE__ */ new Date()).toISOString()} worker-errors.log rotated (exceeded ${WORKER_ERROR_LOG_MAX_BYTES} bytes)
|
|
14520
14411
|
`
|
|
@@ -14523,13 +14414,13 @@ function cleanupWorkerStateFiles(dir) {
|
|
|
14523
14414
|
} catch {
|
|
14524
14415
|
}
|
|
14525
14416
|
try {
|
|
14526
|
-
const queueDir =
|
|
14417
|
+
const queueDir = path11.dirname(dirtyQueuePathFor(dir));
|
|
14527
14418
|
const cutoff = Date.now() - CORRUPT_QUARANTINE_MAX_AGE_MS;
|
|
14528
|
-
for (const file of
|
|
14419
|
+
for (const file of fs14.readdirSync(queueDir)) {
|
|
14529
14420
|
if (!file.includes(".corrupt-")) continue;
|
|
14530
|
-
const full =
|
|
14421
|
+
const full = path11.join(queueDir, file);
|
|
14531
14422
|
try {
|
|
14532
|
-
if (
|
|
14423
|
+
if (fs14.statSync(full).mtimeMs < cutoff) fs14.unlinkSync(full);
|
|
14533
14424
|
} catch {
|
|
14534
14425
|
}
|
|
14535
14426
|
}
|
|
@@ -14563,7 +14454,7 @@ registerReset(() => {
|
|
|
14563
14454
|
function embedFileSerialized(absPath, dbPath, sha) {
|
|
14564
14455
|
const key = foldPath(absPath);
|
|
14565
14456
|
const prior = inFlightEmbeddings.get(key);
|
|
14566
|
-
const dir =
|
|
14457
|
+
const dir = path11.dirname(dbPath);
|
|
14567
14458
|
const onEmbedError = (err) => {
|
|
14568
14459
|
const message = extractErrorMessage(err);
|
|
14569
14460
|
appendWorkerErrorLog(dir, `${(/* @__PURE__ */ new Date()).toISOString()} indexFileEmbeddings failed for ${absPath}: ${message}
|
|
@@ -14600,7 +14491,7 @@ function oneLogLine(line) {
|
|
|
14600
14491
|
}
|
|
14601
14492
|
function appendWorkerErrorLog(dir, line) {
|
|
14602
14493
|
try {
|
|
14603
|
-
|
|
14494
|
+
fs14.appendFileSync(workerErrorLogPath(dir), oneLogLine(line));
|
|
14604
14495
|
} catch {
|
|
14605
14496
|
}
|
|
14606
14497
|
}
|
|
@@ -14617,7 +14508,7 @@ function logTransientReadFailure(dir, absPath) {
|
|
|
14617
14508
|
);
|
|
14618
14509
|
}
|
|
14619
14510
|
function bumpAndCheckRetry(dir, absPath) {
|
|
14620
|
-
const dbPath =
|
|
14511
|
+
const dbPath = path11.join(dir, "global.db");
|
|
14621
14512
|
const attempts = bumpRetryCount(dbPath, absPath);
|
|
14622
14513
|
if (attempts > MAX_TRANSIENT_RETRIES) {
|
|
14623
14514
|
if (attempts === MAX_TRANSIENT_RETRIES + 1) {
|
|
@@ -14634,14 +14525,14 @@ function bumpAndCheckRetry(dir, absPath) {
|
|
|
14634
14525
|
function appendToDirtyQueue(dir, absPath) {
|
|
14635
14526
|
const queuePath = dirtyQueuePathFor(dir);
|
|
14636
14527
|
try {
|
|
14637
|
-
ensureDirSync(
|
|
14528
|
+
ensureDirSync(path11.dirname(queuePath));
|
|
14638
14529
|
let leadingNewline = "";
|
|
14639
14530
|
try {
|
|
14640
|
-
const existing =
|
|
14531
|
+
const existing = fs14.readFileSync(queuePath, "utf8");
|
|
14641
14532
|
if (existing.length > 0 && !existing.endsWith("\n")) leadingNewline = "\n";
|
|
14642
14533
|
} catch {
|
|
14643
14534
|
}
|
|
14644
|
-
|
|
14535
|
+
fs14.appendFileSync(queuePath, `${leadingNewline}${encodeDirtyQueueLine(absPath)}
|
|
14645
14536
|
`);
|
|
14646
14537
|
} catch {
|
|
14647
14538
|
}
|
|
@@ -14650,7 +14541,7 @@ function requeueDirtyPath(dir, absPath) {
|
|
|
14650
14541
|
if (bumpAndCheckRetry(dir, absPath)) appendToDirtyQueue(dir, absPath);
|
|
14651
14542
|
}
|
|
14652
14543
|
function makeIndexer(dbPath) {
|
|
14653
|
-
const dir =
|
|
14544
|
+
const dir = path11.dirname(dbPath);
|
|
14654
14545
|
return (absPath, sha) => {
|
|
14655
14546
|
try {
|
|
14656
14547
|
const ixCfgForSkip = loadConfig().indexing;
|
|
@@ -14711,14 +14602,14 @@ function processDirtyBatch(paths, index = makeIndexer(globalDbPath()), remove =
|
|
|
14711
14602
|
continue;
|
|
14712
14603
|
}
|
|
14713
14604
|
try {
|
|
14714
|
-
const
|
|
14605
|
+
const dirname14 = path11.dirname(p);
|
|
14715
14606
|
let root;
|
|
14716
|
-
if (projectRootCache.has(
|
|
14717
|
-
root = projectRootCache.get(
|
|
14607
|
+
if (projectRootCache.has(dirname14)) {
|
|
14608
|
+
root = projectRootCache.get(dirname14) ?? null;
|
|
14718
14609
|
} else {
|
|
14719
|
-
const project = findProject(
|
|
14610
|
+
const project = findProject(dirname14);
|
|
14720
14611
|
root = project?.root ?? null;
|
|
14721
|
-
projectRootCache.set(
|
|
14612
|
+
projectRootCache.set(dirname14, root);
|
|
14722
14613
|
}
|
|
14723
14614
|
if (root) lastKnownProjectRoots.set(dir, root);
|
|
14724
14615
|
} catch {
|
|
@@ -14728,7 +14619,7 @@ function processDirtyBatch(paths, index = makeIndexer(globalDbPath()), remove =
|
|
|
14728
14619
|
requeue(dir, p);
|
|
14729
14620
|
continue;
|
|
14730
14621
|
}
|
|
14731
|
-
clearRetryCount(
|
|
14622
|
+
clearRetryCount(path11.join(dir, "global.db"), p);
|
|
14732
14623
|
if (result !== false) indexed += 1;
|
|
14733
14624
|
}
|
|
14734
14625
|
return indexed;
|
|
@@ -14742,7 +14633,7 @@ function sleepSyncMs(ms) {
|
|
|
14742
14633
|
function drainOnce(dir, index, remove) {
|
|
14743
14634
|
const queuePath = dirtyQueuePathFor(dir);
|
|
14744
14635
|
const draining = `${queuePath}.draining`;
|
|
14745
|
-
const dbPath =
|
|
14636
|
+
const dbPath = path11.join(dir, "global.db");
|
|
14746
14637
|
const indexFn = index ?? makeIndexer(dbPath);
|
|
14747
14638
|
const removeFn = remove ?? makeRemover(dbPath);
|
|
14748
14639
|
let processed = 0;
|
|
@@ -14759,7 +14650,7 @@ function drainOnce(dir, index, remove) {
|
|
|
14759
14650
|
let drainingContent = null;
|
|
14760
14651
|
for (let attempt = 0; attempt < DRAINING_READ_ATTEMPTS; attempt++) {
|
|
14761
14652
|
try {
|
|
14762
|
-
drainingContent =
|
|
14653
|
+
drainingContent = fs14.readFileSync(drainingFile, "utf8");
|
|
14763
14654
|
break;
|
|
14764
14655
|
} catch {
|
|
14765
14656
|
if (attempt < DRAINING_READ_ATTEMPTS - 1) sleepSyncMs(DRAINING_READ_RETRY_DELAY_MS);
|
|
@@ -14767,7 +14658,7 @@ function drainOnce(dir, index, remove) {
|
|
|
14767
14658
|
}
|
|
14768
14659
|
if (drainingContent === null) {
|
|
14769
14660
|
try {
|
|
14770
|
-
|
|
14661
|
+
fs14.renameSync(drainingFile, `${drainingFile}.corrupt-${Date.now()}`);
|
|
14771
14662
|
unclearedDrainingSnapshots.delete(drainingFile);
|
|
14772
14663
|
} catch {
|
|
14773
14664
|
}
|
|
@@ -14777,23 +14668,23 @@ function drainOnce(dir, index, remove) {
|
|
|
14777
14668
|
processed += processDirtyBatch(parseDirtyQueueLines(drainingContent), indexFn, removeFn, dir, requeueFn);
|
|
14778
14669
|
}
|
|
14779
14670
|
try {
|
|
14780
|
-
|
|
14671
|
+
fs14.rmSync(drainingFile, { force: true });
|
|
14781
14672
|
unclearedDrainingSnapshots.delete(drainingFile);
|
|
14782
14673
|
} catch {
|
|
14783
14674
|
try {
|
|
14784
|
-
|
|
14675
|
+
fs14.renameSync(drainingFile, `${drainingFile}.corrupt-${Date.now()}`);
|
|
14785
14676
|
unclearedDrainingSnapshots.delete(drainingFile);
|
|
14786
14677
|
} catch {
|
|
14787
14678
|
unclearedDrainingSnapshots.set(drainingFile, drainingSnapshotStamp(drainingFile, drainingContent));
|
|
14788
14679
|
}
|
|
14789
14680
|
}
|
|
14790
14681
|
}
|
|
14791
|
-
if (
|
|
14792
|
-
const claimTarget =
|
|
14682
|
+
if (fs14.existsSync(queuePath)) {
|
|
14683
|
+
const claimTarget = fs14.existsSync(draining) ? `${draining}.alt-${Date.now()}` : draining;
|
|
14793
14684
|
let claimed = false;
|
|
14794
14685
|
for (let attempt = 0; attempt < 5; attempt++) {
|
|
14795
14686
|
try {
|
|
14796
|
-
|
|
14687
|
+
fs14.renameSync(queuePath, claimTarget);
|
|
14797
14688
|
claimed = true;
|
|
14798
14689
|
break;
|
|
14799
14690
|
} catch {
|
|
@@ -14804,14 +14695,14 @@ function drainOnce(dir, index, remove) {
|
|
|
14804
14695
|
let claimedContent = "";
|
|
14805
14696
|
let readOk = false;
|
|
14806
14697
|
try {
|
|
14807
|
-
claimedContent =
|
|
14698
|
+
claimedContent = fs14.readFileSync(claimTarget, "utf8");
|
|
14808
14699
|
readOk = true;
|
|
14809
14700
|
} catch {
|
|
14810
14701
|
}
|
|
14811
14702
|
if (readOk) {
|
|
14812
14703
|
processed += processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn);
|
|
14813
14704
|
try {
|
|
14814
|
-
const recheck =
|
|
14705
|
+
const recheck = fs14.readFileSync(claimTarget, "utf8");
|
|
14815
14706
|
if (recheck !== claimedContent) {
|
|
14816
14707
|
const extra = recheck.startsWith(claimedContent) ? recheck.slice(claimedContent.length) : recheck;
|
|
14817
14708
|
for (const p of parseDirtyQueueLines(extra)) appendToDirtyQueue(dir, p);
|
|
@@ -14819,10 +14710,10 @@ function drainOnce(dir, index, remove) {
|
|
|
14819
14710
|
} catch {
|
|
14820
14711
|
}
|
|
14821
14712
|
try {
|
|
14822
|
-
|
|
14713
|
+
fs14.rmSync(claimTarget, { force: true });
|
|
14823
14714
|
} catch {
|
|
14824
14715
|
try {
|
|
14825
|
-
|
|
14716
|
+
fs14.renameSync(claimTarget, `${claimTarget}.corrupt-${Date.now()}`);
|
|
14826
14717
|
} catch {
|
|
14827
14718
|
unclearedDrainingSnapshots.set(claimTarget, drainingSnapshotStamp(claimTarget, claimedContent));
|
|
14828
14719
|
}
|
|
@@ -14855,7 +14746,7 @@ function pidAlive(pid) {
|
|
|
14855
14746
|
}
|
|
14856
14747
|
function readPidFile(dir) {
|
|
14857
14748
|
try {
|
|
14858
|
-
const raw =
|
|
14749
|
+
const raw = fs14.readFileSync(workerPidPath(dir), "utf8").trim();
|
|
14859
14750
|
if (!/^\d+$/.test(raw)) return null;
|
|
14860
14751
|
return parseInt(raw, 10);
|
|
14861
14752
|
} catch {
|
|
@@ -14868,20 +14759,20 @@ function isWorkerRunning(dir = dataDir()) {
|
|
|
14868
14759
|
return pidAlive(pid) && hasFreshWorkerHeartbeat(dir, pid);
|
|
14869
14760
|
}
|
|
14870
14761
|
function workerHealthCheckMarkerPath(dir) {
|
|
14871
|
-
return
|
|
14762
|
+
return path11.join(dir, "worker-healthcheck.marker");
|
|
14872
14763
|
}
|
|
14873
14764
|
var WORKER_HEALTHCHECK_MIN_INTERVAL_MS = 5 * 60 * 1e3;
|
|
14874
14765
|
function ensureWorkerAlive(dir = dataDir()) {
|
|
14875
14766
|
if (process.env["TOKEN_GOAT_NO_WORKER_SPAWN"] === "1") return;
|
|
14876
14767
|
const markerPath = workerHealthCheckMarkerPath(dir);
|
|
14877
14768
|
try {
|
|
14878
|
-
const stat2 =
|
|
14769
|
+
const stat2 = fs14.statSync(markerPath);
|
|
14879
14770
|
if (Date.now() - stat2.mtimeMs < WORKER_HEALTHCHECK_MIN_INTERVAL_MS) return;
|
|
14880
14771
|
} catch {
|
|
14881
14772
|
}
|
|
14882
14773
|
try {
|
|
14883
14774
|
ensureDirSync(dir);
|
|
14884
|
-
|
|
14775
|
+
fs14.writeFileSync(markerPath, "");
|
|
14885
14776
|
} catch {
|
|
14886
14777
|
}
|
|
14887
14778
|
if (isWorkerRunning(dir)) return;
|
|
@@ -14911,7 +14802,7 @@ function stopWorker(dir = dataDir()) {
|
|
|
14911
14802
|
}
|
|
14912
14803
|
if (readPidFile(dir) === pid) {
|
|
14913
14804
|
try {
|
|
14914
|
-
|
|
14805
|
+
fs14.rmSync(workerPidPath(dir), { force: true });
|
|
14915
14806
|
} catch {
|
|
14916
14807
|
}
|
|
14917
14808
|
}
|
|
@@ -14926,7 +14817,7 @@ var WorkerAlreadyRunningError = class extends Error {
|
|
|
14926
14817
|
function claimWorkerPidFile(dir, pid) {
|
|
14927
14818
|
const pidPath = workerPidPath(dir);
|
|
14928
14819
|
try {
|
|
14929
|
-
|
|
14820
|
+
fs14.writeFileSync(pidPath, `${pid}
|
|
14930
14821
|
`, { flag: "wx" });
|
|
14931
14822
|
return true;
|
|
14932
14823
|
} catch (e) {
|
|
@@ -14943,11 +14834,11 @@ function claimWorkerPidFile(dir, pid) {
|
|
|
14943
14834
|
}
|
|
14944
14835
|
}
|
|
14945
14836
|
try {
|
|
14946
|
-
|
|
14837
|
+
fs14.rmSync(pidPath, { force: true });
|
|
14947
14838
|
} catch {
|
|
14948
14839
|
}
|
|
14949
14840
|
try {
|
|
14950
|
-
|
|
14841
|
+
fs14.writeFileSync(pidPath, `${pid}
|
|
14951
14842
|
`, { flag: "wx" });
|
|
14952
14843
|
return true;
|
|
14953
14844
|
} catch (e2) {
|
|
@@ -14957,9 +14848,9 @@ function claimWorkerPidFile(dir, pid) {
|
|
|
14957
14848
|
}
|
|
14958
14849
|
function daemonEntryScript() {
|
|
14959
14850
|
const self = fileURLToPath(import.meta.url);
|
|
14960
|
-
const launcher =
|
|
14851
|
+
const launcher = path11.join(path11.dirname(self), "token-goat.mjs");
|
|
14961
14852
|
try {
|
|
14962
|
-
if (
|
|
14853
|
+
if (fs14.existsSync(launcher)) return launcher;
|
|
14963
14854
|
} catch {
|
|
14964
14855
|
}
|
|
14965
14856
|
return self;
|
|
@@ -14970,9 +14861,9 @@ function startDetachedWorker(opts) {
|
|
|
14970
14861
|
try {
|
|
14971
14862
|
ensureDirSync(dir);
|
|
14972
14863
|
} catch (e) {
|
|
14973
|
-
if (e.code !== "EEXIST" || !
|
|
14864
|
+
if (e.code !== "EEXIST" || !fs14.existsSync(dir)) throw e;
|
|
14974
14865
|
}
|
|
14975
|
-
const child =
|
|
14866
|
+
const child = spawn(
|
|
14976
14867
|
process.execPath,
|
|
14977
14868
|
[daemonEntryScript(), "--worker-daemon"],
|
|
14978
14869
|
{
|
|
@@ -15008,7 +14899,7 @@ async function runWorkerLoop(dir, pollIntervalMs, shouldStop = () => false) {
|
|
|
15008
14899
|
let lastKnownRootsSweepMs = 0;
|
|
15009
14900
|
let ownedPidFile = false;
|
|
15010
14901
|
while (!shouldStop()) {
|
|
15011
|
-
if (!
|
|
14902
|
+
if (!fs14.existsSync(dir)) break;
|
|
15012
14903
|
const pidOwner = readPidFile(dir);
|
|
15013
14904
|
if (pidOwner === process.pid) ownedPidFile = true;
|
|
15014
14905
|
else if (ownedPidFile && pidOwner !== null) break;
|
|
@@ -15033,7 +14924,7 @@ async function runWorkerLoop(dir, pollIntervalMs, shouldStop = () => false) {
|
|
|
15033
14924
|
}
|
|
15034
14925
|
if (Date.now() - lastKnownRootsSweepMs >= KNOWN_ROOTS_SWEEP_INTERVAL_MS) {
|
|
15035
14926
|
try {
|
|
15036
|
-
const result = sweepKnownRoots(
|
|
14927
|
+
const result = sweepKnownRoots(path11.join(dir, "global.db"));
|
|
15037
14928
|
if (result.flaggedRoots.length > 0) {
|
|
15038
14929
|
appendWorkerErrorLog(
|
|
15039
14930
|
dir,
|
|
@@ -15069,7 +14960,7 @@ function runDetachedWorkerDaemon() {
|
|
|
15069
14960
|
process.on("exit", () => {
|
|
15070
14961
|
if (readPidFile(dir) === process.pid) {
|
|
15071
14962
|
try {
|
|
15072
|
-
|
|
14963
|
+
fs14.rmSync(workerPidPath(dir), { force: true });
|
|
15073
14964
|
} catch {
|
|
15074
14965
|
}
|
|
15075
14966
|
}
|
|
@@ -15080,26 +14971,26 @@ function runDetachedWorkerDaemon() {
|
|
|
15080
14971
|
|
|
15081
14972
|
// src/hooks_index.ts
|
|
15082
14973
|
init_define_import_meta_env();
|
|
15083
|
-
import * as
|
|
15084
|
-
import * as
|
|
14974
|
+
import * as fs15 from "node:fs";
|
|
14975
|
+
import * as path12 from "node:path";
|
|
15085
14976
|
function dirtyQueuePath() {
|
|
15086
|
-
return
|
|
14977
|
+
return path12.join(dataDir(), "queue", "dirty.txt");
|
|
15087
14978
|
}
|
|
15088
14979
|
function appendDirtyPath(normalizedPath) {
|
|
15089
14980
|
const queuePath = dirtyQueuePath();
|
|
15090
|
-
const dir =
|
|
14981
|
+
const dir = path12.dirname(queuePath);
|
|
15091
14982
|
try {
|
|
15092
14983
|
ensureDirSync(dir);
|
|
15093
14984
|
} catch (e) {
|
|
15094
|
-
if (e.code !== "EEXIST" || !
|
|
14985
|
+
if (e.code !== "EEXIST" || !fs15.existsSync(dir)) throw e;
|
|
15095
14986
|
}
|
|
15096
14987
|
let leadingNewline = "";
|
|
15097
14988
|
try {
|
|
15098
|
-
const existing =
|
|
14989
|
+
const existing = fs15.readFileSync(queuePath, "utf8");
|
|
15099
14990
|
if (existing.length > 0 && !existing.endsWith("\n")) leadingNewline = "\n";
|
|
15100
14991
|
} catch {
|
|
15101
14992
|
}
|
|
15102
|
-
|
|
14993
|
+
fs15.appendFileSync(queuePath, `${leadingNewline}${encodeDirtyQueueLine(normalizedPath)}
|
|
15103
14994
|
`);
|
|
15104
14995
|
}
|
|
15105
14996
|
function enqueueDirtyPathSafe(filePath, opts) {
|
|
@@ -15117,7 +15008,7 @@ function getDirtyPaths() {
|
|
|
15117
15008
|
const queuePath = dirtyQueuePath();
|
|
15118
15009
|
let raw;
|
|
15119
15010
|
try {
|
|
15120
|
-
raw =
|
|
15011
|
+
raw = fs15.readFileSync(queuePath, "utf8");
|
|
15121
15012
|
} catch {
|
|
15122
15013
|
return [];
|
|
15123
15014
|
}
|
|
@@ -15126,9 +15017,9 @@ function getDirtyPaths() {
|
|
|
15126
15017
|
function preCompactIndexHandler(_event) {
|
|
15127
15018
|
const paths = getDirtyPaths();
|
|
15128
15019
|
if (paths.length > 0) {
|
|
15129
|
-
const sidecar =
|
|
15020
|
+
const sidecar = path12.join(dataDir(), "queue", "pending.txt");
|
|
15130
15021
|
try {
|
|
15131
|
-
ensureDirSync(
|
|
15022
|
+
ensureDirSync(path12.dirname(sidecar));
|
|
15132
15023
|
atomicWriteBytes(sidecar, Buffer.from(`${paths.join("\n")}
|
|
15133
15024
|
`, "utf8"));
|
|
15134
15025
|
} catch {
|
|
@@ -15140,7 +15031,7 @@ registerHook("pre_compact", preCompactIndexHandler, { advisory: true });
|
|
|
15140
15031
|
|
|
15141
15032
|
// src/section_reader.ts
|
|
15142
15033
|
init_define_import_meta_env();
|
|
15143
|
-
import { readFileSync as
|
|
15034
|
+
import { readFileSync as readFileSync11 } from "node:fs";
|
|
15144
15035
|
function parseHeadingSpec(spec, headers) {
|
|
15145
15036
|
const m = /^([^#\r\n]+)#(\d+)$/.exec(spec);
|
|
15146
15037
|
if (m !== null && m[1] !== void 0 && m[2] !== void 0) {
|
|
@@ -15438,7 +15329,7 @@ function readTextForSections(filePath, readFn) {
|
|
|
15438
15329
|
text = read;
|
|
15439
15330
|
} else {
|
|
15440
15331
|
try {
|
|
15441
|
-
text = decodeSource(
|
|
15332
|
+
text = decodeSource(readFileSync11(filePath));
|
|
15442
15333
|
} catch {
|
|
15443
15334
|
return null;
|
|
15444
15335
|
}
|
|
@@ -15499,8 +15390,8 @@ function listSections(filePath, readFn) {
|
|
|
15499
15390
|
|
|
15500
15391
|
// src/hooks_read.ts
|
|
15501
15392
|
init_define_import_meta_env();
|
|
15502
|
-
import * as
|
|
15503
|
-
import * as
|
|
15393
|
+
import * as fs20 from "node:fs";
|
|
15394
|
+
import * as path17 from "node:path";
|
|
15504
15395
|
|
|
15505
15396
|
// src/served_lines.ts
|
|
15506
15397
|
init_define_import_meta_env();
|
|
@@ -15594,8 +15485,8 @@ function planServedElisions(rows, bodies) {
|
|
|
15594
15485
|
|
|
15595
15486
|
// src/compact.ts
|
|
15596
15487
|
init_define_import_meta_env();
|
|
15597
|
-
import * as
|
|
15598
|
-
import * as
|
|
15488
|
+
import * as fs16 from "node:fs";
|
|
15489
|
+
import * as path13 from "node:path";
|
|
15599
15490
|
var CONTEXT_AUTOCOMPACT_TOKENS = 66e4;
|
|
15600
15491
|
var CATALOG_TOKENS = 10800;
|
|
15601
15492
|
var CONTEXT_TIER_WARM = 0.5;
|
|
@@ -15785,7 +15676,7 @@ function inferSessionGoal(cache, maxTokens = 80) {
|
|
|
15785
15676
|
const dirCounts = new Counter();
|
|
15786
15677
|
for (const fpath of editedPaths) {
|
|
15787
15678
|
try {
|
|
15788
|
-
let parent =
|
|
15679
|
+
let parent = path13.dirname(fpath);
|
|
15789
15680
|
if (parent === ".") {
|
|
15790
15681
|
parent = "root";
|
|
15791
15682
|
} else if (parent.startsWith("./")) {
|
|
@@ -15863,11 +15754,11 @@ function isNoisePath(inputPath) {
|
|
|
15863
15754
|
}
|
|
15864
15755
|
function findLatestSessionId() {
|
|
15865
15756
|
try {
|
|
15866
|
-
const sessionsDir =
|
|
15867
|
-
if (!
|
|
15757
|
+
const sessionsDir = path13.join(tokenGoatHome(), "sessions");
|
|
15758
|
+
if (!fs16.existsSync(sessionsDir)) {
|
|
15868
15759
|
return null;
|
|
15869
15760
|
}
|
|
15870
|
-
const files =
|
|
15761
|
+
const files = fs16.readdirSync(sessionsDir);
|
|
15871
15762
|
const jsonFiles = files.filter((f) => f.endsWith(".json") && !f.includes(AGENT_SALT_MARKER));
|
|
15872
15763
|
if (jsonFiles.length === 0) {
|
|
15873
15764
|
return null;
|
|
@@ -15877,9 +15768,9 @@ function findLatestSessionId() {
|
|
|
15877
15768
|
return null;
|
|
15878
15769
|
}
|
|
15879
15770
|
let latestFile = firstFile;
|
|
15880
|
-
let latestMtime =
|
|
15771
|
+
let latestMtime = fs16.statSync(path13.join(sessionsDir, firstFile)).mtimeMs;
|
|
15881
15772
|
for (const file of jsonFiles) {
|
|
15882
|
-
const mtime =
|
|
15773
|
+
const mtime = fs16.statSync(path13.join(sessionsDir, file)).mtimeMs;
|
|
15883
15774
|
if (mtime > latestMtime) {
|
|
15884
15775
|
latestFile = file;
|
|
15885
15776
|
latestMtime = mtime;
|
|
@@ -15901,37 +15792,37 @@ function eventCount(cache) {
|
|
|
15901
15792
|
function writeSessionManifest(projectHash, sessionId, manifestJson) {
|
|
15902
15793
|
const safeSessionId = sanitizeIdForFilename(sessionId, 64);
|
|
15903
15794
|
if (!safeSessionId) return;
|
|
15904
|
-
const sessionsDir =
|
|
15905
|
-
if (!
|
|
15795
|
+
const sessionsDir = path13.join(dataDir(), "projects", projectHash, "sessions");
|
|
15796
|
+
if (!fs16.existsSync(sessionsDir)) {
|
|
15906
15797
|
ensureDirSync(sessionsDir);
|
|
15907
15798
|
}
|
|
15908
|
-
const dest =
|
|
15799
|
+
const dest = path13.join(sessionsDir, `${safeSessionId}.json`);
|
|
15909
15800
|
atomicWriteText(dest, JSON.stringify(manifestJson));
|
|
15910
15801
|
}
|
|
15911
15802
|
function readAllSessionManifests(projectHash, maxAgeSecs = 3600) {
|
|
15912
|
-
const sessionsDir =
|
|
15913
|
-
if (!
|
|
15803
|
+
const sessionsDir = path13.join(dataDir(), "projects", projectHash, "sessions");
|
|
15804
|
+
if (!fs16.existsSync(sessionsDir)) {
|
|
15914
15805
|
return [];
|
|
15915
15806
|
}
|
|
15916
15807
|
const now = Date.now() / 1e3;
|
|
15917
15808
|
const results = [];
|
|
15918
15809
|
try {
|
|
15919
|
-
const files =
|
|
15810
|
+
const files = fs16.readdirSync(sessionsDir);
|
|
15920
15811
|
for (const file of files) {
|
|
15921
15812
|
if (!file.endsWith(".json")) {
|
|
15922
15813
|
continue;
|
|
15923
15814
|
}
|
|
15924
15815
|
try {
|
|
15925
|
-
const fullPath =
|
|
15926
|
-
const stat2 =
|
|
15816
|
+
const fullPath = path13.join(sessionsDir, file);
|
|
15817
|
+
const stat2 = fs16.statSync(fullPath);
|
|
15927
15818
|
if (now - stat2.mtimeMs / 1e3 > maxAgeSecs) {
|
|
15928
15819
|
try {
|
|
15929
|
-
|
|
15820
|
+
fs16.unlinkSync(fullPath);
|
|
15930
15821
|
} catch {
|
|
15931
15822
|
}
|
|
15932
15823
|
continue;
|
|
15933
15824
|
}
|
|
15934
|
-
const text =
|
|
15825
|
+
const text = fs16.readFileSync(fullPath, "utf8");
|
|
15935
15826
|
const data = JSON.parse(text);
|
|
15936
15827
|
if (typeof data === "object" && data !== null && "files" in data) {
|
|
15937
15828
|
results.push(data);
|
|
@@ -16133,11 +16024,11 @@ function buildPackageManifestHint(options) {
|
|
|
16133
16024
|
return null;
|
|
16134
16025
|
}
|
|
16135
16026
|
}
|
|
16136
|
-
function _sanitizeHintPath(
|
|
16137
|
-
if (typeof
|
|
16027
|
+
function _sanitizeHintPath(path22) {
|
|
16028
|
+
if (typeof path22 !== "string") {
|
|
16138
16029
|
return "???";
|
|
16139
16030
|
}
|
|
16140
|
-
return
|
|
16031
|
+
return path22.replace(/[\x00]/g, "").slice(0, 200);
|
|
16141
16032
|
}
|
|
16142
16033
|
|
|
16143
16034
|
// src/hints/file_type_handler.ts
|
|
@@ -16152,9 +16043,16 @@ var FILE_TYPE_THRESHOLDS = {
|
|
|
16152
16043
|
xml: 2e4,
|
|
16153
16044
|
csv: 1e4,
|
|
16154
16045
|
tsv: 1e4,
|
|
16046
|
+
json: 2e4,
|
|
16047
|
+
yaml: 2e4,
|
|
16048
|
+
jsonl: 2e4,
|
|
16155
16049
|
transcript: 1e4,
|
|
16156
16050
|
office: 0,
|
|
16157
16051
|
// always block (.docx etc)
|
|
16052
|
+
sqlite: 0,
|
|
16053
|
+
// always block (.db, .sqlite, etc)
|
|
16054
|
+
parquet: 0,
|
|
16055
|
+
// always block (.parquet)
|
|
16158
16056
|
generic: 1e5
|
|
16159
16057
|
// catch-all for unrecognized large files
|
|
16160
16058
|
};
|
|
@@ -16349,6 +16247,7 @@ function handleDocx(filePath) {
|
|
|
16349
16247
|
message: [
|
|
16350
16248
|
`Word file \u2014 Read cannot return document content; this is not retryable with different Read parameters.`,
|
|
16351
16249
|
`See headings: token-goat docx-outline "${filePath}"`,
|
|
16250
|
+
`Extract tables: token-goat docx-tables "${filePath}"`,
|
|
16352
16251
|
`Read full text: token-goat docx-text "${filePath}"`
|
|
16353
16252
|
].join("\n")
|
|
16354
16253
|
};
|
|
@@ -16415,6 +16314,106 @@ function handleTranscript(filePath, content, contentLengthHint) {
|
|
|
16415
16314
|
].filter(Boolean).join("\n")
|
|
16416
16315
|
};
|
|
16417
16316
|
}
|
|
16317
|
+
function handleSqlite(filePath) {
|
|
16318
|
+
return {
|
|
16319
|
+
shouldBlock: true,
|
|
16320
|
+
message: [
|
|
16321
|
+
`SQLite database \u2014 Read cannot return binary database content; this is not retryable with different Read parameters.`,
|
|
16322
|
+
`See tables: token-goat sqlite-tables "${filePath}"`,
|
|
16323
|
+
`See schema: token-goat sqlite-schema "${filePath}"`,
|
|
16324
|
+
`Query data: token-goat sqlite-query "${filePath}" "SELECT ... FROM ... LIMIT 20"`
|
|
16325
|
+
].join("\n")
|
|
16326
|
+
};
|
|
16327
|
+
}
|
|
16328
|
+
function handleParquet(filePath) {
|
|
16329
|
+
return {
|
|
16330
|
+
shouldBlock: true,
|
|
16331
|
+
message: [
|
|
16332
|
+
`Parquet file \u2014 Read cannot return binary columnar content; this is not retryable with different Read parameters.`,
|
|
16333
|
+
`Query with DuckDB: duckdb -c "SELECT * FROM read_parquet('${filePath}') LIMIT 10"`
|
|
16334
|
+
].join("\n")
|
|
16335
|
+
};
|
|
16336
|
+
}
|
|
16337
|
+
function handleJson(filePath, content, contentLengthHint) {
|
|
16338
|
+
const length = contentLengthHint ?? content.length;
|
|
16339
|
+
if (length < FILE_TYPE_THRESHOLDS.json) return { shouldBlock: false, message: "" };
|
|
16340
|
+
if (previewUnavailable(content, length)) {
|
|
16341
|
+
return {
|
|
16342
|
+
shouldBlock: true,
|
|
16343
|
+
message: [
|
|
16344
|
+
`Large JSON file (${formatBytes(length)}) \u2014 too large to preview (exceeds the in-hook scan cap).`,
|
|
16345
|
+
`See structure: token-goat json-outline "${filePath}"`,
|
|
16346
|
+
`Query subtree: token-goat json-query "${filePath}" '<path>'`
|
|
16347
|
+
].join("\n")
|
|
16348
|
+
};
|
|
16349
|
+
}
|
|
16350
|
+
let summary = "";
|
|
16351
|
+
try {
|
|
16352
|
+
const parsed = JSON.parse(content);
|
|
16353
|
+
if (Array.isArray(parsed)) {
|
|
16354
|
+
summary = `JSON array (${parsed.length} items)`;
|
|
16355
|
+
if (parsed.length > 0 && typeof parsed[0] === "object" && parsed[0] !== null) {
|
|
16356
|
+
const itemKeys = Object.keys(parsed[0]).slice(0, 10);
|
|
16357
|
+
summary += `, item keys: ${itemKeys.join(", ")}${Object.keys(parsed[0]).length > 10 ? "..." : ""}`;
|
|
16358
|
+
}
|
|
16359
|
+
} else if (typeof parsed === "object" && parsed !== null) {
|
|
16360
|
+
const keys = Object.keys(parsed).slice(0, 15);
|
|
16361
|
+
summary = `Top-level keys (${Object.keys(parsed).length}): ${keys.join(", ")}${Object.keys(parsed).length > 15 ? "..." : ""}`;
|
|
16362
|
+
}
|
|
16363
|
+
} catch {
|
|
16364
|
+
}
|
|
16365
|
+
return {
|
|
16366
|
+
shouldBlock: true,
|
|
16367
|
+
message: [
|
|
16368
|
+
`Large JSON file (${formatBytes(length)}).`,
|
|
16369
|
+
summary ? fenceUntrustedFileContent(summary) : "",
|
|
16370
|
+
`See structure: token-goat json-outline "${filePath}"`,
|
|
16371
|
+
`Query subtree: token-goat json-query "${filePath}" '<path>'`
|
|
16372
|
+
].filter(Boolean).join("\n")
|
|
16373
|
+
};
|
|
16374
|
+
}
|
|
16375
|
+
function handleYaml(filePath, content, contentLengthHint) {
|
|
16376
|
+
const length = contentLengthHint ?? content.length;
|
|
16377
|
+
if (length < FILE_TYPE_THRESHOLDS.yaml) return { shouldBlock: false, message: "" };
|
|
16378
|
+
return {
|
|
16379
|
+
shouldBlock: true,
|
|
16380
|
+
message: [
|
|
16381
|
+
`Large YAML file (${formatBytes(length)}).`,
|
|
16382
|
+
`See structure: token-goat yaml-outline "${filePath}"`,
|
|
16383
|
+
`Query subtree: token-goat yaml-query "${filePath}" '<path>'`
|
|
16384
|
+
].join("\n")
|
|
16385
|
+
};
|
|
16386
|
+
}
|
|
16387
|
+
function handleJsonl(filePath, content, contentLengthHint) {
|
|
16388
|
+
const length = contentLengthHint ?? content.length;
|
|
16389
|
+
if (length < FILE_TYPE_THRESHOLDS.jsonl) return { shouldBlock: false, message: "" };
|
|
16390
|
+
if (previewUnavailable(content, length)) {
|
|
16391
|
+
return {
|
|
16392
|
+
shouldBlock: true,
|
|
16393
|
+
message: `Large JSON Lines file (${formatBytes(length)}) \u2014 too large to preview (exceeds the in-hook scan cap). Slice records with Read using offset and limit (each line is a single JSON record).`
|
|
16394
|
+
};
|
|
16395
|
+
}
|
|
16396
|
+
const lines2 = content.split(/\r?\n/).filter((l) => l.trim().length > 0);
|
|
16397
|
+
let schema = "";
|
|
16398
|
+
if (lines2.length > 0) {
|
|
16399
|
+
try {
|
|
16400
|
+
const firstObj = JSON.parse(lines2[0]);
|
|
16401
|
+
if (typeof firstObj === "object" && firstObj !== null && !Array.isArray(firstObj)) {
|
|
16402
|
+
const keys = Object.keys(firstObj).slice(0, 15);
|
|
16403
|
+
schema = `Record keys: ${keys.join(", ")}${Object.keys(firstObj).length > 15 ? "..." : ""}`;
|
|
16404
|
+
}
|
|
16405
|
+
} catch {
|
|
16406
|
+
}
|
|
16407
|
+
}
|
|
16408
|
+
return {
|
|
16409
|
+
shouldBlock: true,
|
|
16410
|
+
message: [
|
|
16411
|
+
`Large JSON Lines file (${formatBytes(length)}, ~${lines2.length} records).`,
|
|
16412
|
+
schema ? fenceUntrustedFileContent(schema) : "",
|
|
16413
|
+
`Slice records with Read using offset and limit (each line is a single JSON record), e.g. offset=1, limit=5.`
|
|
16414
|
+
].filter(Boolean).join("\n")
|
|
16415
|
+
};
|
|
16416
|
+
}
|
|
16418
16417
|
function handleGenericLarge(filePath, contentLength) {
|
|
16419
16418
|
if (contentLength < FILE_TYPE_THRESHOLDS.generic) return { shouldBlock: false, message: "" };
|
|
16420
16419
|
return {
|
|
@@ -16435,40 +16434,45 @@ function dispatchFileTypeHandler(filePath, content, contentLengthHint) {
|
|
|
16435
16434
|
if (ext === "pptx") return handlePptx(filePath);
|
|
16436
16435
|
if (ext === "docx") return handleDocx(filePath);
|
|
16437
16436
|
if (["odt", "ods", "ott", "odp"].includes(ext)) return handleOfficeBinary(filePath);
|
|
16437
|
+
if (["sqlite", "db", "sqlite3", "db3"].includes(ext)) return handleSqlite(filePath);
|
|
16438
|
+
if (ext === "parquet") return handleParquet(filePath);
|
|
16438
16439
|
if (ext === "csv" || ext === "tsv") return handleCsv(filePath, content, effectiveLength);
|
|
16439
16440
|
if (ext === "vtt" || ext === "srt") return handleTranscript(filePath, content, effectiveLength);
|
|
16441
|
+
if (ext === "json") return handleJson(filePath, content, effectiveLength);
|
|
16442
|
+
if (["yaml", "yml"].includes(ext)) return handleYaml(filePath, content, effectiveLength);
|
|
16443
|
+
if (ext === "jsonl") return handleJsonl(filePath, content, effectiveLength);
|
|
16440
16444
|
return handleGenericLarge(filePath, effectiveLength);
|
|
16441
16445
|
}
|
|
16442
16446
|
|
|
16443
16447
|
// src/doc_compact.ts
|
|
16444
16448
|
init_define_import_meta_env();
|
|
16445
|
-
import * as
|
|
16446
|
-
import * as
|
|
16449
|
+
import * as fs17 from "fs";
|
|
16450
|
+
import * as path14 from "path";
|
|
16447
16451
|
var defaultSentencesPerSection = 2;
|
|
16448
16452
|
var headerPrefix = "<!-- token-goat doc-compact source-hash:";
|
|
16449
16453
|
var headerRegex = /^<!-- token-goat doc-compact source-hash:(\S+) source:(.+?) -->\r?$/;
|
|
16450
16454
|
var compactSubdir = "doc_compacts";
|
|
16451
16455
|
function sourceHash(filePath) {
|
|
16452
16456
|
try {
|
|
16453
|
-
return fingerprintContent(
|
|
16457
|
+
return fingerprintContent(fs17.readFileSync(filePath));
|
|
16454
16458
|
} catch {
|
|
16455
16459
|
return "";
|
|
16456
16460
|
}
|
|
16457
16461
|
}
|
|
16458
16462
|
function _compactSlug(absPathStr) {
|
|
16459
16463
|
const h = fingerprintContent(foldPath(absPathStr)).slice(0, 12);
|
|
16460
|
-
const ext =
|
|
16461
|
-
const stem =
|
|
16464
|
+
const ext = path14.extname(absPathStr);
|
|
16465
|
+
const stem = path14.basename(absPathStr, ext);
|
|
16462
16466
|
const safeStem = sanitizeIdForFilename(stem, 32);
|
|
16463
16467
|
return `${h}_${safeStem}`;
|
|
16464
16468
|
}
|
|
16465
16469
|
function compactPathFor(sourcePath) {
|
|
16466
16470
|
const abs = resolveIndexPath(sourcePath);
|
|
16467
|
-
return
|
|
16471
|
+
return path14.join(dataDir(), compactSubdir, `${_compactSlug(abs)}.md`);
|
|
16468
16472
|
}
|
|
16469
16473
|
function readCompactHeader(compactPath) {
|
|
16470
16474
|
try {
|
|
16471
|
-
const text =
|
|
16475
|
+
const text = fs17.readFileSync(compactPath, "utf-8");
|
|
16472
16476
|
const firstLine = text.split("\n")[0] || "";
|
|
16473
16477
|
const m = firstLine.match(headerRegex);
|
|
16474
16478
|
if (!m || !m[1] || !m[2]) return null;
|
|
@@ -16487,8 +16491,8 @@ function isCompactFresh(compactPath, sourcePath) {
|
|
|
16487
16491
|
}
|
|
16488
16492
|
function markCompactStale(compactPath) {
|
|
16489
16493
|
try {
|
|
16490
|
-
if (!
|
|
16491
|
-
const text =
|
|
16494
|
+
if (!fs17.existsSync(compactPath)) return false;
|
|
16495
|
+
const text = fs17.readFileSync(compactPath, "utf-8");
|
|
16492
16496
|
const lines2 = text.split("\n");
|
|
16493
16497
|
if (!lines2[0]) return false;
|
|
16494
16498
|
const m = lines2[0].match(headerRegex);
|
|
@@ -16503,7 +16507,7 @@ function markCompactStale(compactPath) {
|
|
|
16503
16507
|
}
|
|
16504
16508
|
function readCompactBody(compactPath) {
|
|
16505
16509
|
try {
|
|
16506
|
-
const text =
|
|
16510
|
+
const text = fs17.readFileSync(compactPath, "utf-8");
|
|
16507
16511
|
const lines2 = text.split("\n");
|
|
16508
16512
|
if (lines2.length < 2) return null;
|
|
16509
16513
|
const body = lines2.slice(1).join("\n").trimStart();
|
|
@@ -16513,14 +16517,14 @@ function readCompactBody(compactPath) {
|
|
|
16513
16517
|
}
|
|
16514
16518
|
}
|
|
16515
16519
|
function writeCompact(compactPath, sourcePath, compactBody, sourceRel) {
|
|
16516
|
-
const srcPath =
|
|
16520
|
+
const srcPath = path14.resolve(sourcePath);
|
|
16517
16521
|
const sha = sourceHash(srcPath);
|
|
16518
|
-
const displayRel = sourceRel ||
|
|
16522
|
+
const displayRel = sourceRel || path14.basename(srcPath);
|
|
16519
16523
|
const header = `${headerPrefix}${sha} source:${displayRel} -->
|
|
16520
16524
|
`;
|
|
16521
16525
|
const fullText = header + compactBody.trimStart();
|
|
16522
|
-
const dir =
|
|
16523
|
-
if (!
|
|
16526
|
+
const dir = path14.dirname(compactPath);
|
|
16527
|
+
if (!fs17.existsSync(dir)) {
|
|
16524
16528
|
ensureDirSync(dir);
|
|
16525
16529
|
}
|
|
16526
16530
|
atomicWriteText(compactPath, fullText);
|
|
@@ -16644,7 +16648,7 @@ function extractDocCompact(body, heading) {
|
|
|
16644
16648
|
}
|
|
16645
16649
|
function compactDoc(filePath, heading) {
|
|
16646
16650
|
try {
|
|
16647
|
-
const body =
|
|
16651
|
+
const body = fs17.readFileSync(filePath, "utf-8");
|
|
16648
16652
|
const compact = extractDocCompact(body, heading);
|
|
16649
16653
|
return compact || null;
|
|
16650
16654
|
} catch {
|
|
@@ -16655,20 +16659,20 @@ function compactDoc(filePath, heading) {
|
|
|
16655
16659
|
// src/evidence_cache.ts
|
|
16656
16660
|
init_define_import_meta_env();
|
|
16657
16661
|
import crypto from "node:crypto";
|
|
16658
|
-
import
|
|
16659
|
-
import
|
|
16662
|
+
import fs18 from "node:fs";
|
|
16663
|
+
import path15 from "node:path";
|
|
16660
16664
|
var MAX_ENTRIES = 500;
|
|
16661
16665
|
var MAX_TEXT_BYTES = 128 * 1024;
|
|
16662
16666
|
var CACHE_FILE = "workspace-evidence.json";
|
|
16663
16667
|
function cachePath() {
|
|
16664
|
-
return
|
|
16668
|
+
return path15.join(dataDir(), CACHE_FILE);
|
|
16665
16669
|
}
|
|
16666
16670
|
function hash(text) {
|
|
16667
16671
|
return crypto.createHash("sha256").update(text).digest("hex");
|
|
16668
16672
|
}
|
|
16669
16673
|
function load2() {
|
|
16670
16674
|
try {
|
|
16671
|
-
const parsed = JSON.parse(
|
|
16675
|
+
const parsed = JSON.parse(fs18.readFileSync(cachePath(), "utf8"));
|
|
16672
16676
|
if (!Array.isArray(parsed)) return [];
|
|
16673
16677
|
return parsed.filter(
|
|
16674
16678
|
(entry) => typeof entry === "object" && entry !== null && typeof entry.id === "string" && typeof entry.projectRoot === "string" && typeof entry.source === "string" && (entry.representation === "file" || entry.representation === "tool-output") && typeof entry.contentHash === "string" && typeof entry.text === "string" && typeof entry.createdAt === "number" && (entry.embedding === void 0 || typeof entry.embedding === "string")
|
|
@@ -16680,7 +16684,7 @@ function load2() {
|
|
|
16680
16684
|
function save(entries) {
|
|
16681
16685
|
try {
|
|
16682
16686
|
ensureDirSync(dataDir());
|
|
16683
|
-
|
|
16687
|
+
fs18.writeFileSync(cachePath(), JSON.stringify(entries.slice(0, MAX_ENTRIES)), "utf8");
|
|
16684
16688
|
} catch {
|
|
16685
16689
|
}
|
|
16686
16690
|
}
|
|
@@ -16771,7 +16775,7 @@ function buildDeltaCapsule(projectRoot, limit = 8) {
|
|
|
16771
16775
|
const root = normalizePath(projectRoot);
|
|
16772
16776
|
const changed = load2().filter((entry) => entry.projectRoot === root && entry.representation === "file").filter((entry) => {
|
|
16773
16777
|
try {
|
|
16774
|
-
return hash(
|
|
16778
|
+
return hash(fs18.readFileSync(entry.source, "utf8")) !== entry.contentHash;
|
|
16775
16779
|
} catch {
|
|
16776
16780
|
return true;
|
|
16777
16781
|
}
|
|
@@ -16783,8 +16787,8 @@ ${changed.map((entry) => `- ${displaySafePath(entry.source)} (use a fresh surgic
|
|
|
16783
16787
|
|
|
16784
16788
|
// src/notebook_compact.ts
|
|
16785
16789
|
init_define_import_meta_env();
|
|
16786
|
-
import * as
|
|
16787
|
-
import * as
|
|
16790
|
+
import * as fs19 from "node:fs";
|
|
16791
|
+
import * as path16 from "node:path";
|
|
16788
16792
|
var NB_STRIP_MIN_SAVINGS = 4096;
|
|
16789
16793
|
function stripNotebook(nbDict) {
|
|
16790
16794
|
const cells = [];
|
|
@@ -16804,23 +16808,23 @@ function stripNotebook(nbDict) {
|
|
|
16804
16808
|
var SIDECAR_DEFAULT_MAX_COUNT = 200;
|
|
16805
16809
|
var SIDECAR_DEFAULT_MAX_AGE_MS = 24 * 3600 * 1e3;
|
|
16806
16810
|
function pruneSidecars(cacheRoot, maxCount = SIDECAR_DEFAULT_MAX_COUNT, maxAgeMs = SIDECAR_DEFAULT_MAX_AGE_MS) {
|
|
16807
|
-
const nbStripDir =
|
|
16811
|
+
const nbStripDir = path16.join(cacheRoot, "nb_strip");
|
|
16808
16812
|
let removed = 0;
|
|
16809
16813
|
try {
|
|
16810
|
-
if (!
|
|
16814
|
+
if (!fs19.existsSync(nbStripDir)) return 0;
|
|
16811
16815
|
const cutoff = Date.now() - maxAgeMs;
|
|
16812
16816
|
const kept = [];
|
|
16813
|
-
for (const entry of
|
|
16814
|
-
const dir =
|
|
16817
|
+
for (const entry of fs19.readdirSync(nbStripDir)) {
|
|
16818
|
+
const dir = path16.join(nbStripDir, entry);
|
|
16815
16819
|
let mtime;
|
|
16816
16820
|
try {
|
|
16817
|
-
mtime =
|
|
16821
|
+
mtime = fs19.statSync(dir).mtimeMs;
|
|
16818
16822
|
} catch {
|
|
16819
16823
|
continue;
|
|
16820
16824
|
}
|
|
16821
16825
|
if (mtime < cutoff) {
|
|
16822
16826
|
try {
|
|
16823
|
-
|
|
16827
|
+
fs19.rmSync(dir, { recursive: true, force: true });
|
|
16824
16828
|
removed++;
|
|
16825
16829
|
} catch {
|
|
16826
16830
|
continue;
|
|
@@ -16833,7 +16837,7 @@ function pruneSidecars(cacheRoot, maxCount = SIDECAR_DEFAULT_MAX_COUNT, maxAgeMs
|
|
|
16833
16837
|
kept.sort((a, b) => a[1] - b[1]);
|
|
16834
16838
|
for (const [dir] of kept.slice(0, kept.length - maxCount)) {
|
|
16835
16839
|
try {
|
|
16836
|
-
|
|
16840
|
+
fs19.rmSync(dir, { recursive: true, force: true });
|
|
16837
16841
|
removed++;
|
|
16838
16842
|
} catch {
|
|
16839
16843
|
continue;
|
|
@@ -16847,9 +16851,9 @@ function pruneSidecars(cacheRoot, maxCount = SIDECAR_DEFAULT_MAX_COUNT, maxAgeMs
|
|
|
16847
16851
|
}
|
|
16848
16852
|
function getOrCreateSidecar(rawBytes, cacheRoot, opts = {}) {
|
|
16849
16853
|
const sha = fingerprintContent(rawBytes);
|
|
16850
|
-
const sidecarDir =
|
|
16851
|
-
const sidecarPath =
|
|
16852
|
-
if (
|
|
16854
|
+
const sidecarDir = path16.join(cacheRoot, "nb_strip", sha);
|
|
16855
|
+
const sidecarPath = path16.join(sidecarDir, "stripped.ipynb");
|
|
16856
|
+
if (fs19.existsSync(sidecarPath)) {
|
|
16853
16857
|
return [sidecarPath, false];
|
|
16854
16858
|
}
|
|
16855
16859
|
let nb;
|
|
@@ -16866,7 +16870,7 @@ function getOrCreateSidecar(rawBytes, cacheRoot, opts = {}) {
|
|
|
16866
16870
|
try {
|
|
16867
16871
|
ensureDirSync(sidecarDir);
|
|
16868
16872
|
} catch (err) {
|
|
16869
|
-
if (!
|
|
16873
|
+
if (!fs19.existsSync(sidecarDir)) {
|
|
16870
16874
|
throw err;
|
|
16871
16875
|
}
|
|
16872
16876
|
}
|
|
@@ -16954,10 +16958,10 @@ var COMMENT_SYNTAX = new Map([
|
|
|
16954
16958
|
...["py", "rb", "sh", "bash", "zsh", "pl", "r"].map((ext) => [ext, HASH]),
|
|
16955
16959
|
...["sql", "lua", "hs"].map((ext) => [ext, DOUBLE_DASH])
|
|
16956
16960
|
]);
|
|
16957
|
-
function commentSyntaxFor(
|
|
16958
|
-
const dot =
|
|
16961
|
+
function commentSyntaxFor(path22) {
|
|
16962
|
+
const dot = path22.lastIndexOf(".");
|
|
16959
16963
|
if (dot < 0) return null;
|
|
16960
|
-
return COMMENT_SYNTAX.get(
|
|
16964
|
+
return COMMENT_SYNTAX.get(path22.slice(dot + 1).toLowerCase()) ?? null;
|
|
16961
16965
|
}
|
|
16962
16966
|
function planCommentFolds(rows, syntax, commentKeep, commentMinBlock, occupied) {
|
|
16963
16967
|
if (syntax === null || rows.length === 0 || commentKeep < 1 || commentMinBlock < 1) return [];
|
|
@@ -17066,8 +17070,8 @@ function planProseFolds(rows, claimed) {
|
|
|
17066
17070
|
}
|
|
17067
17071
|
return folds;
|
|
17068
17072
|
}
|
|
17069
|
-
function foldDetail(
|
|
17070
|
-
const head = `${
|
|
17073
|
+
function foldDetail(path22, folds, maxLen = MAX_FOLD_DETAIL) {
|
|
17074
|
+
const head = `${path22}::`;
|
|
17071
17075
|
const ids = folds.map((f) => f.kind === "body" ? f.name : `#${f.firstLine}-${f.lastLine}`);
|
|
17072
17076
|
const budget = maxLen - head.length - FOLD_DETAIL_SUFFIX_BUDGET;
|
|
17073
17077
|
const kept = [];
|
|
@@ -17085,7 +17089,7 @@ function foldDetail(path24, folds, maxLen = MAX_FOLD_DETAIL) {
|
|
|
17085
17089
|
|
|
17086
17090
|
// src/fold_delivery.ts
|
|
17087
17091
|
init_define_import_meta_env();
|
|
17088
|
-
import { readFileSync as
|
|
17092
|
+
import { readFileSync as readFileSync14, statSync as statSync13 } from "node:fs";
|
|
17089
17093
|
var BODY_FOLD_KEEP_LINES = 8;
|
|
17090
17094
|
var BODY_FOLD_MIN_SPAN = 20;
|
|
17091
17095
|
var COMMENT_FOLD_KEEP_LINES = 2;
|
|
@@ -17117,10 +17121,10 @@ function commentFoldNotice(firstLine, lastLine, shownPath) {
|
|
|
17117
17121
|
var FOLD_SPAN_PARSE_MAX_BYTES = 4e5;
|
|
17118
17122
|
function parseFoldSpansFromDisk(normalizedPath, rows) {
|
|
17119
17123
|
try {
|
|
17120
|
-
if (
|
|
17124
|
+
if (statSync13(normalizedPath).size > FOLD_SPAN_PARSE_MAX_BYTES) return [];
|
|
17121
17125
|
const language = detectLanguage(normalizedPath);
|
|
17122
17126
|
if (!isTreeSitterAvailable(language)) return [];
|
|
17123
|
-
const fileText =
|
|
17127
|
+
const fileText = readFileSync14(normalizedPath, "utf-8");
|
|
17124
17128
|
const diskLines = fileText.split("\n");
|
|
17125
17129
|
for (const row of rows) {
|
|
17126
17130
|
const disk = diskLines[row.no - 1];
|
|
@@ -17136,7 +17140,7 @@ function parseFoldSpansFromDisk(normalizedPath, rows) {
|
|
|
17136
17140
|
function resolveFoldSpans(normalizedPath, hasCommentSyntax, rows) {
|
|
17137
17141
|
try {
|
|
17138
17142
|
const entry = getFileEntry(normalizedPath);
|
|
17139
|
-
if (entry !== null && entry.sha !== "" && entry.sha === fingerprintFile(normalizedPath) && entry.parserSha === PARSER_FINGERPRINT) {
|
|
17143
|
+
if (entry !== null && entry.sha !== "" && !isVirtualIndexedPath(normalizedPath) && entry.sha === fingerprintFile(normalizedPath) && entry.parserSha === PARSER_FINGERPRINT) {
|
|
17140
17144
|
return querySymbols({ filePath: normalizedPath, limit: BODY_FOLD_SYMBOL_LIMIT });
|
|
17141
17145
|
}
|
|
17142
17146
|
if (hasCommentSyntax) enqueueDirtyPathSafe(normalizedPath);
|
|
@@ -17351,8 +17355,8 @@ function isNodeModulesPath(p) {
|
|
|
17351
17355
|
return check.includes("/node_modules/") || check.includes("\\node_modules\\");
|
|
17352
17356
|
}
|
|
17353
17357
|
function relPathWithinRoot(root, target) {
|
|
17354
|
-
const rel =
|
|
17355
|
-
if (rel.startsWith("..") ||
|
|
17358
|
+
const rel = path17.relative(root, target).replace(/\\/g, "/");
|
|
17359
|
+
if (rel.startsWith("..") || path17.isAbsolute(rel)) return null;
|
|
17356
17360
|
return rel;
|
|
17357
17361
|
}
|
|
17358
17362
|
function _isDocFile(filePath) {
|
|
@@ -17436,7 +17440,7 @@ function scanRequestedSlice(absPath, offset, limit) {
|
|
|
17436
17440
|
const windowEnd = offset + limit;
|
|
17437
17441
|
let fd;
|
|
17438
17442
|
try {
|
|
17439
|
-
fd =
|
|
17443
|
+
fd = fs20.openSync(absPath, "r");
|
|
17440
17444
|
} catch {
|
|
17441
17445
|
return null;
|
|
17442
17446
|
}
|
|
@@ -17450,7 +17454,7 @@ function scanRequestedSlice(absPath, offset, limit) {
|
|
|
17450
17454
|
const nearSingleLine = lineNumber < NEAR_SINGLE_LINE_SCAN_THRESHOLD;
|
|
17451
17455
|
return { bytes: sliceBytes, trustworthy: nearSingleLine, nearSingleLine };
|
|
17452
17456
|
}
|
|
17453
|
-
const bytesRead =
|
|
17457
|
+
const bytesRead = fs20.readSync(fd, buf, 0, buf.length, null);
|
|
17454
17458
|
if (bytesRead === 0) {
|
|
17455
17459
|
return {
|
|
17456
17460
|
bytes: sliceBytes,
|
|
@@ -17469,7 +17473,7 @@ function scanRequestedSlice(absPath, offset, limit) {
|
|
|
17469
17473
|
}
|
|
17470
17474
|
} finally {
|
|
17471
17475
|
try {
|
|
17472
|
-
|
|
17476
|
+
fs20.closeSync(fd);
|
|
17473
17477
|
} catch {
|
|
17474
17478
|
}
|
|
17475
17479
|
}
|
|
@@ -17500,11 +17504,11 @@ function isDiffableSource(basename10) {
|
|
|
17500
17504
|
function isSourceExtension(basename10) {
|
|
17501
17505
|
return languageHasFlag(detectLanguage(basename10), "sourceHints");
|
|
17502
17506
|
}
|
|
17503
|
-
var BINARY_FILE_TYPE_EXTS = /* @__PURE__ */ new Set(["pdf", "docx", "xlsx", "pptx", "odt", "ods", "ott", "odp"]);
|
|
17504
|
-
var TEXT_FILE_TYPE_EXTS = /* @__PURE__ */ new Set(["html", "htm", "xhtml", "txt", "log", "out", "err", "trace", "csv", "tsv", "vtt", "srt", "svg", "xml"]);
|
|
17507
|
+
var BINARY_FILE_TYPE_EXTS = /* @__PURE__ */ new Set(["pdf", "docx", "xlsx", "pptx", "odt", "ods", "ott", "odp", "sqlite", "db", "sqlite3", "db3", "parquet"]);
|
|
17508
|
+
var TEXT_FILE_TYPE_EXTS = /* @__PURE__ */ new Set(["html", "htm", "xhtml", "txt", "log", "out", "err", "trace", "csv", "tsv", "vtt", "srt", "svg", "xml", "json", "yaml", "yml", "jsonl"]);
|
|
17505
17509
|
var DISPATCHED_FILE_TYPE_EXTS = /* @__PURE__ */ new Set([...BINARY_FILE_TYPE_EXTS, ...TEXT_FILE_TYPE_EXTS]);
|
|
17506
17510
|
function isDispatchedFileType(basename10) {
|
|
17507
|
-
return DISPATCHED_FILE_TYPE_EXTS.has(
|
|
17511
|
+
return DISPATCHED_FILE_TYPE_EXTS.has(path17.extname(basename10).slice(1).toLowerCase());
|
|
17508
17512
|
}
|
|
17509
17513
|
function surgicalHint(filePath, basename10, lineCount) {
|
|
17510
17514
|
if (lineCount < loadConfig().hints.min_file_lines_for_hint) return "";
|
|
@@ -17567,7 +17571,7 @@ function loadSnapshotDiff(sessionId, normalized, basename10) {
|
|
|
17567
17571
|
try {
|
|
17568
17572
|
const sz = statSize(normalized);
|
|
17569
17573
|
if (sz === null || sz > 256 * 1024) return { kind: "none" };
|
|
17570
|
-
const currentContent =
|
|
17574
|
+
const currentContent = fs20.readFileSync(normalized, "utf8");
|
|
17571
17575
|
const TRUNC_MARKER = "\n<snapshot truncated at ";
|
|
17572
17576
|
const oldRaw = oldSnap.toString("utf8");
|
|
17573
17577
|
const truncIdx = oldRaw.indexOf(TRUNC_MARKER);
|
|
@@ -17654,7 +17658,7 @@ function preReadHandlerInner(event) {
|
|
|
17654
17658
|
try {
|
|
17655
17659
|
const cwd = getCwd(event) ?? process.cwd();
|
|
17656
17660
|
const project = findProject(cwd) ?? makeProjectAt(cwd);
|
|
17657
|
-
const current =
|
|
17661
|
+
const current = fs20.readFileSync(normalized, "utf8");
|
|
17658
17662
|
const evidence = findVerifiedFileEvidence(project.root, normalized, current);
|
|
17659
17663
|
if (evidence !== null) {
|
|
17660
17664
|
recordStat("evidence_cache_hit", 0);
|
|
@@ -17665,7 +17669,7 @@ function preReadHandlerInner(event) {
|
|
|
17665
17669
|
} catch {
|
|
17666
17670
|
}
|
|
17667
17671
|
}
|
|
17668
|
-
const basename10 = displaySafePath(
|
|
17672
|
+
const basename10 = displaySafePath(path17.basename(normalized));
|
|
17669
17673
|
if (isLockFile(basename10)) {
|
|
17670
17674
|
return denyOutput(
|
|
17671
17675
|
'Lock files are rarely useful to read in full. Use `token-goat section "' + shown + '::<section>"` to extract a specific dependency, or read the relevant manifest instead.'
|
|
@@ -17704,7 +17708,7 @@ function preReadHandlerInner(event) {
|
|
|
17704
17708
|
const skillName = detectSkillFile(normalized);
|
|
17705
17709
|
if (skillName && basename10 === "SKILL.md") {
|
|
17706
17710
|
try {
|
|
17707
|
-
const body =
|
|
17711
|
+
const body = fs20.readFileSync(normalized, "utf-8");
|
|
17708
17712
|
const bodySha = contentHash(body);
|
|
17709
17713
|
const compact = getCompactAnySessionSync(skillName);
|
|
17710
17714
|
const stale = isCompactStale(compact, skillName, bodySha);
|
|
@@ -17736,9 +17740,9 @@ function preReadHandlerInner(event) {
|
|
|
17736
17740
|
const isNotebook = /\.ipynb$/i.test(basename10);
|
|
17737
17741
|
if (event.toolName !== "Grep" && isNotebook) {
|
|
17738
17742
|
try {
|
|
17739
|
-
const rawBytes =
|
|
17743
|
+
const rawBytes = fs20.readFileSync(normalized);
|
|
17740
17744
|
const [sidecarPath] = getOrCreateSidecar(rawBytes, dataDir());
|
|
17741
|
-
const sidecarContent =
|
|
17745
|
+
const sidecarContent = fs20.readFileSync(sidecarPath, "utf-8");
|
|
17742
17746
|
const savedBytes = rawBytes.length - sidecarContent.length;
|
|
17743
17747
|
if (savedBytes >= NB_STRIP_MIN_SAVINGS) {
|
|
17744
17748
|
recordActualRead(event, normalized);
|
|
@@ -17759,7 +17763,7 @@ function preReadHandlerInner(event) {
|
|
|
17759
17763
|
const sz = statSize(normalized);
|
|
17760
17764
|
if (sz !== null && sz >= MARKDOWN_SIZE_THRESHOLD) {
|
|
17761
17765
|
markdownSize = sz;
|
|
17762
|
-
fileContent =
|
|
17766
|
+
fileContent = fs20.readFileSync(normalized, "utf8");
|
|
17763
17767
|
}
|
|
17764
17768
|
} catch {
|
|
17765
17769
|
}
|
|
@@ -18043,7 +18047,7 @@ function preReadHandlerInner(event) {
|
|
|
18043
18047
|
"Note: " + shown + " is large (" + kb + "KB). " + hint + contextPressureAdvisorySuffix()
|
|
18044
18048
|
);
|
|
18045
18049
|
}
|
|
18046
|
-
const fileTypeExt =
|
|
18050
|
+
const fileTypeExt = path17.extname(normalized).slice(1).toLowerCase();
|
|
18047
18051
|
const fileStatSize = size ?? statSize(normalized) ?? 0;
|
|
18048
18052
|
const isKnownFileType = DISPATCHED_FILE_TYPE_EXTS.has(fileTypeExt);
|
|
18049
18053
|
if (event.toolName !== "Grep" && !isImagePath(normalized) && (isKnownFileType || fileStatSize >= FILE_TYPE_THRESHOLDS.generic)) {
|
|
@@ -18052,7 +18056,7 @@ function preReadHandlerInner(event) {
|
|
|
18052
18056
|
let ftContent = "";
|
|
18053
18057
|
if (!BINARY_FILE_TYPE_EXTS.has(fileTypeExt) && fileStatSize <= SLICE_ESTIMATE_SCAN_CAP_BYTES) {
|
|
18054
18058
|
try {
|
|
18055
|
-
ftContent =
|
|
18059
|
+
ftContent = fs20.readFileSync(normalized, "utf8");
|
|
18056
18060
|
} catch {
|
|
18057
18061
|
}
|
|
18058
18062
|
}
|
|
@@ -18084,7 +18088,7 @@ function estimateTruncatedLineCount(normalized) {
|
|
|
18084
18088
|
try {
|
|
18085
18089
|
const sz = statSize(normalized);
|
|
18086
18090
|
if (sz !== null && sz <= SLICE_ESTIMATE_SCAN_CAP_BYTES) {
|
|
18087
|
-
return countTextLines(
|
|
18091
|
+
return countTextLines(fs20.readFileSync(normalized, "utf8"));
|
|
18088
18092
|
}
|
|
18089
18093
|
} catch {
|
|
18090
18094
|
}
|
|
@@ -18107,13 +18111,13 @@ function postReadHandlerInner(event, suppressStructuralHint) {
|
|
|
18107
18111
|
if (isTruncatedReadDelivery(event, respText)) {
|
|
18108
18112
|
markFileTruncated(normalized);
|
|
18109
18113
|
}
|
|
18110
|
-
const postBasename =
|
|
18114
|
+
const postBasename = path17.basename(normalized);
|
|
18111
18115
|
const diffSourcesEnabled = loadConfig().hints.serve_diff_on_reread;
|
|
18112
18116
|
if (/\.(md|mdx|markdown|rst|txt)$/i.test(postBasename) || isSessionArtifactFile(normalized) || diffSourcesEnabled && isDiffableSource(postBasename)) {
|
|
18113
18117
|
try {
|
|
18114
18118
|
const sz = statSize(normalized);
|
|
18115
18119
|
if (sz !== null && sz <= 256 * 1024) {
|
|
18116
|
-
const content =
|
|
18120
|
+
const content = fs20.readFileSync(normalized);
|
|
18117
18121
|
store(sessionStateKey(event), normalized, content);
|
|
18118
18122
|
}
|
|
18119
18123
|
} catch {
|
|
@@ -18123,7 +18127,7 @@ function postReadHandlerInner(event, suppressStructuralHint) {
|
|
|
18123
18127
|
try {
|
|
18124
18128
|
const cwd = getCwd(event) ?? process.cwd();
|
|
18125
18129
|
const project = findProject(cwd) ?? makeProjectAt(cwd);
|
|
18126
|
-
const source = decodeSource(
|
|
18130
|
+
const source = decodeSource(fs20.readFileSync(normalized));
|
|
18127
18131
|
recordEvidence({ projectRoot: project.root, source: normalized, representation: "file", text: source });
|
|
18128
18132
|
} catch {
|
|
18129
18133
|
}
|
|
@@ -18154,7 +18158,7 @@ function postReadHandlerInner(event, suppressStructuralHint) {
|
|
|
18154
18158
|
try {
|
|
18155
18159
|
const sz = statSize(normalized);
|
|
18156
18160
|
if (sz !== null && sz <= SLICE_ESTIMATE_SCAN_CAP_BYTES) {
|
|
18157
|
-
const lineCount = countTextLines(
|
|
18161
|
+
const lineCount = countTextLines(fs20.readFileSync(normalized, "utf8"));
|
|
18158
18162
|
const minLines = loadConfig().post_read_code_compress.min_lines;
|
|
18159
18163
|
if (lineCount >= minLines && meetsSavingsFloor(sz) && !suppressStructuralHint) {
|
|
18160
18164
|
recordStat("session_hint", 0, 0);
|
|
@@ -18171,7 +18175,7 @@ function postReadHandlerInner(event, suppressStructuralHint) {
|
|
|
18171
18175
|
function readWindowFromDisk(event, normalized) {
|
|
18172
18176
|
const size = statSize(normalized);
|
|
18173
18177
|
if (size === null || size > SLICE_ESTIMATE_SCAN_CAP_BYTES) return null;
|
|
18174
|
-
const text = decodeSource(
|
|
18178
|
+
const text = decodeSource(fs20.readFileSync(normalized));
|
|
18175
18179
|
const window2 = readRequestedSliceWindow(event);
|
|
18176
18180
|
const start = window2.offset !== void 0 && window2.offset >= 1 ? window2.offset : 1;
|
|
18177
18181
|
const limit = window2.limit;
|
|
@@ -18372,9 +18376,9 @@ registerHook("post_tool_use", postReadHandler, { toolName: "Read" });
|
|
|
18372
18376
|
|
|
18373
18377
|
// src/cli_context_stats.ts
|
|
18374
18378
|
init_define_import_meta_env();
|
|
18375
|
-
import * as
|
|
18379
|
+
import * as fs22 from "node:fs";
|
|
18376
18380
|
import * as os2 from "node:os";
|
|
18377
|
-
import * as
|
|
18381
|
+
import * as path19 from "node:path";
|
|
18378
18382
|
|
|
18379
18383
|
// src/confirm_apply.ts
|
|
18380
18384
|
init_define_import_meta_env();
|
|
@@ -18430,8 +18434,8 @@ ${diff}
|
|
|
18430
18434
|
|
|
18431
18435
|
// src/memory_prune.ts
|
|
18432
18436
|
init_define_import_meta_env();
|
|
18433
|
-
import * as
|
|
18434
|
-
import * as
|
|
18437
|
+
import * as fs21 from "node:fs";
|
|
18438
|
+
import * as path18 from "node:path";
|
|
18435
18439
|
var ENTRY_RE = /^\s*-\s*\[(?<title>[^\]]+)\]\((?<target>[^)]+?\.md)\)/;
|
|
18436
18440
|
var URL_SCHEME_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
18437
18441
|
function parseIndex(text) {
|
|
@@ -18476,10 +18480,10 @@ function pruneIndex(memoryDir, opts) {
|
|
|
18476
18480
|
changed: false,
|
|
18477
18481
|
tokensSaved: 0
|
|
18478
18482
|
};
|
|
18479
|
-
const memoryMd =
|
|
18483
|
+
const memoryMd = path18.join(memoryDir, "MEMORY.md");
|
|
18480
18484
|
let text;
|
|
18481
18485
|
try {
|
|
18482
|
-
text =
|
|
18486
|
+
text = fs21.readFileSync(memoryMd, "utf-8");
|
|
18483
18487
|
} catch {
|
|
18484
18488
|
return result;
|
|
18485
18489
|
}
|
|
@@ -18490,7 +18494,7 @@ function pruneIndex(memoryDir, opts) {
|
|
|
18490
18494
|
const dups = [];
|
|
18491
18495
|
for (const entry of entries) {
|
|
18492
18496
|
const isUrl = URL_SCHEME_RE.test(entry.target);
|
|
18493
|
-
const targetExists = isUrl ? true :
|
|
18497
|
+
const targetExists = isUrl ? true : path18.isAbsolute(entry.target) ? fs21.existsSync(entry.target) : fs21.existsSync(path18.join(memoryDir, entry.target));
|
|
18494
18498
|
const foldedTarget = foldPath(entry.target);
|
|
18495
18499
|
if (!targetExists) {
|
|
18496
18500
|
dead.push(entry);
|
|
@@ -18544,7 +18548,7 @@ function jaccard(a, b) {
|
|
|
18544
18548
|
function siblingSnippet(filePath) {
|
|
18545
18549
|
let text;
|
|
18546
18550
|
try {
|
|
18547
|
-
text =
|
|
18551
|
+
text = fs21.readFileSync(filePath, { encoding: "utf-8" });
|
|
18548
18552
|
} catch {
|
|
18549
18553
|
return "";
|
|
18550
18554
|
}
|
|
@@ -18634,7 +18638,7 @@ async function tryEmbeddingClusters(siblings, snippets, threshold) {
|
|
|
18634
18638
|
}
|
|
18635
18639
|
async function findContentDuplicates(memoryDir, _opts) {
|
|
18636
18640
|
const threshold = _opts?.threshold ?? 0.92;
|
|
18637
|
-
const siblings =
|
|
18641
|
+
const siblings = fs21.readdirSync(memoryDir).filter((name) => name.toLowerCase().endsWith(".md") && name.toLowerCase() !== "memory.md").map((name) => path18.join(memoryDir, name)).sort();
|
|
18638
18642
|
if (siblings.length < 2) {
|
|
18639
18643
|
return [];
|
|
18640
18644
|
}
|
|
@@ -18658,7 +18662,7 @@ function auditClaudeMd(files) {
|
|
|
18658
18662
|
for (const filePath of files) {
|
|
18659
18663
|
let text;
|
|
18660
18664
|
try {
|
|
18661
|
-
text =
|
|
18665
|
+
text = fs21.readFileSync(filePath, { encoding: "utf-8" });
|
|
18662
18666
|
} catch {
|
|
18663
18667
|
continue;
|
|
18664
18668
|
}
|
|
@@ -18718,7 +18722,7 @@ function auditClaudeMd(files) {
|
|
|
18718
18722
|
const overlaps = [];
|
|
18719
18723
|
for (const [stripped, filesSet] of lineToFiles) {
|
|
18720
18724
|
if (filesSet.has(report.path) && filesSet.size > 1) {
|
|
18721
|
-
const others = Array.from(filesSet).filter((p) => p !== report.path).map((p) =>
|
|
18725
|
+
const others = Array.from(filesSet).filter((p) => p !== report.path).map((p) => path18.basename(p));
|
|
18722
18726
|
if (others.length > 0) {
|
|
18723
18727
|
if (stripped.length > 60) {
|
|
18724
18728
|
overlaps.push(
|
|
@@ -18738,7 +18742,7 @@ function auditClaudeMd(files) {
|
|
|
18738
18742
|
// src/cli_context_stats.ts
|
|
18739
18743
|
function tok(filePath) {
|
|
18740
18744
|
try {
|
|
18741
|
-
const size =
|
|
18745
|
+
const size = fs22.statSync(filePath).size;
|
|
18742
18746
|
return Math.floor(size / 4);
|
|
18743
18747
|
} catch {
|
|
18744
18748
|
return 0;
|
|
@@ -18747,42 +18751,42 @@ function tok(filePath) {
|
|
|
18747
18751
|
function findClaudeMdFiles(projectRoot, homeDir = os2.homedir()) {
|
|
18748
18752
|
const found = [];
|
|
18749
18753
|
const seen = /* @__PURE__ */ new Set();
|
|
18750
|
-
let current =
|
|
18754
|
+
let current = path19.resolve(projectRoot);
|
|
18751
18755
|
while (true) {
|
|
18752
|
-
const candidate =
|
|
18753
|
-
if (!seen.has(candidate) &&
|
|
18756
|
+
const candidate = path19.join(current, "CLAUDE.md");
|
|
18757
|
+
if (!seen.has(candidate) && fs22.existsSync(candidate)) {
|
|
18754
18758
|
found.push(candidate);
|
|
18755
18759
|
seen.add(candidate);
|
|
18756
18760
|
}
|
|
18757
|
-
const parent =
|
|
18761
|
+
const parent = path19.dirname(current);
|
|
18758
18762
|
if (parent === current) break;
|
|
18759
18763
|
current = parent;
|
|
18760
18764
|
}
|
|
18761
|
-
const globalMd =
|
|
18762
|
-
if (!seen.has(globalMd) &&
|
|
18765
|
+
const globalMd = path19.join(homeDir, ".claude", "CLAUDE.md");
|
|
18766
|
+
if (!seen.has(globalMd) && fs22.existsSync(globalMd)) {
|
|
18763
18767
|
found.push(globalMd);
|
|
18764
18768
|
}
|
|
18765
18769
|
return found;
|
|
18766
18770
|
}
|
|
18767
18771
|
function findMemoryMd(projectRoot, homeDir = os2.homedir(), alternateRoots = []) {
|
|
18768
18772
|
try {
|
|
18769
|
-
const projectsDir =
|
|
18770
|
-
if (!
|
|
18771
|
-
const rootStr =
|
|
18773
|
+
const projectsDir = path19.join(homeDir, ".claude", "projects");
|
|
18774
|
+
if (!fs22.existsSync(projectsDir)) return null;
|
|
18775
|
+
const rootStr = path19.resolve(projectRoot);
|
|
18772
18776
|
const candidateRoots = [rootStr];
|
|
18773
18777
|
try {
|
|
18774
|
-
const realRoot =
|
|
18778
|
+
const realRoot = fs22.realpathSync.native(rootStr);
|
|
18775
18779
|
if (realRoot !== rootStr) candidateRoots.push(realRoot);
|
|
18776
18780
|
} catch {
|
|
18777
18781
|
}
|
|
18778
18782
|
for (const alternate of alternateRoots) {
|
|
18779
|
-
const resolved =
|
|
18783
|
+
const resolved = path19.resolve(alternate);
|
|
18780
18784
|
if (!candidateRoots.includes(resolved)) candidateRoots.push(resolved);
|
|
18781
18785
|
}
|
|
18782
18786
|
for (const root of candidateRoots) {
|
|
18783
18787
|
const expectedSlug = root.replace(/[^A-Za-z0-9]/g, "-");
|
|
18784
|
-
const candidate =
|
|
18785
|
-
if (
|
|
18788
|
+
const candidate = path19.join(projectsDir, expectedSlug, "memory", "MEMORY.md");
|
|
18789
|
+
if (fs22.existsSync(candidate)) return candidate;
|
|
18786
18790
|
}
|
|
18787
18791
|
return null;
|
|
18788
18792
|
} catch {
|
|
@@ -18796,10 +18800,10 @@ function buildStats(projectRoot, homeDir = os2.homedir(), alternateRoots = []) {
|
|
|
18796
18800
|
for (const p of claudeMds) {
|
|
18797
18801
|
const t = tok(p);
|
|
18798
18802
|
claudeMdTotal += t;
|
|
18799
|
-
const parentDir =
|
|
18803
|
+
const parentDir = path19.basename(path19.dirname(p));
|
|
18800
18804
|
const label = parentDir === ".claude" ? "~/.claude/CLAUDE.md" : (() => {
|
|
18801
18805
|
try {
|
|
18802
|
-
return
|
|
18806
|
+
return path19.relative(projectRoot, p);
|
|
18803
18807
|
} catch {
|
|
18804
18808
|
return p;
|
|
18805
18809
|
}
|
|
@@ -18863,11 +18867,11 @@ async function runContextStats(opts = {}) {
|
|
|
18863
18867
|
process.stdout.write("[--fix] No MEMORY.md found; nothing to prune.\n");
|
|
18864
18868
|
} else {
|
|
18865
18869
|
const memPath = result.memory_md_path;
|
|
18866
|
-
const pruneResult = pruneIndex(
|
|
18870
|
+
const pruneResult = pruneIndex(path19.dirname(memPath), { dryRun: true });
|
|
18867
18871
|
if (!pruneResult.changed || pruneResult.after === void 0) {
|
|
18868
18872
|
process.stdout.write("[--fix] MEMORY.md already clean; nothing to prune.\n");
|
|
18869
18873
|
} else {
|
|
18870
|
-
const before =
|
|
18874
|
+
const before = fs22.readFileSync(memPath, "utf-8");
|
|
18871
18875
|
const applyResult = await confirmAndApply(
|
|
18872
18876
|
[{ path: memPath, before, after: pruneResult.after, label: "MEMORY.md" }],
|
|
18873
18877
|
opts.yes === true ? { yes: true } : {}
|
|
@@ -18895,8 +18899,8 @@ async function runContextStats(opts = {}) {
|
|
|
18895
18899
|
|
|
18896
18900
|
// src/baseline.ts
|
|
18897
18901
|
init_define_import_meta_env();
|
|
18898
|
-
import * as
|
|
18899
|
-
import * as
|
|
18902
|
+
import * as fs23 from "node:fs";
|
|
18903
|
+
import * as path20 from "node:path";
|
|
18900
18904
|
var SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
18901
18905
|
"node_modules",
|
|
18902
18906
|
".git",
|
|
@@ -18934,12 +18938,12 @@ function walkProject(rootDir, opts = {}) {
|
|
|
18934
18938
|
if (dir === void 0) break;
|
|
18935
18939
|
let entries;
|
|
18936
18940
|
try {
|
|
18937
|
-
entries =
|
|
18941
|
+
entries = fs23.readdirSync(dir, { withFileTypes: true });
|
|
18938
18942
|
} catch {
|
|
18939
18943
|
continue;
|
|
18940
18944
|
}
|
|
18941
18945
|
for (const entry of entries) {
|
|
18942
|
-
const full =
|
|
18946
|
+
const full = path20.join(dir, entry.name);
|
|
18943
18947
|
if (entry.isDirectory()) {
|
|
18944
18948
|
if (SKIP_DIRS.has(entry.name) || extraSkipDirs.includes(entry.name)) continue;
|
|
18945
18949
|
if (entry.name.startsWith(".") && entry.name !== ".") {
|
|
@@ -19021,7 +19025,7 @@ function fetchTopSymbols(limit, dbPath, rootDir) {
|
|
|
19021
19025
|
}
|
|
19022
19026
|
}
|
|
19023
19027
|
function buildProjectMap(rootDir = process.cwd(), opts = {}) {
|
|
19024
|
-
const root =
|
|
19028
|
+
const root = path20.resolve(rootDir);
|
|
19025
19029
|
const config = loadConfig();
|
|
19026
19030
|
const { files, languages, truncated } = walkProject(root, { excludeTests: config.repomap.exclude_tests });
|
|
19027
19031
|
const compact = opts.compact === true || files.length > config.repomap.compact_file_threshold;
|
|
@@ -19030,12 +19034,12 @@ function buildProjectMap(rootDir = process.cwd(), opts = {}) {
|
|
|
19030
19034
|
const recentFiles = files.map((f) => {
|
|
19031
19035
|
let mtime;
|
|
19032
19036
|
try {
|
|
19033
|
-
mtime =
|
|
19037
|
+
mtime = fs23.statSync(f).mtimeMs;
|
|
19034
19038
|
} catch {
|
|
19035
19039
|
mtime = 0;
|
|
19036
19040
|
}
|
|
19037
19041
|
return { f, mtime };
|
|
19038
|
-
}).sort((a, b) => b.mtime - a.mtime).slice(0, compact ? 5 : 15).map((x) =>
|
|
19042
|
+
}).sort((a, b) => b.mtime - a.mtime).slice(0, compact ? 5 : 15).map((x) => path20.relative(root, x.f));
|
|
19039
19043
|
return {
|
|
19040
19044
|
rootDir: root,
|
|
19041
19045
|
fileCount: files.length,
|
|
@@ -19048,7 +19052,7 @@ function buildProjectMap(rootDir = process.cwd(), opts = {}) {
|
|
|
19048
19052
|
}
|
|
19049
19053
|
function formatProjectMap(map, compact = false) {
|
|
19050
19054
|
const lines2 = [];
|
|
19051
|
-
const rel =
|
|
19055
|
+
const rel = path20.basename(map.rootDir);
|
|
19052
19056
|
lines2.push(`# Project map: ${rel}`);
|
|
19053
19057
|
lines2.push(map.fileCountTruncated === true ? `Files: at least ${map.fileCount} (walk stopped at the ${MAX_FILES_SCANNED}-file cap)` : `Files: ${map.fileCount}`);
|
|
19054
19058
|
const langPairs = Object.entries(map.languages).sort((a, b) => b[1] - a[1]);
|
|
@@ -19059,7 +19063,7 @@ function formatProjectMap(map, compact = false) {
|
|
|
19059
19063
|
lines2.push("");
|
|
19060
19064
|
lines2.push("## Top symbols");
|
|
19061
19065
|
for (const s of map.topSymbols) {
|
|
19062
|
-
const loc =
|
|
19066
|
+
const loc = formatSymbolLocation(displaySafeText(toDisplayPath(map.rootDir, s.filePath)), s.lineStart, s.lineEnd);
|
|
19063
19067
|
lines2.push(`- ${displaySafeText(s.name)} (${displaySafeText(s.kind)}) \u2014 ${loc}`);
|
|
19064
19068
|
}
|
|
19065
19069
|
} else {
|
|
@@ -19077,7 +19081,7 @@ function formatProjectMap(map, compact = false) {
|
|
|
19077
19081
|
}
|
|
19078
19082
|
function mapLookupBytesSaved(map, emittedText) {
|
|
19079
19083
|
const referencedFiles = /* @__PURE__ */ new Set([
|
|
19080
|
-
...map.recentFiles.map((f) => normalizePath(
|
|
19084
|
+
...map.recentFiles.map((f) => normalizePath(path20.resolve(map.rootDir, f))),
|
|
19081
19085
|
...map.topSymbols.map((s) => normalizePath(s.filePath))
|
|
19082
19086
|
]);
|
|
19083
19087
|
const listingText = Array.from(referencedFiles).sort().join("\n");
|
|
@@ -19092,14 +19096,14 @@ function findMemSuggestionCandidates(projectRoot) {
|
|
|
19092
19096
|
const claudeMdFiles = findClaudeMdFiles(projectRoot);
|
|
19093
19097
|
const candidateFiles = new Set(claudeMdFiles);
|
|
19094
19098
|
for (const claudeMd of claudeMdFiles) {
|
|
19095
|
-
const agentsMd =
|
|
19096
|
-
if (
|
|
19099
|
+
const agentsMd = path20.join(path20.dirname(claudeMd), "AGENTS.md");
|
|
19100
|
+
if (fs23.existsSync(agentsMd)) candidateFiles.add(agentsMd);
|
|
19097
19101
|
}
|
|
19098
19102
|
const suggestions = [];
|
|
19099
19103
|
for (const filePath of candidateFiles) {
|
|
19100
19104
|
let text;
|
|
19101
19105
|
try {
|
|
19102
|
-
text =
|
|
19106
|
+
text = fs23.readFileSync(filePath, { encoding: "utf-8" });
|
|
19103
19107
|
} catch {
|
|
19104
19108
|
continue;
|
|
19105
19109
|
}
|
|
@@ -19128,7 +19132,7 @@ function formatMemSuggestions(projectRoot) {
|
|
|
19128
19132
|
if (suggestions.length === 0) return "";
|
|
19129
19133
|
const lines2 = ["", "## mem suggestions"];
|
|
19130
19134
|
for (const s of suggestions) {
|
|
19131
|
-
const basename10 =
|
|
19135
|
+
const basename10 = path20.basename(s.path);
|
|
19132
19136
|
lines2.push(
|
|
19133
19137
|
"Consider: mem import --from-md " + s.path + " # migrates " + s.count + " preference-shaped lines from " + basename10 + " as pending facts for review"
|
|
19134
19138
|
);
|
|
@@ -19172,16 +19176,16 @@ function scanAndRecord(text) {
|
|
|
19172
19176
|
|
|
19173
19177
|
// src/repomap.ts
|
|
19174
19178
|
init_define_import_meta_env();
|
|
19175
|
-
import * as
|
|
19176
|
-
import * as
|
|
19179
|
+
import * as fs24 from "fs";
|
|
19180
|
+
import * as path21 from "path";
|
|
19177
19181
|
function getTrackedFiles(cwd = process.cwd()) {
|
|
19178
19182
|
try {
|
|
19179
19183
|
let dir = cwd;
|
|
19180
19184
|
let onlyFile = "";
|
|
19181
19185
|
try {
|
|
19182
|
-
if (
|
|
19183
|
-
dir =
|
|
19184
|
-
onlyFile =
|
|
19186
|
+
if (fs24.statSync(cwd).isFile()) {
|
|
19187
|
+
dir = path21.dirname(cwd);
|
|
19188
|
+
onlyFile = path21.basename(cwd);
|
|
19185
19189
|
}
|
|
19186
19190
|
} catch {
|
|
19187
19191
|
}
|
|
@@ -19190,7 +19194,7 @@ function getTrackedFiles(cwd = process.cwd()) {
|
|
|
19190
19194
|
if (result.exitCode !== 0 || !result.stdout) {
|
|
19191
19195
|
return [];
|
|
19192
19196
|
}
|
|
19193
|
-
return result.stdout.split("\n").filter((line) => line.trim().length > 0).map((rel) =>
|
|
19197
|
+
return result.stdout.split("\n").filter((line) => line.trim().length > 0).map((rel) => path21.join(dir, rel));
|
|
19194
19198
|
} catch {
|
|
19195
19199
|
return [];
|
|
19196
19200
|
}
|
|
@@ -19430,6 +19434,8 @@ export {
|
|
|
19430
19434
|
unzipBounded,
|
|
19431
19435
|
docxOutline,
|
|
19432
19436
|
docxText,
|
|
19437
|
+
docxTables,
|
|
19438
|
+
formatDocxTables,
|
|
19433
19439
|
pptxOutline,
|
|
19434
19440
|
pptxSlideText,
|
|
19435
19441
|
pptxNotesText,
|
|
@@ -19444,6 +19450,8 @@ export {
|
|
|
19444
19450
|
rangeSheet,
|
|
19445
19451
|
formatXlsxRange,
|
|
19446
19452
|
querySheet,
|
|
19453
|
+
xlsxColumns,
|
|
19454
|
+
formatXlsxColumns,
|
|
19447
19455
|
isEmbeddableDocument,
|
|
19448
19456
|
pathEqClause,
|
|
19449
19457
|
detectWalkMode,
|
|
@@ -19509,13 +19517,6 @@ export {
|
|
|
19509
19517
|
getFileServedOutputs,
|
|
19510
19518
|
markFileTruncated,
|
|
19511
19519
|
getSessionId,
|
|
19512
|
-
DEFAULT_MAX_COUNT,
|
|
19513
|
-
DEFAULT_MAX_AGE_MS,
|
|
19514
|
-
isBlobStale,
|
|
19515
|
-
storeBlob,
|
|
19516
|
-
loadBlob,
|
|
19517
|
-
listBlobs,
|
|
19518
|
-
pruneBlobs,
|
|
19519
19520
|
isBuildCommand,
|
|
19520
19521
|
isTestRunnerCommand,
|
|
19521
19522
|
getMonitoringRecallHint,
|
|
@@ -19574,10 +19575,6 @@ export {
|
|
|
19574
19575
|
getSkillFilePath,
|
|
19575
19576
|
installedSkillPath,
|
|
19576
19577
|
pruneSkillOutputs,
|
|
19577
|
-
ocrIntegrityFailed,
|
|
19578
|
-
isOcrEngineAvailable,
|
|
19579
|
-
ocrImage,
|
|
19580
|
-
isTextHeavy,
|
|
19581
19578
|
visionTokens,
|
|
19582
19579
|
visionTokensSaved,
|
|
19583
19580
|
visionTokensSavedByText,
|