token-goat 2.9.11 → 2.9.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/{token-goat-chunk-FDURVZQD.mjs → token-goat-chunk-2WC4ZUXN.mjs} +1 -1
- package/dist/{token-goat-chunk-MRZ555B3.mjs → token-goat-chunk-3ESRORNM.mjs} +188 -97
- package/dist/token-goat-chunk-4NXUKV7D.mjs +56 -0
- package/dist/{token-goat-chunk-Y5VKNLVD.mjs → token-goat-chunk-6B44WLIF.mjs} +5 -5
- package/dist/{token-goat-chunk-2ZUPQYLO.mjs → token-goat-chunk-B3CTCQTH.mjs} +1 -1
- package/dist/{token-goat-chunk-V53Z47ZH.mjs → token-goat-chunk-FQCNJV4V.mjs} +20 -6
- package/dist/{token-goat-chunk-3ZALKJ23.mjs → token-goat-chunk-FZU7GMUS.mjs} +49 -24
- package/dist/{token-goat-chunk-AAEYU2U2.mjs → token-goat-chunk-JOXLE672.mjs} +344 -5699
- package/dist/{token-goat-chunk-HDL77BN3.mjs → token-goat-chunk-JVNPCQB7.mjs} +10 -7
- package/dist/{token-goat-chunk-LHLQFGWQ.mjs → token-goat-chunk-OMNQUUIT.mjs} +9 -27
- package/dist/{token-goat-chunk-PJPFOOGM.mjs → token-goat-chunk-P2PU4CR5.mjs} +8 -6
- package/dist/{token-goat-chunk-YCKCFYTM.mjs → token-goat-chunk-QKXBGBQR.mjs} +110 -21
- package/dist/{token-goat-chunk-MKITQ5RY.mjs → token-goat-chunk-QWSUZWFP.mjs} +44 -33
- package/dist/{token-goat-chunk-4NZUXC4F.mjs → token-goat-chunk-RDITECDL.mjs} +7 -5
- package/dist/{token-goat-chunk-LKOYKSID.mjs → token-goat-chunk-U7X6LQD2.mjs} +271 -69
- package/dist/token-goat-chunk-UM47DRD3.mjs +242 -0
- package/dist/token-goat-chunk-UMXJN7DI.mjs +5521 -0
- package/dist/token-goat-chunk-YOA4N6WA.mjs +308 -0
- package/dist/{token-goat-chunk-SY7WTZMW.mjs → token-goat-chunk-ZZI3IDQZ.mjs} +1016 -995
- package/dist/token-goat-hook.mjs +8 -6
- package/dist/token-goat.core.mjs +10 -7
- package/docs/cli.md +11 -8
- package/docs/security.md +1 -1
- package/package.json +1 -1
|
@@ -2,46 +2,57 @@ import { createRequire as __cjsRequire } from 'node:module';
|
|
|
2
2
|
const require = __cjsRequire(import.meta.url);
|
|
3
3
|
import {
|
|
4
4
|
OUTPUT_FIRST_TOOL_RESPONSE_KEYS,
|
|
5
|
-
SOURCE_HINT,
|
|
6
|
-
TREE_SITTER_LANGUAGES,
|
|
7
5
|
VSCODE_TOOL_NAME_KEY,
|
|
8
6
|
contextOutput,
|
|
9
7
|
denyOutput,
|
|
10
|
-
detectHarness,
|
|
11
|
-
detectLanguage,
|
|
12
|
-
detectLanguageOfFile,
|
|
13
8
|
emitRewrite,
|
|
14
9
|
estimateTokens,
|
|
15
10
|
extractToolResponseField,
|
|
11
|
+
getCwd,
|
|
12
|
+
getFilePath,
|
|
13
|
+
hasUnquotedOperator,
|
|
14
|
+
isRewriteWorthwhile,
|
|
15
|
+
materializeShrunkImageFile,
|
|
16
|
+
passOutput,
|
|
17
|
+
registerHook,
|
|
18
|
+
resolveMinNetSavingsBytes,
|
|
19
|
+
sessionStateKey
|
|
20
|
+
} from "./token-goat-chunk-JOXLE672.mjs";
|
|
21
|
+
import {
|
|
22
|
+
DEFAULT_MAX_AGE_MS,
|
|
23
|
+
DEFAULT_MAX_COUNT,
|
|
24
|
+
blobPath,
|
|
25
|
+
isBlobStale,
|
|
26
|
+
loadBlob,
|
|
27
|
+
storeBlob,
|
|
28
|
+
sweepCacheRoots
|
|
29
|
+
} from "./token-goat-chunk-UM47DRD3.mjs";
|
|
30
|
+
import {
|
|
31
|
+
SOURCE_HINT,
|
|
32
|
+
TREE_SITTER_LANGUAGES,
|
|
33
|
+
detectHarness,
|
|
34
|
+
detectLanguage,
|
|
35
|
+
detectLanguageOfFile,
|
|
16
36
|
fileIsAbsent,
|
|
17
37
|
findProject,
|
|
18
38
|
fingerprintContent,
|
|
19
39
|
fingerprintFile,
|
|
20
|
-
getCwd,
|
|
21
40
|
getDb,
|
|
22
|
-
getFilePath,
|
|
23
41
|
getHarnessName,
|
|
24
|
-
hasUnquotedOperator,
|
|
25
42
|
isAutoTriggerMultiplierExplicit,
|
|
26
|
-
isRewriteWorthwhile,
|
|
27
43
|
isUnderSystemTemp,
|
|
28
44
|
languageHasFlag,
|
|
29
45
|
loadConfig,
|
|
30
46
|
makeProjectAt,
|
|
31
|
-
materializeShrunkImageFile,
|
|
32
|
-
passOutput,
|
|
33
47
|
recordStat,
|
|
34
48
|
redactIfDotenv,
|
|
35
49
|
redactSecrets,
|
|
36
50
|
refineLanguageByContent,
|
|
37
|
-
registerHook,
|
|
38
|
-
resolveMinNetSavingsBytes,
|
|
39
51
|
resolveProjectRoot,
|
|
40
52
|
savedTokensFromBytes,
|
|
41
|
-
sessionStateKey,
|
|
42
53
|
shortFingerprint,
|
|
43
54
|
summarize
|
|
44
|
-
} from "./token-goat-chunk-
|
|
55
|
+
} from "./token-goat-chunk-UMXJN7DI.mjs";
|
|
45
56
|
import {
|
|
46
57
|
registerReset
|
|
47
58
|
} from "./token-goat-chunk-EEIDFMEM.mjs";
|
|
@@ -71,18 +82,20 @@ import {
|
|
|
71
82
|
extractErrorMessage,
|
|
72
83
|
fenceUntrustedContent,
|
|
73
84
|
fenceUntrustedFileContent,
|
|
74
|
-
fenceUntrustedOcrText,
|
|
75
85
|
filtersFilteredToEmptyNotice,
|
|
76
86
|
findHtmlHeadingMatches,
|
|
77
87
|
foldPath,
|
|
78
88
|
foldPathForContainment,
|
|
89
|
+
formatSymbolLocation,
|
|
79
90
|
globalDbPath,
|
|
91
|
+
ipynbToVirtualSource,
|
|
80
92
|
isCaseInsensitiveFs,
|
|
81
93
|
isCodeFenceDelimiter,
|
|
82
94
|
isInsideRoot,
|
|
83
95
|
isTestFile,
|
|
84
96
|
isUncOrDevicePath,
|
|
85
97
|
isUnderBlockedRoot,
|
|
98
|
+
isVirtualIndexedPath,
|
|
86
99
|
isWithinQuietHours,
|
|
87
100
|
normalizePath,
|
|
88
101
|
normalizePathForwardSlash,
|
|
@@ -103,7 +116,7 @@ import {
|
|
|
103
116
|
toKB,
|
|
104
117
|
tokenGoatHome,
|
|
105
118
|
withFileLock
|
|
106
|
-
} from "./token-goat-chunk-
|
|
119
|
+
} from "./token-goat-chunk-QKXBGBQR.mjs";
|
|
107
120
|
import {
|
|
108
121
|
__commonJS,
|
|
109
122
|
__toESM,
|
|
@@ -2258,9 +2271,9 @@ var require_decoder = __commonJS({
|
|
|
2258
2271
|
return a < 0 ? 0 : a > 255 ? 255 : a;
|
|
2259
2272
|
}
|
|
2260
2273
|
constructor.prototype = {
|
|
2261
|
-
load: function load3(
|
|
2274
|
+
load: function load3(path22) {
|
|
2262
2275
|
var xhr = new XMLHttpRequest();
|
|
2263
|
-
xhr.open("GET",
|
|
2276
|
+
xhr.open("GET", path22, true);
|
|
2264
2277
|
xhr.responseType = "arraybuffer";
|
|
2265
2278
|
xhr.onload = (function() {
|
|
2266
2279
|
var data = new Uint8Array(xhr.response || xhr.mozResponseArrayBuffer);
|
|
@@ -2834,6 +2847,18 @@ init_define_import_meta_env();
|
|
|
2834
2847
|
init_define_import_meta_env();
|
|
2835
2848
|
import * as fs from "node:fs";
|
|
2836
2849
|
|
|
2850
|
+
// src/document_refusal.ts
|
|
2851
|
+
init_define_import_meta_env();
|
|
2852
|
+
var MAX_DOCUMENT_WORK_MILLIS = 6e4;
|
|
2853
|
+
var DocumentRefusedError = class extends Error {
|
|
2854
|
+
transient;
|
|
2855
|
+
constructor(message, name, transient = false) {
|
|
2856
|
+
super(message);
|
|
2857
|
+
this.name = name;
|
|
2858
|
+
this.transient = transient;
|
|
2859
|
+
}
|
|
2860
|
+
};
|
|
2861
|
+
|
|
2837
2862
|
// src/lazy_module.ts
|
|
2838
2863
|
init_define_import_meta_env();
|
|
2839
2864
|
function createLazyModuleLoader(load3, errorLabel) {
|
|
@@ -2854,6 +2879,11 @@ function createLazyModuleLoader(load3, errorLabel) {
|
|
|
2854
2879
|
// src/xml_parser.ts
|
|
2855
2880
|
init_define_import_meta_env();
|
|
2856
2881
|
var MAX_XML_DEPTH = 512;
|
|
2882
|
+
var XmlTooDeepError = class extends DocumentRefusedError {
|
|
2883
|
+
constructor(message) {
|
|
2884
|
+
super(message, "XmlTooDeepError");
|
|
2885
|
+
}
|
|
2886
|
+
};
|
|
2857
2887
|
var NAMED_ENTITIES = {
|
|
2858
2888
|
lt: "<",
|
|
2859
2889
|
gt: ">",
|
|
@@ -3051,7 +3081,7 @@ function parseXml(xml) {
|
|
|
3051
3081
|
continue;
|
|
3052
3082
|
}
|
|
3053
3083
|
if (stack.length > MAX_XML_DEPTH) {
|
|
3054
|
-
throw new
|
|
3084
|
+
throw new XmlTooDeepError(`XML nesting deeper than ${MAX_XML_DEPTH} elements; refusing to parse (this file is not something any office application produces)`);
|
|
3055
3085
|
}
|
|
3056
3086
|
stack.push(frame);
|
|
3057
3087
|
}
|
|
@@ -3064,17 +3094,6 @@ function parseXml(xml) {
|
|
|
3064
3094
|
|
|
3065
3095
|
// src/zip_bounds.ts
|
|
3066
3096
|
init_define_import_meta_env();
|
|
3067
|
-
|
|
3068
|
-
// src/document_refusal.ts
|
|
3069
|
-
init_define_import_meta_env();
|
|
3070
|
-
var DocumentRefusedError = class extends Error {
|
|
3071
|
-
constructor(message, name) {
|
|
3072
|
-
super(message);
|
|
3073
|
-
this.name = name;
|
|
3074
|
-
}
|
|
3075
|
-
};
|
|
3076
|
-
|
|
3077
|
-
// src/zip_bounds.ts
|
|
3078
3097
|
var MAX_ZIP_INPUT_BYTES = 50 * 1024 * 1024;
|
|
3079
3098
|
var MAX_ZIP_OUTPUT_BYTES = 500 * 1024 * 1024;
|
|
3080
3099
|
var STREAM_CHUNK_BYTES = 64 * 1024;
|
|
@@ -3147,6 +3166,26 @@ function unzipBounded(mod, data, opts) {
|
|
|
3147
3166
|
}
|
|
3148
3167
|
|
|
3149
3168
|
// src/ooxml_extract.ts
|
|
3169
|
+
var MAX_OOXML_WORK_MILLIS = MAX_DOCUMENT_WORK_MILLIS;
|
|
3170
|
+
var OoxmlTookTooLongError = class extends DocumentRefusedError {
|
|
3171
|
+
constructor(message) {
|
|
3172
|
+
super(message, "OoxmlTookTooLongError", true);
|
|
3173
|
+
}
|
|
3174
|
+
};
|
|
3175
|
+
var NotAnOfficeDocumentError = class extends DocumentRefusedError {
|
|
3176
|
+
constructor(message, cause) {
|
|
3177
|
+
super(message, "NotAnOfficeDocumentError");
|
|
3178
|
+
if (cause !== void 0) this.cause = cause;
|
|
3179
|
+
}
|
|
3180
|
+
};
|
|
3181
|
+
function ooxmlWorkDeadline() {
|
|
3182
|
+
return Date.now() + MAX_OOXML_WORK_MILLIS;
|
|
3183
|
+
}
|
|
3184
|
+
function assertOoxmlWithinDeadline(deadline, hint) {
|
|
3185
|
+
if (Date.now() > deadline) {
|
|
3186
|
+
throw new OoxmlTookTooLongError(`reading this document's slides/sheets passed the ${MAX_OOXML_WORK_MILLIS}ms limit. ${hint}`);
|
|
3187
|
+
}
|
|
3188
|
+
}
|
|
3150
3189
|
var loadFflate = createLazyModuleLoader(
|
|
3151
3190
|
async () => await import("fflate"),
|
|
3152
3191
|
"office-file reading disabled (fflate unavailable)"
|
|
@@ -3165,10 +3204,8 @@ async function readOoxmlZip(filePath, kind) {
|
|
|
3165
3204
|
} catch (err) {
|
|
3166
3205
|
throw new Error(accessFailureMessage(err, filePath), { cause: err });
|
|
3167
3206
|
}
|
|
3168
|
-
if (!stat2.isFile()) throw new
|
|
3169
|
-
if (stat2.size > MAX_ZIP_INPUT_BYTES)
|
|
3170
|
-
throw new Error(`${filePath} is ${Math.round(stat2.size / (1024 * 1024))}MB, over the ${MAX_ZIP_INPUT_BYTES / (1024 * 1024)}MB limit for OOXML files`);
|
|
3171
|
-
}
|
|
3207
|
+
if (!stat2.isFile()) throw new NotAnOfficeDocumentError(`not a valid ${kind} file: ${filePath}`);
|
|
3208
|
+
if (stat2.size > MAX_ZIP_INPUT_BYTES) throw new ZipInputTooLargeError(filePath, stat2.size, MAX_ZIP_INPUT_BYTES);
|
|
3172
3209
|
let data;
|
|
3173
3210
|
try {
|
|
3174
3211
|
data = fs.readFileSync(filePath);
|
|
@@ -3179,12 +3216,30 @@ async function readOoxmlZip(filePath, kind) {
|
|
|
3179
3216
|
return unzipBounded(fflate, new Uint8Array(data), { limitBytes: MAX_ZIP_OUTPUT_BYTES, shouldExtract: () => true });
|
|
3180
3217
|
} catch (err) {
|
|
3181
3218
|
if (err instanceof ZipOutputTooLargeError) throw err;
|
|
3182
|
-
throw new
|
|
3219
|
+
throw new NotAnOfficeDocumentError(`not a valid ${kind} file: ${filePath}`, err);
|
|
3220
|
+
}
|
|
3221
|
+
}
|
|
3222
|
+
var MAX_OOXML_PART_BYTES = 32 * 1024 * 1024;
|
|
3223
|
+
var OoxmlPartTooLargeError = class extends DocumentRefusedError {
|
|
3224
|
+
constructor(entryPath, bytes) {
|
|
3225
|
+
super(`${entryPath} is ${bytes} bytes, past the ${MAX_OOXML_PART_BYTES}-byte limit for one part of an office file. Split the document, or extract from a smaller copy.`, "OoxmlPartTooLargeError");
|
|
3183
3226
|
}
|
|
3227
|
+
};
|
|
3228
|
+
var MAX_OOXML_DOCUMENT_PART_BYTES = 64 * 1024 * 1024;
|
|
3229
|
+
var OoxmlDocumentTooLargeError = class extends DocumentRefusedError {
|
|
3230
|
+
constructor(entryPath, spentBytes, partBytes) {
|
|
3231
|
+
super(`decoding ${entryPath} (${partBytes} bytes, after ${spentBytes} already decoded) would take this office file past the ${MAX_OOXML_DOCUMENT_PART_BYTES}-byte limit on the XML one document may have decoded at once. Split the document, or extract from a smaller copy.`, "OoxmlDocumentTooLargeError");
|
|
3232
|
+
}
|
|
3233
|
+
};
|
|
3234
|
+
function ooxmlPartBudget() {
|
|
3235
|
+
return { spent: 0 };
|
|
3184
3236
|
}
|
|
3185
|
-
function decodeZipEntry(entries, entryPath) {
|
|
3237
|
+
function decodeZipEntry(entries, entryPath, budget) {
|
|
3186
3238
|
const bytes = entries[entryPath];
|
|
3187
3239
|
if (bytes === void 0) return null;
|
|
3240
|
+
if (bytes.length > MAX_OOXML_PART_BYTES) throw new OoxmlPartTooLargeError(entryPath, bytes.length);
|
|
3241
|
+
if (budget.spent + bytes.length > MAX_OOXML_DOCUMENT_PART_BYTES) throw new OoxmlDocumentTooLargeError(entryPath, budget.spent, bytes.length);
|
|
3242
|
+
budget.spent += bytes.length;
|
|
3188
3243
|
return new TextDecoder("utf-8").decode(bytes);
|
|
3189
3244
|
}
|
|
3190
3245
|
async function parseOoxmlPart(xmlText) {
|
|
@@ -3263,8 +3318,8 @@ function headingLevel(styleVal) {
|
|
|
3263
3318
|
}
|
|
3264
3319
|
async function loadDocumentBody(filePath) {
|
|
3265
3320
|
const entries = await readOoxmlZip(filePath, ".docx");
|
|
3266
|
-
const xml = decodeZipEntry(entries, "word/document.xml");
|
|
3267
|
-
if (xml === null) throw new
|
|
3321
|
+
const xml = decodeZipEntry(entries, "word/document.xml", ooxmlPartBudget());
|
|
3322
|
+
if (xml === null) throw new NotAnOfficeDocumentError(`no word/document.xml found in ${filePath} (not a valid .docx?)`);
|
|
3268
3323
|
return parseOoxmlPart(xml);
|
|
3269
3324
|
}
|
|
3270
3325
|
async function docxOutline(filePath) {
|
|
@@ -3285,6 +3340,68 @@ async function docxText(filePath) {
|
|
|
3285
3340
|
const lines2 = paragraphs.map((p) => collectTextRuns(p, "w:t").join("")).filter((t) => t.trim().length > 0);
|
|
3286
3341
|
return lines2.join("\n\n");
|
|
3287
3342
|
}
|
|
3343
|
+
async function docxTables(filePath) {
|
|
3344
|
+
const parsed = await loadDocumentBody(filePath);
|
|
3345
|
+
const tbls = collectElements(parsed, "w:tbl");
|
|
3346
|
+
const out = [];
|
|
3347
|
+
for (let i = 0; i < tbls.length; i++) {
|
|
3348
|
+
const tbl = tbls[i];
|
|
3349
|
+
const trElements = collectElements(tbl, "w:tr");
|
|
3350
|
+
const rows = [];
|
|
3351
|
+
let maxCols = 0;
|
|
3352
|
+
for (const tr of trElements) {
|
|
3353
|
+
const tcElements = collectElements(tr, "w:tc");
|
|
3354
|
+
const rowCells = [];
|
|
3355
|
+
for (const tc of tcElements) {
|
|
3356
|
+
const cellText2 = collectTextRuns(tc, "w:t").join("").replace(/\r?\n/g, " ").trim();
|
|
3357
|
+
rowCells.push(cellText2);
|
|
3358
|
+
}
|
|
3359
|
+
if (rowCells.length > maxCols) maxCols = rowCells.length;
|
|
3360
|
+
rows.push(rowCells);
|
|
3361
|
+
}
|
|
3362
|
+
if (rows.length > 0) {
|
|
3363
|
+
out.push({
|
|
3364
|
+
tableIndex: i + 1,
|
|
3365
|
+
rowCount: rows.length,
|
|
3366
|
+
colCount: maxCols,
|
|
3367
|
+
rows
|
|
3368
|
+
});
|
|
3369
|
+
}
|
|
3370
|
+
}
|
|
3371
|
+
return out;
|
|
3372
|
+
}
|
|
3373
|
+
function formatDocxTables(tables, opts) {
|
|
3374
|
+
if (tables.length === 0) return "no tables found in document";
|
|
3375
|
+
const selected = opts?.tableIndex !== void 0 ? tables.filter((t) => t.tableIndex === opts.tableIndex) : tables;
|
|
3376
|
+
if (selected.length === 0) {
|
|
3377
|
+
return `table ${opts?.tableIndex} not found (document has ${tables.length} table${tables.length === 1 ? "" : "s"})`;
|
|
3378
|
+
}
|
|
3379
|
+
const sections = [];
|
|
3380
|
+
for (const table of selected) {
|
|
3381
|
+
const lines2 = [
|
|
3382
|
+
`Table ${table.tableIndex} (${table.rowCount} row${table.rowCount === 1 ? "" : "s"} x ${table.colCount} col${table.colCount === 1 ? "" : "s"}):`
|
|
3383
|
+
];
|
|
3384
|
+
if (table.rows.length === 0) {
|
|
3385
|
+
lines2.push(" (empty)");
|
|
3386
|
+
sections.push(lines2.join("\n"));
|
|
3387
|
+
continue;
|
|
3388
|
+
}
|
|
3389
|
+
const header = table.rows[0];
|
|
3390
|
+
const padCols = Math.max(table.colCount, 1);
|
|
3391
|
+
const paddedHeader = [...header];
|
|
3392
|
+
while (paddedHeader.length < padCols) paddedHeader.push("");
|
|
3393
|
+
lines2.push("| " + paddedHeader.map((c) => displaySafeText(c.replace(/\|/g, "\\|"))).join(" | ") + " |");
|
|
3394
|
+
lines2.push("| " + Array.from({ length: padCols }, () => "---").join(" | ") + " |");
|
|
3395
|
+
for (let r = 1; r < table.rows.length; r++) {
|
|
3396
|
+
const row = table.rows[r];
|
|
3397
|
+
const paddedRow = [...row];
|
|
3398
|
+
while (paddedRow.length < padCols) paddedRow.push("");
|
|
3399
|
+
lines2.push("| " + paddedRow.map((c) => displaySafeText(c.replace(/\|/g, "\\|"))).join(" | ") + " |");
|
|
3400
|
+
}
|
|
3401
|
+
sections.push(lines2.join("\n"));
|
|
3402
|
+
}
|
|
3403
|
+
return sections.join("\n\n");
|
|
3404
|
+
}
|
|
3288
3405
|
|
|
3289
3406
|
// src/pptx_extract.ts
|
|
3290
3407
|
init_define_import_meta_env();
|
|
@@ -3309,9 +3426,9 @@ function tableRowBlocks(parsedSlide) {
|
|
|
3309
3426
|
}
|
|
3310
3427
|
return blocks;
|
|
3311
3428
|
}
|
|
3312
|
-
async function slidePathsInPresentationOrder(entries) {
|
|
3313
|
-
const presXml = decodeZipEntry(entries, "ppt/presentation.xml");
|
|
3314
|
-
const relsXml = decodeZipEntry(entries, "ppt/_rels/presentation.xml.rels");
|
|
3429
|
+
async function slidePathsInPresentationOrder(entries, budget) {
|
|
3430
|
+
const presXml = decodeZipEntry(entries, "ppt/presentation.xml", budget);
|
|
3431
|
+
const relsXml = decodeZipEntry(entries, "ppt/_rels/presentation.xml.rels", budget);
|
|
3315
3432
|
if (presXml === null || relsXml === null) return null;
|
|
3316
3433
|
const presParsed = await parseOoxmlPart(presXml);
|
|
3317
3434
|
const relsParsed = await parseOoxmlPart(relsXml);
|
|
@@ -3322,24 +3439,28 @@ async function slidePathsInPresentationOrder(entries) {
|
|
|
3322
3439
|
}
|
|
3323
3440
|
}
|
|
3324
3441
|
const ordered = [];
|
|
3442
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3325
3443
|
for (const sldId of collectElements(presParsed, "p:sldId")) {
|
|
3326
3444
|
const rid = sldId["@_r:id"];
|
|
3327
3445
|
if (rid === void 0) continue;
|
|
3328
3446
|
const target = ridToTarget.get(rid);
|
|
3329
3447
|
if (target === void 0) continue;
|
|
3330
3448
|
const normalized = target.startsWith("slides/") ? `ppt/${target}` : `ppt/slides/${target}`;
|
|
3331
|
-
if (entries[normalized]
|
|
3449
|
+
if (entries[normalized] === void 0 || seen.has(normalized)) continue;
|
|
3450
|
+
seen.add(normalized);
|
|
3451
|
+
ordered.push(normalized);
|
|
3332
3452
|
}
|
|
3333
3453
|
return ordered.length > 0 ? ordered : null;
|
|
3334
3454
|
}
|
|
3335
3455
|
async function listSlideParts(filePath) {
|
|
3336
3456
|
const entries = await readOoxmlZip(filePath, ".pptx");
|
|
3337
|
-
const
|
|
3457
|
+
const budget = ooxmlPartBudget();
|
|
3458
|
+
const slidePaths = await slidePathsInPresentationOrder(entries, budget) ?? sortNumberedParts(
|
|
3338
3459
|
Object.keys(entries).filter((p) => /^ppt\/slides\/slide\d+\.xml$/.test(p)),
|
|
3339
3460
|
/slide(\d+)\.xml$/
|
|
3340
3461
|
);
|
|
3341
|
-
if (slidePaths.length === 0) throw new
|
|
3342
|
-
return { entries, slidePaths };
|
|
3462
|
+
if (slidePaths.length === 0) throw new NotAnOfficeDocumentError(`no slides found in ${filePath} (not a valid .pptx?)`);
|
|
3463
|
+
return { entries, slidePaths, budget };
|
|
3343
3464
|
}
|
|
3344
3465
|
function relsPathFor(slidePath) {
|
|
3345
3466
|
const idx = slidePath.lastIndexOf("/");
|
|
@@ -3356,8 +3477,8 @@ function resolveRelativeTarget(basePath, target) {
|
|
|
3356
3477
|
}
|
|
3357
3478
|
return resolved.join("/");
|
|
3358
3479
|
}
|
|
3359
|
-
async function notesPathFor(entries, slidePath) {
|
|
3360
|
-
const relsXml = decodeZipEntry(entries, relsPathFor(slidePath));
|
|
3480
|
+
async function notesPathFor(entries, slidePath, budget) {
|
|
3481
|
+
const relsXml = decodeZipEntry(entries, relsPathFor(slidePath), budget);
|
|
3361
3482
|
if (relsXml === null) return null;
|
|
3362
3483
|
const relsParsed = await parseOoxmlPart(relsXml);
|
|
3363
3484
|
for (const rel of collectElements(relsParsed, "Relationship")) {
|
|
@@ -3367,26 +3488,27 @@ async function notesPathFor(entries, slidePath) {
|
|
|
3367
3488
|
}
|
|
3368
3489
|
return null;
|
|
3369
3490
|
}
|
|
3370
|
-
async function parseSlide(entries,
|
|
3371
|
-
const xml = decodeZipEntry(entries,
|
|
3372
|
-
if (xml === null) throw new Error(`missing part: ${
|
|
3491
|
+
async function parseSlide(entries, path22, budget) {
|
|
3492
|
+
const xml = decodeZipEntry(entries, path22, budget);
|
|
3493
|
+
if (xml === null) throw new Error(`missing part: ${path22}`);
|
|
3373
3494
|
return parseOoxmlPart(xml);
|
|
3374
3495
|
}
|
|
3375
|
-
async function notesTextFor(entries, notesPath) {
|
|
3496
|
+
async function notesTextFor(entries, notesPath, budget) {
|
|
3376
3497
|
if (notesPath === null) return "";
|
|
3377
|
-
const xml = decodeZipEntry(entries, notesPath);
|
|
3498
|
+
const xml = decodeZipEntry(entries, notesPath, budget);
|
|
3378
3499
|
if (xml === null) return "";
|
|
3379
3500
|
const parsed = await parseOoxmlPart(xml);
|
|
3380
3501
|
const shapes = collectElements(parsed, "p:sp");
|
|
3381
3502
|
const bodyShape = shapes.find((s) => shapePlaceholderType(s) === "body");
|
|
3382
3503
|
return bodyShape !== void 0 ? shapeText(bodyShape) : collectTextRuns(parsed, "a:t").join(" ").trim();
|
|
3383
3504
|
}
|
|
3384
|
-
async function pptxOutline(filePath) {
|
|
3385
|
-
const { entries, slidePaths } = await listSlideParts(filePath);
|
|
3505
|
+
async function pptxOutline(filePath, deadline = ooxmlWorkDeadline()) {
|
|
3506
|
+
const { entries, slidePaths, budget } = await listSlideParts(filePath);
|
|
3386
3507
|
const out = [];
|
|
3387
3508
|
for (let i = 0; i < slidePaths.length; i++) {
|
|
3388
|
-
|
|
3389
|
-
const
|
|
3509
|
+
assertOoxmlWithinDeadline(deadline, "Narrow the read to specific slides with pptx-slide, or use a smaller deck.");
|
|
3510
|
+
const path22 = slidePaths[i];
|
|
3511
|
+
const parsed = await parseSlide(entries, path22, budget);
|
|
3390
3512
|
const shapes = slideShapes(parsed);
|
|
3391
3513
|
const titleShape = shapes.find((s) => {
|
|
3392
3514
|
const t = shapePlaceholderType(s);
|
|
@@ -3395,49 +3517,63 @@ async function pptxOutline(filePath) {
|
|
|
3395
3517
|
const title = titleShape !== void 0 ? shapeText(titleShape) : "";
|
|
3396
3518
|
const allText = collectTextRuns(parsed, "a:t").join(" ");
|
|
3397
3519
|
const bodyChars = Math.max(0, allText.length - title.length);
|
|
3398
|
-
const hasNotes = (await notesTextFor(entries, await notesPathFor(entries,
|
|
3520
|
+
const hasNotes = (await notesTextFor(entries, await notesPathFor(entries, path22, budget), budget)).length > 0;
|
|
3399
3521
|
out.push({ slide: i + 1, title, bodyChars, hasNotes });
|
|
3400
3522
|
}
|
|
3401
3523
|
return out;
|
|
3402
3524
|
}
|
|
3403
|
-
async function
|
|
3404
|
-
const { entries, slidePaths } = await listSlideParts(filePath);
|
|
3525
|
+
async function slideTextFromParts(entries, slidePaths, slideNumber, includeNotes, budget) {
|
|
3405
3526
|
if (slideNumber < 1 || slideNumber > slidePaths.length) {
|
|
3406
3527
|
throw new Error(`slide ${slideNumber} out of range (this deck has ${slidePaths.length} slides)`);
|
|
3407
3528
|
}
|
|
3408
|
-
const
|
|
3409
|
-
const parsed = await parseSlide(entries,
|
|
3529
|
+
const path22 = slidePaths[slideNumber - 1];
|
|
3530
|
+
const parsed = await parseSlide(entries, path22, budget);
|
|
3410
3531
|
const shapes = slideShapes(parsed);
|
|
3411
3532
|
const blocks = [...shapes.map(shapeText).filter((t) => t.length > 0), ...tableRowBlocks(parsed)];
|
|
3412
3533
|
const lines2 = [`# Slide ${slideNumber}`, ...blocks];
|
|
3413
3534
|
if (includeNotes) {
|
|
3414
|
-
const notes = await notesTextFor(entries, await notesPathFor(entries,
|
|
3535
|
+
const notes = await notesTextFor(entries, await notesPathFor(entries, path22, budget), budget);
|
|
3415
3536
|
if (notes.length > 0) lines2.push("", "## Speaker notes", notes);
|
|
3416
3537
|
}
|
|
3417
3538
|
return lines2.join("\n\n");
|
|
3418
3539
|
}
|
|
3419
|
-
async function
|
|
3420
|
-
const { entries, slidePaths } = await listSlideParts(filePath);
|
|
3540
|
+
async function pptxSlideText(filePath, slideNumber, includeNotes) {
|
|
3541
|
+
const { entries, slidePaths, budget } = await listSlideParts(filePath);
|
|
3542
|
+
return slideTextFromParts(entries, slidePaths, slideNumber, includeNotes, budget);
|
|
3543
|
+
}
|
|
3544
|
+
async function pptxAllSlidesText(filePath, includeNotes, deadline = ooxmlWorkDeadline()) {
|
|
3545
|
+
const { entries, slidePaths, budget } = await listSlideParts(filePath);
|
|
3546
|
+
const slideTexts = [];
|
|
3547
|
+
for (let i = 1; i <= slidePaths.length; i++) {
|
|
3548
|
+
assertOoxmlWithinDeadline(deadline, "Narrow the read to specific slides with pptx-slide, or use a smaller deck.");
|
|
3549
|
+
slideTexts.push(await slideTextFromParts(entries, slidePaths, i, includeNotes, budget));
|
|
3550
|
+
}
|
|
3551
|
+
return slideTexts.join("\n\n");
|
|
3552
|
+
}
|
|
3553
|
+
async function pptxNotesText(filePath, slideNumber, deadline = ooxmlWorkDeadline()) {
|
|
3554
|
+
const { entries, slidePaths, budget } = await listSlideParts(filePath);
|
|
3421
3555
|
const targets = slideNumber !== void 0 ? [slideNumber] : slidePaths.map((_, i) => i + 1);
|
|
3422
3556
|
const sections = [];
|
|
3423
3557
|
for (const n of targets) {
|
|
3558
|
+
assertOoxmlWithinDeadline(deadline, "Narrow the read to one slide with the slide-number argument, or use a smaller deck.");
|
|
3424
3559
|
if (n < 1 || n > slidePaths.length) throw new Error(`slide ${n} out of range (this deck has ${slidePaths.length} slides)`);
|
|
3425
|
-
const notesPath = await notesPathFor(entries, slidePaths[n - 1]);
|
|
3426
|
-
const text = await notesTextFor(entries, notesPath);
|
|
3560
|
+
const notesPath = await notesPathFor(entries, slidePaths[n - 1], budget);
|
|
3561
|
+
const text = await notesTextFor(entries, notesPath, budget);
|
|
3427
3562
|
if (text.length > 0) sections.push(`# Slide ${n} notes
|
|
3428
3563
|
|
|
3429
3564
|
${text}`);
|
|
3430
3565
|
}
|
|
3431
3566
|
return sections.join("\n\n");
|
|
3432
3567
|
}
|
|
3433
|
-
async function pptxTextGrep(filePath, pattern) {
|
|
3434
|
-
const { entries, slidePaths } = await listSlideParts(filePath);
|
|
3568
|
+
async function pptxTextGrep(filePath, pattern, deadline = ooxmlWorkDeadline()) {
|
|
3569
|
+
const { entries, slidePaths, budget } = await listSlideParts(filePath);
|
|
3435
3570
|
const guarded = compileGuardedRegex(pattern, "i");
|
|
3436
3571
|
if (!guarded.ok) throw new Error(`invalid --grep pattern: ${pattern} -- the pattern ${guarded.reason}`);
|
|
3437
3572
|
const re = guarded.re;
|
|
3438
3573
|
const out = [];
|
|
3439
3574
|
for (let i = 0; i < slidePaths.length; i++) {
|
|
3440
|
-
|
|
3575
|
+
assertOoxmlWithinDeadline(deadline, "Narrow the read to specific slides with pptx-slide, or use a smaller deck.");
|
|
3576
|
+
const parsed = await parseSlide(entries, slidePaths[i], budget);
|
|
3441
3577
|
const text = collectTextRuns(parsed, "a:t").join(" ");
|
|
3442
3578
|
if (re.test(text)) {
|
|
3443
3579
|
const idx = text.search(re);
|
|
@@ -5176,13 +5312,30 @@ var parse = function(data, opts = {}) {
|
|
|
5176
5312
|
};
|
|
5177
5313
|
|
|
5178
5314
|
// src/csv_query.ts
|
|
5315
|
+
var DELIMITER_CANDIDATES = [",", " ", ";", "|"];
|
|
5316
|
+
var DELIMITER_SAMPLE_ROWS = 5;
|
|
5317
|
+
function detectDelimiter(content) {
|
|
5318
|
+
const slice = content.slice(0, 1e4);
|
|
5319
|
+
if (slice.trim() === "") return ",";
|
|
5320
|
+
for (const cand of DELIMITER_CANDIDATES) {
|
|
5321
|
+
let rows;
|
|
5322
|
+
try {
|
|
5323
|
+
rows = parse(slice, { columns: false, skip_empty_lines: true, trim: true, delimiter: cand, bom: true, relax_quotes: true, to: DELIMITER_SAMPLE_ROWS });
|
|
5324
|
+
} catch {
|
|
5325
|
+
continue;
|
|
5326
|
+
}
|
|
5327
|
+
if ((rows[0]?.length ?? 0) < 2) continue;
|
|
5328
|
+
return cand;
|
|
5329
|
+
}
|
|
5330
|
+
return ",";
|
|
5331
|
+
}
|
|
5179
5332
|
function parseRecords(content, opts) {
|
|
5180
|
-
const delimiter = opts.delimiter ??
|
|
5333
|
+
const delimiter = opts.delimiter ?? detectDelimiter(content);
|
|
5181
5334
|
if (opts.noHeader === true) {
|
|
5182
|
-
const rows = parse(content, { columns: false, skip_empty_lines: true, trim: true, delimiter, bom: true, relax_column_count: true });
|
|
5335
|
+
const rows = parse(content, { columns: false, skip_empty_lines: true, trim: true, delimiter, bom: true, relax_column_count: true, relax_quotes: true });
|
|
5183
5336
|
return rows.map((row) => Object.fromEntries(row.map((cell, i) => [`col${i + 1}`, cell])));
|
|
5184
5337
|
}
|
|
5185
|
-
const header = csvHeader(content, opts);
|
|
5338
|
+
const header = csvHeader(content, { ...opts, delimiter });
|
|
5186
5339
|
const dupes = header.filter((name, i) => name !== "" && header.indexOf(name) !== i);
|
|
5187
5340
|
if (dupes.length > 0) {
|
|
5188
5341
|
const unique = [...new Set(dupes)];
|
|
@@ -5190,13 +5343,13 @@ function parseRecords(content, opts) {
|
|
|
5190
5343
|
`duplicate column ${unique.length === 1 ? "name" : "names"} in header: ${unique.join(", ")} \u2014 rename the duplicates or pass --no-header to address columns positionally as col1, col2, \u2026`
|
|
5191
5344
|
);
|
|
5192
5345
|
}
|
|
5193
|
-
return parse(content, { columns: true, skip_empty_lines: true, trim: true, delimiter, bom: true, relax_column_count: true });
|
|
5346
|
+
return parse(content, { columns: true, skip_empty_lines: true, trim: true, delimiter, bom: true, relax_column_count: true, relax_quotes: true });
|
|
5194
5347
|
}
|
|
5195
5348
|
function csvHeader(content, opts) {
|
|
5196
5349
|
if (opts.noHeader === true) return [];
|
|
5197
|
-
const delimiter = opts.delimiter ??
|
|
5350
|
+
const delimiter = opts.delimiter ?? detectDelimiter(content);
|
|
5198
5351
|
try {
|
|
5199
|
-
const rows = parse(content, { columns: false, skip_empty_lines: true, trim: true, delimiter, bom: true, to: 1 });
|
|
5352
|
+
const rows = parse(content, { columns: false, skip_empty_lines: true, trim: true, delimiter, bom: true, to: 1, relax_quotes: true });
|
|
5200
5353
|
return rows[0] ?? [];
|
|
5201
5354
|
} catch {
|
|
5202
5355
|
return [];
|
|
@@ -5580,30 +5733,39 @@ function makeWorksheet(name, data) {
|
|
|
5580
5733
|
}
|
|
5581
5734
|
};
|
|
5582
5735
|
}
|
|
5583
|
-
async function readXlsxWorkbook(filePath) {
|
|
5736
|
+
async function readXlsxWorkbook(filePath, deadline = ooxmlWorkDeadline()) {
|
|
5584
5737
|
const entries = await readOoxmlZip(filePath, ".xlsx");
|
|
5585
|
-
const
|
|
5586
|
-
|
|
5738
|
+
const budget = ooxmlPartBudget();
|
|
5739
|
+
const workbookXml = decodeZipEntry(entries, "xl/workbook.xml", budget);
|
|
5740
|
+
if (workbookXml === null) throw new NotAnOfficeDocumentError(`not a valid .xlsx file: ${filePath}`);
|
|
5587
5741
|
const workbookRoot = await parseOoxmlPart(workbookXml);
|
|
5588
5742
|
const wbNode = workbookRoot?.["workbook"];
|
|
5589
5743
|
if (wbNode === void 0 || wbNode === null || typeof wbNode !== "object") {
|
|
5590
|
-
throw new
|
|
5744
|
+
throw new NotAnOfficeDocumentError(`not a valid .xlsx file: ${filePath}`);
|
|
5591
5745
|
}
|
|
5592
5746
|
const wb = wbNode;
|
|
5593
5747
|
const date1904 = isTruthyAttr(attr(wb["workbookPr"], "date1904"));
|
|
5594
|
-
const relsXml = decodeZipEntry(entries, "xl/_rels/workbook.xml.rels");
|
|
5748
|
+
const relsXml = decodeZipEntry(entries, "xl/_rels/workbook.xml.rels", budget);
|
|
5595
5749
|
const rels = parseWorkbookRels(relsXml === null ? null : await parseOoxmlPart(relsXml));
|
|
5596
|
-
const sharedXml = decodeZipEntry(entries, "xl/sharedStrings.xml");
|
|
5750
|
+
const sharedXml = decodeZipEntry(entries, "xl/sharedStrings.xml", budget);
|
|
5597
5751
|
const shared = sharedXml === null ? [] : parseSharedStrings(await parseOoxmlPart(sharedXml));
|
|
5598
|
-
const stylesXml = decodeZipEntry(entries, "xl/styles.xml");
|
|
5752
|
+
const stylesXml = decodeZipEntry(entries, "xl/styles.xml", budget);
|
|
5599
5753
|
const styles = parseStyles(stylesXml, stylesXml === null ? null : await parseOoxmlPart(stylesXml));
|
|
5600
5754
|
const worksheets = [];
|
|
5755
|
+
const parsedByPart = /* @__PURE__ */ new Map();
|
|
5601
5756
|
for (const sheet of asArray(wb["sheets"]?.["sheet"])) {
|
|
5602
5757
|
const name = attr(sheet, "name") ?? "";
|
|
5603
5758
|
const rid = attr(sheet, "r:id") ?? attr(sheet, "relationshipId");
|
|
5604
5759
|
const partPath = rid === void 0 ? void 0 : rels.get(rid);
|
|
5605
|
-
const
|
|
5760
|
+
const cached = partPath === void 0 ? void 0 : parsedByPart.get(partPath);
|
|
5761
|
+
if (cached !== void 0) {
|
|
5762
|
+
worksheets.push(makeWorksheet(name, cached));
|
|
5763
|
+
continue;
|
|
5764
|
+
}
|
|
5765
|
+
assertOoxmlWithinDeadline(deadline, "Narrow the read to specific sheets with xlsx-head, or use a smaller workbook.");
|
|
5766
|
+
const sheetXml = partPath === void 0 ? null : decodeZipEntry(entries, partPath, budget);
|
|
5606
5767
|
const data = sheetXml === null ? { cells: /* @__PURE__ */ new Map(), rowCount: 0, columnCount: 0, populatedRows: 0 } : parseSheet(await parseOoxmlPart(sheetXml), shared, styles, date1904);
|
|
5768
|
+
if (partPath !== void 0) parsedByPart.set(partPath, data);
|
|
5607
5769
|
worksheets.push(makeWorksheet(name, data));
|
|
5608
5770
|
}
|
|
5609
5771
|
return {
|
|
@@ -5615,17 +5777,27 @@ async function readXlsxWorkbook(filePath) {
|
|
|
5615
5777
|
// src/xlsx_extract.ts
|
|
5616
5778
|
var loadWorkbook = readXlsxWorkbook;
|
|
5617
5779
|
function requireSheet(wb, sheetName) {
|
|
5618
|
-
|
|
5619
|
-
|
|
5620
|
-
|
|
5780
|
+
if (sheetName !== void 0 && sheetName.trim() !== "") {
|
|
5781
|
+
const ws = wb.getWorksheet(sheetName);
|
|
5782
|
+
if (ws === void 0) {
|
|
5783
|
+
throw new Error(`unknown sheet: ${sheetName} (available: ${wb.worksheets.map((s) => s.name).join(", ")})`);
|
|
5784
|
+
}
|
|
5785
|
+
return ws;
|
|
5786
|
+
}
|
|
5787
|
+
const first = wb.worksheets[0];
|
|
5788
|
+
if (first === void 0) {
|
|
5789
|
+
throw new Error("workbook contains no worksheets");
|
|
5621
5790
|
}
|
|
5622
|
-
return
|
|
5791
|
+
return first;
|
|
5623
5792
|
}
|
|
5624
5793
|
var MAX_XLSX_SCAN_CELLS = 2e7;
|
|
5625
|
-
|
|
5626
|
-
|
|
5627
|
-
|
|
5794
|
+
var XlsxScanTooLargeError = class extends DocumentRefusedError {
|
|
5795
|
+
constructor(extent, cells, hint) {
|
|
5796
|
+
super(`${extent} (${cells.toLocaleString()} cells), over the ${MAX_XLSX_SCAN_CELLS.toLocaleString()}-cell scan limit; ${hint}`, "XlsxScanTooLargeError");
|
|
5628
5797
|
}
|
|
5798
|
+
};
|
|
5799
|
+
function assertCellCount(cells, extent, hint) {
|
|
5800
|
+
if (cells > MAX_XLSX_SCAN_CELLS) throw new XlsxScanTooLargeError(extent, cells, hint);
|
|
5629
5801
|
}
|
|
5630
5802
|
function assertScannableExtent(ws) {
|
|
5631
5803
|
assertCellCount(
|
|
@@ -5716,44 +5888,76 @@ function usedRange(ws) {
|
|
|
5716
5888
|
}
|
|
5717
5889
|
return { ref: `A1:${indexToColLetters(cols)}${rows}`, rows, cols };
|
|
5718
5890
|
}
|
|
5719
|
-
async function listSheets(filePath) {
|
|
5720
|
-
const wb = await loadWorkbook(filePath);
|
|
5891
|
+
async function listSheets(filePath, deadline = ooxmlWorkDeadline()) {
|
|
5892
|
+
const wb = await loadWorkbook(filePath, deadline);
|
|
5721
5893
|
return wb.worksheets.map((ws) => {
|
|
5894
|
+
assertOoxmlWithinDeadline(deadline, "Narrow the read to specific sheets with xlsx-head, or use a smaller workbook.");
|
|
5722
5895
|
const { ref: ref2, rows, cols } = usedRange(ws);
|
|
5723
5896
|
return { name: ws.name, ref: ref2, rows, cols };
|
|
5724
5897
|
});
|
|
5725
5898
|
}
|
|
5726
|
-
|
|
5727
|
-
const wb = await loadWorkbook(filePath);
|
|
5728
|
-
const ws = requireSheet(wb, sheetName);
|
|
5899
|
+
function headSheetFromWorksheet(ws, rows, columns) {
|
|
5729
5900
|
assertScannableExtent(ws);
|
|
5730
5901
|
const rowCount = ws.rowCount || 0;
|
|
5902
|
+
const sheetCols = ws.columnCount || 0;
|
|
5903
|
+
const scanRows = Math.min(rowCount, rows + 1);
|
|
5731
5904
|
const aoa = [];
|
|
5732
|
-
let
|
|
5733
|
-
for (let r = 1; r <= rowCount; r++) {
|
|
5734
|
-
const row = ws.getRow(r);
|
|
5735
|
-
let maxCol = 0;
|
|
5736
|
-
row.eachCell({ includeEmpty: false }, (_c, colNumber) => {
|
|
5737
|
-
if (colNumber > maxCol) maxCol = colNumber;
|
|
5738
|
-
});
|
|
5739
|
-
if (maxCol > sheetCols) sheetCols = maxCol;
|
|
5905
|
+
for (let r = 1; r <= scanRows; r++) {
|
|
5740
5906
|
const rowVals = [];
|
|
5741
|
-
for (let c = 1; c <=
|
|
5907
|
+
for (let c = 1; c <= sheetCols; c++) {
|
|
5742
5908
|
rowVals.push(cellText(ws.getCell(encodeCell({ r, c }))));
|
|
5743
5909
|
}
|
|
5744
5910
|
aoa.push(rowVals);
|
|
5745
5911
|
}
|
|
5746
|
-
|
|
5747
|
-
|
|
5912
|
+
let header = Array.from({ length: sheetCols }, (_, i) => String(aoa[0]?.[i] ?? ""));
|
|
5913
|
+
let dataRows = aoa.slice(1, 1 + rows).map(
|
|
5748
5914
|
(r) => Array.from({ length: sheetCols }, (_, i) => String(r[i] ?? ""))
|
|
5749
5915
|
);
|
|
5916
|
+
if (columns !== void 0 && columns.length > 0) {
|
|
5917
|
+
const normCols = columns.map((c) => c.trim()).filter(Boolean);
|
|
5918
|
+
const selectedIndices = [];
|
|
5919
|
+
for (const req of normCols) {
|
|
5920
|
+
let idx = header.findIndex((h) => h === req);
|
|
5921
|
+
if (idx === -1) {
|
|
5922
|
+
idx = header.findIndex((h) => h.toLowerCase() === req.toLowerCase());
|
|
5923
|
+
}
|
|
5924
|
+
if (idx === -1 && /^[A-Za-z]+$/.test(req)) {
|
|
5925
|
+
const colNum = colLettersToIndex(req);
|
|
5926
|
+
if (colNum >= 1 && colNum <= sheetCols) {
|
|
5927
|
+
idx = colNum - 1;
|
|
5928
|
+
}
|
|
5929
|
+
}
|
|
5930
|
+
if (idx !== -1 && !selectedIndices.includes(idx)) {
|
|
5931
|
+
selectedIndices.push(idx);
|
|
5932
|
+
} else if (idx === -1) {
|
|
5933
|
+
throw new Error(`unknown column: ${req} (available: ${header.filter(Boolean).join(", ")})`);
|
|
5934
|
+
}
|
|
5935
|
+
}
|
|
5936
|
+
header = selectedIndices.map((i) => header[i] ?? "");
|
|
5937
|
+
dataRows = dataRows.map((r) => selectedIndices.map((i) => r[i] ?? ""));
|
|
5938
|
+
}
|
|
5750
5939
|
const lines2 = [header.map(quoteCsvCell).join(",")];
|
|
5751
5940
|
for (const r of dataRows) lines2.push(r.map(quoteCsvCell).join(","));
|
|
5752
|
-
if (
|
|
5753
|
-
lines2.push(`...(${
|
|
5941
|
+
if (rowCount - 1 > dataRows.length) {
|
|
5942
|
+
lines2.push(`...(${rowCount - 1 - dataRows.length} more rows elided; use --rows to see more, or xlsx-query for filtering)`);
|
|
5754
5943
|
}
|
|
5755
5944
|
return lines2.join("\n");
|
|
5756
5945
|
}
|
|
5946
|
+
async function headSheet(filePath, sheetName, rows = 20, columns) {
|
|
5947
|
+
const wb = await loadWorkbook(filePath);
|
|
5948
|
+
const ws = requireSheet(wb, sheetName);
|
|
5949
|
+
return headSheetFromWorksheet(ws, rows, columns);
|
|
5950
|
+
}
|
|
5951
|
+
async function allSheetsHeadText(filePath, rows, deadline = ooxmlWorkDeadline()) {
|
|
5952
|
+
const wb = await loadWorkbook(filePath, deadline);
|
|
5953
|
+
const sheetTexts = [];
|
|
5954
|
+
for (const ws of wb.worksheets) {
|
|
5955
|
+
assertOoxmlWithinDeadline(deadline, "Narrow the read to specific sheets with xlsx-head, or use a smaller workbook.");
|
|
5956
|
+
sheetTexts.push(`# Sheet: ${ws.name}
|
|
5957
|
+
${headSheetFromWorksheet(ws, rows)}`);
|
|
5958
|
+
}
|
|
5959
|
+
return sheetTexts.join("\n\n");
|
|
5960
|
+
}
|
|
5757
5961
|
async function rangeSheet(filePath, sheetName, rangeSpec, showFormulas) {
|
|
5758
5962
|
const wb = await loadWorkbook(filePath);
|
|
5759
5963
|
const ws = requireSheet(wb, sheetName);
|
|
@@ -5802,6 +6006,60 @@ async function querySheet(filePath, sheetName, opts) {
|
|
|
5802
6006
|
const csvText = await sheetToCsv(ws);
|
|
5803
6007
|
return queryCsv(csvText, opts);
|
|
5804
6008
|
}
|
|
6009
|
+
async function xlsxColumns(filePath, sheetName, sampleLimit = 100) {
|
|
6010
|
+
const wb = await loadWorkbook(filePath);
|
|
6011
|
+
const ws = requireSheet(wb, sheetName);
|
|
6012
|
+
assertScannableExtent(ws);
|
|
6013
|
+
const rowCount = ws.rowCount || 0;
|
|
6014
|
+
const { cols: sheetCols } = usedRange(ws);
|
|
6015
|
+
if (rowCount === 0 || sheetCols === 0) {
|
|
6016
|
+
return { sheetName: ws.name, totalSheetRows: 0, sampleRows: 0, columns: [] };
|
|
6017
|
+
}
|
|
6018
|
+
const sampleRows = Math.min(Math.max(1, sampleLimit), Math.max(1, rowCount - 1));
|
|
6019
|
+
const headers = [];
|
|
6020
|
+
for (let c = 1; c <= sheetCols; c++) {
|
|
6021
|
+
const val = cellText(ws.getCell(encodeCell({ r: 1, c })));
|
|
6022
|
+
headers.push(val || `(col ${indexToColLetters(c)})`);
|
|
6023
|
+
}
|
|
6024
|
+
const colSummaries = headers.map((name, i) => ({
|
|
6025
|
+
letter: indexToColLetters(i + 1),
|
|
6026
|
+
name,
|
|
6027
|
+
index: i + 1,
|
|
6028
|
+
nonEmptyRows: 0,
|
|
6029
|
+
sampleRows,
|
|
6030
|
+
sampleValues: []
|
|
6031
|
+
}));
|
|
6032
|
+
for (let r = 2; r <= 1 + sampleRows; r++) {
|
|
6033
|
+
for (let c = 1; c <= sheetCols; c++) {
|
|
6034
|
+
const txt = cellText(ws.getCell(encodeCell({ r, c })));
|
|
6035
|
+
if (txt !== "") {
|
|
6036
|
+
const summary = colSummaries[c - 1];
|
|
6037
|
+
summary.nonEmptyRows++;
|
|
6038
|
+
if (summary.sampleValues.length < 3 && !summary.sampleValues.includes(txt)) {
|
|
6039
|
+
summary.sampleValues.push(txt);
|
|
6040
|
+
}
|
|
6041
|
+
}
|
|
6042
|
+
}
|
|
6043
|
+
}
|
|
6044
|
+
return {
|
|
6045
|
+
sheetName: ws.name,
|
|
6046
|
+
totalSheetRows: Math.max(0, rowCount - 1),
|
|
6047
|
+
sampleRows,
|
|
6048
|
+
columns: colSummaries
|
|
6049
|
+
};
|
|
6050
|
+
}
|
|
6051
|
+
function formatXlsxColumns(result) {
|
|
6052
|
+
if (result.columns.length === 0) return `Sheet "${displaySafeText(result.sheetName)}" is empty`;
|
|
6053
|
+
const lines2 = [
|
|
6054
|
+
`Sheet: ${displaySafeText(result.sheetName)} (${result.totalSheetRows} data rows, ${result.columns.length} columns; sampled first ${result.sampleRows} rows)`
|
|
6055
|
+
];
|
|
6056
|
+
for (const c of result.columns) {
|
|
6057
|
+
const pct = result.sampleRows > 0 ? Math.round(c.nonEmptyRows / c.sampleRows * 100) : 0;
|
|
6058
|
+
const samples = c.sampleValues.length > 0 ? ` (e.g. ${c.sampleValues.map((v) => JSON.stringify(v.length > 30 ? v.slice(0, 27) + "..." : v)).join(", ")})` : " (all empty)";
|
|
6059
|
+
lines2.push(` ${c.letter.padEnd(4)} ${displaySafeText(c.name).padEnd(25)} ${c.nonEmptyRows}/${c.sampleRows} (${pct}%)${samples}`);
|
|
6060
|
+
}
|
|
6061
|
+
return lines2.join("\n");
|
|
6062
|
+
}
|
|
5805
6063
|
|
|
5806
6064
|
// src/doc_embed_extract.ts
|
|
5807
6065
|
init_define_import_meta_env();
|
|
@@ -5827,7 +6085,7 @@ var MAX_OUTLINE_ENTRIES = 2e4;
|
|
|
5827
6085
|
var MAX_OUTLINE_TITLE_CHARS = 500;
|
|
5828
6086
|
var MAX_LOCATE_MATCHES = 1e3;
|
|
5829
6087
|
var MAX_LOCATE_CONTEXT_CHARS = 4e3;
|
|
5830
|
-
var MAX_PDF_WORK_MILLIS =
|
|
6088
|
+
var MAX_PDF_WORK_MILLIS = MAX_DOCUMENT_WORK_MILLIS;
|
|
5831
6089
|
var PdfRefusedError = class extends DocumentRefusedError {
|
|
5832
6090
|
};
|
|
5833
6091
|
var PdfTooLargeError = class extends PdfRefusedError {
|
|
@@ -5837,7 +6095,7 @@ var PdfTooLargeError = class extends PdfRefusedError {
|
|
|
5837
6095
|
};
|
|
5838
6096
|
var PdfTookTooLongError = class extends PdfRefusedError {
|
|
5839
6097
|
constructor(message) {
|
|
5840
|
-
super(message, "PdfTookTooLongError");
|
|
6098
|
+
super(message, "PdfTookTooLongError", true);
|
|
5841
6099
|
}
|
|
5842
6100
|
};
|
|
5843
6101
|
function pdfWorkDeadline() {
|
|
@@ -5950,9 +6208,13 @@ function reconstructLayout(items, deadline) {
|
|
|
5950
6208
|
for (const item of items) {
|
|
5951
6209
|
if ((checked++ & 1023) === 0 && Date.now() > deadline) throw pdfWorkTookTooLong();
|
|
5952
6210
|
const y = item.transform[5];
|
|
6211
|
+
if (!Number.isFinite(y)) {
|
|
6212
|
+
rows.push([item]);
|
|
6213
|
+
continue;
|
|
6214
|
+
}
|
|
5953
6215
|
const home = bucketOf(y);
|
|
5954
6216
|
let found = -1;
|
|
5955
|
-
for (
|
|
6217
|
+
for (const bucket of [home - 1, home, home + 1]) {
|
|
5956
6218
|
for (const row of byBucket.get(bucket) ?? []) {
|
|
5957
6219
|
const last = rows[row][rows[row].length - 1];
|
|
5958
6220
|
if (Math.abs(last.transform[5] - y) < Y_EPSILON && (found === -1 || row < found)) found = row;
|
|
@@ -5973,7 +6235,16 @@ function reconstructLayout(items, deadline) {
|
|
|
5973
6235
|
place(home, found);
|
|
5974
6236
|
}
|
|
5975
6237
|
}
|
|
5976
|
-
rows.sort((a, b) =>
|
|
6238
|
+
rows.sort((a, b) => {
|
|
6239
|
+
const ya = a[0].transform[5];
|
|
6240
|
+
const yb = b[0].transform[5];
|
|
6241
|
+
const finiteA = Number.isFinite(ya);
|
|
6242
|
+
const finiteB = Number.isFinite(yb);
|
|
6243
|
+
if (finiteA && finiteB) return yb - ya;
|
|
6244
|
+
if (finiteA) return -1;
|
|
6245
|
+
if (finiteB) return 1;
|
|
6246
|
+
return 0;
|
|
6247
|
+
});
|
|
5977
6248
|
const lines2 = [];
|
|
5978
6249
|
for (const row of rows) {
|
|
5979
6250
|
row.sort((a, b) => a.transform[4] - b.transform[4]);
|
|
@@ -6103,16 +6374,17 @@ function locateSnippet(text, index, matchLen, context) {
|
|
|
6103
6374
|
const snippet = text.slice(from, to).replace(/\s+/g, " ").trim();
|
|
6104
6375
|
return matchLen > kept ? `${snippet}...` : snippet;
|
|
6105
6376
|
}
|
|
6106
|
-
async function resolveDestPage(doc, dest) {
|
|
6377
|
+
async function resolveDestPage(doc, dest, deadline) {
|
|
6107
6378
|
let explicitDest = dest;
|
|
6108
6379
|
if (typeof explicitDest === "string") {
|
|
6109
|
-
explicitDest = await doc.getDestination(explicitDest);
|
|
6380
|
+
explicitDest = await raceDeadline(doc.getDestination(explicitDest), deadline);
|
|
6110
6381
|
}
|
|
6111
6382
|
if (!Array.isArray(explicitDest) || explicitDest.length === 0) return null;
|
|
6112
6383
|
try {
|
|
6113
|
-
const pageIndex = await doc.getPageIndex(explicitDest[0]);
|
|
6384
|
+
const pageIndex = await raceDeadline(doc.getPageIndex(explicitDest[0]), deadline);
|
|
6114
6385
|
return pageIndex + 1;
|
|
6115
|
-
} catch {
|
|
6386
|
+
} catch (err) {
|
|
6387
|
+
if (err instanceof PdfTookTooLongError) throw err;
|
|
6116
6388
|
return null;
|
|
6117
6389
|
}
|
|
6118
6390
|
}
|
|
@@ -6129,7 +6401,7 @@ async function extractPdfOutline(data) {
|
|
|
6129
6401
|
for (const item of items) {
|
|
6130
6402
|
if (entries.length >= MAX_OUTLINE_ENTRIES) return;
|
|
6131
6403
|
if (Date.now() > deadline) throw pdfWorkTookTooLong();
|
|
6132
|
-
const page = await resolveDestPage(doc, item.dest);
|
|
6404
|
+
const page = await resolveDestPage(doc, item.dest, deadline);
|
|
6133
6405
|
const title = item.title.trim();
|
|
6134
6406
|
entries.push({ level, title: title.length > MAX_OUTLINE_TITLE_CHARS ? `${title.slice(0, MAX_OUTLINE_TITLE_CHARS)}...` : title, page });
|
|
6135
6407
|
if (item.items.length > 0) await walk(item.items, level + 1);
|
|
@@ -6182,6 +6454,9 @@ function isEmbeddableDocument(filePath) {
|
|
|
6182
6454
|
function isDocumentRefusal(err) {
|
|
6183
6455
|
return err instanceof DocumentRefusedError;
|
|
6184
6456
|
}
|
|
6457
|
+
function isTransientDocumentRefusal(err) {
|
|
6458
|
+
return err instanceof DocumentRefusedError && err.transient;
|
|
6459
|
+
}
|
|
6185
6460
|
async function extractEmbeddableDocumentText(filePath) {
|
|
6186
6461
|
switch (path.extname(filePath).toLowerCase()) {
|
|
6187
6462
|
case ".pdf": {
|
|
@@ -6190,24 +6465,10 @@ async function extractEmbeddableDocumentText(filePath) {
|
|
|
6190
6465
|
}
|
|
6191
6466
|
case ".docx":
|
|
6192
6467
|
return await docxText(filePath);
|
|
6193
|
-
case ".pptx":
|
|
6194
|
-
|
|
6195
|
-
|
|
6196
|
-
|
|
6197
|
-
slideTexts.push(await pptxSlideText(filePath, i, true));
|
|
6198
|
-
}
|
|
6199
|
-
return slideTexts.join("\n\n");
|
|
6200
|
-
}
|
|
6201
|
-
case ".xlsx": {
|
|
6202
|
-
const sheets = await listSheets(filePath);
|
|
6203
|
-
const sheetTexts = [];
|
|
6204
|
-
for (const sheet of sheets) {
|
|
6205
|
-
const body = await headSheet(filePath, sheet.name, XLSX_SHEET_ROW_CAP);
|
|
6206
|
-
sheetTexts.push(`# Sheet: ${sheet.name}
|
|
6207
|
-
${body}`);
|
|
6208
|
-
}
|
|
6209
|
-
return sheetTexts.join("\n\n");
|
|
6210
|
-
}
|
|
6468
|
+
case ".pptx":
|
|
6469
|
+
return await pptxAllSlidesText(filePath, true);
|
|
6470
|
+
case ".xlsx":
|
|
6471
|
+
return await allSheetsHeadText(filePath, XLSX_SHEET_ROW_CAP);
|
|
6211
6472
|
default:
|
|
6212
6473
|
return null;
|
|
6213
6474
|
}
|
|
@@ -7151,20 +7412,26 @@ function queryRefsByContext(context, filePath, dbPath = globalDbPath()) {
|
|
|
7151
7412
|
const rows = db.prepare(sql).all(context, foldPath(filePath));
|
|
7152
7413
|
return rows.map(toRefEntry);
|
|
7153
7414
|
}
|
|
7415
|
+
var REF_COUNT_BATCH = 900;
|
|
7154
7416
|
function queryRefCounts(names, dbPath = globalDbPath(), rootDir) {
|
|
7155
7417
|
const counts = /* @__PURE__ */ new Map();
|
|
7156
7418
|
if (names.length === 0) return counts;
|
|
7157
7419
|
const db = getDb(dbPath);
|
|
7158
|
-
const placeholders = names.map(() => "?").join(", ");
|
|
7159
|
-
const params = [...names];
|
|
7160
7420
|
const scopeWhere = [];
|
|
7161
|
-
|
|
7421
|
+
const scopeParams = [];
|
|
7422
|
+
applyRootDirScope(rootDir, "file_path", scopeWhere, scopeParams);
|
|
7162
7423
|
const scopeSql = scopeWhere.length > 0 ? ` AND ${scopeWhere.join(" AND ")}` : "";
|
|
7163
|
-
const
|
|
7164
|
-
const
|
|
7165
|
-
|
|
7166
|
-
|
|
7167
|
-
|
|
7424
|
+
const sqlFor = (width) => `SELECT name, COUNT(*) as c FROM refs WHERE name IN (${Array.from({ length: width }, () => "?").join(", ")})${scopeSql} GROUP BY name`;
|
|
7425
|
+
const full = names.length >= REF_COUNT_BATCH ? db.prepare(sqlFor(REF_COUNT_BATCH)) : void 0;
|
|
7426
|
+
db.transaction(() => {
|
|
7427
|
+
for (let start = 0; start < names.length; start += REF_COUNT_BATCH) {
|
|
7428
|
+
const batch = names.slice(start, start + REF_COUNT_BATCH);
|
|
7429
|
+
const stmt = batch.length === REF_COUNT_BATCH && full !== void 0 ? full : db.prepare(sqlFor(batch.length));
|
|
7430
|
+
for (const row of stmt.all(...batch, ...scopeParams)) {
|
|
7431
|
+
counts.set(row.name, row.c);
|
|
7432
|
+
}
|
|
7433
|
+
}
|
|
7434
|
+
})();
|
|
7168
7435
|
return counts;
|
|
7169
7436
|
}
|
|
7170
7437
|
function getProjectFileEntries(rootDir, dbPath = globalDbPath()) {
|
|
@@ -7201,9 +7468,9 @@ function getFileEntry(filePath, dbPath = globalDbPath()) {
|
|
|
7201
7468
|
parserSha: row.parser_sha ?? ""
|
|
7202
7469
|
};
|
|
7203
7470
|
}
|
|
7204
|
-
function sanitizeFtsQuery(query,
|
|
7471
|
+
function sanitizeFtsQuery(query, join16 = "AND") {
|
|
7205
7472
|
const terms = query.split(/\s+/).filter((t) => t.length > 0).map((t) => '"' + t.replace(/"/g, '""') + '"');
|
|
7206
|
-
return terms.join(
|
|
7473
|
+
return terms.join(join16 === "OR" ? " OR " : " ");
|
|
7207
7474
|
}
|
|
7208
7475
|
function runFtsQuery(db, match, limit, scope, rootDir) {
|
|
7209
7476
|
const sql = `SELECT s.file_path, s.name, s.kind, s.line_start, s.line_end, s.body, s.docstring, s.parent FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid WHERE symbols_fts MATCH ?${scope !== void 0 ? ` AND ${scope.clause}` : ""} ORDER BY bm25(symbols_fts) LIMIT ?`;
|
|
@@ -7233,230 +7500,14 @@ function searchSymbolsFts(query, limit = 50, dbPath = globalDbPath(), rootDir) {
|
|
|
7233
7500
|
|
|
7234
7501
|
// src/recall_index.ts
|
|
7235
7502
|
init_define_import_meta_env();
|
|
7236
|
-
import * as fs7 from "node:fs";
|
|
7237
|
-
|
|
7238
|
-
// src/disk_cache.ts
|
|
7239
|
-
init_define_import_meta_env();
|
|
7240
7503
|
import * as fs6 from "node:fs";
|
|
7241
|
-
import * as path3 from "node:path";
|
|
7242
|
-
var DEFAULT_MAX_COUNT = 200;
|
|
7243
|
-
var DEFAULT_MAX_AGE_MS = 24 * 3600 * 1e3;
|
|
7244
|
-
function sanitizeId(id) {
|
|
7245
|
-
return sanitizeIdForFilename(id, 64);
|
|
7246
|
-
}
|
|
7247
|
-
function blobDir(subdir) {
|
|
7248
|
-
return path3.join(tokenGoatHome(), subdir);
|
|
7249
|
-
}
|
|
7250
|
-
function blobPath(subdir, id) {
|
|
7251
|
-
const safe = sanitizeId(id);
|
|
7252
|
-
if (!safe) return null;
|
|
7253
|
-
const dir = blobDir(subdir);
|
|
7254
|
-
const candidate = path3.join(dir, `${safe}.json`);
|
|
7255
|
-
try {
|
|
7256
|
-
const rel = path3.relative(dir, candidate);
|
|
7257
|
-
if (rel.startsWith("..")) return null;
|
|
7258
|
-
} catch {
|
|
7259
|
-
return null;
|
|
7260
|
-
}
|
|
7261
|
-
return candidate;
|
|
7262
|
-
}
|
|
7263
|
-
function isBlobStale(subdir, id) {
|
|
7264
|
-
const p = blobPath(subdir, id);
|
|
7265
|
-
if (p === null) return false;
|
|
7266
|
-
try {
|
|
7267
|
-
const stat2 = fs6.statSync(p);
|
|
7268
|
-
return Date.now() - stat2.mtimeMs > DEFAULT_MAX_AGE_MS;
|
|
7269
|
-
} catch {
|
|
7270
|
-
return false;
|
|
7271
|
-
}
|
|
7272
|
-
}
|
|
7273
|
-
function subdirCacheDefaults(subdir) {
|
|
7274
|
-
try {
|
|
7275
|
-
if (subdir === "bash_outputs") {
|
|
7276
|
-
const bc = loadConfig().bash_compress;
|
|
7277
|
-
return { maxCount: bc.cache_max_file_count, maxBytes: bc.cache_max_bytes, maxBytesPerItem: bc.cache_max_bytes_per_output };
|
|
7278
|
-
}
|
|
7279
|
-
if (subdir === "web_outputs") {
|
|
7280
|
-
const wf = loadConfig().webfetch;
|
|
7281
|
-
return { maxCount: wf.max_file_count, maxBytes: wf.max_bytes, maxBytesPerItem: Number.POSITIVE_INFINITY };
|
|
7282
|
-
}
|
|
7283
|
-
} catch {
|
|
7284
|
-
}
|
|
7285
|
-
return { maxCount: DEFAULT_MAX_COUNT, maxBytes: Number.POSITIVE_INFINITY, maxBytesPerItem: Number.POSITIVE_INFINITY };
|
|
7286
|
-
}
|
|
7287
|
-
function storeBlob(subdir, id, value, opts = {}) {
|
|
7288
|
-
const p = blobPath(subdir, id);
|
|
7289
|
-
if (!p) return false;
|
|
7290
|
-
const defaults = subdirCacheDefaults(subdir);
|
|
7291
|
-
const maxBytesPerItem = opts.maxBytesPerItem ?? defaults.maxBytesPerItem;
|
|
7292
|
-
let rawJson;
|
|
7293
|
-
try {
|
|
7294
|
-
rawJson = JSON.stringify(value);
|
|
7295
|
-
} catch {
|
|
7296
|
-
return false;
|
|
7297
|
-
}
|
|
7298
|
-
let json;
|
|
7299
|
-
try {
|
|
7300
|
-
const result = redactSecrets(rawJson);
|
|
7301
|
-
json = result.text;
|
|
7302
|
-
if (result.count > 0) recordStat("secret_redacted", 0, result.count, void 0, subdir);
|
|
7303
|
-
} catch {
|
|
7304
|
-
return false;
|
|
7305
|
-
}
|
|
7306
|
-
if (Number.isFinite(maxBytesPerItem) && Buffer.byteLength(json, "utf-8") > maxBytesPerItem) return false;
|
|
7307
|
-
try {
|
|
7308
|
-
const dir = path3.dirname(p);
|
|
7309
|
-
if (!fs6.existsSync(dir)) ensureDirSync(dir);
|
|
7310
|
-
atomicWriteText(p, json);
|
|
7311
|
-
} catch {
|
|
7312
|
-
return false;
|
|
7313
|
-
}
|
|
7314
|
-
pruneBlobs(
|
|
7315
|
-
subdir,
|
|
7316
|
-
opts.maxCount ?? defaults.maxCount,
|
|
7317
|
-
opts.maxAgeMs ?? DEFAULT_MAX_AGE_MS,
|
|
7318
|
-
opts.maxBytes ?? defaults.maxBytes,
|
|
7319
|
-
p
|
|
7320
|
-
);
|
|
7321
|
-
return true;
|
|
7322
|
-
}
|
|
7323
|
-
function loadBlob(subdir, id) {
|
|
7324
|
-
const p = blobPath(subdir, id);
|
|
7325
|
-
if (!p) return null;
|
|
7326
|
-
try {
|
|
7327
|
-
if (!fs6.existsSync(p)) return null;
|
|
7328
|
-
return JSON.parse(fs6.readFileSync(p, "utf8"));
|
|
7329
|
-
} catch {
|
|
7330
|
-
return null;
|
|
7331
|
-
}
|
|
7332
|
-
}
|
|
7333
|
-
function listBlobs(subdir) {
|
|
7334
|
-
const dir = blobDir(subdir);
|
|
7335
|
-
const out = [];
|
|
7336
|
-
try {
|
|
7337
|
-
if (!fs6.existsSync(dir)) return out;
|
|
7338
|
-
for (const file of fs6.readdirSync(dir)) {
|
|
7339
|
-
if (!file.endsWith(".json")) continue;
|
|
7340
|
-
const id = file.slice(0, -5);
|
|
7341
|
-
let mtime = 0;
|
|
7342
|
-
try {
|
|
7343
|
-
mtime = fs6.statSync(path3.join(dir, file)).mtimeMs;
|
|
7344
|
-
} catch {
|
|
7345
|
-
}
|
|
7346
|
-
const value = loadBlob(subdir, id);
|
|
7347
|
-
if (value !== null) out.push({ id, mtime, value });
|
|
7348
|
-
}
|
|
7349
|
-
} catch {
|
|
7350
|
-
return out;
|
|
7351
|
-
}
|
|
7352
|
-
return out;
|
|
7353
|
-
}
|
|
7354
|
-
function pruneBlobs(subdir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_AGE_MS, maxBytes = Number.POSITIVE_INFINITY, protectedPath) {
|
|
7355
|
-
return pruneBlobDir(blobDir(subdir), maxCount, maxAgeMs, maxBytes, protectedPath);
|
|
7356
|
-
}
|
|
7357
|
-
function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_AGE_MS, maxBytes = Number.POSITIVE_INFINITY, protectedPath) {
|
|
7358
|
-
let removed = 0;
|
|
7359
|
-
try {
|
|
7360
|
-
if (!fs6.existsSync(dir)) return 0;
|
|
7361
|
-
const cutoff = Date.now() - maxAgeMs;
|
|
7362
|
-
let kept = [];
|
|
7363
|
-
let protectedEntry;
|
|
7364
|
-
for (const file of fs6.readdirSync(dir)) {
|
|
7365
|
-
const full = path3.join(dir, file);
|
|
7366
|
-
let stat2;
|
|
7367
|
-
try {
|
|
7368
|
-
stat2 = fs6.statSync(full);
|
|
7369
|
-
} catch {
|
|
7370
|
-
continue;
|
|
7371
|
-
}
|
|
7372
|
-
if (!stat2.isFile()) continue;
|
|
7373
|
-
if (protectedPath !== void 0 && full === protectedPath) {
|
|
7374
|
-
protectedEntry = [full, stat2.mtimeMs, stat2.size];
|
|
7375
|
-
continue;
|
|
7376
|
-
}
|
|
7377
|
-
if (stat2.mtimeMs < cutoff) {
|
|
7378
|
-
try {
|
|
7379
|
-
fs6.unlinkSync(full);
|
|
7380
|
-
removed++;
|
|
7381
|
-
} catch {
|
|
7382
|
-
continue;
|
|
7383
|
-
}
|
|
7384
|
-
} else if (file.endsWith(".json")) {
|
|
7385
|
-
kept.push([full, stat2.mtimeMs, stat2.size]);
|
|
7386
|
-
}
|
|
7387
|
-
}
|
|
7388
|
-
const countBudget = protectedEntry ? Math.max(0, maxCount - 1) : maxCount;
|
|
7389
|
-
if (kept.length > countBudget) {
|
|
7390
|
-
kept.sort((a, b) => a[1] - b[1]);
|
|
7391
|
-
const excess = kept.slice(0, kept.length - countBudget);
|
|
7392
|
-
kept = kept.slice(kept.length - countBudget);
|
|
7393
|
-
for (const [full] of excess) {
|
|
7394
|
-
try {
|
|
7395
|
-
fs6.unlinkSync(full);
|
|
7396
|
-
removed++;
|
|
7397
|
-
} catch {
|
|
7398
|
-
continue;
|
|
7399
|
-
}
|
|
7400
|
-
}
|
|
7401
|
-
}
|
|
7402
|
-
if (Number.isFinite(maxBytes)) {
|
|
7403
|
-
kept.sort((a, b) => a[1] - b[1]);
|
|
7404
|
-
let total = kept.reduce((sum, [, , size]) => sum + size, 0) + (protectedEntry ? protectedEntry[2] : 0);
|
|
7405
|
-
while (total > maxBytes && kept.length > 0) {
|
|
7406
|
-
const oldest = kept.shift();
|
|
7407
|
-
if (!oldest) break;
|
|
7408
|
-
const [full, , size] = oldest;
|
|
7409
|
-
try {
|
|
7410
|
-
fs6.unlinkSync(full);
|
|
7411
|
-
removed++;
|
|
7412
|
-
total -= size;
|
|
7413
|
-
} catch {
|
|
7414
|
-
break;
|
|
7415
|
-
}
|
|
7416
|
-
}
|
|
7417
|
-
}
|
|
7418
|
-
} catch {
|
|
7419
|
-
return removed;
|
|
7420
|
-
}
|
|
7421
|
-
return removed;
|
|
7422
|
-
}
|
|
7423
|
-
var SWEEPABLE_CACHE_SUBDIRS = [
|
|
7424
|
-
{ subdir: "bash_outputs", countCapped: true },
|
|
7425
|
-
{ subdir: "web_outputs", countCapped: true },
|
|
7426
|
-
{ subdir: "mcp_outputs", countCapped: true },
|
|
7427
|
-
{ subdir: "sessions", countCapped: false }
|
|
7428
|
-
];
|
|
7429
|
-
function sweepCacheRoots(extraRoots = []) {
|
|
7430
|
-
let removed = 0;
|
|
7431
|
-
const roots = [tokenGoatHome(), ...extraRoots];
|
|
7432
|
-
const seen = /* @__PURE__ */ new Set();
|
|
7433
|
-
for (const root of roots) {
|
|
7434
|
-
if (!root || seen.has(root)) continue;
|
|
7435
|
-
seen.add(root);
|
|
7436
|
-
for (const { subdir, countCapped } of SWEEPABLE_CACHE_SUBDIRS) {
|
|
7437
|
-
try {
|
|
7438
|
-
const defaults = subdirCacheDefaults(subdir);
|
|
7439
|
-
removed += pruneBlobDir(
|
|
7440
|
-
path3.join(root, subdir),
|
|
7441
|
-
countCapped ? defaults.maxCount : Number.POSITIVE_INFINITY,
|
|
7442
|
-
DEFAULT_MAX_AGE_MS,
|
|
7443
|
-
countCapped ? defaults.maxBytes : Number.POSITIVE_INFINITY
|
|
7444
|
-
);
|
|
7445
|
-
} catch {
|
|
7446
|
-
}
|
|
7447
|
-
}
|
|
7448
|
-
}
|
|
7449
|
-
return removed;
|
|
7450
|
-
}
|
|
7451
|
-
|
|
7452
|
-
// src/recall_index.ts
|
|
7453
7504
|
function subdirForCacheType(cacheType) {
|
|
7454
7505
|
return cacheType === "web" ? "web_outputs" : "bash_outputs";
|
|
7455
7506
|
}
|
|
7456
7507
|
function blobStillExists(cacheType, id) {
|
|
7457
7508
|
try {
|
|
7458
7509
|
const p = blobPath(subdirForCacheType(cacheType), id);
|
|
7459
|
-
return p !== null &&
|
|
7510
|
+
return p !== null && fs6.existsSync(p);
|
|
7460
7511
|
} catch {
|
|
7461
7512
|
return false;
|
|
7462
7513
|
}
|
|
@@ -7607,7 +7658,7 @@ function searchRecall(query, opts = {}) {
|
|
|
7607
7658
|
|
|
7608
7659
|
// src/bash_output_cache.ts
|
|
7609
7660
|
init_define_import_meta_env();
|
|
7610
|
-
import { readdirSync
|
|
7661
|
+
import { readdirSync, readFileSync as readFileSync2, statSync as statSync3 } from "fs";
|
|
7611
7662
|
import { resolve } from "path";
|
|
7612
7663
|
|
|
7613
7664
|
// src/hints/lang_patterns.ts
|
|
@@ -7953,11 +8004,11 @@ function gitStateFingerprintSync(cwd) {
|
|
|
7953
8004
|
}
|
|
7954
8005
|
}
|
|
7955
8006
|
var DIR_FINGERPRINT_LISTING_CAP_ENTRIES = 1e4;
|
|
7956
|
-
function dirStateFingerprintSync(
|
|
8007
|
+
function dirStateFingerprintSync(path22) {
|
|
7957
8008
|
try {
|
|
7958
|
-
const stat2 =
|
|
8009
|
+
const stat2 = statSync3(path22);
|
|
7959
8010
|
if (!stat2.isDirectory()) return null;
|
|
7960
|
-
const entries =
|
|
8011
|
+
const entries = readdirSync(path22);
|
|
7961
8012
|
if (entries.length <= DIR_FINGERPRINT_LISTING_CAP_ENTRIES) {
|
|
7962
8013
|
return shortFingerprint(entries.slice().sort().join("\0"));
|
|
7963
8014
|
}
|
|
@@ -7967,12 +8018,12 @@ function dirStateFingerprintSync(path24) {
|
|
|
7967
8018
|
}
|
|
7968
8019
|
}
|
|
7969
8020
|
var FILE_FINGERPRINT_CONTENT_CAP_BYTES = 2 * 1024 * 1024;
|
|
7970
|
-
function fileStateFingerprintSync(
|
|
8021
|
+
function fileStateFingerprintSync(path22) {
|
|
7971
8022
|
try {
|
|
7972
|
-
const stat2 =
|
|
8023
|
+
const stat2 = statSync3(path22);
|
|
7973
8024
|
if (!stat2.isFile()) return null;
|
|
7974
8025
|
if (stat2.size <= FILE_FINGERPRINT_CONTENT_CAP_BYTES) {
|
|
7975
|
-
return shortFingerprint(
|
|
8026
|
+
return shortFingerprint(readFileSync2(path22));
|
|
7976
8027
|
}
|
|
7977
8028
|
return shortFingerprint(`${stat2.mtimeMs}\0${stat2.size}`);
|
|
7978
8029
|
} catch {
|
|
@@ -7988,7 +8039,7 @@ function depLockfileFingerprintSync(cmd, cwd) {
|
|
|
7988
8039
|
if (!candidates) return null;
|
|
7989
8040
|
for (const lockfile of candidates) {
|
|
7990
8041
|
try {
|
|
7991
|
-
const content =
|
|
8042
|
+
const content = readFileSync2(resolve(cwd, lockfile));
|
|
7992
8043
|
return shortFingerprint(content);
|
|
7993
8044
|
} catch {
|
|
7994
8045
|
continue;
|
|
@@ -8241,8 +8292,8 @@ registerReset(() => {
|
|
|
8241
8292
|
// src/session_store.ts
|
|
8242
8293
|
init_define_import_meta_env();
|
|
8243
8294
|
import { createHash } from "node:crypto";
|
|
8244
|
-
import * as
|
|
8245
|
-
import * as
|
|
8295
|
+
import * as fs7 from "node:fs";
|
|
8296
|
+
import * as path3 from "node:path";
|
|
8246
8297
|
var MAX_FILES = 500;
|
|
8247
8298
|
var SESSIONS_SUBDIR = "sessions";
|
|
8248
8299
|
var AGENT_SALT_MARKER = sanitizeIdForFilename(":agent:");
|
|
@@ -8267,10 +8318,10 @@ function sessionSidecarPath(sessionId, suffix) {
|
|
|
8267
8318
|
const safe = sessionFileStem(sessionId);
|
|
8268
8319
|
if (!safe) return null;
|
|
8269
8320
|
if (suffix.includes("/") || suffix.includes("\\") || suffix.includes("..")) return null;
|
|
8270
|
-
const dir =
|
|
8271
|
-
const candidate =
|
|
8321
|
+
const dir = path3.join(tokenGoatHome(), SESSIONS_SUBDIR);
|
|
8322
|
+
const candidate = path3.join(dir, `${safe}${suffix}`);
|
|
8272
8323
|
try {
|
|
8273
|
-
const rel =
|
|
8324
|
+
const rel = path3.relative(dir, candidate);
|
|
8274
8325
|
if (rel.startsWith("..")) return null;
|
|
8275
8326
|
} catch {
|
|
8276
8327
|
return null;
|
|
@@ -8544,8 +8595,8 @@ function capFiles(s, max) {
|
|
|
8544
8595
|
}
|
|
8545
8596
|
function readDiskState(p) {
|
|
8546
8597
|
try {
|
|
8547
|
-
if (!
|
|
8548
|
-
return coerce(JSON.parse(
|
|
8598
|
+
if (!fs7.existsSync(p)) return null;
|
|
8599
|
+
return coerce(JSON.parse(fs7.readFileSync(p, "utf8")));
|
|
8549
8600
|
} catch {
|
|
8550
8601
|
return null;
|
|
8551
8602
|
}
|
|
@@ -8559,13 +8610,13 @@ function listSiblingSessionStates(sessionId) {
|
|
|
8559
8610
|
if (!sessionId) return [];
|
|
8560
8611
|
const prefix = saltedStemPrefix(sessionId);
|
|
8561
8612
|
if (prefix === AGENT_SALT_MARKER) return [];
|
|
8562
|
-
const dir =
|
|
8613
|
+
const dir = path3.join(tokenGoatHome(), SESSIONS_SUBDIR);
|
|
8563
8614
|
const out = [];
|
|
8564
8615
|
try {
|
|
8565
|
-
if (!
|
|
8566
|
-
for (const file of
|
|
8616
|
+
if (!fs7.existsSync(dir)) return out;
|
|
8617
|
+
for (const file of fs7.readdirSync(dir)) {
|
|
8567
8618
|
if (!file.endsWith(".json") || !file.startsWith(prefix)) continue;
|
|
8568
|
-
const state = readDiskState(
|
|
8619
|
+
const state = readDiskState(path3.join(dir, file));
|
|
8569
8620
|
if (state !== null) out.push(state);
|
|
8570
8621
|
}
|
|
8571
8622
|
} catch {
|
|
@@ -8584,8 +8635,8 @@ function saveSessionState(sessionId) {
|
|
|
8584
8635
|
const p = sessionPath(sessionId);
|
|
8585
8636
|
if (!p) return;
|
|
8586
8637
|
try {
|
|
8587
|
-
const dir =
|
|
8588
|
-
if (!
|
|
8638
|
+
const dir = path3.dirname(p);
|
|
8639
|
+
if (!fs7.existsSync(dir)) ensureDirSync(dir);
|
|
8589
8640
|
const mem = exportSessionState();
|
|
8590
8641
|
const writeMerged = () => {
|
|
8591
8642
|
const disk = readDiskState(p);
|
|
@@ -8607,10 +8658,10 @@ function saveSessionState(sessionId) {
|
|
|
8607
8658
|
|
|
8608
8659
|
// src/skill_cache.ts
|
|
8609
8660
|
init_define_import_meta_env();
|
|
8610
|
-
import * as
|
|
8661
|
+
import * as fs8 from "fs/promises";
|
|
8611
8662
|
import { resolve as resolve2 } from "path";
|
|
8612
8663
|
import { homedir } from "os";
|
|
8613
|
-
import { readdirSync as
|
|
8664
|
+
import { readdirSync as readdirSync3, readFileSync as readFileSync4, existsSync as existsSync5, statSync as statSync4, unlinkSync } from "node:fs";
|
|
8614
8665
|
var COMPACT_END_MARKER = "<!-- COMPACT_END -->";
|
|
8615
8666
|
var SKILLS_OUTPUT_SUBDIR = "skills";
|
|
8616
8667
|
var _skillOutputsDirOverride = null;
|
|
@@ -8775,14 +8826,14 @@ function extractChecklistSection(body) {
|
|
|
8775
8826
|
async function listOutputs() {
|
|
8776
8827
|
try {
|
|
8777
8828
|
const dir = skillOutputsDir();
|
|
8778
|
-
const entries = await
|
|
8829
|
+
const entries = await fs8.readdir(dir, { withFileTypes: true });
|
|
8779
8830
|
const metas = [];
|
|
8780
8831
|
for (const entry of entries) {
|
|
8781
8832
|
if (!entry.isFile() || !entry.name.endsWith(".meta")) {
|
|
8782
8833
|
continue;
|
|
8783
8834
|
}
|
|
8784
8835
|
try {
|
|
8785
|
-
const content = await
|
|
8836
|
+
const content = await fs8.readFile(resolve2(dir, entry.name), "utf-8");
|
|
8786
8837
|
const meta = JSON.parse(content);
|
|
8787
8838
|
metas.push(meta);
|
|
8788
8839
|
} catch {
|
|
@@ -8833,7 +8884,7 @@ async function findCrossSessionEntry(skillName, contentSha) {
|
|
|
8833
8884
|
const dir = skillOutputsDir();
|
|
8834
8885
|
const bodyPath = resolve2(dir, `${meta.outputId}.txt`);
|
|
8835
8886
|
try {
|
|
8836
|
-
const bodyExists = await
|
|
8887
|
+
const bodyExists = await fs8.access(bodyPath).then(() => true).catch(() => false);
|
|
8837
8888
|
if (bodyExists) {
|
|
8838
8889
|
return meta;
|
|
8839
8890
|
}
|
|
@@ -8936,11 +8987,11 @@ function getCompactAnySessionSync(skillName) {
|
|
|
8936
8987
|
const suffix = compactSessionSuffix(skillName);
|
|
8937
8988
|
if (!suffix) return null;
|
|
8938
8989
|
const dir = skillOutputsDir();
|
|
8939
|
-
const entries =
|
|
8990
|
+
const entries = readdirSync3(dir, { withFileTypes: true });
|
|
8940
8991
|
for (const entry of entries) {
|
|
8941
8992
|
if (!matchesCompactSuffix(entry.name, entry.isFile(), suffix)) continue;
|
|
8942
8993
|
try {
|
|
8943
|
-
const text =
|
|
8994
|
+
const text = readFileSync4(resolve2(dir, entry.name), "utf-8");
|
|
8944
8995
|
if (text.trim()) return text;
|
|
8945
8996
|
} catch {
|
|
8946
8997
|
continue;
|
|
@@ -8969,7 +9020,7 @@ async function readSkillHits(skillName) {
|
|
|
8969
9020
|
if (!name) return { count: 0, lastTs: 0 };
|
|
8970
9021
|
const dir = skillOutputsDir();
|
|
8971
9022
|
const hitsFile = resolve2(dir, `${sanitizeSkillId(name)}.hits`);
|
|
8972
|
-
const content = await
|
|
9023
|
+
const content = await fs8.readFile(hitsFile, "utf-8").catch(() => null);
|
|
8973
9024
|
if (content) {
|
|
8974
9025
|
const parsed = JSON.parse(content);
|
|
8975
9026
|
return parsed;
|
|
@@ -8997,14 +9048,14 @@ function runExclusiveInProcess(key, fn) {
|
|
|
8997
9048
|
async function acquireSkillHitLock(lockPath) {
|
|
8998
9049
|
for (let attempt = 0; attempt < SKILL_HIT_LOCK_MAX_ATTEMPTS; attempt++) {
|
|
8999
9050
|
try {
|
|
9000
|
-
await
|
|
9051
|
+
await fs8.mkdir(lockPath);
|
|
9001
9052
|
return true;
|
|
9002
9053
|
} catch (err) {
|
|
9003
9054
|
if (err.code !== "EEXIST") return false;
|
|
9004
9055
|
try {
|
|
9005
|
-
const st = await
|
|
9056
|
+
const st = await fs8.stat(lockPath);
|
|
9006
9057
|
if (Date.now() - st.mtimeMs > SKILL_HIT_LOCK_STALE_MS) {
|
|
9007
|
-
await
|
|
9058
|
+
await fs8.rmdir(lockPath).catch(() => {
|
|
9008
9059
|
});
|
|
9009
9060
|
continue;
|
|
9010
9061
|
}
|
|
@@ -9033,7 +9084,7 @@ async function incrementSkillHit(skillName) {
|
|
|
9033
9084
|
hits.lastTs = Date.now();
|
|
9034
9085
|
await atomicWriteText(hitsFile, JSON.stringify(hits, null, 2));
|
|
9035
9086
|
} finally {
|
|
9036
|
-
await
|
|
9087
|
+
await fs8.rmdir(lockPath).catch(() => {
|
|
9037
9088
|
});
|
|
9038
9089
|
}
|
|
9039
9090
|
});
|
|
@@ -9069,14 +9120,14 @@ async function listSkills(sessionId) {
|
|
|
9069
9120
|
let compactLen = 0;
|
|
9070
9121
|
let compactText = "";
|
|
9071
9122
|
try {
|
|
9072
|
-
const stat2 = await
|
|
9123
|
+
const stat2 = await fs8.stat(resolve2(dir, compactFileId));
|
|
9073
9124
|
compactLen = stat2.size;
|
|
9074
|
-
compactText = await
|
|
9125
|
+
compactText = await fs8.readFile(resolve2(dir, compactFileId), "utf-8").catch(() => "");
|
|
9075
9126
|
} catch {
|
|
9076
9127
|
compactLen = 0;
|
|
9077
9128
|
}
|
|
9078
9129
|
const hasMarker = extractCompactFromMarker(
|
|
9079
|
-
await
|
|
9130
|
+
await fs8.readFile(resolve2(dir, `${meta.outputId}.txt`), "utf-8").catch(() => "")
|
|
9080
9131
|
) !== null;
|
|
9081
9132
|
const compactStale = isCompactStale(compactText, meta.skillName, meta.contentSha);
|
|
9082
9133
|
const { count: hitCount } = await readSkillHits(meta.skillName);
|
|
@@ -9115,7 +9166,7 @@ async function resolvePluginSkillPath(pluginName, skillSlug) {
|
|
|
9115
9166
|
const manifestPath = _pluginsManifestPathOverride ?? resolve2(homedir(), ".claude", "plugins", "installed_plugins.json");
|
|
9116
9167
|
let raw;
|
|
9117
9168
|
try {
|
|
9118
|
-
raw = await
|
|
9169
|
+
raw = await fs8.readFile(manifestPath, "utf8");
|
|
9119
9170
|
} catch {
|
|
9120
9171
|
return null;
|
|
9121
9172
|
}
|
|
@@ -9137,7 +9188,7 @@ async function resolvePluginSkillPath(pluginName, skillSlug) {
|
|
|
9137
9188
|
if (typeof installPath !== "string" || installPath === "") continue;
|
|
9138
9189
|
const diskPath = resolve2(installPath, "skills", skillSlug, "SKILL.md");
|
|
9139
9190
|
try {
|
|
9140
|
-
await
|
|
9191
|
+
await fs8.access(diskPath);
|
|
9141
9192
|
return diskPath;
|
|
9142
9193
|
} catch {
|
|
9143
9194
|
continue;
|
|
@@ -9156,7 +9207,7 @@ async function installedSkillPath(skillName) {
|
|
|
9156
9207
|
}
|
|
9157
9208
|
const diskPath = resolve2(skillsSourceDir(), name, "SKILL.md");
|
|
9158
9209
|
try {
|
|
9159
|
-
await
|
|
9210
|
+
await fs8.access(diskPath);
|
|
9160
9211
|
return diskPath;
|
|
9161
9212
|
} catch {
|
|
9162
9213
|
return null;
|
|
@@ -9166,19 +9217,19 @@ function pruneSkillOutputs(maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
|
|
|
9166
9217
|
const dir = skillOutputsDir();
|
|
9167
9218
|
let removed = 0;
|
|
9168
9219
|
try {
|
|
9169
|
-
if (!
|
|
9220
|
+
if (!existsSync5(dir)) return 0;
|
|
9170
9221
|
const cutoff = Date.now() - maxAgeMs;
|
|
9171
9222
|
const entries = [];
|
|
9172
|
-
for (const file of
|
|
9223
|
+
for (const file of readdirSync3(dir)) {
|
|
9173
9224
|
if (!file.endsWith(".meta")) continue;
|
|
9174
9225
|
const outputId = file.slice(0, -".meta".length);
|
|
9175
9226
|
let ts;
|
|
9176
9227
|
try {
|
|
9177
|
-
const parsed = JSON.parse(
|
|
9228
|
+
const parsed = JSON.parse(readFileSync4(resolve2(dir, file), "utf-8"));
|
|
9178
9229
|
ts = parsed.ts;
|
|
9179
9230
|
} catch {
|
|
9180
9231
|
try {
|
|
9181
|
-
ts =
|
|
9232
|
+
ts = statSync4(resolve2(dir, file)).mtimeMs;
|
|
9182
9233
|
} catch {
|
|
9183
9234
|
continue;
|
|
9184
9235
|
}
|
|
@@ -9188,7 +9239,7 @@ function pruneSkillOutputs(maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
|
|
|
9188
9239
|
const removeEntry = (outputId) => {
|
|
9189
9240
|
for (const ext of [".meta", ".txt"]) {
|
|
9190
9241
|
try {
|
|
9191
|
-
|
|
9242
|
+
unlinkSync(resolve2(dir, `${outputId}${ext}`));
|
|
9192
9243
|
} catch {
|
|
9193
9244
|
}
|
|
9194
9245
|
}
|
|
@@ -9217,9 +9268,9 @@ function pruneSkillOutputs(maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
|
|
|
9217
9268
|
// src/image_shrink.ts
|
|
9218
9269
|
init_define_import_meta_env();
|
|
9219
9270
|
var import_omggif2 = __toESM(require_omggif(), 1);
|
|
9220
|
-
import { createHash as
|
|
9221
|
-
import * as
|
|
9222
|
-
import * as
|
|
9271
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
9272
|
+
import * as fs9 from "node:fs";
|
|
9273
|
+
import * as path5 from "node:path";
|
|
9223
9274
|
|
|
9224
9275
|
// src/image_engine.ts
|
|
9225
9276
|
init_define_import_meta_env();
|
|
@@ -9779,10 +9830,10 @@ function calculateFitInside(width, height, maxDimension) {
|
|
|
9779
9830
|
|
|
9780
9831
|
// src/vscode_path_gate.ts
|
|
9781
9832
|
init_define_import_meta_env();
|
|
9782
|
-
import * as
|
|
9833
|
+
import * as path4 from "node:path";
|
|
9783
9834
|
function resolvesToUncPath(target, cwd) {
|
|
9784
9835
|
try {
|
|
9785
|
-
return isUncOrDevicePath(
|
|
9836
|
+
return isUncOrDevicePath(path4.resolve(cwd ?? process.cwd(), target));
|
|
9786
9837
|
} catch {
|
|
9787
9838
|
return false;
|
|
9788
9839
|
}
|
|
@@ -9790,8 +9841,8 @@ function resolvesToUncPath(target, cwd) {
|
|
|
9790
9841
|
function vscodePathAllowed(filePath, workspace) {
|
|
9791
9842
|
if (isUncOrDevicePath(filePath)) return false;
|
|
9792
9843
|
if (workspace === void 0 || isUncOrDevicePath(workspace)) return false;
|
|
9793
|
-
const resolvedRoot =
|
|
9794
|
-
const resolvedTarget =
|
|
9844
|
+
const resolvedRoot = path4.resolve(workspace);
|
|
9845
|
+
const resolvedTarget = path4.resolve(workspace, filePath);
|
|
9795
9846
|
const root = foldPathForContainment(resolvedRoot.replace(/\\/g, "/"));
|
|
9796
9847
|
const target = foldPathForContainment(resolvedTarget.replace(/\\/g, "/"));
|
|
9797
9848
|
if (target !== root && !target.startsWith(root.endsWith("/") ? root : root + "/")) return false;
|
|
@@ -9808,163 +9859,6 @@ function preToolPathDeclined(event, target) {
|
|
|
9808
9859
|
return escapesOntoNetworkThroughLinks(target, cwd);
|
|
9809
9860
|
}
|
|
9810
9861
|
|
|
9811
|
-
// src/image_ocr.ts
|
|
9812
|
-
init_define_import_meta_env();
|
|
9813
|
-
import { spawn } from "node:child_process";
|
|
9814
|
-
import { createHash as createHash2 } from "node:crypto";
|
|
9815
|
-
import * as fs10 from "node:fs";
|
|
9816
|
-
import { createRequire } from "node:module";
|
|
9817
|
-
import * as path6 from "node:path";
|
|
9818
|
-
var _ocrTimeoutMs = 12e3;
|
|
9819
|
-
var OCR_LANG_FILE = "eng.traineddata";
|
|
9820
|
-
var OCR_LANG_PATH = "https://cdn.jsdelivr.net/npm/@tesseract.js-data/eng@1.0.0/4.0.0_best_int";
|
|
9821
|
-
var OCR_LANG_SHA256 = "5dc5d8d640a212c9d6184921ba103b186f50e0fed9ee716c53e6b312b400d747";
|
|
9822
|
-
function verifyOcrLangCache() {
|
|
9823
|
-
const file = path6.join(ocrCacheDir(), OCR_LANG_FILE);
|
|
9824
|
-
let bytes;
|
|
9825
|
-
try {
|
|
9826
|
-
if (!fs10.existsSync(file)) return "absent";
|
|
9827
|
-
bytes = fs10.readFileSync(file);
|
|
9828
|
-
} catch {
|
|
9829
|
-
return "unreadable";
|
|
9830
|
-
}
|
|
9831
|
-
return createHash2("sha256").update(bytes).digest("hex") === OCR_LANG_SHA256 ? "ok" : "mismatch";
|
|
9832
|
-
}
|
|
9833
|
-
function quarantineOcrLangCache() {
|
|
9834
|
-
try {
|
|
9835
|
-
fs10.rmSync(path6.join(ocrCacheDir(), OCR_LANG_FILE), { force: true });
|
|
9836
|
-
} catch {
|
|
9837
|
-
}
|
|
9838
|
-
}
|
|
9839
|
-
var _ocrIntegrityFailed = false;
|
|
9840
|
-
function ocrIntegrityFailed() {
|
|
9841
|
-
return _ocrIntegrityFailed;
|
|
9842
|
-
}
|
|
9843
|
-
function ocrBlockedOffline() {
|
|
9844
|
-
if (!loadConfig().network.offline) return false;
|
|
9845
|
-
return !fs10.existsSync(path6.join(ocrCacheDir(), OCR_LANG_FILE));
|
|
9846
|
-
}
|
|
9847
|
-
function ocrCacheDir() {
|
|
9848
|
-
return path6.join(tokenGoatHome(), "ocr-cache");
|
|
9849
|
-
}
|
|
9850
|
-
function ensureOcrCacheDir() {
|
|
9851
|
-
try {
|
|
9852
|
-
ensureDirSync(ocrCacheDir());
|
|
9853
|
-
} catch {
|
|
9854
|
-
}
|
|
9855
|
-
}
|
|
9856
|
-
var _require = createRequire(import.meta.url);
|
|
9857
|
-
var _tesseractEntryPath;
|
|
9858
|
-
function resolveTesseractEntry() {
|
|
9859
|
-
if (_tesseractEntryPath !== void 0) return _tesseractEntryPath;
|
|
9860
|
-
try {
|
|
9861
|
-
_tesseractEntryPath = _require.resolve("tesseract.js");
|
|
9862
|
-
} catch {
|
|
9863
|
-
_tesseractEntryPath = null;
|
|
9864
|
-
}
|
|
9865
|
-
return _tesseractEntryPath;
|
|
9866
|
-
}
|
|
9867
|
-
function isOcrEngineAvailable() {
|
|
9868
|
-
return !_ocrUnavailableThisProcess && resolveTesseractEntry() !== null;
|
|
9869
|
-
}
|
|
9870
|
-
var _ocrUnavailableThisProcess = false;
|
|
9871
|
-
function buildChildScript(entryPath, cacheDir) {
|
|
9872
|
-
return [
|
|
9873
|
-
`const { createWorker } = require(${JSON.stringify(entryPath)});`,
|
|
9874
|
-
"const chunks = [];",
|
|
9875
|
-
"process.stdin.on('data', (c) => chunks.push(c));",
|
|
9876
|
-
"process.stdin.on('end', async () => {",
|
|
9877
|
-
" try {",
|
|
9878
|
-
" const buf = Buffer.concat(chunks);",
|
|
9879
|
-
` const worker = await createWorker('eng', 1, { cachePath: ${JSON.stringify(cacheDir)}, langPath: ${JSON.stringify(OCR_LANG_PATH)}, errorHandler: () => {} });`,
|
|
9880
|
-
" const { data } = await worker.recognize(buf);",
|
|
9881
|
-
" process.stdout.write(JSON.stringify({ text: data.text || '', confidence: data.confidence || 0 }));",
|
|
9882
|
-
" await worker.terminate();",
|
|
9883
|
-
" process.exit(0);",
|
|
9884
|
-
" } catch (e) {",
|
|
9885
|
-
" process.exit(1);",
|
|
9886
|
-
" }",
|
|
9887
|
-
"});"
|
|
9888
|
-
].join("\n");
|
|
9889
|
-
}
|
|
9890
|
-
async function ocrImage(input) {
|
|
9891
|
-
if (_ocrUnavailableThisProcess) return null;
|
|
9892
|
-
const entryPath = resolveTesseractEntry();
|
|
9893
|
-
if (entryPath === null) return null;
|
|
9894
|
-
if (ocrBlockedOffline()) return null;
|
|
9895
|
-
if (_ocrIntegrityFailed) return null;
|
|
9896
|
-
if (verifyOcrLangCache() === "mismatch") {
|
|
9897
|
-
_ocrIntegrityFailed = true;
|
|
9898
|
-
quarantineOcrLangCache();
|
|
9899
|
-
return null;
|
|
9900
|
-
}
|
|
9901
|
-
return new Promise((resolve8) => {
|
|
9902
|
-
let settled = false;
|
|
9903
|
-
let child;
|
|
9904
|
-
try {
|
|
9905
|
-
ensureOcrCacheDir();
|
|
9906
|
-
child = spawn(process.execPath, ["-e", buildChildScript(entryPath, ocrCacheDir())], {
|
|
9907
|
-
stdio: ["pipe", "pipe", "ignore"]
|
|
9908
|
-
});
|
|
9909
|
-
} catch {
|
|
9910
|
-
_ocrUnavailableThisProcess = true;
|
|
9911
|
-
resolve8(null);
|
|
9912
|
-
return;
|
|
9913
|
-
}
|
|
9914
|
-
const chunks = [];
|
|
9915
|
-
const finish = (result, subprocessBroken) => {
|
|
9916
|
-
if (settled) return;
|
|
9917
|
-
settled = true;
|
|
9918
|
-
clearTimeout(timer);
|
|
9919
|
-
if (subprocessBroken) _ocrUnavailableThisProcess = true;
|
|
9920
|
-
try {
|
|
9921
|
-
child.kill();
|
|
9922
|
-
} catch {
|
|
9923
|
-
}
|
|
9924
|
-
resolve8(result);
|
|
9925
|
-
};
|
|
9926
|
-
const timer = setTimeout(() => finish(null, true), _ocrTimeoutMs);
|
|
9927
|
-
child.stdout?.on("data", (c) => chunks.push(c));
|
|
9928
|
-
child.on("error", () => finish(null, true));
|
|
9929
|
-
child.on("close", (code) => {
|
|
9930
|
-
if (code !== 0) {
|
|
9931
|
-
finish(null, false);
|
|
9932
|
-
return;
|
|
9933
|
-
}
|
|
9934
|
-
if (verifyOcrLangCache() === "mismatch") {
|
|
9935
|
-
_ocrIntegrityFailed = true;
|
|
9936
|
-
quarantineOcrLangCache();
|
|
9937
|
-
finish(null, false);
|
|
9938
|
-
return;
|
|
9939
|
-
}
|
|
9940
|
-
try {
|
|
9941
|
-
const raw = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
9942
|
-
const parsed = raw;
|
|
9943
|
-
const text = typeof parsed.text === "string" ? parsed.text.trim() : "";
|
|
9944
|
-
const confidence = typeof parsed.confidence === "number" ? parsed.confidence : 0;
|
|
9945
|
-
finish({ text, confidence }, false);
|
|
9946
|
-
} catch {
|
|
9947
|
-
finish(null, false);
|
|
9948
|
-
}
|
|
9949
|
-
});
|
|
9950
|
-
try {
|
|
9951
|
-
child.stdin?.end(input);
|
|
9952
|
-
} catch {
|
|
9953
|
-
finish(null, false);
|
|
9954
|
-
}
|
|
9955
|
-
});
|
|
9956
|
-
}
|
|
9957
|
-
function isTextHeavy(result, minConfidence, minChars = 40) {
|
|
9958
|
-
return result.confidence >= minConfidence && result.text.length >= minChars;
|
|
9959
|
-
}
|
|
9960
|
-
function formatOcrSummary(result, subject, originalBytes) {
|
|
9961
|
-
const kb = Math.round(originalBytes / 1024);
|
|
9962
|
-
const summary = `token-goat OCR'd ${subject} instead of shrinking it: text-heavy image detected (${Math.round(result.confidence)}% confidence), extracted ${result.text.length} chars of text from ${kb}kb of pixels.`;
|
|
9963
|
-
return `${summary}
|
|
9964
|
-
|
|
9965
|
-
${fenceUntrustedOcrText(result.text)}`;
|
|
9966
|
-
}
|
|
9967
|
-
|
|
9968
9862
|
// src/image_shrink.ts
|
|
9969
9863
|
var IMAGE_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
9970
9864
|
".png",
|
|
@@ -10036,7 +9930,7 @@ function formatShrinkSummary(result, subject) {
|
|
|
10036
9930
|
return { summary, dataUrl };
|
|
10037
9931
|
}
|
|
10038
9932
|
function isImagePath(p) {
|
|
10039
|
-
return IMAGE_EXTENSIONS.has(
|
|
9933
|
+
return IMAGE_EXTENSIONS.has(path5.extname(p).toLowerCase());
|
|
10040
9934
|
}
|
|
10041
9935
|
var ImageDecodeError = class extends Error {
|
|
10042
9936
|
};
|
|
@@ -10151,17 +10045,17 @@ async function shrinkImage(input, opts) {
|
|
|
10151
10045
|
}
|
|
10152
10046
|
function statInfo(absPath) {
|
|
10153
10047
|
try {
|
|
10154
|
-
const st =
|
|
10048
|
+
const st = fs9.statSync(absPath);
|
|
10155
10049
|
return st.isFile() ? { size: st.size, mtimeMs: st.mtimeMs } : null;
|
|
10156
10050
|
} catch {
|
|
10157
10051
|
return null;
|
|
10158
10052
|
}
|
|
10159
10053
|
}
|
|
10160
10054
|
function imageShrinkCacheDir() {
|
|
10161
|
-
return
|
|
10055
|
+
return path5.join(tokenGoatHome(), "image_shrink_cache");
|
|
10162
10056
|
}
|
|
10163
10057
|
function shrinkCacheKey(originalPath, size, mtimeMs, quality) {
|
|
10164
|
-
return
|
|
10058
|
+
return createHash2("sha256").update(`${originalPath}:${size}:${mtimeMs}:${quality}`).digest("hex").slice(0, 16);
|
|
10165
10059
|
}
|
|
10166
10060
|
var SHRINK_CACHE_EXTENSION = /* @__PURE__ */ new Map([
|
|
10167
10061
|
["jpeg", ".jpg"],
|
|
@@ -10179,7 +10073,7 @@ function findCachedShrink(originalPath, size, mtimeMs, quality) {
|
|
|
10179
10073
|
const dir = imageShrinkCacheDir();
|
|
10180
10074
|
let entries;
|
|
10181
10075
|
try {
|
|
10182
|
-
entries =
|
|
10076
|
+
entries = fs9.readdirSync(dir);
|
|
10183
10077
|
} catch {
|
|
10184
10078
|
return null;
|
|
10185
10079
|
}
|
|
@@ -10190,7 +10084,7 @@ function findCachedShrink(originalPath, size, mtimeMs, quality) {
|
|
|
10190
10084
|
const format = SHRINK_CACHE_FORMAT.get(m[3] ?? "");
|
|
10191
10085
|
if (format === void 0) continue;
|
|
10192
10086
|
return {
|
|
10193
|
-
filePath:
|
|
10087
|
+
filePath: path5.join(dir, file),
|
|
10194
10088
|
format,
|
|
10195
10089
|
originalWidth: Number(m[1]),
|
|
10196
10090
|
originalHeight: Number(m[2])
|
|
@@ -10205,7 +10099,7 @@ function writeCachedShrink(originalPath, result, mtimeMs, quality) {
|
|
|
10205
10099
|
const key = shrinkCacheKey(originalPath, result.originalBytes, mtimeMs, quality);
|
|
10206
10100
|
const ext = SHRINK_CACHE_EXTENSION.get(result.format);
|
|
10207
10101
|
if (ext === void 0) return;
|
|
10208
|
-
atomicWriteBytes(
|
|
10102
|
+
atomicWriteBytes(path5.join(dir, `token-goat-shrink-${key}-${result.originalWidth}x${result.originalHeight}${ext}`), result.data);
|
|
10209
10103
|
} catch {
|
|
10210
10104
|
}
|
|
10211
10105
|
}
|
|
@@ -10217,14 +10111,14 @@ function pruneShrinkCache() {
|
|
|
10217
10111
|
lastShrinkCachePruneAtMs = now;
|
|
10218
10112
|
try {
|
|
10219
10113
|
const dir = imageShrinkCacheDir();
|
|
10220
|
-
if (!
|
|
10114
|
+
if (!fs9.existsSync(dir)) return;
|
|
10221
10115
|
const cutoff = now - DEFAULT_MAX_AGE_MS;
|
|
10222
|
-
for (const file of
|
|
10116
|
+
for (const file of fs9.readdirSync(dir)) {
|
|
10223
10117
|
if (!file.startsWith("token-goat-shrink-")) continue;
|
|
10224
|
-
const full =
|
|
10118
|
+
const full = path5.join(dir, file);
|
|
10225
10119
|
try {
|
|
10226
|
-
const st =
|
|
10227
|
-
if (st.mtimeMs < cutoff)
|
|
10120
|
+
const st = fs9.statSync(full);
|
|
10121
|
+
if (st.mtimeMs < cutoff) fs9.unlinkSync(full);
|
|
10228
10122
|
} catch {
|
|
10229
10123
|
}
|
|
10230
10124
|
}
|
|
@@ -10233,7 +10127,7 @@ function pruneShrinkCache() {
|
|
|
10233
10127
|
}
|
|
10234
10128
|
var HOST_MATERIALIZED_HARNESSES = /* @__PURE__ */ new Set(["copilot_cli", "pi", "opencode", "openclaw"]);
|
|
10235
10129
|
async function finalizeShrinkResult(result, filePath, event) {
|
|
10236
|
-
const basename10 = displaySafePath(
|
|
10130
|
+
const basename10 = displaySafePath(path5.basename(filePath));
|
|
10237
10131
|
const shrinkSaved = result.originalBytes - result.shrunkBytes;
|
|
10238
10132
|
const tier = loadConfig().image_shrink.vision_tier;
|
|
10239
10133
|
const shrinkTokens = visionTokensSaved(result.originalWidth, result.originalHeight, result.width, result.height, tier);
|
|
@@ -10251,6 +10145,7 @@ ${dataUrl2}`);
|
|
|
10251
10145
|
}
|
|
10252
10146
|
recordStat("image_shrink", shrinkSaved, shrinkTokens, void 0, basename10);
|
|
10253
10147
|
if (loadConfig().image_shrink.ocr_enabled) {
|
|
10148
|
+
const { ocrImage, isTextHeavy, formatOcrSummary } = await import("./token-goat-chunk-4NXUKV7D.mjs");
|
|
10254
10149
|
const ocr = await ocrImage(result.data);
|
|
10255
10150
|
if (ocr !== null && isTextHeavy(ocr, loadConfig().image_shrink.ocr_min_confidence)) {
|
|
10256
10151
|
const emitted = formatOcrSummary(ocr, basename10, result.originalBytes);
|
|
@@ -10281,7 +10176,7 @@ async function preReadImageHandler(event) {
|
|
|
10281
10176
|
if (cached !== null) {
|
|
10282
10177
|
let cachedData;
|
|
10283
10178
|
try {
|
|
10284
|
-
cachedData =
|
|
10179
|
+
cachedData = fs9.readFileSync(cached.filePath);
|
|
10285
10180
|
} catch {
|
|
10286
10181
|
cachedData = null;
|
|
10287
10182
|
}
|
|
@@ -10307,13 +10202,13 @@ async function preReadImageHandler(event) {
|
|
|
10307
10202
|
return finalizeShrinkResult(result2, filePath, event);
|
|
10308
10203
|
}
|
|
10309
10204
|
try {
|
|
10310
|
-
|
|
10205
|
+
fs9.unlinkSync(cached.filePath);
|
|
10311
10206
|
} catch {
|
|
10312
10207
|
}
|
|
10313
10208
|
}
|
|
10314
10209
|
let input;
|
|
10315
10210
|
try {
|
|
10316
|
-
input =
|
|
10211
|
+
input = fs9.readFileSync(filePath);
|
|
10317
10212
|
} catch {
|
|
10318
10213
|
return passOutput();
|
|
10319
10214
|
}
|
|
@@ -10333,10 +10228,10 @@ init_define_import_meta_env();
|
|
|
10333
10228
|
|
|
10334
10229
|
// src/embed_model.ts
|
|
10335
10230
|
init_define_import_meta_env();
|
|
10336
|
-
import { createHash as
|
|
10337
|
-
import * as
|
|
10338
|
-
import { createRequire
|
|
10339
|
-
import * as
|
|
10231
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
10232
|
+
import * as fs10 from "node:fs";
|
|
10233
|
+
import { createRequire } from "node:module";
|
|
10234
|
+
import * as path6 from "node:path";
|
|
10340
10235
|
import { pipeline } from "node:stream/promises";
|
|
10341
10236
|
|
|
10342
10237
|
// src/embed_tokenizer.ts
|
|
@@ -10351,20 +10246,20 @@ var UnsupportedTokenizerError = class extends Error {
|
|
|
10351
10246
|
function fail(message) {
|
|
10352
10247
|
throw new UnsupportedTokenizerError(message);
|
|
10353
10248
|
}
|
|
10354
|
-
function asRecord(value,
|
|
10355
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${
|
|
10249
|
+
function asRecord(value, path22) {
|
|
10250
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${path22} is not an object`);
|
|
10356
10251
|
return value;
|
|
10357
10252
|
}
|
|
10358
|
-
function asString(value,
|
|
10359
|
-
if (typeof value !== "string") fail(`${
|
|
10253
|
+
function asString(value, path22) {
|
|
10254
|
+
if (typeof value !== "string") fail(`${path22} is not a string`);
|
|
10360
10255
|
return value;
|
|
10361
10256
|
}
|
|
10362
|
-
function asArray2(value,
|
|
10363
|
-
if (!Array.isArray(value)) fail(`${
|
|
10257
|
+
function asArray2(value, path22) {
|
|
10258
|
+
if (!Array.isArray(value)) fail(`${path22} is not an array`);
|
|
10364
10259
|
return value;
|
|
10365
10260
|
}
|
|
10366
|
-
function expectValue(actual, wanted,
|
|
10367
|
-
if (actual !== wanted) fail(`${
|
|
10261
|
+
function expectValue(actual, wanted, path22) {
|
|
10262
|
+
if (actual !== wanted) fail(`${path22} is ${JSON.stringify(actual)}, expected ${JSON.stringify(wanted)}`);
|
|
10368
10263
|
}
|
|
10369
10264
|
function isChinese(cp) {
|
|
10370
10265
|
return cp >= 19968 && cp <= 40959 || cp >= 13312 && cp <= 19903 || cp >= 131072 && cp <= 173791 || cp >= 173824 && cp <= 177983 || cp >= 177984 && cp <= 178207 || cp >= 178208 && cp <= 183983 || cp >= 63744 && cp <= 64255 || cp >= 194560 && cp <= 195103;
|
|
@@ -10547,7 +10442,7 @@ var BertWordPiece = class _BertWordPiece {
|
|
|
10547
10442
|
};
|
|
10548
10443
|
|
|
10549
10444
|
// src/embed_model.ts
|
|
10550
|
-
var
|
|
10445
|
+
var _require = createRequire(import.meta.url);
|
|
10551
10446
|
var DEFAULT_MODEL = "Xenova/bge-small-en-v1.5";
|
|
10552
10447
|
var DEFAULT_DIM = 384;
|
|
10553
10448
|
var DEFAULT_EMBED_THREADS = 2;
|
|
@@ -10565,7 +10460,7 @@ var MODEL_FILES = [
|
|
|
10565
10460
|
}
|
|
10566
10461
|
];
|
|
10567
10462
|
function modelDir() {
|
|
10568
|
-
return
|
|
10463
|
+
return path6.join(dataDir(), "models", ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
10569
10464
|
}
|
|
10570
10465
|
function downloadUrl(file) {
|
|
10571
10466
|
return `https://huggingface.co/${DEFAULT_MODEL}/resolve/${PINNED_MODEL_REVISION}/${file.name}`;
|
|
@@ -10573,44 +10468,44 @@ function downloadUrl(file) {
|
|
|
10573
10468
|
function sharedModelCacheDir() {
|
|
10574
10469
|
const raw = process.env["TOKEN_GOAT_MODEL_CACHE_DIR"]?.trim();
|
|
10575
10470
|
if (!raw) return null;
|
|
10576
|
-
return
|
|
10471
|
+
return path6.join(raw, ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
10577
10472
|
}
|
|
10578
10473
|
async function copyFromSharedCache(shared, file, target) {
|
|
10579
|
-
const source =
|
|
10474
|
+
const source = path6.join(shared, file.name);
|
|
10580
10475
|
const temp = `${target}.${process.pid}.shared`;
|
|
10581
10476
|
try {
|
|
10582
|
-
const info =
|
|
10477
|
+
const info = fs10.lstatSync(source, { throwIfNoEntry: false });
|
|
10583
10478
|
if (!info?.isFile() || info.size !== file.bytes) return false;
|
|
10584
|
-
|
|
10479
|
+
fs10.copyFileSync(source, temp);
|
|
10585
10480
|
if (await sha256Of(temp) !== file.sha256) {
|
|
10586
|
-
|
|
10587
|
-
|
|
10481
|
+
fs10.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
10482
|
+
fs10.rmSync(source, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
10588
10483
|
return false;
|
|
10589
10484
|
}
|
|
10590
|
-
|
|
10485
|
+
fs10.renameSync(temp, target);
|
|
10591
10486
|
return true;
|
|
10592
10487
|
} catch {
|
|
10593
10488
|
try {
|
|
10594
|
-
|
|
10489
|
+
fs10.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
10595
10490
|
} catch {
|
|
10596
10491
|
}
|
|
10597
10492
|
return false;
|
|
10598
10493
|
}
|
|
10599
10494
|
}
|
|
10600
10495
|
function publishToSharedCache(shared, file, target) {
|
|
10601
|
-
const destination =
|
|
10496
|
+
const destination = path6.join(shared, file.name);
|
|
10602
10497
|
const temp = `${destination}.${process.pid}.partial`;
|
|
10603
10498
|
let created = false;
|
|
10604
10499
|
try {
|
|
10605
|
-
if (
|
|
10606
|
-
ensureDirSync(
|
|
10607
|
-
|
|
10500
|
+
if (fs10.existsSync(destination)) return;
|
|
10501
|
+
ensureDirSync(path6.dirname(destination));
|
|
10502
|
+
fs10.copyFileSync(target, temp, fs10.constants.COPYFILE_EXCL);
|
|
10608
10503
|
created = true;
|
|
10609
|
-
|
|
10504
|
+
fs10.renameSync(temp, destination);
|
|
10610
10505
|
} catch {
|
|
10611
10506
|
if (!created) return;
|
|
10612
10507
|
try {
|
|
10613
|
-
|
|
10508
|
+
fs10.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
10614
10509
|
} catch {
|
|
10615
10510
|
}
|
|
10616
10511
|
}
|
|
@@ -10622,7 +10517,7 @@ function ensureRuntimeLoaded() {
|
|
|
10622
10517
|
if (_ortLoadAttempted) return;
|
|
10623
10518
|
_ortLoadAttempted = true;
|
|
10624
10519
|
try {
|
|
10625
|
-
_ort =
|
|
10520
|
+
_ort = _require("onnxruntime-node");
|
|
10626
10521
|
} catch (e) {
|
|
10627
10522
|
_ortError = e instanceof Error ? e : new Error(String(e));
|
|
10628
10523
|
}
|
|
@@ -10639,14 +10534,14 @@ function runtimeVersion() {
|
|
|
10639
10534
|
ensureRuntimeLoaded();
|
|
10640
10535
|
if (_ort === null) return "unknown";
|
|
10641
10536
|
try {
|
|
10642
|
-
let dir =
|
|
10537
|
+
let dir = path6.dirname(_require.resolve("onnxruntime-node"));
|
|
10643
10538
|
for (let depth = 0; depth < 6; depth++) {
|
|
10644
|
-
const manifest =
|
|
10645
|
-
if (
|
|
10646
|
-
const parsed = JSON.parse(
|
|
10539
|
+
const manifest = path6.join(dir, "package.json");
|
|
10540
|
+
if (fs10.existsSync(manifest)) {
|
|
10541
|
+
const parsed = JSON.parse(fs10.readFileSync(manifest, "utf8"));
|
|
10647
10542
|
if (parsed.name === "onnxruntime-node" && typeof parsed.version === "string") return parsed.version;
|
|
10648
10543
|
}
|
|
10649
|
-
const up =
|
|
10544
|
+
const up = path6.dirname(dir);
|
|
10650
10545
|
if (up === dir) break;
|
|
10651
10546
|
dir = up;
|
|
10652
10547
|
}
|
|
@@ -10656,8 +10551,8 @@ function runtimeVersion() {
|
|
|
10656
10551
|
}
|
|
10657
10552
|
function sha256Of(filePath) {
|
|
10658
10553
|
return new Promise((resolve8, reject) => {
|
|
10659
|
-
const hash2 =
|
|
10660
|
-
const stream =
|
|
10554
|
+
const hash2 = createHash3("sha256");
|
|
10555
|
+
const stream = fs10.createReadStream(filePath);
|
|
10661
10556
|
stream.on("error", reject);
|
|
10662
10557
|
stream.on("data", (chunk) => hash2.update(chunk));
|
|
10663
10558
|
stream.on("end", () => resolve8(hash2.digest("hex")));
|
|
@@ -10669,9 +10564,9 @@ async function download(file, target) {
|
|
|
10669
10564
|
if (!response.ok) throw new Error(`GET ${url} returned ${response.status} ${response.statusText}`);
|
|
10670
10565
|
if (!response.body) throw new Error(`GET ${url} returned no body`);
|
|
10671
10566
|
const temp = `${target}.${process.pid}.partial`;
|
|
10672
|
-
const hash2 =
|
|
10567
|
+
const hash2 = createHash3("sha256");
|
|
10673
10568
|
let written = 0;
|
|
10674
|
-
const out =
|
|
10569
|
+
const out = fs10.createWriteStream(temp);
|
|
10675
10570
|
try {
|
|
10676
10571
|
await pipeline(async function* () {
|
|
10677
10572
|
for await (const chunk of response.body) {
|
|
@@ -10688,14 +10583,14 @@ async function download(file, target) {
|
|
|
10688
10583
|
if (digest !== file.sha256) {
|
|
10689
10584
|
throw new Error(`${file.name} has sha256 ${digest}, expected the pinned ${file.sha256}`);
|
|
10690
10585
|
}
|
|
10691
|
-
|
|
10586
|
+
fs10.renameSync(temp, target);
|
|
10692
10587
|
} catch (e) {
|
|
10693
10588
|
await new Promise((resolve8) => {
|
|
10694
10589
|
if (out.closed) resolve8();
|
|
10695
10590
|
else out.once("close", () => resolve8());
|
|
10696
10591
|
});
|
|
10697
10592
|
try {
|
|
10698
|
-
|
|
10593
|
+
fs10.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
10699
10594
|
} catch {
|
|
10700
10595
|
}
|
|
10701
10596
|
throw e;
|
|
@@ -10712,14 +10607,14 @@ async function ensureModelFiles(modelName = DEFAULT_MODEL) {
|
|
|
10712
10607
|
const offline = loadConfig().network.offline;
|
|
10713
10608
|
const shared = sharedModelCacheDir();
|
|
10714
10609
|
for (const file of MODEL_FILES) {
|
|
10715
|
-
const target =
|
|
10716
|
-
if (
|
|
10610
|
+
const target = path6.join(dir, file.name);
|
|
10611
|
+
if (fs10.existsSync(target)) {
|
|
10717
10612
|
const digest = await sha256Of(target);
|
|
10718
10613
|
if (digest === file.sha256) continue;
|
|
10719
|
-
|
|
10614
|
+
fs10.rmSync(target, { force: true });
|
|
10720
10615
|
}
|
|
10721
10616
|
if (shared) {
|
|
10722
|
-
ensureDirSync(
|
|
10617
|
+
ensureDirSync(path6.dirname(target));
|
|
10723
10618
|
if (await copyFromSharedCache(shared, file, target)) continue;
|
|
10724
10619
|
}
|
|
10725
10620
|
if (offline) {
|
|
@@ -10727,7 +10622,7 @@ async function ensureModelFiles(modelName = DEFAULT_MODEL) {
|
|
|
10727
10622
|
`Offline mode is on (network.offline): refusing to download ${file.name} for the embedding model. Copy the pinned files into ${dir} on a connected machine to use semantic search here.`
|
|
10728
10623
|
);
|
|
10729
10624
|
}
|
|
10730
|
-
ensureDirSync(
|
|
10625
|
+
ensureDirSync(path6.dirname(target));
|
|
10731
10626
|
console.warn(
|
|
10732
10627
|
`Downloading the embedding model, once (${file.name}, ${Math.round(file.bytes / 1024 / 1024)} MB) into ${dir}`
|
|
10733
10628
|
);
|
|
@@ -10768,10 +10663,10 @@ var EmbeddingModel = class _EmbeddingModel {
|
|
|
10768
10663
|
throw new Error(`onnxruntime-node is not available: ${_ortError?.message ?? "unknown error"}`);
|
|
10769
10664
|
}
|
|
10770
10665
|
const dir = await ensureModelFiles(modelName);
|
|
10771
|
-
const tokenizer = BertWordPiece.fromJson(
|
|
10666
|
+
const tokenizer = BertWordPiece.fromJson(fs10.readFileSync(path6.join(dir, "tokenizer.json"), "utf8"));
|
|
10772
10667
|
const ort = _ort;
|
|
10773
10668
|
const threads = loadConfig().worker.embed_threads ?? DEFAULT_EMBED_THREADS;
|
|
10774
|
-
const session = await ort.InferenceSession.create(
|
|
10669
|
+
const session = await ort.InferenceSession.create(path6.join(dir, "onnx", "model_quantized.onnx"), {
|
|
10775
10670
|
intraOpNumThreads: threads,
|
|
10776
10671
|
interOpNumThreads: 1
|
|
10777
10672
|
});
|
|
@@ -10884,6 +10779,7 @@ var _TOKEN_RE = /\w+/g;
|
|
|
10884
10779
|
var _MIN_TOKEN_LEN = 3;
|
|
10885
10780
|
var OVER_FETCH_FACTOR = 4;
|
|
10886
10781
|
var MAX_OVER_FETCH = 100;
|
|
10782
|
+
var VEC_MAX_K = 4096;
|
|
10887
10783
|
function isAvailable() {
|
|
10888
10784
|
return isRuntimeAvailable();
|
|
10889
10785
|
}
|
|
@@ -11186,6 +11082,34 @@ function fetchScopedHits(db, queryVec, k, maxDistance, rootDir) {
|
|
|
11186
11082
|
}
|
|
11187
11083
|
return { hits, candidateCount: rows.length };
|
|
11188
11084
|
}
|
|
11085
|
+
function fetchScopedExactHits(db, queryVec, limit, maxDistance, rootDir) {
|
|
11086
|
+
const scope = projectScopeClause("c.file_path");
|
|
11087
|
+
const rows = db.prepare(
|
|
11088
|
+
`SELECT file_path, start_line, end_line, text, kind, distance FROM (
|
|
11089
|
+
SELECT c.id AS id, c.file_path AS file_path, c.start_line AS start_line, c.end_line AS end_line, c.text AS text, c.kind AS kind, vec_distance_L2(v.embedding, ?) AS distance
|
|
11090
|
+
FROM chunks c JOIN chunk_vectors v ON v.rowid = c.id
|
|
11091
|
+
WHERE ${scope.clause}
|
|
11092
|
+
)
|
|
11093
|
+
WHERE distance IS NOT NULL AND distance <= ?
|
|
11094
|
+
ORDER BY distance ASC, file_path ASC, start_line ASC, id ASC
|
|
11095
|
+
LIMIT ?`
|
|
11096
|
+
).all(packVec(queryVec), ...scope.params(rootDir), maxDistance, limit);
|
|
11097
|
+
const hits = [];
|
|
11098
|
+
for (const row of rows) {
|
|
11099
|
+
if (!row || typeof row.distance !== "number" || !Number.isFinite(row.distance) || row.distance > maxDistance) {
|
|
11100
|
+
continue;
|
|
11101
|
+
}
|
|
11102
|
+
hits.push({
|
|
11103
|
+
filePath: row.file_path,
|
|
11104
|
+
startLine: row.start_line,
|
|
11105
|
+
endLine: row.end_line,
|
|
11106
|
+
kind: row.kind,
|
|
11107
|
+
distance: row.distance,
|
|
11108
|
+
text: row.text
|
|
11109
|
+
});
|
|
11110
|
+
}
|
|
11111
|
+
return hits;
|
|
11112
|
+
}
|
|
11189
11113
|
async function searchSemantic(db, query, topK = 8, modelName = DEFAULT_MODEL, maxDistance = DEFAULT_DISTANCE_THRESHOLD, rootDir) {
|
|
11190
11114
|
if (!isAvailable()) {
|
|
11191
11115
|
return [];
|
|
@@ -11209,16 +11133,19 @@ async function searchSemantic(db, query, topK = 8, modelName = DEFAULT_MODEL, ma
|
|
|
11209
11133
|
MAX_OVER_FETCH,
|
|
11210
11134
|
Math.ceil(topK * OVER_FETCH_FACTOR)
|
|
11211
11135
|
);
|
|
11212
|
-
let k = overFetchK;
|
|
11136
|
+
let k = Math.min(overFetchK, VEC_MAX_K);
|
|
11213
11137
|
let pass = fetchScopedHits(db, queryVec, k, maxDistance, rootDir);
|
|
11214
11138
|
let hits = pass.hits;
|
|
11215
11139
|
if (rootDir !== void 0) {
|
|
11216
|
-
const ceiling = countStoredVectors(db);
|
|
11140
|
+
const ceiling = Math.min(countStoredVectors(db), VEC_MAX_K);
|
|
11217
11141
|
while (hits.length < topK && pass.candidateCount === k && k < ceiling) {
|
|
11218
11142
|
k = Math.min(ceiling, k * BACKFILL_MULTIPLIER);
|
|
11219
11143
|
pass = fetchScopedHits(db, queryVec, k, maxDistance, rootDir);
|
|
11220
11144
|
hits = pass.hits;
|
|
11221
11145
|
}
|
|
11146
|
+
if (hits.length < topK && pass.candidateCount === k && k >= ceiling && countStoredVectors(db) > VEC_MAX_K) {
|
|
11147
|
+
hits = fetchScopedExactHits(db, queryVec, VEC_MAX_K, maxDistance, rootDir);
|
|
11148
|
+
}
|
|
11222
11149
|
}
|
|
11223
11150
|
return rerankHits(hits, query, topK);
|
|
11224
11151
|
}
|
|
@@ -11422,13 +11349,13 @@ function _pathPriorityPenalty(filePath) {
|
|
|
11422
11349
|
|
|
11423
11350
|
// src/parser_fingerprint.ts
|
|
11424
11351
|
init_define_import_meta_env();
|
|
11425
|
-
var PARSER_FINGERPRINT = "
|
|
11352
|
+
var PARSER_FINGERPRINT = "9c0e5a5680783f75";
|
|
11426
11353
|
|
|
11427
11354
|
// src/parser.ts
|
|
11428
11355
|
init_define_import_meta_env();
|
|
11429
|
-
import * as
|
|
11430
|
-
import { createRequire as
|
|
11431
|
-
import * as
|
|
11356
|
+
import * as fs11 from "node:fs";
|
|
11357
|
+
import { createRequire as createRequire2 } from "node:module";
|
|
11358
|
+
import * as path8 from "node:path";
|
|
11432
11359
|
|
|
11433
11360
|
// src/markdown_lines.ts
|
|
11434
11361
|
init_define_import_meta_env();
|
|
@@ -11551,14 +11478,14 @@ function extractChangelogVersionHint(content, filePath) {
|
|
|
11551
11478
|
|
|
11552
11479
|
// src/languages/salesforce_frontend.ts
|
|
11553
11480
|
init_define_import_meta_env();
|
|
11554
|
-
import * as
|
|
11481
|
+
import * as path7 from "node:path";
|
|
11555
11482
|
function lines(content) {
|
|
11556
11483
|
return content.split("\n");
|
|
11557
11484
|
}
|
|
11558
11485
|
function bundleName(filePath) {
|
|
11559
11486
|
const normalized = filePath.replaceAll("\\", "/");
|
|
11560
|
-
const parent =
|
|
11561
|
-
const base =
|
|
11487
|
+
const parent = path7.posix.basename(path7.posix.dirname(normalized));
|
|
11488
|
+
const base = path7.posix.basename(normalized).replace(/\.[^.]+$/, "");
|
|
11562
11489
|
return parent === "lwc" || parent === "aura" ? base : parent;
|
|
11563
11490
|
}
|
|
11564
11491
|
function lwcTagAlias(name) {
|
|
@@ -11662,7 +11589,7 @@ var MARKUP_KIND = {
|
|
|
11662
11589
|
};
|
|
11663
11590
|
function markupArtifactName(filePath, extension) {
|
|
11664
11591
|
if (extension === ".page" || extension === ".component" || extension === ".email") {
|
|
11665
|
-
return
|
|
11592
|
+
return path7.posix.basename(filePath.replaceAll("\\", "/")).replace(new RegExp(`${extension.replace(".", "\\.")}$`, "i"), "");
|
|
11666
11593
|
}
|
|
11667
11594
|
return bundleName(filePath);
|
|
11668
11595
|
}
|
|
@@ -11682,7 +11609,7 @@ function attributeRefs(refs, content, filePath, attribute, split = false) {
|
|
|
11682
11609
|
}
|
|
11683
11610
|
function extractSalesforceMarkup(content, filePath) {
|
|
11684
11611
|
const normalized = filePath.replaceAll("\\", "/");
|
|
11685
|
-
const extension =
|
|
11612
|
+
const extension = path7.posix.extname(normalized).toLowerCase();
|
|
11686
11613
|
const kind = MARKUP_KIND[extension] ?? "salesforce_markup";
|
|
11687
11614
|
const symbols = [
|
|
11688
11615
|
symbol(filePath, markupArtifactName(normalized, extension), kind, 1, countContentLines(content))
|
|
@@ -11713,56 +11640,8 @@ function extractSalesforceMarkup(content, filePath) {
|
|
|
11713
11640
|
};
|
|
11714
11641
|
}
|
|
11715
11642
|
|
|
11716
|
-
// src/languages/ipynb_idx.ts
|
|
11717
|
-
init_define_import_meta_env();
|
|
11718
|
-
function normalizeSource(source) {
|
|
11719
|
-
if (typeof source === "string") return source;
|
|
11720
|
-
if (!Array.isArray(source)) return "";
|
|
11721
|
-
const lines2 = source.filter((s) => typeof s === "string");
|
|
11722
|
-
if (lines2.length === 0) return "";
|
|
11723
|
-
const noneEndWithNewline = lines2.every((l) => !l.endsWith("\n"));
|
|
11724
|
-
return noneEndWithNewline ? lines2.join("\n") : lines2.join("");
|
|
11725
|
-
}
|
|
11726
|
-
function resolveKernelLanguage(metadata) {
|
|
11727
|
-
const kernelspecLang = metadata?.kernelspec?.language;
|
|
11728
|
-
if (typeof kernelspecLang === "string" && kernelspecLang !== "") return kernelspecLang.toLowerCase();
|
|
11729
|
-
const infoLang = metadata?.language_info?.name;
|
|
11730
|
-
if (typeof infoLang === "string" && infoLang !== "") return infoLang.toLowerCase();
|
|
11731
|
-
return void 0;
|
|
11732
|
-
}
|
|
11733
|
-
function ipynbToVirtualSource(raw) {
|
|
11734
|
-
let data;
|
|
11735
|
-
try {
|
|
11736
|
-
data = JSON.parse(raw);
|
|
11737
|
-
} catch {
|
|
11738
|
-
return { content: "", cellLanguage: null };
|
|
11739
|
-
}
|
|
11740
|
-
if (typeof data !== "object" || data === null) return { content: "", cellLanguage: null };
|
|
11741
|
-
const nb = data;
|
|
11742
|
-
if (!Array.isArray(nb.cells)) return { content: "", cellLanguage: null };
|
|
11743
|
-
const declared = resolveKernelLanguage(nb.metadata);
|
|
11744
|
-
const isPython = declared === void 0 || declared === "python" || declared === "python3";
|
|
11745
|
-
if (!isPython) return { content: "", cellLanguage: null };
|
|
11746
|
-
const blocks = [];
|
|
11747
|
-
nb.cells.forEach((cell, index) => {
|
|
11748
|
-
if (typeof cell !== "object" || cell === null) return;
|
|
11749
|
-
const c = cell;
|
|
11750
|
-
if (c.cell_type === "code") {
|
|
11751
|
-
const src = normalizeSource(c.source).replace(/\n+$/, "");
|
|
11752
|
-
blocks.push(`# %% cell ${index}
|
|
11753
|
-
${src}`);
|
|
11754
|
-
} else if (c.cell_type === "markdown") {
|
|
11755
|
-
const src = normalizeSource(c.source).replace(/\n+$/, "");
|
|
11756
|
-
const commented = src.split("\n").map((line) => line === "" ? "#" : `# ${line}`).join("\n");
|
|
11757
|
-
blocks.push(`# %% [markdown] cell ${index}
|
|
11758
|
-
${commented}`);
|
|
11759
|
-
}
|
|
11760
|
-
});
|
|
11761
|
-
return { content: blocks.join("\n\n"), cellLanguage: "python" };
|
|
11762
|
-
}
|
|
11763
|
-
|
|
11764
11643
|
// src/parser.ts
|
|
11765
|
-
var
|
|
11644
|
+
var _require2 = createRequire2(import.meta.url);
|
|
11766
11645
|
var COMPILER_ARTIFACT_REF_NAMES = /* @__PURE__ */ new Set(["__name"]);
|
|
11767
11646
|
var _parserCtor;
|
|
11768
11647
|
var _parserCtorError = null;
|
|
@@ -11773,7 +11652,7 @@ function loadParserCtor() {
|
|
|
11773
11652
|
if (_parserCtorOverride !== void 0) return _parserCtorOverride;
|
|
11774
11653
|
if (_parserCtor !== void 0) return _parserCtor;
|
|
11775
11654
|
try {
|
|
11776
|
-
_parserCtor =
|
|
11655
|
+
_parserCtor = _require2("tree-sitter");
|
|
11777
11656
|
} catch (e) {
|
|
11778
11657
|
_parserCtor = null;
|
|
11779
11658
|
_parserCtorError = e instanceof Error ? e : new Error(String(e));
|
|
@@ -11862,32 +11741,32 @@ function countNewlines(s) {
|
|
|
11862
11741
|
return n;
|
|
11863
11742
|
}
|
|
11864
11743
|
function loadGrammar(lang, filePath, content) {
|
|
11865
|
-
const useTsx = lang === "typescript" && filePath !== void 0 &&
|
|
11866
|
-
const useCppHeader = lang === "c" && filePath !== void 0 &&
|
|
11744
|
+
const useTsx = lang === "typescript" && filePath !== void 0 && path8.extname(filePath).toLowerCase() === ".tsx";
|
|
11745
|
+
const useCppHeader = lang === "c" && filePath !== void 0 && path8.extname(filePath).toLowerCase() === ".h" && content !== void 0 && CPP_HEADER_SNIFF_RE.test(content);
|
|
11867
11746
|
const cacheKey = useTsx ? "typescript:tsx" : useCppHeader ? "c:cpp-header" : lang;
|
|
11868
11747
|
const cached = _grammarCache.get(cacheKey);
|
|
11869
11748
|
if (cached !== void 0) return cached;
|
|
11870
11749
|
let grammar = null;
|
|
11871
11750
|
try {
|
|
11872
11751
|
if (lang === "typescript") {
|
|
11873
|
-
const mod =
|
|
11752
|
+
const mod = _require2("tree-sitter-typescript");
|
|
11874
11753
|
grammar = useTsx ? mod.tsx : mod.typescript;
|
|
11875
11754
|
} else if (lang === "javascript") {
|
|
11876
|
-
grammar =
|
|
11755
|
+
grammar = _require2("tree-sitter-javascript");
|
|
11877
11756
|
} else if (lang === "python") {
|
|
11878
|
-
grammar =
|
|
11757
|
+
grammar = _require2("tree-sitter-python");
|
|
11879
11758
|
} else if (lang === "go") {
|
|
11880
|
-
grammar =
|
|
11759
|
+
grammar = _require2("tree-sitter-go");
|
|
11881
11760
|
} else if (lang === "rust") {
|
|
11882
|
-
grammar =
|
|
11761
|
+
grammar = _require2("tree-sitter-rust");
|
|
11883
11762
|
} else if (lang === "ruby") {
|
|
11884
|
-
grammar =
|
|
11763
|
+
grammar = _require2("tree-sitter-ruby");
|
|
11885
11764
|
} else if (lang === "java") {
|
|
11886
|
-
grammar =
|
|
11765
|
+
grammar = _require2("tree-sitter-java");
|
|
11887
11766
|
} else if (lang === "c") {
|
|
11888
|
-
grammar = useCppHeader ?
|
|
11767
|
+
grammar = useCppHeader ? _require2("tree-sitter-cpp") : _require2("tree-sitter-c");
|
|
11889
11768
|
} else if (lang === "cpp") {
|
|
11890
|
-
grammar =
|
|
11769
|
+
grammar = _require2("tree-sitter-cpp");
|
|
11891
11770
|
}
|
|
11892
11771
|
} catch {
|
|
11893
11772
|
grammar = null;
|
|
@@ -11913,7 +11792,7 @@ var TREE_SITTER_GRAMMAR_PACKAGES = [
|
|
|
11913
11792
|
function missingTreeSitterGrammarPackages() {
|
|
11914
11793
|
return TREE_SITTER_GRAMMAR_PACKAGES.filter((pkg) => {
|
|
11915
11794
|
try {
|
|
11916
|
-
|
|
11795
|
+
_require2.resolve(pkg);
|
|
11917
11796
|
return false;
|
|
11918
11797
|
} catch {
|
|
11919
11798
|
return true;
|
|
@@ -12554,7 +12433,7 @@ var CALL_TYPES_BY_LANG = /* @__PURE__ */ new Map([
|
|
|
12554
12433
|
["rust", /* @__PURE__ */ new Set(["call_expression", "macro_invocation"])],
|
|
12555
12434
|
["java", /* @__PURE__ */ new Set(["method_invocation", "object_creation_expression"])],
|
|
12556
12435
|
["c", /* @__PURE__ */ new Set(["call_expression"])],
|
|
12557
|
-
["cpp", /* @__PURE__ */ new Set(["call_expression"])],
|
|
12436
|
+
["cpp", /* @__PURE__ */ new Set(["call_expression", "new_expression"])],
|
|
12558
12437
|
["ruby", /* @__PURE__ */ new Set(["call"])]
|
|
12559
12438
|
]);
|
|
12560
12439
|
var REF_NOISE_BY_LANG = /* @__PURE__ */ new Map([
|
|
@@ -12746,6 +12625,38 @@ function cFunctionName(node) {
|
|
|
12746
12625
|
}
|
|
12747
12626
|
return null;
|
|
12748
12627
|
}
|
|
12628
|
+
function cppCalleeName(node) {
|
|
12629
|
+
switch (node.type) {
|
|
12630
|
+
case "identifier":
|
|
12631
|
+
case "field_identifier":
|
|
12632
|
+
case "type_identifier":
|
|
12633
|
+
return node.text;
|
|
12634
|
+
// An overloaded operator's name is its own node type, and it is the spelling the extractor gives the operator's definition too, so returning the text here is what makes the two match. Measured: `r.template operator()<int>(3)` parses as `dependent_name(template, template_method(operator_name "operator()", template_argument_list "<int>"))`, so without this case the name resolved to null and the field_expression fallback below recorded `template operator()<int>` -- a spelling no symbol row ever holds, leaving the call permanently unresolvable.
|
|
12635
|
+
case "operator_name":
|
|
12636
|
+
return node.text;
|
|
12637
|
+
case "template_function":
|
|
12638
|
+
case "template_method":
|
|
12639
|
+
case "template_type": {
|
|
12640
|
+
const name = node.childForFieldName("name");
|
|
12641
|
+
return name !== null ? cppCalleeName(name) : null;
|
|
12642
|
+
}
|
|
12643
|
+
case "qualified_identifier": {
|
|
12644
|
+
const name = node.childForFieldName("name");
|
|
12645
|
+
return name !== null ? cppCalleeName(name) : lastSegment(node.text);
|
|
12646
|
+
}
|
|
12647
|
+
case "dependent_name": {
|
|
12648
|
+
const last = node.namedChildren[node.namedChildren.length - 1];
|
|
12649
|
+
return last !== void 0 ? cppCalleeName(last) : null;
|
|
12650
|
+
}
|
|
12651
|
+
case "field_expression": {
|
|
12652
|
+
const field = node.childForFieldName("field");
|
|
12653
|
+
if (field === null) return null;
|
|
12654
|
+
return cppCalleeName(field) ?? field.text;
|
|
12655
|
+
}
|
|
12656
|
+
default:
|
|
12657
|
+
return null;
|
|
12658
|
+
}
|
|
12659
|
+
}
|
|
12749
12660
|
function calleeName(call, language) {
|
|
12750
12661
|
switch (language) {
|
|
12751
12662
|
case "typescript":
|
|
@@ -12797,12 +12708,9 @@ function calleeName(call, language) {
|
|
|
12797
12708
|
}
|
|
12798
12709
|
case "c":
|
|
12799
12710
|
case "cpp": {
|
|
12800
|
-
const fn = call.childForFieldName("function");
|
|
12711
|
+
const fn = call.type === "new_expression" ? call.childForFieldName("type") : call.childForFieldName("function");
|
|
12801
12712
|
if (fn === null) return null;
|
|
12802
|
-
|
|
12803
|
-
if (fn.type === "field_expression") return fn.childForFieldName("field")?.text ?? null;
|
|
12804
|
-
if (fn.type === "qualified_identifier") return lastSegment(fn.text);
|
|
12805
|
-
return null;
|
|
12713
|
+
return cppCalleeName(fn);
|
|
12806
12714
|
}
|
|
12807
12715
|
case "ruby": {
|
|
12808
12716
|
const m = call.childForFieldName("method");
|
|
@@ -13570,7 +13478,7 @@ function parseSourceSymbolsTreeSitterOnly(content, filePath, language) {
|
|
|
13570
13478
|
var REGEX_ADAPTERS_SLOT = /* @__PURE__ */ Symbol.for("token-goat.regex-adapters");
|
|
13571
13479
|
var adapterSlot = globalThis;
|
|
13572
13480
|
async function loadRegexExtractors() {
|
|
13573
|
-
adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-
|
|
13481
|
+
adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-6B44WLIF.mjs");
|
|
13574
13482
|
}
|
|
13575
13483
|
function regexAdapters() {
|
|
13576
13484
|
const loaded = adapterSlot[REGEX_ADAPTERS_SLOT];
|
|
@@ -13632,9 +13540,9 @@ function isUnderSkipDir(filePath, skipDirs) {
|
|
|
13632
13540
|
}
|
|
13633
13541
|
function isParseSkipEligible(filePath, cfg) {
|
|
13634
13542
|
if (isUnderSkipDir(filePath, cfg.skip_dirs)) return true;
|
|
13635
|
-
if (cfg.skip_files.includes(
|
|
13543
|
+
if (cfg.skip_files.includes(path8.basename(filePath))) return true;
|
|
13636
13544
|
try {
|
|
13637
|
-
const stat2 =
|
|
13545
|
+
const stat2 = fs11.statSync(filePath);
|
|
13638
13546
|
if (stat2.size > cfg.large_file_skip_kb * 1024) return true;
|
|
13639
13547
|
} catch {
|
|
13640
13548
|
}
|
|
@@ -13693,7 +13601,7 @@ function indexFileSync(rawPath, dbPath = globalDbPath(), preReadBytes) {
|
|
|
13693
13601
|
raw = preReadBytes;
|
|
13694
13602
|
} else {
|
|
13695
13603
|
try {
|
|
13696
|
-
raw =
|
|
13604
|
+
raw = fs11.readFileSync(filePath);
|
|
13697
13605
|
} catch (err) {
|
|
13698
13606
|
if (err.code === "ENOENT") return;
|
|
13699
13607
|
throw err;
|
|
@@ -13729,32 +13637,36 @@ var OVERSIZE_EMBED_SHA_PREFIX = "oversize:";
|
|
|
13729
13637
|
function oversizeEmbedSha(sha, symbolOnlyKb) {
|
|
13730
13638
|
return `${OVERSIZE_EMBED_SHA_PREFIX}${symbolOnlyKb}:${sha}`;
|
|
13731
13639
|
}
|
|
13640
|
+
var TIMEOUT_EMBED_SHA_PREFIX = "timeout:";
|
|
13641
|
+
function timeoutEmbedSha(sha, workMillis) {
|
|
13642
|
+
return `${TIMEOUT_EMBED_SHA_PREFIX}${workMillis}:${sha}`;
|
|
13643
|
+
}
|
|
13732
13644
|
function canonicalizeIndexPath(absPath) {
|
|
13733
13645
|
if (!isCaseInsensitiveFs()) return absPath;
|
|
13734
13646
|
let real;
|
|
13735
13647
|
try {
|
|
13736
|
-
real =
|
|
13648
|
+
real = fs11.realpathSync.native(absPath);
|
|
13737
13649
|
} catch {
|
|
13738
13650
|
return absPath;
|
|
13739
13651
|
}
|
|
13740
13652
|
const cut = Math.max(absPath.lastIndexOf("/"), absPath.lastIndexOf("\\"));
|
|
13741
13653
|
const base = absPath.slice(cut + 1);
|
|
13742
13654
|
const realNorm = normalizePath(real);
|
|
13743
|
-
const realBase =
|
|
13655
|
+
const realBase = path8.basename(realNorm);
|
|
13744
13656
|
if (base === realBase) return absPath;
|
|
13745
13657
|
if (foldPath(base) !== foldPath(realBase)) return absPath;
|
|
13746
|
-
const callerDir = cut < 0 ? normalizePath(
|
|
13747
|
-
if (foldPath(callerDir) !== foldPath(
|
|
13658
|
+
const callerDir = cut < 0 ? normalizePath(path8.resolve(".")) : normalizePath(absPath.slice(0, cut));
|
|
13659
|
+
if (foldPath(callerDir) !== foldPath(path8.dirname(realNorm))) return absPath;
|
|
13748
13660
|
return absPath.slice(0, cut + 1) + realBase;
|
|
13749
13661
|
}
|
|
13750
13662
|
function indexedPathSpellingIsStale(storedPath, absPath) {
|
|
13751
13663
|
if (!isCaseInsensitiveFs()) return false;
|
|
13752
13664
|
const stored = normalizePath(storedPath);
|
|
13753
|
-
const candidate = normalizePath(
|
|
13665
|
+
const candidate = normalizePath(path8.resolve(absPath));
|
|
13754
13666
|
if (foldPath(stored) !== foldPath(candidate)) return false;
|
|
13755
13667
|
let real;
|
|
13756
13668
|
try {
|
|
13757
|
-
real = normalizePath(
|
|
13669
|
+
real = normalizePath(fs11.realpathSync.native(absPath));
|
|
13758
13670
|
} catch {
|
|
13759
13671
|
return false;
|
|
13760
13672
|
}
|
|
@@ -13782,6 +13694,7 @@ function isEmbedFresh(storedEmbedSha, sha, embeddingsEnabled, depsAvailable, sym
|
|
|
13782
13694
|
if (storedEmbedSha === sha) return true;
|
|
13783
13695
|
if (!depsAvailable && storedEmbedSha === unavailableEmbedSha(sha)) return true;
|
|
13784
13696
|
if (storedEmbedSha === oversizeEmbedSha(sha, symbolOnlyKb)) return true;
|
|
13697
|
+
if (storedEmbedSha === timeoutEmbedSha(sha, MAX_DOCUMENT_WORK_MILLIS)) return true;
|
|
13785
13698
|
return false;
|
|
13786
13699
|
}
|
|
13787
13700
|
async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onError) {
|
|
@@ -13799,6 +13712,7 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
13799
13712
|
}
|
|
13800
13713
|
if (isEmbeddableDocument(filePath)) {
|
|
13801
13714
|
let extracted;
|
|
13715
|
+
let refusedOnTheClock = false;
|
|
13802
13716
|
try {
|
|
13803
13717
|
extracted = await extractEmbeddableDocumentText(filePath);
|
|
13804
13718
|
} catch (err) {
|
|
@@ -13806,12 +13720,13 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
13806
13720
|
onError?.(err);
|
|
13807
13721
|
return;
|
|
13808
13722
|
}
|
|
13723
|
+
refusedOnTheClock = isTransientDocumentRefusal(err);
|
|
13809
13724
|
extracted = null;
|
|
13810
13725
|
}
|
|
13811
13726
|
if (extracted === null || extracted.trim().length === 0) {
|
|
13812
13727
|
const db = getDb(dbPath);
|
|
13813
13728
|
deleteFileEmbeddings(db, filePath);
|
|
13814
|
-
stampEmbedSha(db, filePath, sha, (s) => s);
|
|
13729
|
+
stampEmbedSha(db, filePath, sha, (s) => refusedOnTheClock ? timeoutEmbedSha(s, MAX_DOCUMENT_WORK_MILLIS) : s);
|
|
13815
13730
|
return;
|
|
13816
13731
|
}
|
|
13817
13732
|
if (extracted.length > ixCfg.large_file_symbol_only_kb * 1024) {
|
|
@@ -13831,7 +13746,7 @@ async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onErro
|
|
|
13831
13746
|
}
|
|
13832
13747
|
let content;
|
|
13833
13748
|
try {
|
|
13834
|
-
content = decodeSource(await
|
|
13749
|
+
content = decodeSource(await fs11.promises.readFile(filePath));
|
|
13835
13750
|
} catch {
|
|
13836
13751
|
return;
|
|
13837
13752
|
}
|
|
@@ -13877,7 +13792,7 @@ function stampEmbedSha(db, filePath, sha, makeValue) {
|
|
|
13877
13792
|
}
|
|
13878
13793
|
function safeMtime(filePath) {
|
|
13879
13794
|
try {
|
|
13880
|
-
return
|
|
13795
|
+
return fs11.statSync(filePath).mtimeMs / 1e3;
|
|
13881
13796
|
} catch {
|
|
13882
13797
|
return 0;
|
|
13883
13798
|
}
|
|
@@ -13885,8 +13800,8 @@ function safeMtime(filePath) {
|
|
|
13885
13800
|
|
|
13886
13801
|
// src/index_prune.ts
|
|
13887
13802
|
init_define_import_meta_env();
|
|
13888
|
-
import * as
|
|
13889
|
-
import * as
|
|
13803
|
+
import * as fs12 from "node:fs";
|
|
13804
|
+
import * as path9 from "node:path";
|
|
13890
13805
|
function removeFileFromIndex(db, filePath) {
|
|
13891
13806
|
const tx = db.transaction(() => {
|
|
13892
13807
|
deleteFileRows(db, filePath);
|
|
@@ -13923,7 +13838,7 @@ function findDeletablePaths(rootPrefix, dbPath) {
|
|
|
13923
13838
|
for (const p of foldedPathsUnderRoot(rootPrefix, dbPath)) {
|
|
13924
13839
|
let stillExists;
|
|
13925
13840
|
try {
|
|
13926
|
-
const st =
|
|
13841
|
+
const st = fs12.statSync(p, { throwIfNoEntry: false });
|
|
13927
13842
|
stillExists = st !== void 0 && st.isFile();
|
|
13928
13843
|
} catch {
|
|
13929
13844
|
continue;
|
|
@@ -13951,7 +13866,7 @@ function removeDeletedFilesBestEffort(db, paths) {
|
|
|
13951
13866
|
for (const p of paths) {
|
|
13952
13867
|
let gone;
|
|
13953
13868
|
try {
|
|
13954
|
-
const st =
|
|
13869
|
+
const st = fs12.statSync(p, { throwIfNoEntry: false });
|
|
13955
13870
|
gone = st === void 0 || !st.isFile();
|
|
13956
13871
|
} catch {
|
|
13957
13872
|
continue;
|
|
@@ -14018,7 +13933,7 @@ function pruneOrphanedChunks(dbPath = globalDbPath()) {
|
|
|
14018
13933
|
return removed;
|
|
14019
13934
|
}
|
|
14020
13935
|
function recordKnownRoot(filePath, dbPath = globalDbPath()) {
|
|
14021
|
-
const project = findProject(
|
|
13936
|
+
const project = findProject(path9.dirname(filePath));
|
|
14022
13937
|
if (project === null || isTooShallowToPrune(project.root)) return;
|
|
14023
13938
|
const db = getDb(dbPath);
|
|
14024
13939
|
db.prepare(
|
|
@@ -14041,7 +13956,7 @@ function sweepKnownRoots(dbPath = globalDbPath(), opts) {
|
|
|
14041
13956
|
if (isTooShallowToPrune(root)) continue;
|
|
14042
13957
|
let reachable;
|
|
14043
13958
|
try {
|
|
14044
|
-
reachable =
|
|
13959
|
+
reachable = fs12.statSync(root, { throwIfNoEntry: false })?.isDirectory() === true;
|
|
14045
13960
|
} catch {
|
|
14046
13961
|
reachable = false;
|
|
14047
13962
|
}
|
|
@@ -14076,19 +13991,19 @@ function sweepKnownRoots(dbPath = globalDbPath(), opts) {
|
|
|
14076
13991
|
}
|
|
14077
13992
|
var KNOWN_ROOT_RECORD_MIN_INTERVAL_MS = 60 * 60 * 1e3;
|
|
14078
13993
|
function knownRootRecordMarkerPath(dir, filePath) {
|
|
14079
|
-
return
|
|
13994
|
+
return path9.join(dir, `known-root-record-${shortFingerprint(path9.dirname(filePath))}.marker`);
|
|
14080
13995
|
}
|
|
14081
13996
|
var KNOWN_ROOT_MARKER_PREFIX = "known-root-record-";
|
|
14082
13997
|
function sweepExpiredKnownRootMarkers(dir = dataDir()) {
|
|
14083
13998
|
let removed = 0;
|
|
14084
13999
|
try {
|
|
14085
14000
|
const cutoff = Date.now() - KNOWN_ROOT_RECORD_MIN_INTERVAL_MS;
|
|
14086
|
-
for (const file of
|
|
14001
|
+
for (const file of fs12.readdirSync(dir)) {
|
|
14087
14002
|
if (!file.startsWith(KNOWN_ROOT_MARKER_PREFIX) || !file.endsWith(".marker")) continue;
|
|
14088
|
-
const full =
|
|
14003
|
+
const full = path9.join(dir, file);
|
|
14089
14004
|
try {
|
|
14090
|
-
if (
|
|
14091
|
-
|
|
14005
|
+
if (fs12.statSync(full).mtimeMs < cutoff) {
|
|
14006
|
+
fs12.unlinkSync(full);
|
|
14092
14007
|
removed += 1;
|
|
14093
14008
|
}
|
|
14094
14009
|
} catch {
|
|
@@ -14101,13 +14016,13 @@ function sweepExpiredKnownRootMarkers(dir = dataDir()) {
|
|
|
14101
14016
|
function recordKnownRootThrottled(filePath, dir = dataDir(), dbPath = globalDbPath()) {
|
|
14102
14017
|
const markerPath = knownRootRecordMarkerPath(dir, filePath);
|
|
14103
14018
|
try {
|
|
14104
|
-
const stat2 =
|
|
14019
|
+
const stat2 = fs12.statSync(markerPath);
|
|
14105
14020
|
if (Date.now() - stat2.mtimeMs < KNOWN_ROOT_RECORD_MIN_INTERVAL_MS) return;
|
|
14106
14021
|
} catch {
|
|
14107
14022
|
}
|
|
14108
14023
|
try {
|
|
14109
14024
|
ensureDirSync(dir);
|
|
14110
|
-
|
|
14025
|
+
fs12.writeFileSync(markerPath, "");
|
|
14111
14026
|
} catch {
|
|
14112
14027
|
}
|
|
14113
14028
|
recordKnownRoot(filePath, dbPath);
|
|
@@ -14137,15 +14052,15 @@ function applyIndexingPriority() {
|
|
|
14137
14052
|
|
|
14138
14053
|
// src/worker.ts
|
|
14139
14054
|
init_define_import_meta_env();
|
|
14140
|
-
import { spawn
|
|
14141
|
-
import * as
|
|
14142
|
-
import * as
|
|
14055
|
+
import { spawn } from "node:child_process";
|
|
14056
|
+
import * as fs14 from "node:fs";
|
|
14057
|
+
import * as path11 from "node:path";
|
|
14143
14058
|
import { fileURLToPath } from "node:url";
|
|
14144
14059
|
|
|
14145
14060
|
// src/snapshots.ts
|
|
14146
14061
|
init_define_import_meta_env();
|
|
14147
|
-
import * as
|
|
14148
|
-
import * as
|
|
14062
|
+
import * as fs13 from "node:fs";
|
|
14063
|
+
import * as path10 from "node:path";
|
|
14149
14064
|
var MAX_SNAPSHOTS_PER_SESSION = 150;
|
|
14150
14065
|
var MAX_SNAPSHOT_BYTES = 256 * 1024;
|
|
14151
14066
|
var SNAPSHOT_TRUNCATE_BYTES = 50 * 1024;
|
|
@@ -14155,10 +14070,10 @@ var VALID_KINDS = /* @__PURE__ */ new Set([KIND_READ, KIND_PREDICTIVE]);
|
|
|
14155
14070
|
function sessionDir(sessionId) {
|
|
14156
14071
|
if (!sessionId) return null;
|
|
14157
14072
|
const safe = sanitizeIdForFilename(sessionId, 64, "anon");
|
|
14158
|
-
const base =
|
|
14159
|
-
const candidate =
|
|
14073
|
+
const base = path10.join(tokenGoatHome(), "session_snapshots");
|
|
14074
|
+
const candidate = path10.join(base, safe);
|
|
14160
14075
|
try {
|
|
14161
|
-
const rel =
|
|
14076
|
+
const rel = path10.relative(base, candidate);
|
|
14162
14077
|
if (rel.startsWith("..")) return null;
|
|
14163
14078
|
} catch {
|
|
14164
14079
|
return null;
|
|
@@ -14171,7 +14086,7 @@ function pathKey(filePath) {
|
|
|
14171
14086
|
function snapshot_path(sessionId, filePath) {
|
|
14172
14087
|
const d = sessionDir(sessionId);
|
|
14173
14088
|
if (!d) return null;
|
|
14174
|
-
return
|
|
14089
|
+
return path10.join(d, `${pathKey(filePath)}.bin`);
|
|
14175
14090
|
}
|
|
14176
14091
|
function kindSidecarPath(snapshotPath) {
|
|
14177
14092
|
return snapshotPath + ".kind";
|
|
@@ -14179,11 +14094,11 @@ function kindSidecarPath(snapshotPath) {
|
|
|
14179
14094
|
function writeSnapshotKind(sidecarPath, kind) {
|
|
14180
14095
|
try {
|
|
14181
14096
|
const safeKind = VALID_KINDS.has(kind) ? kind : KIND_READ;
|
|
14182
|
-
const dir =
|
|
14183
|
-
if (!
|
|
14097
|
+
const dir = path10.dirname(sidecarPath);
|
|
14098
|
+
if (!fs13.existsSync(dir)) {
|
|
14184
14099
|
ensureDirSync(dir);
|
|
14185
14100
|
}
|
|
14186
|
-
|
|
14101
|
+
fs13.writeFileSync(sidecarPath, safeKind, "utf8");
|
|
14187
14102
|
return true;
|
|
14188
14103
|
} catch {
|
|
14189
14104
|
return false;
|
|
@@ -14192,12 +14107,12 @@ function writeSnapshotKind(sidecarPath, kind) {
|
|
|
14192
14107
|
function evictOldest(d, maxCount) {
|
|
14193
14108
|
try {
|
|
14194
14109
|
const entries = [];
|
|
14195
|
-
const files =
|
|
14110
|
+
const files = fs13.readdirSync(d);
|
|
14196
14111
|
for (const file of files) {
|
|
14197
|
-
const fullPath =
|
|
14112
|
+
const fullPath = path10.join(d, file);
|
|
14198
14113
|
if (!file.endsWith(".bin")) continue;
|
|
14199
14114
|
try {
|
|
14200
|
-
const stat2 =
|
|
14115
|
+
const stat2 = fs13.statSync(fullPath);
|
|
14201
14116
|
entries.push([fullPath, stat2.mtimeMs]);
|
|
14202
14117
|
} catch {
|
|
14203
14118
|
continue;
|
|
@@ -14209,10 +14124,10 @@ function evictOldest(d, maxCount) {
|
|
|
14209
14124
|
const over = entries.length - maxCount;
|
|
14210
14125
|
for (const [p] of entries.slice(0, over)) {
|
|
14211
14126
|
try {
|
|
14212
|
-
|
|
14127
|
+
fs13.unlinkSync(p);
|
|
14213
14128
|
removed++;
|
|
14214
14129
|
try {
|
|
14215
|
-
|
|
14130
|
+
fs13.unlinkSync(kindSidecarPath(p));
|
|
14216
14131
|
} catch {
|
|
14217
14132
|
}
|
|
14218
14133
|
} catch {
|
|
@@ -14241,10 +14156,10 @@ function store(sessionId, filePath, content, opts = {}) {
|
|
|
14241
14156
|
if (!p) return null;
|
|
14242
14157
|
const sha = fingerprintContent(stored);
|
|
14243
14158
|
try {
|
|
14244
|
-
const isNewEntry = !
|
|
14159
|
+
const isNewEntry = !fs13.existsSync(p);
|
|
14245
14160
|
if (!isNewEntry) {
|
|
14246
14161
|
try {
|
|
14247
|
-
const existing =
|
|
14162
|
+
const existing = fs13.readFileSync(p);
|
|
14248
14163
|
if (Buffer.from(existing).equals(stored)) {
|
|
14249
14164
|
return {
|
|
14250
14165
|
path: p,
|
|
@@ -14255,8 +14170,8 @@ function store(sessionId, filePath, content, opts = {}) {
|
|
|
14255
14170
|
} catch {
|
|
14256
14171
|
}
|
|
14257
14172
|
}
|
|
14258
|
-
const dir =
|
|
14259
|
-
if (!
|
|
14173
|
+
const dir = path10.dirname(p);
|
|
14174
|
+
if (!fs13.existsSync(dir)) {
|
|
14260
14175
|
ensureDirSync(dir);
|
|
14261
14176
|
}
|
|
14262
14177
|
if (isNewEntry) {
|
|
@@ -14276,9 +14191,9 @@ function store(sessionId, filePath, content, opts = {}) {
|
|
|
14276
14191
|
}
|
|
14277
14192
|
function load(sessionId, filePath, opts = {}) {
|
|
14278
14193
|
const p = snapshot_path(sessionId, filePath);
|
|
14279
|
-
if (!p || !
|
|
14194
|
+
if (!p || !fs13.existsSync(p)) return null;
|
|
14280
14195
|
try {
|
|
14281
|
-
const stat2 =
|
|
14196
|
+
const stat2 = fs13.statSync(p);
|
|
14282
14197
|
if (stat2.size > MAX_SNAPSHOT_BYTES) {
|
|
14283
14198
|
return null;
|
|
14284
14199
|
}
|
|
@@ -14286,7 +14201,7 @@ function load(sessionId, filePath, opts = {}) {
|
|
|
14286
14201
|
return null;
|
|
14287
14202
|
}
|
|
14288
14203
|
try {
|
|
14289
|
-
const data =
|
|
14204
|
+
const data = fs13.readFileSync(p);
|
|
14290
14205
|
if (opts.expected_sha) {
|
|
14291
14206
|
const actualSha = fingerprintContent(data);
|
|
14292
14207
|
if (actualSha.toLowerCase() !== opts.expected_sha.toLowerCase()) {
|
|
@@ -14299,31 +14214,31 @@ function load(sessionId, filePath, opts = {}) {
|
|
|
14299
14214
|
}
|
|
14300
14215
|
}
|
|
14301
14216
|
function removeEligibleSnapshotFile(fullPath, file, cutoff) {
|
|
14302
|
-
const stat2 =
|
|
14217
|
+
const stat2 = fs13.lstatSync(fullPath);
|
|
14303
14218
|
if ((stat2.mode & 61440) === 40960) return false;
|
|
14304
14219
|
if (cutoff !== void 0 && stat2.mtimeMs >= cutoff) return false;
|
|
14305
|
-
|
|
14220
|
+
fs13.unlinkSync(fullPath);
|
|
14306
14221
|
return file.endsWith(".bin");
|
|
14307
14222
|
}
|
|
14308
14223
|
function cleanup_stale(maxAgeHours = 24) {
|
|
14309
|
-
const base =
|
|
14310
|
-
if (!
|
|
14224
|
+
const base = path10.join(tokenGoatHome(), "session_snapshots");
|
|
14225
|
+
if (!fs13.existsSync(base)) return 0;
|
|
14311
14226
|
const cutoff = Date.now() - maxAgeHours * 3600 * 1e3;
|
|
14312
14227
|
let removed = 0;
|
|
14313
14228
|
try {
|
|
14314
|
-
const sessionDirs =
|
|
14229
|
+
const sessionDirs = fs13.readdirSync(base);
|
|
14315
14230
|
for (const sessionDir2 of sessionDirs) {
|
|
14316
|
-
const sessionPath2 =
|
|
14231
|
+
const sessionPath2 = path10.join(base, sessionDir2);
|
|
14317
14232
|
try {
|
|
14318
|
-
const stat2 =
|
|
14233
|
+
const stat2 = fs13.statSync(sessionPath2);
|
|
14319
14234
|
if (!stat2.isDirectory()) continue;
|
|
14320
14235
|
} catch {
|
|
14321
14236
|
continue;
|
|
14322
14237
|
}
|
|
14323
14238
|
try {
|
|
14324
|
-
const files =
|
|
14239
|
+
const files = fs13.readdirSync(sessionPath2);
|
|
14325
14240
|
for (const file of files) {
|
|
14326
|
-
const fullPath =
|
|
14241
|
+
const fullPath = path10.join(sessionPath2, file);
|
|
14327
14242
|
try {
|
|
14328
14243
|
if (removeEligibleSnapshotFile(fullPath, file, cutoff)) removed++;
|
|
14329
14244
|
} catch {
|
|
@@ -14331,7 +14246,7 @@ function cleanup_stale(maxAgeHours = 24) {
|
|
|
14331
14246
|
}
|
|
14332
14247
|
}
|
|
14333
14248
|
try {
|
|
14334
|
-
|
|
14249
|
+
fs13.rmdirSync(sessionPath2);
|
|
14335
14250
|
} catch {
|
|
14336
14251
|
}
|
|
14337
14252
|
} catch {
|
|
@@ -14363,7 +14278,7 @@ var unclearedDrainingSnapshots = /* @__PURE__ */ new Map();
|
|
|
14363
14278
|
function drainingSnapshotStamp(file, content) {
|
|
14364
14279
|
let identity = "unknown";
|
|
14365
14280
|
try {
|
|
14366
|
-
const stat2 =
|
|
14281
|
+
const stat2 = fs14.statSync(file);
|
|
14367
14282
|
identity = `${stat2.mtimeMs}:${stat2.size}:${stat2.ino}`;
|
|
14368
14283
|
} catch {
|
|
14369
14284
|
}
|
|
@@ -14372,15 +14287,15 @@ function drainingSnapshotStamp(file, content) {
|
|
|
14372
14287
|
var DRAINING_READ_ATTEMPTS = 5;
|
|
14373
14288
|
var DRAINING_READ_RETRY_DELAY_MS = 50;
|
|
14374
14289
|
function listDrainingFiles(queuePath) {
|
|
14375
|
-
const dir =
|
|
14376
|
-
const base = `${
|
|
14290
|
+
const dir = path11.dirname(queuePath);
|
|
14291
|
+
const base = `${path11.basename(queuePath)}.draining`;
|
|
14377
14292
|
let entries;
|
|
14378
14293
|
try {
|
|
14379
|
-
entries =
|
|
14294
|
+
entries = fs14.readdirSync(dir);
|
|
14380
14295
|
} catch {
|
|
14381
14296
|
return [];
|
|
14382
14297
|
}
|
|
14383
|
-
return entries.filter((name) => name === base || name.startsWith(`${base}.alt-`) && !name.includes(".corrupt-")).sort().map((name) =>
|
|
14298
|
+
return entries.filter((name) => name === base || name.startsWith(`${base}.alt-`) && !name.includes(".corrupt-")).sort().map((name) => path11.join(dir, name));
|
|
14384
14299
|
}
|
|
14385
14300
|
var MAX_TRANSIENT_RETRIES = 5;
|
|
14386
14301
|
function bumpRetryCount(dbPath, absPath) {
|
|
@@ -14408,17 +14323,17 @@ function clearRetryCount(dbPath, absPath) {
|
|
|
14408
14323
|
}
|
|
14409
14324
|
}
|
|
14410
14325
|
function dirtyQueuePathFor(dir) {
|
|
14411
|
-
return
|
|
14326
|
+
return path11.join(dir, "queue", "dirty.txt");
|
|
14412
14327
|
}
|
|
14413
14328
|
function drainHeartbeatPathFor(dir) {
|
|
14414
|
-
return
|
|
14329
|
+
return path11.join(dir, "queue", "drain-heartbeat");
|
|
14415
14330
|
}
|
|
14416
14331
|
function writeDrainHeartbeat(dir, force = false) {
|
|
14417
14332
|
const now = Date.now();
|
|
14418
14333
|
if (!force && now - (heartbeatWriteTimes.get(dir) ?? 0) < WORKER_HEARTBEAT_REFRESH_MS) return;
|
|
14419
14334
|
try {
|
|
14420
|
-
ensureDirSync(
|
|
14421
|
-
|
|
14335
|
+
ensureDirSync(path11.dirname(drainHeartbeatPathFor(dir)));
|
|
14336
|
+
fs14.writeFileSync(drainHeartbeatPathFor(dir), `${process.pid}
|
|
14422
14337
|
`);
|
|
14423
14338
|
heartbeatWriteTimes.set(dir, now);
|
|
14424
14339
|
} catch {
|
|
@@ -14427,15 +14342,15 @@ function writeDrainHeartbeat(dir, force = false) {
|
|
|
14427
14342
|
function hasFreshWorkerHeartbeat(dir, pid) {
|
|
14428
14343
|
try {
|
|
14429
14344
|
const heartbeatPath = drainHeartbeatPathFor(dir);
|
|
14430
|
-
if (Date.now() -
|
|
14431
|
-
return
|
|
14345
|
+
if (Date.now() - fs14.statSync(heartbeatPath).mtimeMs > WORKER_HEARTBEAT_STALE_MS) return false;
|
|
14346
|
+
return fs14.readFileSync(heartbeatPath, "utf8").trim() === String(pid);
|
|
14432
14347
|
} catch {
|
|
14433
14348
|
return false;
|
|
14434
14349
|
}
|
|
14435
14350
|
}
|
|
14436
14351
|
function pidFileIsWithinStartupGrace(dir) {
|
|
14437
14352
|
try {
|
|
14438
|
-
return Date.now() -
|
|
14353
|
+
return Date.now() - fs14.statSync(workerPidPath(dir)).mtimeMs < WORKER_STARTUP_GRACE_MS;
|
|
14439
14354
|
} catch {
|
|
14440
14355
|
return false;
|
|
14441
14356
|
}
|
|
@@ -14469,28 +14384,28 @@ function parseDirtyQueueLines(raw) {
|
|
|
14469
14384
|
return out;
|
|
14470
14385
|
}
|
|
14471
14386
|
function workerPidPath(dir = dataDir()) {
|
|
14472
|
-
return
|
|
14387
|
+
return path11.join(dir, "worker.pid");
|
|
14473
14388
|
}
|
|
14474
14389
|
function getDirtyPathsFor(dir) {
|
|
14475
14390
|
let raw;
|
|
14476
14391
|
try {
|
|
14477
|
-
raw =
|
|
14392
|
+
raw = fs14.readFileSync(dirtyQueuePathFor(dir), "utf8");
|
|
14478
14393
|
} catch {
|
|
14479
14394
|
return [];
|
|
14480
14395
|
}
|
|
14481
14396
|
return parseDirtyQueueLines(raw);
|
|
14482
14397
|
}
|
|
14483
14398
|
function workerErrorLogPath(dir) {
|
|
14484
|
-
return
|
|
14399
|
+
return path11.join(dir, "worker-errors.log");
|
|
14485
14400
|
}
|
|
14486
14401
|
var WORKER_ERROR_LOG_MAX_BYTES = 5 * 1024 * 1024;
|
|
14487
14402
|
var CORRUPT_QUARANTINE_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1e3;
|
|
14488
14403
|
function cleanupWorkerStateFiles(dir) {
|
|
14489
14404
|
try {
|
|
14490
14405
|
const logPath = workerErrorLogPath(dir);
|
|
14491
|
-
const stat2 =
|
|
14406
|
+
const stat2 = fs14.statSync(logPath);
|
|
14492
14407
|
if (stat2.size > WORKER_ERROR_LOG_MAX_BYTES) {
|
|
14493
|
-
|
|
14408
|
+
fs14.writeFileSync(
|
|
14494
14409
|
logPath,
|
|
14495
14410
|
`${(/* @__PURE__ */ new Date()).toISOString()} worker-errors.log rotated (exceeded ${WORKER_ERROR_LOG_MAX_BYTES} bytes)
|
|
14496
14411
|
`
|
|
@@ -14499,13 +14414,13 @@ function cleanupWorkerStateFiles(dir) {
|
|
|
14499
14414
|
} catch {
|
|
14500
14415
|
}
|
|
14501
14416
|
try {
|
|
14502
|
-
const queueDir =
|
|
14417
|
+
const queueDir = path11.dirname(dirtyQueuePathFor(dir));
|
|
14503
14418
|
const cutoff = Date.now() - CORRUPT_QUARANTINE_MAX_AGE_MS;
|
|
14504
|
-
for (const file of
|
|
14419
|
+
for (const file of fs14.readdirSync(queueDir)) {
|
|
14505
14420
|
if (!file.includes(".corrupt-")) continue;
|
|
14506
|
-
const full =
|
|
14421
|
+
const full = path11.join(queueDir, file);
|
|
14507
14422
|
try {
|
|
14508
|
-
if (
|
|
14423
|
+
if (fs14.statSync(full).mtimeMs < cutoff) fs14.unlinkSync(full);
|
|
14509
14424
|
} catch {
|
|
14510
14425
|
}
|
|
14511
14426
|
}
|
|
@@ -14539,7 +14454,7 @@ registerReset(() => {
|
|
|
14539
14454
|
function embedFileSerialized(absPath, dbPath, sha) {
|
|
14540
14455
|
const key = foldPath(absPath);
|
|
14541
14456
|
const prior = inFlightEmbeddings.get(key);
|
|
14542
|
-
const dir =
|
|
14457
|
+
const dir = path11.dirname(dbPath);
|
|
14543
14458
|
const onEmbedError = (err) => {
|
|
14544
14459
|
const message = extractErrorMessage(err);
|
|
14545
14460
|
appendWorkerErrorLog(dir, `${(/* @__PURE__ */ new Date()).toISOString()} indexFileEmbeddings failed for ${absPath}: ${message}
|
|
@@ -14576,7 +14491,7 @@ function oneLogLine(line) {
|
|
|
14576
14491
|
}
|
|
14577
14492
|
function appendWorkerErrorLog(dir, line) {
|
|
14578
14493
|
try {
|
|
14579
|
-
|
|
14494
|
+
fs14.appendFileSync(workerErrorLogPath(dir), oneLogLine(line));
|
|
14580
14495
|
} catch {
|
|
14581
14496
|
}
|
|
14582
14497
|
}
|
|
@@ -14593,7 +14508,7 @@ function logTransientReadFailure(dir, absPath) {
|
|
|
14593
14508
|
);
|
|
14594
14509
|
}
|
|
14595
14510
|
function bumpAndCheckRetry(dir, absPath) {
|
|
14596
|
-
const dbPath =
|
|
14511
|
+
const dbPath = path11.join(dir, "global.db");
|
|
14597
14512
|
const attempts = bumpRetryCount(dbPath, absPath);
|
|
14598
14513
|
if (attempts > MAX_TRANSIENT_RETRIES) {
|
|
14599
14514
|
if (attempts === MAX_TRANSIENT_RETRIES + 1) {
|
|
@@ -14610,14 +14525,14 @@ function bumpAndCheckRetry(dir, absPath) {
|
|
|
14610
14525
|
function appendToDirtyQueue(dir, absPath) {
|
|
14611
14526
|
const queuePath = dirtyQueuePathFor(dir);
|
|
14612
14527
|
try {
|
|
14613
|
-
ensureDirSync(
|
|
14528
|
+
ensureDirSync(path11.dirname(queuePath));
|
|
14614
14529
|
let leadingNewline = "";
|
|
14615
14530
|
try {
|
|
14616
|
-
const existing =
|
|
14531
|
+
const existing = fs14.readFileSync(queuePath, "utf8");
|
|
14617
14532
|
if (existing.length > 0 && !existing.endsWith("\n")) leadingNewline = "\n";
|
|
14618
14533
|
} catch {
|
|
14619
14534
|
}
|
|
14620
|
-
|
|
14535
|
+
fs14.appendFileSync(queuePath, `${leadingNewline}${encodeDirtyQueueLine(absPath)}
|
|
14621
14536
|
`);
|
|
14622
14537
|
} catch {
|
|
14623
14538
|
}
|
|
@@ -14626,7 +14541,7 @@ function requeueDirtyPath(dir, absPath) {
|
|
|
14626
14541
|
if (bumpAndCheckRetry(dir, absPath)) appendToDirtyQueue(dir, absPath);
|
|
14627
14542
|
}
|
|
14628
14543
|
function makeIndexer(dbPath) {
|
|
14629
|
-
const dir =
|
|
14544
|
+
const dir = path11.dirname(dbPath);
|
|
14630
14545
|
return (absPath, sha) => {
|
|
14631
14546
|
try {
|
|
14632
14547
|
const ixCfgForSkip = loadConfig().indexing;
|
|
@@ -14687,14 +14602,14 @@ function processDirtyBatch(paths, index = makeIndexer(globalDbPath()), remove =
|
|
|
14687
14602
|
continue;
|
|
14688
14603
|
}
|
|
14689
14604
|
try {
|
|
14690
|
-
const
|
|
14605
|
+
const dirname14 = path11.dirname(p);
|
|
14691
14606
|
let root;
|
|
14692
|
-
if (projectRootCache.has(
|
|
14693
|
-
root = projectRootCache.get(
|
|
14607
|
+
if (projectRootCache.has(dirname14)) {
|
|
14608
|
+
root = projectRootCache.get(dirname14) ?? null;
|
|
14694
14609
|
} else {
|
|
14695
|
-
const project = findProject(
|
|
14610
|
+
const project = findProject(dirname14);
|
|
14696
14611
|
root = project?.root ?? null;
|
|
14697
|
-
projectRootCache.set(
|
|
14612
|
+
projectRootCache.set(dirname14, root);
|
|
14698
14613
|
}
|
|
14699
14614
|
if (root) lastKnownProjectRoots.set(dir, root);
|
|
14700
14615
|
} catch {
|
|
@@ -14704,7 +14619,7 @@ function processDirtyBatch(paths, index = makeIndexer(globalDbPath()), remove =
|
|
|
14704
14619
|
requeue(dir, p);
|
|
14705
14620
|
continue;
|
|
14706
14621
|
}
|
|
14707
|
-
clearRetryCount(
|
|
14622
|
+
clearRetryCount(path11.join(dir, "global.db"), p);
|
|
14708
14623
|
if (result !== false) indexed += 1;
|
|
14709
14624
|
}
|
|
14710
14625
|
return indexed;
|
|
@@ -14718,7 +14633,7 @@ function sleepSyncMs(ms) {
|
|
|
14718
14633
|
function drainOnce(dir, index, remove) {
|
|
14719
14634
|
const queuePath = dirtyQueuePathFor(dir);
|
|
14720
14635
|
const draining = `${queuePath}.draining`;
|
|
14721
|
-
const dbPath =
|
|
14636
|
+
const dbPath = path11.join(dir, "global.db");
|
|
14722
14637
|
const indexFn = index ?? makeIndexer(dbPath);
|
|
14723
14638
|
const removeFn = remove ?? makeRemover(dbPath);
|
|
14724
14639
|
let processed = 0;
|
|
@@ -14735,7 +14650,7 @@ function drainOnce(dir, index, remove) {
|
|
|
14735
14650
|
let drainingContent = null;
|
|
14736
14651
|
for (let attempt = 0; attempt < DRAINING_READ_ATTEMPTS; attempt++) {
|
|
14737
14652
|
try {
|
|
14738
|
-
drainingContent =
|
|
14653
|
+
drainingContent = fs14.readFileSync(drainingFile, "utf8");
|
|
14739
14654
|
break;
|
|
14740
14655
|
} catch {
|
|
14741
14656
|
if (attempt < DRAINING_READ_ATTEMPTS - 1) sleepSyncMs(DRAINING_READ_RETRY_DELAY_MS);
|
|
@@ -14743,7 +14658,7 @@ function drainOnce(dir, index, remove) {
|
|
|
14743
14658
|
}
|
|
14744
14659
|
if (drainingContent === null) {
|
|
14745
14660
|
try {
|
|
14746
|
-
|
|
14661
|
+
fs14.renameSync(drainingFile, `${drainingFile}.corrupt-${Date.now()}`);
|
|
14747
14662
|
unclearedDrainingSnapshots.delete(drainingFile);
|
|
14748
14663
|
} catch {
|
|
14749
14664
|
}
|
|
@@ -14753,23 +14668,23 @@ function drainOnce(dir, index, remove) {
|
|
|
14753
14668
|
processed += processDirtyBatch(parseDirtyQueueLines(drainingContent), indexFn, removeFn, dir, requeueFn);
|
|
14754
14669
|
}
|
|
14755
14670
|
try {
|
|
14756
|
-
|
|
14671
|
+
fs14.rmSync(drainingFile, { force: true });
|
|
14757
14672
|
unclearedDrainingSnapshots.delete(drainingFile);
|
|
14758
14673
|
} catch {
|
|
14759
14674
|
try {
|
|
14760
|
-
|
|
14675
|
+
fs14.renameSync(drainingFile, `${drainingFile}.corrupt-${Date.now()}`);
|
|
14761
14676
|
unclearedDrainingSnapshots.delete(drainingFile);
|
|
14762
14677
|
} catch {
|
|
14763
14678
|
unclearedDrainingSnapshots.set(drainingFile, drainingSnapshotStamp(drainingFile, drainingContent));
|
|
14764
14679
|
}
|
|
14765
14680
|
}
|
|
14766
14681
|
}
|
|
14767
|
-
if (
|
|
14768
|
-
const claimTarget =
|
|
14682
|
+
if (fs14.existsSync(queuePath)) {
|
|
14683
|
+
const claimTarget = fs14.existsSync(draining) ? `${draining}.alt-${Date.now()}` : draining;
|
|
14769
14684
|
let claimed = false;
|
|
14770
14685
|
for (let attempt = 0; attempt < 5; attempt++) {
|
|
14771
14686
|
try {
|
|
14772
|
-
|
|
14687
|
+
fs14.renameSync(queuePath, claimTarget);
|
|
14773
14688
|
claimed = true;
|
|
14774
14689
|
break;
|
|
14775
14690
|
} catch {
|
|
@@ -14780,14 +14695,14 @@ function drainOnce(dir, index, remove) {
|
|
|
14780
14695
|
let claimedContent = "";
|
|
14781
14696
|
let readOk = false;
|
|
14782
14697
|
try {
|
|
14783
|
-
claimedContent =
|
|
14698
|
+
claimedContent = fs14.readFileSync(claimTarget, "utf8");
|
|
14784
14699
|
readOk = true;
|
|
14785
14700
|
} catch {
|
|
14786
14701
|
}
|
|
14787
14702
|
if (readOk) {
|
|
14788
14703
|
processed += processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn);
|
|
14789
14704
|
try {
|
|
14790
|
-
const recheck =
|
|
14705
|
+
const recheck = fs14.readFileSync(claimTarget, "utf8");
|
|
14791
14706
|
if (recheck !== claimedContent) {
|
|
14792
14707
|
const extra = recheck.startsWith(claimedContent) ? recheck.slice(claimedContent.length) : recheck;
|
|
14793
14708
|
for (const p of parseDirtyQueueLines(extra)) appendToDirtyQueue(dir, p);
|
|
@@ -14795,10 +14710,10 @@ function drainOnce(dir, index, remove) {
|
|
|
14795
14710
|
} catch {
|
|
14796
14711
|
}
|
|
14797
14712
|
try {
|
|
14798
|
-
|
|
14713
|
+
fs14.rmSync(claimTarget, { force: true });
|
|
14799
14714
|
} catch {
|
|
14800
14715
|
try {
|
|
14801
|
-
|
|
14716
|
+
fs14.renameSync(claimTarget, `${claimTarget}.corrupt-${Date.now()}`);
|
|
14802
14717
|
} catch {
|
|
14803
14718
|
unclearedDrainingSnapshots.set(claimTarget, drainingSnapshotStamp(claimTarget, claimedContent));
|
|
14804
14719
|
}
|
|
@@ -14831,7 +14746,7 @@ function pidAlive(pid) {
|
|
|
14831
14746
|
}
|
|
14832
14747
|
function readPidFile(dir) {
|
|
14833
14748
|
try {
|
|
14834
|
-
const raw =
|
|
14749
|
+
const raw = fs14.readFileSync(workerPidPath(dir), "utf8").trim();
|
|
14835
14750
|
if (!/^\d+$/.test(raw)) return null;
|
|
14836
14751
|
return parseInt(raw, 10);
|
|
14837
14752
|
} catch {
|
|
@@ -14844,20 +14759,20 @@ function isWorkerRunning(dir = dataDir()) {
|
|
|
14844
14759
|
return pidAlive(pid) && hasFreshWorkerHeartbeat(dir, pid);
|
|
14845
14760
|
}
|
|
14846
14761
|
function workerHealthCheckMarkerPath(dir) {
|
|
14847
|
-
return
|
|
14762
|
+
return path11.join(dir, "worker-healthcheck.marker");
|
|
14848
14763
|
}
|
|
14849
14764
|
var WORKER_HEALTHCHECK_MIN_INTERVAL_MS = 5 * 60 * 1e3;
|
|
14850
14765
|
function ensureWorkerAlive(dir = dataDir()) {
|
|
14851
14766
|
if (process.env["TOKEN_GOAT_NO_WORKER_SPAWN"] === "1") return;
|
|
14852
14767
|
const markerPath = workerHealthCheckMarkerPath(dir);
|
|
14853
14768
|
try {
|
|
14854
|
-
const stat2 =
|
|
14769
|
+
const stat2 = fs14.statSync(markerPath);
|
|
14855
14770
|
if (Date.now() - stat2.mtimeMs < WORKER_HEALTHCHECK_MIN_INTERVAL_MS) return;
|
|
14856
14771
|
} catch {
|
|
14857
14772
|
}
|
|
14858
14773
|
try {
|
|
14859
14774
|
ensureDirSync(dir);
|
|
14860
|
-
|
|
14775
|
+
fs14.writeFileSync(markerPath, "");
|
|
14861
14776
|
} catch {
|
|
14862
14777
|
}
|
|
14863
14778
|
if (isWorkerRunning(dir)) return;
|
|
@@ -14887,7 +14802,7 @@ function stopWorker(dir = dataDir()) {
|
|
|
14887
14802
|
}
|
|
14888
14803
|
if (readPidFile(dir) === pid) {
|
|
14889
14804
|
try {
|
|
14890
|
-
|
|
14805
|
+
fs14.rmSync(workerPidPath(dir), { force: true });
|
|
14891
14806
|
} catch {
|
|
14892
14807
|
}
|
|
14893
14808
|
}
|
|
@@ -14902,7 +14817,7 @@ var WorkerAlreadyRunningError = class extends Error {
|
|
|
14902
14817
|
function claimWorkerPidFile(dir, pid) {
|
|
14903
14818
|
const pidPath = workerPidPath(dir);
|
|
14904
14819
|
try {
|
|
14905
|
-
|
|
14820
|
+
fs14.writeFileSync(pidPath, `${pid}
|
|
14906
14821
|
`, { flag: "wx" });
|
|
14907
14822
|
return true;
|
|
14908
14823
|
} catch (e) {
|
|
@@ -14919,11 +14834,11 @@ function claimWorkerPidFile(dir, pid) {
|
|
|
14919
14834
|
}
|
|
14920
14835
|
}
|
|
14921
14836
|
try {
|
|
14922
|
-
|
|
14837
|
+
fs14.rmSync(pidPath, { force: true });
|
|
14923
14838
|
} catch {
|
|
14924
14839
|
}
|
|
14925
14840
|
try {
|
|
14926
|
-
|
|
14841
|
+
fs14.writeFileSync(pidPath, `${pid}
|
|
14927
14842
|
`, { flag: "wx" });
|
|
14928
14843
|
return true;
|
|
14929
14844
|
} catch (e2) {
|
|
@@ -14933,9 +14848,9 @@ function claimWorkerPidFile(dir, pid) {
|
|
|
14933
14848
|
}
|
|
14934
14849
|
function daemonEntryScript() {
|
|
14935
14850
|
const self = fileURLToPath(import.meta.url);
|
|
14936
|
-
const launcher =
|
|
14851
|
+
const launcher = path11.join(path11.dirname(self), "token-goat.mjs");
|
|
14937
14852
|
try {
|
|
14938
|
-
if (
|
|
14853
|
+
if (fs14.existsSync(launcher)) return launcher;
|
|
14939
14854
|
} catch {
|
|
14940
14855
|
}
|
|
14941
14856
|
return self;
|
|
@@ -14946,9 +14861,9 @@ function startDetachedWorker(opts) {
|
|
|
14946
14861
|
try {
|
|
14947
14862
|
ensureDirSync(dir);
|
|
14948
14863
|
} catch (e) {
|
|
14949
|
-
if (e.code !== "EEXIST" || !
|
|
14864
|
+
if (e.code !== "EEXIST" || !fs14.existsSync(dir)) throw e;
|
|
14950
14865
|
}
|
|
14951
|
-
const child =
|
|
14866
|
+
const child = spawn(
|
|
14952
14867
|
process.execPath,
|
|
14953
14868
|
[daemonEntryScript(), "--worker-daemon"],
|
|
14954
14869
|
{
|
|
@@ -14984,7 +14899,7 @@ async function runWorkerLoop(dir, pollIntervalMs, shouldStop = () => false) {
|
|
|
14984
14899
|
let lastKnownRootsSweepMs = 0;
|
|
14985
14900
|
let ownedPidFile = false;
|
|
14986
14901
|
while (!shouldStop()) {
|
|
14987
|
-
if (!
|
|
14902
|
+
if (!fs14.existsSync(dir)) break;
|
|
14988
14903
|
const pidOwner = readPidFile(dir);
|
|
14989
14904
|
if (pidOwner === process.pid) ownedPidFile = true;
|
|
14990
14905
|
else if (ownedPidFile && pidOwner !== null) break;
|
|
@@ -15009,7 +14924,7 @@ async function runWorkerLoop(dir, pollIntervalMs, shouldStop = () => false) {
|
|
|
15009
14924
|
}
|
|
15010
14925
|
if (Date.now() - lastKnownRootsSweepMs >= KNOWN_ROOTS_SWEEP_INTERVAL_MS) {
|
|
15011
14926
|
try {
|
|
15012
|
-
const result = sweepKnownRoots(
|
|
14927
|
+
const result = sweepKnownRoots(path11.join(dir, "global.db"));
|
|
15013
14928
|
if (result.flaggedRoots.length > 0) {
|
|
15014
14929
|
appendWorkerErrorLog(
|
|
15015
14930
|
dir,
|
|
@@ -15045,7 +14960,7 @@ function runDetachedWorkerDaemon() {
|
|
|
15045
14960
|
process.on("exit", () => {
|
|
15046
14961
|
if (readPidFile(dir) === process.pid) {
|
|
15047
14962
|
try {
|
|
15048
|
-
|
|
14963
|
+
fs14.rmSync(workerPidPath(dir), { force: true });
|
|
15049
14964
|
} catch {
|
|
15050
14965
|
}
|
|
15051
14966
|
}
|
|
@@ -15056,26 +14971,26 @@ function runDetachedWorkerDaemon() {
|
|
|
15056
14971
|
|
|
15057
14972
|
// src/hooks_index.ts
|
|
15058
14973
|
init_define_import_meta_env();
|
|
15059
|
-
import * as
|
|
15060
|
-
import * as
|
|
14974
|
+
import * as fs15 from "node:fs";
|
|
14975
|
+
import * as path12 from "node:path";
|
|
15061
14976
|
function dirtyQueuePath() {
|
|
15062
|
-
return
|
|
14977
|
+
return path12.join(dataDir(), "queue", "dirty.txt");
|
|
15063
14978
|
}
|
|
15064
14979
|
function appendDirtyPath(normalizedPath) {
|
|
15065
14980
|
const queuePath = dirtyQueuePath();
|
|
15066
|
-
const dir =
|
|
14981
|
+
const dir = path12.dirname(queuePath);
|
|
15067
14982
|
try {
|
|
15068
14983
|
ensureDirSync(dir);
|
|
15069
14984
|
} catch (e) {
|
|
15070
|
-
if (e.code !== "EEXIST" || !
|
|
14985
|
+
if (e.code !== "EEXIST" || !fs15.existsSync(dir)) throw e;
|
|
15071
14986
|
}
|
|
15072
14987
|
let leadingNewline = "";
|
|
15073
14988
|
try {
|
|
15074
|
-
const existing =
|
|
14989
|
+
const existing = fs15.readFileSync(queuePath, "utf8");
|
|
15075
14990
|
if (existing.length > 0 && !existing.endsWith("\n")) leadingNewline = "\n";
|
|
15076
14991
|
} catch {
|
|
15077
14992
|
}
|
|
15078
|
-
|
|
14993
|
+
fs15.appendFileSync(queuePath, `${leadingNewline}${encodeDirtyQueueLine(normalizedPath)}
|
|
15079
14994
|
`);
|
|
15080
14995
|
}
|
|
15081
14996
|
function enqueueDirtyPathSafe(filePath, opts) {
|
|
@@ -15093,7 +15008,7 @@ function getDirtyPaths() {
|
|
|
15093
15008
|
const queuePath = dirtyQueuePath();
|
|
15094
15009
|
let raw;
|
|
15095
15010
|
try {
|
|
15096
|
-
raw =
|
|
15011
|
+
raw = fs15.readFileSync(queuePath, "utf8");
|
|
15097
15012
|
} catch {
|
|
15098
15013
|
return [];
|
|
15099
15014
|
}
|
|
@@ -15102,9 +15017,9 @@ function getDirtyPaths() {
|
|
|
15102
15017
|
function preCompactIndexHandler(_event) {
|
|
15103
15018
|
const paths = getDirtyPaths();
|
|
15104
15019
|
if (paths.length > 0) {
|
|
15105
|
-
const sidecar =
|
|
15020
|
+
const sidecar = path12.join(dataDir(), "queue", "pending.txt");
|
|
15106
15021
|
try {
|
|
15107
|
-
ensureDirSync(
|
|
15022
|
+
ensureDirSync(path12.dirname(sidecar));
|
|
15108
15023
|
atomicWriteBytes(sidecar, Buffer.from(`${paths.join("\n")}
|
|
15109
15024
|
`, "utf8"));
|
|
15110
15025
|
} catch {
|
|
@@ -15116,7 +15031,7 @@ registerHook("pre_compact", preCompactIndexHandler, { advisory: true });
|
|
|
15116
15031
|
|
|
15117
15032
|
// src/section_reader.ts
|
|
15118
15033
|
init_define_import_meta_env();
|
|
15119
|
-
import { readFileSync as
|
|
15034
|
+
import { readFileSync as readFileSync11 } from "node:fs";
|
|
15120
15035
|
function parseHeadingSpec(spec, headers) {
|
|
15121
15036
|
const m = /^([^#\r\n]+)#(\d+)$/.exec(spec);
|
|
15122
15037
|
if (m !== null && m[1] !== void 0 && m[2] !== void 0) {
|
|
@@ -15414,7 +15329,7 @@ function readTextForSections(filePath, readFn) {
|
|
|
15414
15329
|
text = read;
|
|
15415
15330
|
} else {
|
|
15416
15331
|
try {
|
|
15417
|
-
text = decodeSource(
|
|
15332
|
+
text = decodeSource(readFileSync11(filePath));
|
|
15418
15333
|
} catch {
|
|
15419
15334
|
return null;
|
|
15420
15335
|
}
|
|
@@ -15475,8 +15390,8 @@ function listSections(filePath, readFn) {
|
|
|
15475
15390
|
|
|
15476
15391
|
// src/hooks_read.ts
|
|
15477
15392
|
init_define_import_meta_env();
|
|
15478
|
-
import * as
|
|
15479
|
-
import * as
|
|
15393
|
+
import * as fs20 from "node:fs";
|
|
15394
|
+
import * as path17 from "node:path";
|
|
15480
15395
|
|
|
15481
15396
|
// src/served_lines.ts
|
|
15482
15397
|
init_define_import_meta_env();
|
|
@@ -15570,8 +15485,8 @@ function planServedElisions(rows, bodies) {
|
|
|
15570
15485
|
|
|
15571
15486
|
// src/compact.ts
|
|
15572
15487
|
init_define_import_meta_env();
|
|
15573
|
-
import * as
|
|
15574
|
-
import * as
|
|
15488
|
+
import * as fs16 from "node:fs";
|
|
15489
|
+
import * as path13 from "node:path";
|
|
15575
15490
|
var CONTEXT_AUTOCOMPACT_TOKENS = 66e4;
|
|
15576
15491
|
var CATALOG_TOKENS = 10800;
|
|
15577
15492
|
var CONTEXT_TIER_WARM = 0.5;
|
|
@@ -15761,7 +15676,7 @@ function inferSessionGoal(cache, maxTokens = 80) {
|
|
|
15761
15676
|
const dirCounts = new Counter();
|
|
15762
15677
|
for (const fpath of editedPaths) {
|
|
15763
15678
|
try {
|
|
15764
|
-
let parent =
|
|
15679
|
+
let parent = path13.dirname(fpath);
|
|
15765
15680
|
if (parent === ".") {
|
|
15766
15681
|
parent = "root";
|
|
15767
15682
|
} else if (parent.startsWith("./")) {
|
|
@@ -15839,11 +15754,11 @@ function isNoisePath(inputPath) {
|
|
|
15839
15754
|
}
|
|
15840
15755
|
function findLatestSessionId() {
|
|
15841
15756
|
try {
|
|
15842
|
-
const sessionsDir =
|
|
15843
|
-
if (!
|
|
15757
|
+
const sessionsDir = path13.join(tokenGoatHome(), "sessions");
|
|
15758
|
+
if (!fs16.existsSync(sessionsDir)) {
|
|
15844
15759
|
return null;
|
|
15845
15760
|
}
|
|
15846
|
-
const files =
|
|
15761
|
+
const files = fs16.readdirSync(sessionsDir);
|
|
15847
15762
|
const jsonFiles = files.filter((f) => f.endsWith(".json") && !f.includes(AGENT_SALT_MARKER));
|
|
15848
15763
|
if (jsonFiles.length === 0) {
|
|
15849
15764
|
return null;
|
|
@@ -15853,9 +15768,9 @@ function findLatestSessionId() {
|
|
|
15853
15768
|
return null;
|
|
15854
15769
|
}
|
|
15855
15770
|
let latestFile = firstFile;
|
|
15856
|
-
let latestMtime =
|
|
15771
|
+
let latestMtime = fs16.statSync(path13.join(sessionsDir, firstFile)).mtimeMs;
|
|
15857
15772
|
for (const file of jsonFiles) {
|
|
15858
|
-
const mtime =
|
|
15773
|
+
const mtime = fs16.statSync(path13.join(sessionsDir, file)).mtimeMs;
|
|
15859
15774
|
if (mtime > latestMtime) {
|
|
15860
15775
|
latestFile = file;
|
|
15861
15776
|
latestMtime = mtime;
|
|
@@ -15877,37 +15792,37 @@ function eventCount(cache) {
|
|
|
15877
15792
|
function writeSessionManifest(projectHash, sessionId, manifestJson) {
|
|
15878
15793
|
const safeSessionId = sanitizeIdForFilename(sessionId, 64);
|
|
15879
15794
|
if (!safeSessionId) return;
|
|
15880
|
-
const sessionsDir =
|
|
15881
|
-
if (!
|
|
15795
|
+
const sessionsDir = path13.join(dataDir(), "projects", projectHash, "sessions");
|
|
15796
|
+
if (!fs16.existsSync(sessionsDir)) {
|
|
15882
15797
|
ensureDirSync(sessionsDir);
|
|
15883
15798
|
}
|
|
15884
|
-
const dest =
|
|
15799
|
+
const dest = path13.join(sessionsDir, `${safeSessionId}.json`);
|
|
15885
15800
|
atomicWriteText(dest, JSON.stringify(manifestJson));
|
|
15886
15801
|
}
|
|
15887
15802
|
function readAllSessionManifests(projectHash, maxAgeSecs = 3600) {
|
|
15888
|
-
const sessionsDir =
|
|
15889
|
-
if (!
|
|
15803
|
+
const sessionsDir = path13.join(dataDir(), "projects", projectHash, "sessions");
|
|
15804
|
+
if (!fs16.existsSync(sessionsDir)) {
|
|
15890
15805
|
return [];
|
|
15891
15806
|
}
|
|
15892
15807
|
const now = Date.now() / 1e3;
|
|
15893
15808
|
const results = [];
|
|
15894
15809
|
try {
|
|
15895
|
-
const files =
|
|
15810
|
+
const files = fs16.readdirSync(sessionsDir);
|
|
15896
15811
|
for (const file of files) {
|
|
15897
15812
|
if (!file.endsWith(".json")) {
|
|
15898
15813
|
continue;
|
|
15899
15814
|
}
|
|
15900
15815
|
try {
|
|
15901
|
-
const fullPath =
|
|
15902
|
-
const stat2 =
|
|
15816
|
+
const fullPath = path13.join(sessionsDir, file);
|
|
15817
|
+
const stat2 = fs16.statSync(fullPath);
|
|
15903
15818
|
if (now - stat2.mtimeMs / 1e3 > maxAgeSecs) {
|
|
15904
15819
|
try {
|
|
15905
|
-
|
|
15820
|
+
fs16.unlinkSync(fullPath);
|
|
15906
15821
|
} catch {
|
|
15907
15822
|
}
|
|
15908
15823
|
continue;
|
|
15909
15824
|
}
|
|
15910
|
-
const text =
|
|
15825
|
+
const text = fs16.readFileSync(fullPath, "utf8");
|
|
15911
15826
|
const data = JSON.parse(text);
|
|
15912
15827
|
if (typeof data === "object" && data !== null && "files" in data) {
|
|
15913
15828
|
results.push(data);
|
|
@@ -16109,11 +16024,11 @@ function buildPackageManifestHint(options) {
|
|
|
16109
16024
|
return null;
|
|
16110
16025
|
}
|
|
16111
16026
|
}
|
|
16112
|
-
function _sanitizeHintPath(
|
|
16113
|
-
if (typeof
|
|
16027
|
+
function _sanitizeHintPath(path22) {
|
|
16028
|
+
if (typeof path22 !== "string") {
|
|
16114
16029
|
return "???";
|
|
16115
16030
|
}
|
|
16116
|
-
return
|
|
16031
|
+
return path22.replace(/[\x00]/g, "").slice(0, 200);
|
|
16117
16032
|
}
|
|
16118
16033
|
|
|
16119
16034
|
// src/hints/file_type_handler.ts
|
|
@@ -16128,9 +16043,16 @@ var FILE_TYPE_THRESHOLDS = {
|
|
|
16128
16043
|
xml: 2e4,
|
|
16129
16044
|
csv: 1e4,
|
|
16130
16045
|
tsv: 1e4,
|
|
16046
|
+
json: 2e4,
|
|
16047
|
+
yaml: 2e4,
|
|
16048
|
+
jsonl: 2e4,
|
|
16131
16049
|
transcript: 1e4,
|
|
16132
16050
|
office: 0,
|
|
16133
16051
|
// always block (.docx etc)
|
|
16052
|
+
sqlite: 0,
|
|
16053
|
+
// always block (.db, .sqlite, etc)
|
|
16054
|
+
parquet: 0,
|
|
16055
|
+
// always block (.parquet)
|
|
16134
16056
|
generic: 1e5
|
|
16135
16057
|
// catch-all for unrecognized large files
|
|
16136
16058
|
};
|
|
@@ -16325,6 +16247,7 @@ function handleDocx(filePath) {
|
|
|
16325
16247
|
message: [
|
|
16326
16248
|
`Word file \u2014 Read cannot return document content; this is not retryable with different Read parameters.`,
|
|
16327
16249
|
`See headings: token-goat docx-outline "${filePath}"`,
|
|
16250
|
+
`Extract tables: token-goat docx-tables "${filePath}"`,
|
|
16328
16251
|
`Read full text: token-goat docx-text "${filePath}"`
|
|
16329
16252
|
].join("\n")
|
|
16330
16253
|
};
|
|
@@ -16391,6 +16314,106 @@ function handleTranscript(filePath, content, contentLengthHint) {
|
|
|
16391
16314
|
].filter(Boolean).join("\n")
|
|
16392
16315
|
};
|
|
16393
16316
|
}
|
|
16317
|
+
function handleSqlite(filePath) {
|
|
16318
|
+
return {
|
|
16319
|
+
shouldBlock: true,
|
|
16320
|
+
message: [
|
|
16321
|
+
`SQLite database \u2014 Read cannot return binary database content; this is not retryable with different Read parameters.`,
|
|
16322
|
+
`See tables: token-goat sqlite-tables "${filePath}"`,
|
|
16323
|
+
`See schema: token-goat sqlite-schema "${filePath}"`,
|
|
16324
|
+
`Query data: token-goat sqlite-query "${filePath}" "SELECT ... FROM ... LIMIT 20"`
|
|
16325
|
+
].join("\n")
|
|
16326
|
+
};
|
|
16327
|
+
}
|
|
16328
|
+
function handleParquet(filePath) {
|
|
16329
|
+
return {
|
|
16330
|
+
shouldBlock: true,
|
|
16331
|
+
message: [
|
|
16332
|
+
`Parquet file \u2014 Read cannot return binary columnar content; this is not retryable with different Read parameters.`,
|
|
16333
|
+
`Query with DuckDB: duckdb -c "SELECT * FROM read_parquet('${filePath}') LIMIT 10"`
|
|
16334
|
+
].join("\n")
|
|
16335
|
+
};
|
|
16336
|
+
}
|
|
16337
|
+
function handleJson(filePath, content, contentLengthHint) {
|
|
16338
|
+
const length = contentLengthHint ?? content.length;
|
|
16339
|
+
if (length < FILE_TYPE_THRESHOLDS.json) return { shouldBlock: false, message: "" };
|
|
16340
|
+
if (previewUnavailable(content, length)) {
|
|
16341
|
+
return {
|
|
16342
|
+
shouldBlock: true,
|
|
16343
|
+
message: [
|
|
16344
|
+
`Large JSON file (${formatBytes(length)}) \u2014 too large to preview (exceeds the in-hook scan cap).`,
|
|
16345
|
+
`See structure: token-goat json-outline "${filePath}"`,
|
|
16346
|
+
`Query subtree: token-goat json-query "${filePath}" '<path>'`
|
|
16347
|
+
].join("\n")
|
|
16348
|
+
};
|
|
16349
|
+
}
|
|
16350
|
+
let summary = "";
|
|
16351
|
+
try {
|
|
16352
|
+
const parsed = JSON.parse(content);
|
|
16353
|
+
if (Array.isArray(parsed)) {
|
|
16354
|
+
summary = `JSON array (${parsed.length} items)`;
|
|
16355
|
+
if (parsed.length > 0 && typeof parsed[0] === "object" && parsed[0] !== null) {
|
|
16356
|
+
const itemKeys = Object.keys(parsed[0]).slice(0, 10);
|
|
16357
|
+
summary += `, item keys: ${itemKeys.join(", ")}${Object.keys(parsed[0]).length > 10 ? "..." : ""}`;
|
|
16358
|
+
}
|
|
16359
|
+
} else if (typeof parsed === "object" && parsed !== null) {
|
|
16360
|
+
const keys = Object.keys(parsed).slice(0, 15);
|
|
16361
|
+
summary = `Top-level keys (${Object.keys(parsed).length}): ${keys.join(", ")}${Object.keys(parsed).length > 15 ? "..." : ""}`;
|
|
16362
|
+
}
|
|
16363
|
+
} catch {
|
|
16364
|
+
}
|
|
16365
|
+
return {
|
|
16366
|
+
shouldBlock: true,
|
|
16367
|
+
message: [
|
|
16368
|
+
`Large JSON file (${formatBytes(length)}).`,
|
|
16369
|
+
summary ? fenceUntrustedFileContent(summary) : "",
|
|
16370
|
+
`See structure: token-goat json-outline "${filePath}"`,
|
|
16371
|
+
`Query subtree: token-goat json-query "${filePath}" '<path>'`
|
|
16372
|
+
].filter(Boolean).join("\n")
|
|
16373
|
+
};
|
|
16374
|
+
}
|
|
16375
|
+
function handleYaml(filePath, content, contentLengthHint) {
|
|
16376
|
+
const length = contentLengthHint ?? content.length;
|
|
16377
|
+
if (length < FILE_TYPE_THRESHOLDS.yaml) return { shouldBlock: false, message: "" };
|
|
16378
|
+
return {
|
|
16379
|
+
shouldBlock: true,
|
|
16380
|
+
message: [
|
|
16381
|
+
`Large YAML file (${formatBytes(length)}).`,
|
|
16382
|
+
`See structure: token-goat yaml-outline "${filePath}"`,
|
|
16383
|
+
`Query subtree: token-goat yaml-query "${filePath}" '<path>'`
|
|
16384
|
+
].join("\n")
|
|
16385
|
+
};
|
|
16386
|
+
}
|
|
16387
|
+
function handleJsonl(filePath, content, contentLengthHint) {
|
|
16388
|
+
const length = contentLengthHint ?? content.length;
|
|
16389
|
+
if (length < FILE_TYPE_THRESHOLDS.jsonl) return { shouldBlock: false, message: "" };
|
|
16390
|
+
if (previewUnavailable(content, length)) {
|
|
16391
|
+
return {
|
|
16392
|
+
shouldBlock: true,
|
|
16393
|
+
message: `Large JSON Lines file (${formatBytes(length)}) \u2014 too large to preview (exceeds the in-hook scan cap). Slice records with Read using offset and limit (each line is a single JSON record).`
|
|
16394
|
+
};
|
|
16395
|
+
}
|
|
16396
|
+
const lines2 = content.split(/\r?\n/).filter((l) => l.trim().length > 0);
|
|
16397
|
+
let schema = "";
|
|
16398
|
+
if (lines2.length > 0) {
|
|
16399
|
+
try {
|
|
16400
|
+
const firstObj = JSON.parse(lines2[0]);
|
|
16401
|
+
if (typeof firstObj === "object" && firstObj !== null && !Array.isArray(firstObj)) {
|
|
16402
|
+
const keys = Object.keys(firstObj).slice(0, 15);
|
|
16403
|
+
schema = `Record keys: ${keys.join(", ")}${Object.keys(firstObj).length > 15 ? "..." : ""}`;
|
|
16404
|
+
}
|
|
16405
|
+
} catch {
|
|
16406
|
+
}
|
|
16407
|
+
}
|
|
16408
|
+
return {
|
|
16409
|
+
shouldBlock: true,
|
|
16410
|
+
message: [
|
|
16411
|
+
`Large JSON Lines file (${formatBytes(length)}, ~${lines2.length} records).`,
|
|
16412
|
+
schema ? fenceUntrustedFileContent(schema) : "",
|
|
16413
|
+
`Slice records with Read using offset and limit (each line is a single JSON record), e.g. offset=1, limit=5.`
|
|
16414
|
+
].filter(Boolean).join("\n")
|
|
16415
|
+
};
|
|
16416
|
+
}
|
|
16394
16417
|
function handleGenericLarge(filePath, contentLength) {
|
|
16395
16418
|
if (contentLength < FILE_TYPE_THRESHOLDS.generic) return { shouldBlock: false, message: "" };
|
|
16396
16419
|
return {
|
|
@@ -16411,40 +16434,45 @@ function dispatchFileTypeHandler(filePath, content, contentLengthHint) {
|
|
|
16411
16434
|
if (ext === "pptx") return handlePptx(filePath);
|
|
16412
16435
|
if (ext === "docx") return handleDocx(filePath);
|
|
16413
16436
|
if (["odt", "ods", "ott", "odp"].includes(ext)) return handleOfficeBinary(filePath);
|
|
16437
|
+
if (["sqlite", "db", "sqlite3", "db3"].includes(ext)) return handleSqlite(filePath);
|
|
16438
|
+
if (ext === "parquet") return handleParquet(filePath);
|
|
16414
16439
|
if (ext === "csv" || ext === "tsv") return handleCsv(filePath, content, effectiveLength);
|
|
16415
16440
|
if (ext === "vtt" || ext === "srt") return handleTranscript(filePath, content, effectiveLength);
|
|
16441
|
+
if (ext === "json") return handleJson(filePath, content, effectiveLength);
|
|
16442
|
+
if (["yaml", "yml"].includes(ext)) return handleYaml(filePath, content, effectiveLength);
|
|
16443
|
+
if (ext === "jsonl") return handleJsonl(filePath, content, effectiveLength);
|
|
16416
16444
|
return handleGenericLarge(filePath, effectiveLength);
|
|
16417
16445
|
}
|
|
16418
16446
|
|
|
16419
16447
|
// src/doc_compact.ts
|
|
16420
16448
|
init_define_import_meta_env();
|
|
16421
|
-
import * as
|
|
16422
|
-
import * as
|
|
16449
|
+
import * as fs17 from "fs";
|
|
16450
|
+
import * as path14 from "path";
|
|
16423
16451
|
var defaultSentencesPerSection = 2;
|
|
16424
16452
|
var headerPrefix = "<!-- token-goat doc-compact source-hash:";
|
|
16425
16453
|
var headerRegex = /^<!-- token-goat doc-compact source-hash:(\S+) source:(.+?) -->\r?$/;
|
|
16426
16454
|
var compactSubdir = "doc_compacts";
|
|
16427
16455
|
function sourceHash(filePath) {
|
|
16428
16456
|
try {
|
|
16429
|
-
return fingerprintContent(
|
|
16457
|
+
return fingerprintContent(fs17.readFileSync(filePath));
|
|
16430
16458
|
} catch {
|
|
16431
16459
|
return "";
|
|
16432
16460
|
}
|
|
16433
16461
|
}
|
|
16434
16462
|
function _compactSlug(absPathStr) {
|
|
16435
16463
|
const h = fingerprintContent(foldPath(absPathStr)).slice(0, 12);
|
|
16436
|
-
const ext =
|
|
16437
|
-
const stem =
|
|
16464
|
+
const ext = path14.extname(absPathStr);
|
|
16465
|
+
const stem = path14.basename(absPathStr, ext);
|
|
16438
16466
|
const safeStem = sanitizeIdForFilename(stem, 32);
|
|
16439
16467
|
return `${h}_${safeStem}`;
|
|
16440
16468
|
}
|
|
16441
16469
|
function compactPathFor(sourcePath) {
|
|
16442
16470
|
const abs = resolveIndexPath(sourcePath);
|
|
16443
|
-
return
|
|
16471
|
+
return path14.join(dataDir(), compactSubdir, `${_compactSlug(abs)}.md`);
|
|
16444
16472
|
}
|
|
16445
16473
|
function readCompactHeader(compactPath) {
|
|
16446
16474
|
try {
|
|
16447
|
-
const text =
|
|
16475
|
+
const text = fs17.readFileSync(compactPath, "utf-8");
|
|
16448
16476
|
const firstLine = text.split("\n")[0] || "";
|
|
16449
16477
|
const m = firstLine.match(headerRegex);
|
|
16450
16478
|
if (!m || !m[1] || !m[2]) return null;
|
|
@@ -16463,8 +16491,8 @@ function isCompactFresh(compactPath, sourcePath) {
|
|
|
16463
16491
|
}
|
|
16464
16492
|
function markCompactStale(compactPath) {
|
|
16465
16493
|
try {
|
|
16466
|
-
if (!
|
|
16467
|
-
const text =
|
|
16494
|
+
if (!fs17.existsSync(compactPath)) return false;
|
|
16495
|
+
const text = fs17.readFileSync(compactPath, "utf-8");
|
|
16468
16496
|
const lines2 = text.split("\n");
|
|
16469
16497
|
if (!lines2[0]) return false;
|
|
16470
16498
|
const m = lines2[0].match(headerRegex);
|
|
@@ -16479,7 +16507,7 @@ function markCompactStale(compactPath) {
|
|
|
16479
16507
|
}
|
|
16480
16508
|
function readCompactBody(compactPath) {
|
|
16481
16509
|
try {
|
|
16482
|
-
const text =
|
|
16510
|
+
const text = fs17.readFileSync(compactPath, "utf-8");
|
|
16483
16511
|
const lines2 = text.split("\n");
|
|
16484
16512
|
if (lines2.length < 2) return null;
|
|
16485
16513
|
const body = lines2.slice(1).join("\n").trimStart();
|
|
@@ -16489,14 +16517,14 @@ function readCompactBody(compactPath) {
|
|
|
16489
16517
|
}
|
|
16490
16518
|
}
|
|
16491
16519
|
function writeCompact(compactPath, sourcePath, compactBody, sourceRel) {
|
|
16492
|
-
const srcPath =
|
|
16520
|
+
const srcPath = path14.resolve(sourcePath);
|
|
16493
16521
|
const sha = sourceHash(srcPath);
|
|
16494
|
-
const displayRel = sourceRel ||
|
|
16522
|
+
const displayRel = sourceRel || path14.basename(srcPath);
|
|
16495
16523
|
const header = `${headerPrefix}${sha} source:${displayRel} -->
|
|
16496
16524
|
`;
|
|
16497
16525
|
const fullText = header + compactBody.trimStart();
|
|
16498
|
-
const dir =
|
|
16499
|
-
if (!
|
|
16526
|
+
const dir = path14.dirname(compactPath);
|
|
16527
|
+
if (!fs17.existsSync(dir)) {
|
|
16500
16528
|
ensureDirSync(dir);
|
|
16501
16529
|
}
|
|
16502
16530
|
atomicWriteText(compactPath, fullText);
|
|
@@ -16620,7 +16648,7 @@ function extractDocCompact(body, heading) {
|
|
|
16620
16648
|
}
|
|
16621
16649
|
function compactDoc(filePath, heading) {
|
|
16622
16650
|
try {
|
|
16623
|
-
const body =
|
|
16651
|
+
const body = fs17.readFileSync(filePath, "utf-8");
|
|
16624
16652
|
const compact = extractDocCompact(body, heading);
|
|
16625
16653
|
return compact || null;
|
|
16626
16654
|
} catch {
|
|
@@ -16631,20 +16659,20 @@ function compactDoc(filePath, heading) {
|
|
|
16631
16659
|
// src/evidence_cache.ts
|
|
16632
16660
|
init_define_import_meta_env();
|
|
16633
16661
|
import crypto from "node:crypto";
|
|
16634
|
-
import
|
|
16635
|
-
import
|
|
16662
|
+
import fs18 from "node:fs";
|
|
16663
|
+
import path15 from "node:path";
|
|
16636
16664
|
var MAX_ENTRIES = 500;
|
|
16637
16665
|
var MAX_TEXT_BYTES = 128 * 1024;
|
|
16638
16666
|
var CACHE_FILE = "workspace-evidence.json";
|
|
16639
16667
|
function cachePath() {
|
|
16640
|
-
return
|
|
16668
|
+
return path15.join(dataDir(), CACHE_FILE);
|
|
16641
16669
|
}
|
|
16642
16670
|
function hash(text) {
|
|
16643
16671
|
return crypto.createHash("sha256").update(text).digest("hex");
|
|
16644
16672
|
}
|
|
16645
16673
|
function load2() {
|
|
16646
16674
|
try {
|
|
16647
|
-
const parsed = JSON.parse(
|
|
16675
|
+
const parsed = JSON.parse(fs18.readFileSync(cachePath(), "utf8"));
|
|
16648
16676
|
if (!Array.isArray(parsed)) return [];
|
|
16649
16677
|
return parsed.filter(
|
|
16650
16678
|
(entry) => typeof entry === "object" && entry !== null && typeof entry.id === "string" && typeof entry.projectRoot === "string" && typeof entry.source === "string" && (entry.representation === "file" || entry.representation === "tool-output") && typeof entry.contentHash === "string" && typeof entry.text === "string" && typeof entry.createdAt === "number" && (entry.embedding === void 0 || typeof entry.embedding === "string")
|
|
@@ -16656,7 +16684,7 @@ function load2() {
|
|
|
16656
16684
|
function save(entries) {
|
|
16657
16685
|
try {
|
|
16658
16686
|
ensureDirSync(dataDir());
|
|
16659
|
-
|
|
16687
|
+
fs18.writeFileSync(cachePath(), JSON.stringify(entries.slice(0, MAX_ENTRIES)), "utf8");
|
|
16660
16688
|
} catch {
|
|
16661
16689
|
}
|
|
16662
16690
|
}
|
|
@@ -16747,7 +16775,7 @@ function buildDeltaCapsule(projectRoot, limit = 8) {
|
|
|
16747
16775
|
const root = normalizePath(projectRoot);
|
|
16748
16776
|
const changed = load2().filter((entry) => entry.projectRoot === root && entry.representation === "file").filter((entry) => {
|
|
16749
16777
|
try {
|
|
16750
|
-
return hash(
|
|
16778
|
+
return hash(fs18.readFileSync(entry.source, "utf8")) !== entry.contentHash;
|
|
16751
16779
|
} catch {
|
|
16752
16780
|
return true;
|
|
16753
16781
|
}
|
|
@@ -16759,8 +16787,8 @@ ${changed.map((entry) => `- ${displaySafePath(entry.source)} (use a fresh surgic
|
|
|
16759
16787
|
|
|
16760
16788
|
// src/notebook_compact.ts
|
|
16761
16789
|
init_define_import_meta_env();
|
|
16762
|
-
import * as
|
|
16763
|
-
import * as
|
|
16790
|
+
import * as fs19 from "node:fs";
|
|
16791
|
+
import * as path16 from "node:path";
|
|
16764
16792
|
var NB_STRIP_MIN_SAVINGS = 4096;
|
|
16765
16793
|
function stripNotebook(nbDict) {
|
|
16766
16794
|
const cells = [];
|
|
@@ -16780,23 +16808,23 @@ function stripNotebook(nbDict) {
|
|
|
16780
16808
|
var SIDECAR_DEFAULT_MAX_COUNT = 200;
|
|
16781
16809
|
var SIDECAR_DEFAULT_MAX_AGE_MS = 24 * 3600 * 1e3;
|
|
16782
16810
|
function pruneSidecars(cacheRoot, maxCount = SIDECAR_DEFAULT_MAX_COUNT, maxAgeMs = SIDECAR_DEFAULT_MAX_AGE_MS) {
|
|
16783
|
-
const nbStripDir =
|
|
16811
|
+
const nbStripDir = path16.join(cacheRoot, "nb_strip");
|
|
16784
16812
|
let removed = 0;
|
|
16785
16813
|
try {
|
|
16786
|
-
if (!
|
|
16814
|
+
if (!fs19.existsSync(nbStripDir)) return 0;
|
|
16787
16815
|
const cutoff = Date.now() - maxAgeMs;
|
|
16788
16816
|
const kept = [];
|
|
16789
|
-
for (const entry of
|
|
16790
|
-
const dir =
|
|
16817
|
+
for (const entry of fs19.readdirSync(nbStripDir)) {
|
|
16818
|
+
const dir = path16.join(nbStripDir, entry);
|
|
16791
16819
|
let mtime;
|
|
16792
16820
|
try {
|
|
16793
|
-
mtime =
|
|
16821
|
+
mtime = fs19.statSync(dir).mtimeMs;
|
|
16794
16822
|
} catch {
|
|
16795
16823
|
continue;
|
|
16796
16824
|
}
|
|
16797
16825
|
if (mtime < cutoff) {
|
|
16798
16826
|
try {
|
|
16799
|
-
|
|
16827
|
+
fs19.rmSync(dir, { recursive: true, force: true });
|
|
16800
16828
|
removed++;
|
|
16801
16829
|
} catch {
|
|
16802
16830
|
continue;
|
|
@@ -16809,7 +16837,7 @@ function pruneSidecars(cacheRoot, maxCount = SIDECAR_DEFAULT_MAX_COUNT, maxAgeMs
|
|
|
16809
16837
|
kept.sort((a, b) => a[1] - b[1]);
|
|
16810
16838
|
for (const [dir] of kept.slice(0, kept.length - maxCount)) {
|
|
16811
16839
|
try {
|
|
16812
|
-
|
|
16840
|
+
fs19.rmSync(dir, { recursive: true, force: true });
|
|
16813
16841
|
removed++;
|
|
16814
16842
|
} catch {
|
|
16815
16843
|
continue;
|
|
@@ -16823,9 +16851,9 @@ function pruneSidecars(cacheRoot, maxCount = SIDECAR_DEFAULT_MAX_COUNT, maxAgeMs
|
|
|
16823
16851
|
}
|
|
16824
16852
|
function getOrCreateSidecar(rawBytes, cacheRoot, opts = {}) {
|
|
16825
16853
|
const sha = fingerprintContent(rawBytes);
|
|
16826
|
-
const sidecarDir =
|
|
16827
|
-
const sidecarPath =
|
|
16828
|
-
if (
|
|
16854
|
+
const sidecarDir = path16.join(cacheRoot, "nb_strip", sha);
|
|
16855
|
+
const sidecarPath = path16.join(sidecarDir, "stripped.ipynb");
|
|
16856
|
+
if (fs19.existsSync(sidecarPath)) {
|
|
16829
16857
|
return [sidecarPath, false];
|
|
16830
16858
|
}
|
|
16831
16859
|
let nb;
|
|
@@ -16842,7 +16870,7 @@ function getOrCreateSidecar(rawBytes, cacheRoot, opts = {}) {
|
|
|
16842
16870
|
try {
|
|
16843
16871
|
ensureDirSync(sidecarDir);
|
|
16844
16872
|
} catch (err) {
|
|
16845
|
-
if (!
|
|
16873
|
+
if (!fs19.existsSync(sidecarDir)) {
|
|
16846
16874
|
throw err;
|
|
16847
16875
|
}
|
|
16848
16876
|
}
|
|
@@ -16930,10 +16958,10 @@ var COMMENT_SYNTAX = new Map([
|
|
|
16930
16958
|
...["py", "rb", "sh", "bash", "zsh", "pl", "r"].map((ext) => [ext, HASH]),
|
|
16931
16959
|
...["sql", "lua", "hs"].map((ext) => [ext, DOUBLE_DASH])
|
|
16932
16960
|
]);
|
|
16933
|
-
function commentSyntaxFor(
|
|
16934
|
-
const dot =
|
|
16961
|
+
function commentSyntaxFor(path22) {
|
|
16962
|
+
const dot = path22.lastIndexOf(".");
|
|
16935
16963
|
if (dot < 0) return null;
|
|
16936
|
-
return COMMENT_SYNTAX.get(
|
|
16964
|
+
return COMMENT_SYNTAX.get(path22.slice(dot + 1).toLowerCase()) ?? null;
|
|
16937
16965
|
}
|
|
16938
16966
|
function planCommentFolds(rows, syntax, commentKeep, commentMinBlock, occupied) {
|
|
16939
16967
|
if (syntax === null || rows.length === 0 || commentKeep < 1 || commentMinBlock < 1) return [];
|
|
@@ -17042,8 +17070,8 @@ function planProseFolds(rows, claimed) {
|
|
|
17042
17070
|
}
|
|
17043
17071
|
return folds;
|
|
17044
17072
|
}
|
|
17045
|
-
function foldDetail(
|
|
17046
|
-
const head = `${
|
|
17073
|
+
function foldDetail(path22, folds, maxLen = MAX_FOLD_DETAIL) {
|
|
17074
|
+
const head = `${path22}::`;
|
|
17047
17075
|
const ids = folds.map((f) => f.kind === "body" ? f.name : `#${f.firstLine}-${f.lastLine}`);
|
|
17048
17076
|
const budget = maxLen - head.length - FOLD_DETAIL_SUFFIX_BUDGET;
|
|
17049
17077
|
const kept = [];
|
|
@@ -17061,7 +17089,7 @@ function foldDetail(path24, folds, maxLen = MAX_FOLD_DETAIL) {
|
|
|
17061
17089
|
|
|
17062
17090
|
// src/fold_delivery.ts
|
|
17063
17091
|
init_define_import_meta_env();
|
|
17064
|
-
import { readFileSync as
|
|
17092
|
+
import { readFileSync as readFileSync14, statSync as statSync13 } from "node:fs";
|
|
17065
17093
|
var BODY_FOLD_KEEP_LINES = 8;
|
|
17066
17094
|
var BODY_FOLD_MIN_SPAN = 20;
|
|
17067
17095
|
var COMMENT_FOLD_KEEP_LINES = 2;
|
|
@@ -17093,10 +17121,10 @@ function commentFoldNotice(firstLine, lastLine, shownPath) {
|
|
|
17093
17121
|
var FOLD_SPAN_PARSE_MAX_BYTES = 4e5;
|
|
17094
17122
|
function parseFoldSpansFromDisk(normalizedPath, rows) {
|
|
17095
17123
|
try {
|
|
17096
|
-
if (
|
|
17124
|
+
if (statSync13(normalizedPath).size > FOLD_SPAN_PARSE_MAX_BYTES) return [];
|
|
17097
17125
|
const language = detectLanguage(normalizedPath);
|
|
17098
17126
|
if (!isTreeSitterAvailable(language)) return [];
|
|
17099
|
-
const fileText =
|
|
17127
|
+
const fileText = readFileSync14(normalizedPath, "utf-8");
|
|
17100
17128
|
const diskLines = fileText.split("\n");
|
|
17101
17129
|
for (const row of rows) {
|
|
17102
17130
|
const disk = diskLines[row.no - 1];
|
|
@@ -17112,7 +17140,7 @@ function parseFoldSpansFromDisk(normalizedPath, rows) {
|
|
|
17112
17140
|
function resolveFoldSpans(normalizedPath, hasCommentSyntax, rows) {
|
|
17113
17141
|
try {
|
|
17114
17142
|
const entry = getFileEntry(normalizedPath);
|
|
17115
|
-
if (entry !== null && entry.sha !== "" && entry.sha === fingerprintFile(normalizedPath) && entry.parserSha === PARSER_FINGERPRINT) {
|
|
17143
|
+
if (entry !== null && entry.sha !== "" && !isVirtualIndexedPath(normalizedPath) && entry.sha === fingerprintFile(normalizedPath) && entry.parserSha === PARSER_FINGERPRINT) {
|
|
17116
17144
|
return querySymbols({ filePath: normalizedPath, limit: BODY_FOLD_SYMBOL_LIMIT });
|
|
17117
17145
|
}
|
|
17118
17146
|
if (hasCommentSyntax) enqueueDirtyPathSafe(normalizedPath);
|
|
@@ -17327,8 +17355,8 @@ function isNodeModulesPath(p) {
|
|
|
17327
17355
|
return check.includes("/node_modules/") || check.includes("\\node_modules\\");
|
|
17328
17356
|
}
|
|
17329
17357
|
function relPathWithinRoot(root, target) {
|
|
17330
|
-
const rel =
|
|
17331
|
-
if (rel.startsWith("..") ||
|
|
17358
|
+
const rel = path17.relative(root, target).replace(/\\/g, "/");
|
|
17359
|
+
if (rel.startsWith("..") || path17.isAbsolute(rel)) return null;
|
|
17332
17360
|
return rel;
|
|
17333
17361
|
}
|
|
17334
17362
|
function _isDocFile(filePath) {
|
|
@@ -17412,7 +17440,7 @@ function scanRequestedSlice(absPath, offset, limit) {
|
|
|
17412
17440
|
const windowEnd = offset + limit;
|
|
17413
17441
|
let fd;
|
|
17414
17442
|
try {
|
|
17415
|
-
fd =
|
|
17443
|
+
fd = fs20.openSync(absPath, "r");
|
|
17416
17444
|
} catch {
|
|
17417
17445
|
return null;
|
|
17418
17446
|
}
|
|
@@ -17426,7 +17454,7 @@ function scanRequestedSlice(absPath, offset, limit) {
|
|
|
17426
17454
|
const nearSingleLine = lineNumber < NEAR_SINGLE_LINE_SCAN_THRESHOLD;
|
|
17427
17455
|
return { bytes: sliceBytes, trustworthy: nearSingleLine, nearSingleLine };
|
|
17428
17456
|
}
|
|
17429
|
-
const bytesRead =
|
|
17457
|
+
const bytesRead = fs20.readSync(fd, buf, 0, buf.length, null);
|
|
17430
17458
|
if (bytesRead === 0) {
|
|
17431
17459
|
return {
|
|
17432
17460
|
bytes: sliceBytes,
|
|
@@ -17445,7 +17473,7 @@ function scanRequestedSlice(absPath, offset, limit) {
|
|
|
17445
17473
|
}
|
|
17446
17474
|
} finally {
|
|
17447
17475
|
try {
|
|
17448
|
-
|
|
17476
|
+
fs20.closeSync(fd);
|
|
17449
17477
|
} catch {
|
|
17450
17478
|
}
|
|
17451
17479
|
}
|
|
@@ -17476,11 +17504,11 @@ function isDiffableSource(basename10) {
|
|
|
17476
17504
|
function isSourceExtension(basename10) {
|
|
17477
17505
|
return languageHasFlag(detectLanguage(basename10), "sourceHints");
|
|
17478
17506
|
}
|
|
17479
|
-
var BINARY_FILE_TYPE_EXTS = /* @__PURE__ */ new Set(["pdf", "docx", "xlsx", "pptx", "odt", "ods", "ott", "odp"]);
|
|
17480
|
-
var TEXT_FILE_TYPE_EXTS = /* @__PURE__ */ new Set(["html", "htm", "xhtml", "txt", "log", "out", "err", "trace", "csv", "tsv", "vtt", "srt", "svg", "xml"]);
|
|
17507
|
+
var BINARY_FILE_TYPE_EXTS = /* @__PURE__ */ new Set(["pdf", "docx", "xlsx", "pptx", "odt", "ods", "ott", "odp", "sqlite", "db", "sqlite3", "db3", "parquet"]);
|
|
17508
|
+
var TEXT_FILE_TYPE_EXTS = /* @__PURE__ */ new Set(["html", "htm", "xhtml", "txt", "log", "out", "err", "trace", "csv", "tsv", "vtt", "srt", "svg", "xml", "json", "yaml", "yml", "jsonl"]);
|
|
17481
17509
|
var DISPATCHED_FILE_TYPE_EXTS = /* @__PURE__ */ new Set([...BINARY_FILE_TYPE_EXTS, ...TEXT_FILE_TYPE_EXTS]);
|
|
17482
17510
|
function isDispatchedFileType(basename10) {
|
|
17483
|
-
return DISPATCHED_FILE_TYPE_EXTS.has(
|
|
17511
|
+
return DISPATCHED_FILE_TYPE_EXTS.has(path17.extname(basename10).slice(1).toLowerCase());
|
|
17484
17512
|
}
|
|
17485
17513
|
function surgicalHint(filePath, basename10, lineCount) {
|
|
17486
17514
|
if (lineCount < loadConfig().hints.min_file_lines_for_hint) return "";
|
|
@@ -17543,7 +17571,7 @@ function loadSnapshotDiff(sessionId, normalized, basename10) {
|
|
|
17543
17571
|
try {
|
|
17544
17572
|
const sz = statSize(normalized);
|
|
17545
17573
|
if (sz === null || sz > 256 * 1024) return { kind: "none" };
|
|
17546
|
-
const currentContent =
|
|
17574
|
+
const currentContent = fs20.readFileSync(normalized, "utf8");
|
|
17547
17575
|
const TRUNC_MARKER = "\n<snapshot truncated at ";
|
|
17548
17576
|
const oldRaw = oldSnap.toString("utf8");
|
|
17549
17577
|
const truncIdx = oldRaw.indexOf(TRUNC_MARKER);
|
|
@@ -17630,7 +17658,7 @@ function preReadHandlerInner(event) {
|
|
|
17630
17658
|
try {
|
|
17631
17659
|
const cwd = getCwd(event) ?? process.cwd();
|
|
17632
17660
|
const project = findProject(cwd) ?? makeProjectAt(cwd);
|
|
17633
|
-
const current =
|
|
17661
|
+
const current = fs20.readFileSync(normalized, "utf8");
|
|
17634
17662
|
const evidence = findVerifiedFileEvidence(project.root, normalized, current);
|
|
17635
17663
|
if (evidence !== null) {
|
|
17636
17664
|
recordStat("evidence_cache_hit", 0);
|
|
@@ -17641,7 +17669,7 @@ function preReadHandlerInner(event) {
|
|
|
17641
17669
|
} catch {
|
|
17642
17670
|
}
|
|
17643
17671
|
}
|
|
17644
|
-
const basename10 = displaySafePath(
|
|
17672
|
+
const basename10 = displaySafePath(path17.basename(normalized));
|
|
17645
17673
|
if (isLockFile(basename10)) {
|
|
17646
17674
|
return denyOutput(
|
|
17647
17675
|
'Lock files are rarely useful to read in full. Use `token-goat section "' + shown + '::<section>"` to extract a specific dependency, or read the relevant manifest instead.'
|
|
@@ -17680,7 +17708,7 @@ function preReadHandlerInner(event) {
|
|
|
17680
17708
|
const skillName = detectSkillFile(normalized);
|
|
17681
17709
|
if (skillName && basename10 === "SKILL.md") {
|
|
17682
17710
|
try {
|
|
17683
|
-
const body =
|
|
17711
|
+
const body = fs20.readFileSync(normalized, "utf-8");
|
|
17684
17712
|
const bodySha = contentHash(body);
|
|
17685
17713
|
const compact = getCompactAnySessionSync(skillName);
|
|
17686
17714
|
const stale = isCompactStale(compact, skillName, bodySha);
|
|
@@ -17712,9 +17740,9 @@ function preReadHandlerInner(event) {
|
|
|
17712
17740
|
const isNotebook = /\.ipynb$/i.test(basename10);
|
|
17713
17741
|
if (event.toolName !== "Grep" && isNotebook) {
|
|
17714
17742
|
try {
|
|
17715
|
-
const rawBytes =
|
|
17743
|
+
const rawBytes = fs20.readFileSync(normalized);
|
|
17716
17744
|
const [sidecarPath] = getOrCreateSidecar(rawBytes, dataDir());
|
|
17717
|
-
const sidecarContent =
|
|
17745
|
+
const sidecarContent = fs20.readFileSync(sidecarPath, "utf-8");
|
|
17718
17746
|
const savedBytes = rawBytes.length - sidecarContent.length;
|
|
17719
17747
|
if (savedBytes >= NB_STRIP_MIN_SAVINGS) {
|
|
17720
17748
|
recordActualRead(event, normalized);
|
|
@@ -17735,7 +17763,7 @@ function preReadHandlerInner(event) {
|
|
|
17735
17763
|
const sz = statSize(normalized);
|
|
17736
17764
|
if (sz !== null && sz >= MARKDOWN_SIZE_THRESHOLD) {
|
|
17737
17765
|
markdownSize = sz;
|
|
17738
|
-
fileContent =
|
|
17766
|
+
fileContent = fs20.readFileSync(normalized, "utf8");
|
|
17739
17767
|
}
|
|
17740
17768
|
} catch {
|
|
17741
17769
|
}
|
|
@@ -18019,7 +18047,7 @@ function preReadHandlerInner(event) {
|
|
|
18019
18047
|
"Note: " + shown + " is large (" + kb + "KB). " + hint + contextPressureAdvisorySuffix()
|
|
18020
18048
|
);
|
|
18021
18049
|
}
|
|
18022
|
-
const fileTypeExt =
|
|
18050
|
+
const fileTypeExt = path17.extname(normalized).slice(1).toLowerCase();
|
|
18023
18051
|
const fileStatSize = size ?? statSize(normalized) ?? 0;
|
|
18024
18052
|
const isKnownFileType = DISPATCHED_FILE_TYPE_EXTS.has(fileTypeExt);
|
|
18025
18053
|
if (event.toolName !== "Grep" && !isImagePath(normalized) && (isKnownFileType || fileStatSize >= FILE_TYPE_THRESHOLDS.generic)) {
|
|
@@ -18028,7 +18056,7 @@ function preReadHandlerInner(event) {
|
|
|
18028
18056
|
let ftContent = "";
|
|
18029
18057
|
if (!BINARY_FILE_TYPE_EXTS.has(fileTypeExt) && fileStatSize <= SLICE_ESTIMATE_SCAN_CAP_BYTES) {
|
|
18030
18058
|
try {
|
|
18031
|
-
ftContent =
|
|
18059
|
+
ftContent = fs20.readFileSync(normalized, "utf8");
|
|
18032
18060
|
} catch {
|
|
18033
18061
|
}
|
|
18034
18062
|
}
|
|
@@ -18060,7 +18088,7 @@ function estimateTruncatedLineCount(normalized) {
|
|
|
18060
18088
|
try {
|
|
18061
18089
|
const sz = statSize(normalized);
|
|
18062
18090
|
if (sz !== null && sz <= SLICE_ESTIMATE_SCAN_CAP_BYTES) {
|
|
18063
|
-
return countTextLines(
|
|
18091
|
+
return countTextLines(fs20.readFileSync(normalized, "utf8"));
|
|
18064
18092
|
}
|
|
18065
18093
|
} catch {
|
|
18066
18094
|
}
|
|
@@ -18083,13 +18111,13 @@ function postReadHandlerInner(event, suppressStructuralHint) {
|
|
|
18083
18111
|
if (isTruncatedReadDelivery(event, respText)) {
|
|
18084
18112
|
markFileTruncated(normalized);
|
|
18085
18113
|
}
|
|
18086
|
-
const postBasename =
|
|
18114
|
+
const postBasename = path17.basename(normalized);
|
|
18087
18115
|
const diffSourcesEnabled = loadConfig().hints.serve_diff_on_reread;
|
|
18088
18116
|
if (/\.(md|mdx|markdown|rst|txt)$/i.test(postBasename) || isSessionArtifactFile(normalized) || diffSourcesEnabled && isDiffableSource(postBasename)) {
|
|
18089
18117
|
try {
|
|
18090
18118
|
const sz = statSize(normalized);
|
|
18091
18119
|
if (sz !== null && sz <= 256 * 1024) {
|
|
18092
|
-
const content =
|
|
18120
|
+
const content = fs20.readFileSync(normalized);
|
|
18093
18121
|
store(sessionStateKey(event), normalized, content);
|
|
18094
18122
|
}
|
|
18095
18123
|
} catch {
|
|
@@ -18099,7 +18127,7 @@ function postReadHandlerInner(event, suppressStructuralHint) {
|
|
|
18099
18127
|
try {
|
|
18100
18128
|
const cwd = getCwd(event) ?? process.cwd();
|
|
18101
18129
|
const project = findProject(cwd) ?? makeProjectAt(cwd);
|
|
18102
|
-
const source = decodeSource(
|
|
18130
|
+
const source = decodeSource(fs20.readFileSync(normalized));
|
|
18103
18131
|
recordEvidence({ projectRoot: project.root, source: normalized, representation: "file", text: source });
|
|
18104
18132
|
} catch {
|
|
18105
18133
|
}
|
|
@@ -18130,7 +18158,7 @@ function postReadHandlerInner(event, suppressStructuralHint) {
|
|
|
18130
18158
|
try {
|
|
18131
18159
|
const sz = statSize(normalized);
|
|
18132
18160
|
if (sz !== null && sz <= SLICE_ESTIMATE_SCAN_CAP_BYTES) {
|
|
18133
|
-
const lineCount = countTextLines(
|
|
18161
|
+
const lineCount = countTextLines(fs20.readFileSync(normalized, "utf8"));
|
|
18134
18162
|
const minLines = loadConfig().post_read_code_compress.min_lines;
|
|
18135
18163
|
if (lineCount >= minLines && meetsSavingsFloor(sz) && !suppressStructuralHint) {
|
|
18136
18164
|
recordStat("session_hint", 0, 0);
|
|
@@ -18147,7 +18175,7 @@ function postReadHandlerInner(event, suppressStructuralHint) {
|
|
|
18147
18175
|
function readWindowFromDisk(event, normalized) {
|
|
18148
18176
|
const size = statSize(normalized);
|
|
18149
18177
|
if (size === null || size > SLICE_ESTIMATE_SCAN_CAP_BYTES) return null;
|
|
18150
|
-
const text = decodeSource(
|
|
18178
|
+
const text = decodeSource(fs20.readFileSync(normalized));
|
|
18151
18179
|
const window2 = readRequestedSliceWindow(event);
|
|
18152
18180
|
const start = window2.offset !== void 0 && window2.offset >= 1 ? window2.offset : 1;
|
|
18153
18181
|
const limit = window2.limit;
|
|
@@ -18348,9 +18376,9 @@ registerHook("post_tool_use", postReadHandler, { toolName: "Read" });
|
|
|
18348
18376
|
|
|
18349
18377
|
// src/cli_context_stats.ts
|
|
18350
18378
|
init_define_import_meta_env();
|
|
18351
|
-
import * as
|
|
18379
|
+
import * as fs22 from "node:fs";
|
|
18352
18380
|
import * as os2 from "node:os";
|
|
18353
|
-
import * as
|
|
18381
|
+
import * as path19 from "node:path";
|
|
18354
18382
|
|
|
18355
18383
|
// src/confirm_apply.ts
|
|
18356
18384
|
init_define_import_meta_env();
|
|
@@ -18406,8 +18434,8 @@ ${diff}
|
|
|
18406
18434
|
|
|
18407
18435
|
// src/memory_prune.ts
|
|
18408
18436
|
init_define_import_meta_env();
|
|
18409
|
-
import * as
|
|
18410
|
-
import * as
|
|
18437
|
+
import * as fs21 from "node:fs";
|
|
18438
|
+
import * as path18 from "node:path";
|
|
18411
18439
|
var ENTRY_RE = /^\s*-\s*\[(?<title>[^\]]+)\]\((?<target>[^)]+?\.md)\)/;
|
|
18412
18440
|
var URL_SCHEME_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
18413
18441
|
function parseIndex(text) {
|
|
@@ -18452,10 +18480,10 @@ function pruneIndex(memoryDir, opts) {
|
|
|
18452
18480
|
changed: false,
|
|
18453
18481
|
tokensSaved: 0
|
|
18454
18482
|
};
|
|
18455
|
-
const memoryMd =
|
|
18483
|
+
const memoryMd = path18.join(memoryDir, "MEMORY.md");
|
|
18456
18484
|
let text;
|
|
18457
18485
|
try {
|
|
18458
|
-
text =
|
|
18486
|
+
text = fs21.readFileSync(memoryMd, "utf-8");
|
|
18459
18487
|
} catch {
|
|
18460
18488
|
return result;
|
|
18461
18489
|
}
|
|
@@ -18466,7 +18494,7 @@ function pruneIndex(memoryDir, opts) {
|
|
|
18466
18494
|
const dups = [];
|
|
18467
18495
|
for (const entry of entries) {
|
|
18468
18496
|
const isUrl = URL_SCHEME_RE.test(entry.target);
|
|
18469
|
-
const targetExists = isUrl ? true :
|
|
18497
|
+
const targetExists = isUrl ? true : path18.isAbsolute(entry.target) ? fs21.existsSync(entry.target) : fs21.existsSync(path18.join(memoryDir, entry.target));
|
|
18470
18498
|
const foldedTarget = foldPath(entry.target);
|
|
18471
18499
|
if (!targetExists) {
|
|
18472
18500
|
dead.push(entry);
|
|
@@ -18520,7 +18548,7 @@ function jaccard(a, b) {
|
|
|
18520
18548
|
function siblingSnippet(filePath) {
|
|
18521
18549
|
let text;
|
|
18522
18550
|
try {
|
|
18523
|
-
text =
|
|
18551
|
+
text = fs21.readFileSync(filePath, { encoding: "utf-8" });
|
|
18524
18552
|
} catch {
|
|
18525
18553
|
return "";
|
|
18526
18554
|
}
|
|
@@ -18610,7 +18638,7 @@ async function tryEmbeddingClusters(siblings, snippets, threshold) {
|
|
|
18610
18638
|
}
|
|
18611
18639
|
async function findContentDuplicates(memoryDir, _opts) {
|
|
18612
18640
|
const threshold = _opts?.threshold ?? 0.92;
|
|
18613
|
-
const siblings =
|
|
18641
|
+
const siblings = fs21.readdirSync(memoryDir).filter((name) => name.toLowerCase().endsWith(".md") && name.toLowerCase() !== "memory.md").map((name) => path18.join(memoryDir, name)).sort();
|
|
18614
18642
|
if (siblings.length < 2) {
|
|
18615
18643
|
return [];
|
|
18616
18644
|
}
|
|
@@ -18634,7 +18662,7 @@ function auditClaudeMd(files) {
|
|
|
18634
18662
|
for (const filePath of files) {
|
|
18635
18663
|
let text;
|
|
18636
18664
|
try {
|
|
18637
|
-
text =
|
|
18665
|
+
text = fs21.readFileSync(filePath, { encoding: "utf-8" });
|
|
18638
18666
|
} catch {
|
|
18639
18667
|
continue;
|
|
18640
18668
|
}
|
|
@@ -18694,7 +18722,7 @@ function auditClaudeMd(files) {
|
|
|
18694
18722
|
const overlaps = [];
|
|
18695
18723
|
for (const [stripped, filesSet] of lineToFiles) {
|
|
18696
18724
|
if (filesSet.has(report.path) && filesSet.size > 1) {
|
|
18697
|
-
const others = Array.from(filesSet).filter((p) => p !== report.path).map((p) =>
|
|
18725
|
+
const others = Array.from(filesSet).filter((p) => p !== report.path).map((p) => path18.basename(p));
|
|
18698
18726
|
if (others.length > 0) {
|
|
18699
18727
|
if (stripped.length > 60) {
|
|
18700
18728
|
overlaps.push(
|
|
@@ -18714,7 +18742,7 @@ function auditClaudeMd(files) {
|
|
|
18714
18742
|
// src/cli_context_stats.ts
|
|
18715
18743
|
function tok(filePath) {
|
|
18716
18744
|
try {
|
|
18717
|
-
const size =
|
|
18745
|
+
const size = fs22.statSync(filePath).size;
|
|
18718
18746
|
return Math.floor(size / 4);
|
|
18719
18747
|
} catch {
|
|
18720
18748
|
return 0;
|
|
@@ -18723,42 +18751,42 @@ function tok(filePath) {
|
|
|
18723
18751
|
function findClaudeMdFiles(projectRoot, homeDir = os2.homedir()) {
|
|
18724
18752
|
const found = [];
|
|
18725
18753
|
const seen = /* @__PURE__ */ new Set();
|
|
18726
|
-
let current =
|
|
18754
|
+
let current = path19.resolve(projectRoot);
|
|
18727
18755
|
while (true) {
|
|
18728
|
-
const candidate =
|
|
18729
|
-
if (!seen.has(candidate) &&
|
|
18756
|
+
const candidate = path19.join(current, "CLAUDE.md");
|
|
18757
|
+
if (!seen.has(candidate) && fs22.existsSync(candidate)) {
|
|
18730
18758
|
found.push(candidate);
|
|
18731
18759
|
seen.add(candidate);
|
|
18732
18760
|
}
|
|
18733
|
-
const parent =
|
|
18761
|
+
const parent = path19.dirname(current);
|
|
18734
18762
|
if (parent === current) break;
|
|
18735
18763
|
current = parent;
|
|
18736
18764
|
}
|
|
18737
|
-
const globalMd =
|
|
18738
|
-
if (!seen.has(globalMd) &&
|
|
18765
|
+
const globalMd = path19.join(homeDir, ".claude", "CLAUDE.md");
|
|
18766
|
+
if (!seen.has(globalMd) && fs22.existsSync(globalMd)) {
|
|
18739
18767
|
found.push(globalMd);
|
|
18740
18768
|
}
|
|
18741
18769
|
return found;
|
|
18742
18770
|
}
|
|
18743
18771
|
function findMemoryMd(projectRoot, homeDir = os2.homedir(), alternateRoots = []) {
|
|
18744
18772
|
try {
|
|
18745
|
-
const projectsDir =
|
|
18746
|
-
if (!
|
|
18747
|
-
const rootStr =
|
|
18773
|
+
const projectsDir = path19.join(homeDir, ".claude", "projects");
|
|
18774
|
+
if (!fs22.existsSync(projectsDir)) return null;
|
|
18775
|
+
const rootStr = path19.resolve(projectRoot);
|
|
18748
18776
|
const candidateRoots = [rootStr];
|
|
18749
18777
|
try {
|
|
18750
|
-
const realRoot =
|
|
18778
|
+
const realRoot = fs22.realpathSync.native(rootStr);
|
|
18751
18779
|
if (realRoot !== rootStr) candidateRoots.push(realRoot);
|
|
18752
18780
|
} catch {
|
|
18753
18781
|
}
|
|
18754
18782
|
for (const alternate of alternateRoots) {
|
|
18755
|
-
const resolved =
|
|
18783
|
+
const resolved = path19.resolve(alternate);
|
|
18756
18784
|
if (!candidateRoots.includes(resolved)) candidateRoots.push(resolved);
|
|
18757
18785
|
}
|
|
18758
18786
|
for (const root of candidateRoots) {
|
|
18759
18787
|
const expectedSlug = root.replace(/[^A-Za-z0-9]/g, "-");
|
|
18760
|
-
const candidate =
|
|
18761
|
-
if (
|
|
18788
|
+
const candidate = path19.join(projectsDir, expectedSlug, "memory", "MEMORY.md");
|
|
18789
|
+
if (fs22.existsSync(candidate)) return candidate;
|
|
18762
18790
|
}
|
|
18763
18791
|
return null;
|
|
18764
18792
|
} catch {
|
|
@@ -18772,10 +18800,10 @@ function buildStats(projectRoot, homeDir = os2.homedir(), alternateRoots = []) {
|
|
|
18772
18800
|
for (const p of claudeMds) {
|
|
18773
18801
|
const t = tok(p);
|
|
18774
18802
|
claudeMdTotal += t;
|
|
18775
|
-
const parentDir =
|
|
18803
|
+
const parentDir = path19.basename(path19.dirname(p));
|
|
18776
18804
|
const label = parentDir === ".claude" ? "~/.claude/CLAUDE.md" : (() => {
|
|
18777
18805
|
try {
|
|
18778
|
-
return
|
|
18806
|
+
return path19.relative(projectRoot, p);
|
|
18779
18807
|
} catch {
|
|
18780
18808
|
return p;
|
|
18781
18809
|
}
|
|
@@ -18839,11 +18867,11 @@ async function runContextStats(opts = {}) {
|
|
|
18839
18867
|
process.stdout.write("[--fix] No MEMORY.md found; nothing to prune.\n");
|
|
18840
18868
|
} else {
|
|
18841
18869
|
const memPath = result.memory_md_path;
|
|
18842
|
-
const pruneResult = pruneIndex(
|
|
18870
|
+
const pruneResult = pruneIndex(path19.dirname(memPath), { dryRun: true });
|
|
18843
18871
|
if (!pruneResult.changed || pruneResult.after === void 0) {
|
|
18844
18872
|
process.stdout.write("[--fix] MEMORY.md already clean; nothing to prune.\n");
|
|
18845
18873
|
} else {
|
|
18846
|
-
const before =
|
|
18874
|
+
const before = fs22.readFileSync(memPath, "utf-8");
|
|
18847
18875
|
const applyResult = await confirmAndApply(
|
|
18848
18876
|
[{ path: memPath, before, after: pruneResult.after, label: "MEMORY.md" }],
|
|
18849
18877
|
opts.yes === true ? { yes: true } : {}
|
|
@@ -18871,8 +18899,8 @@ async function runContextStats(opts = {}) {
|
|
|
18871
18899
|
|
|
18872
18900
|
// src/baseline.ts
|
|
18873
18901
|
init_define_import_meta_env();
|
|
18874
|
-
import * as
|
|
18875
|
-
import * as
|
|
18902
|
+
import * as fs23 from "node:fs";
|
|
18903
|
+
import * as path20 from "node:path";
|
|
18876
18904
|
var SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
18877
18905
|
"node_modules",
|
|
18878
18906
|
".git",
|
|
@@ -18910,12 +18938,12 @@ function walkProject(rootDir, opts = {}) {
|
|
|
18910
18938
|
if (dir === void 0) break;
|
|
18911
18939
|
let entries;
|
|
18912
18940
|
try {
|
|
18913
|
-
entries =
|
|
18941
|
+
entries = fs23.readdirSync(dir, { withFileTypes: true });
|
|
18914
18942
|
} catch {
|
|
18915
18943
|
continue;
|
|
18916
18944
|
}
|
|
18917
18945
|
for (const entry of entries) {
|
|
18918
|
-
const full =
|
|
18946
|
+
const full = path20.join(dir, entry.name);
|
|
18919
18947
|
if (entry.isDirectory()) {
|
|
18920
18948
|
if (SKIP_DIRS.has(entry.name) || extraSkipDirs.includes(entry.name)) continue;
|
|
18921
18949
|
if (entry.name.startsWith(".") && entry.name !== ".") {
|
|
@@ -18997,7 +19025,7 @@ function fetchTopSymbols(limit, dbPath, rootDir) {
|
|
|
18997
19025
|
}
|
|
18998
19026
|
}
|
|
18999
19027
|
function buildProjectMap(rootDir = process.cwd(), opts = {}) {
|
|
19000
|
-
const root =
|
|
19028
|
+
const root = path20.resolve(rootDir);
|
|
19001
19029
|
const config = loadConfig();
|
|
19002
19030
|
const { files, languages, truncated } = walkProject(root, { excludeTests: config.repomap.exclude_tests });
|
|
19003
19031
|
const compact = opts.compact === true || files.length > config.repomap.compact_file_threshold;
|
|
@@ -19006,12 +19034,12 @@ function buildProjectMap(rootDir = process.cwd(), opts = {}) {
|
|
|
19006
19034
|
const recentFiles = files.map((f) => {
|
|
19007
19035
|
let mtime;
|
|
19008
19036
|
try {
|
|
19009
|
-
mtime =
|
|
19037
|
+
mtime = fs23.statSync(f).mtimeMs;
|
|
19010
19038
|
} catch {
|
|
19011
19039
|
mtime = 0;
|
|
19012
19040
|
}
|
|
19013
19041
|
return { f, mtime };
|
|
19014
|
-
}).sort((a, b) => b.mtime - a.mtime).slice(0, compact ? 5 : 15).map((x) =>
|
|
19042
|
+
}).sort((a, b) => b.mtime - a.mtime).slice(0, compact ? 5 : 15).map((x) => path20.relative(root, x.f));
|
|
19015
19043
|
return {
|
|
19016
19044
|
rootDir: root,
|
|
19017
19045
|
fileCount: files.length,
|
|
@@ -19024,7 +19052,7 @@ function buildProjectMap(rootDir = process.cwd(), opts = {}) {
|
|
|
19024
19052
|
}
|
|
19025
19053
|
function formatProjectMap(map, compact = false) {
|
|
19026
19054
|
const lines2 = [];
|
|
19027
|
-
const rel =
|
|
19055
|
+
const rel = path20.basename(map.rootDir);
|
|
19028
19056
|
lines2.push(`# Project map: ${rel}`);
|
|
19029
19057
|
lines2.push(map.fileCountTruncated === true ? `Files: at least ${map.fileCount} (walk stopped at the ${MAX_FILES_SCANNED}-file cap)` : `Files: ${map.fileCount}`);
|
|
19030
19058
|
const langPairs = Object.entries(map.languages).sort((a, b) => b[1] - a[1]);
|
|
@@ -19035,7 +19063,7 @@ function formatProjectMap(map, compact = false) {
|
|
|
19035
19063
|
lines2.push("");
|
|
19036
19064
|
lines2.push("## Top symbols");
|
|
19037
19065
|
for (const s of map.topSymbols) {
|
|
19038
|
-
const loc =
|
|
19066
|
+
const loc = formatSymbolLocation(displaySafeText(toDisplayPath(map.rootDir, s.filePath)), s.lineStart, s.lineEnd);
|
|
19039
19067
|
lines2.push(`- ${displaySafeText(s.name)} (${displaySafeText(s.kind)}) \u2014 ${loc}`);
|
|
19040
19068
|
}
|
|
19041
19069
|
} else {
|
|
@@ -19053,7 +19081,7 @@ function formatProjectMap(map, compact = false) {
|
|
|
19053
19081
|
}
|
|
19054
19082
|
function mapLookupBytesSaved(map, emittedText) {
|
|
19055
19083
|
const referencedFiles = /* @__PURE__ */ new Set([
|
|
19056
|
-
...map.recentFiles.map((f) => normalizePath(
|
|
19084
|
+
...map.recentFiles.map((f) => normalizePath(path20.resolve(map.rootDir, f))),
|
|
19057
19085
|
...map.topSymbols.map((s) => normalizePath(s.filePath))
|
|
19058
19086
|
]);
|
|
19059
19087
|
const listingText = Array.from(referencedFiles).sort().join("\n");
|
|
@@ -19068,14 +19096,14 @@ function findMemSuggestionCandidates(projectRoot) {
|
|
|
19068
19096
|
const claudeMdFiles = findClaudeMdFiles(projectRoot);
|
|
19069
19097
|
const candidateFiles = new Set(claudeMdFiles);
|
|
19070
19098
|
for (const claudeMd of claudeMdFiles) {
|
|
19071
|
-
const agentsMd =
|
|
19072
|
-
if (
|
|
19099
|
+
const agentsMd = path20.join(path20.dirname(claudeMd), "AGENTS.md");
|
|
19100
|
+
if (fs23.existsSync(agentsMd)) candidateFiles.add(agentsMd);
|
|
19073
19101
|
}
|
|
19074
19102
|
const suggestions = [];
|
|
19075
19103
|
for (const filePath of candidateFiles) {
|
|
19076
19104
|
let text;
|
|
19077
19105
|
try {
|
|
19078
|
-
text =
|
|
19106
|
+
text = fs23.readFileSync(filePath, { encoding: "utf-8" });
|
|
19079
19107
|
} catch {
|
|
19080
19108
|
continue;
|
|
19081
19109
|
}
|
|
@@ -19104,7 +19132,7 @@ function formatMemSuggestions(projectRoot) {
|
|
|
19104
19132
|
if (suggestions.length === 0) return "";
|
|
19105
19133
|
const lines2 = ["", "## mem suggestions"];
|
|
19106
19134
|
for (const s of suggestions) {
|
|
19107
|
-
const basename10 =
|
|
19135
|
+
const basename10 = path20.basename(s.path);
|
|
19108
19136
|
lines2.push(
|
|
19109
19137
|
"Consider: mem import --from-md " + s.path + " # migrates " + s.count + " preference-shaped lines from " + basename10 + " as pending facts for review"
|
|
19110
19138
|
);
|
|
@@ -19148,16 +19176,16 @@ function scanAndRecord(text) {
|
|
|
19148
19176
|
|
|
19149
19177
|
// src/repomap.ts
|
|
19150
19178
|
init_define_import_meta_env();
|
|
19151
|
-
import * as
|
|
19152
|
-
import * as
|
|
19179
|
+
import * as fs24 from "fs";
|
|
19180
|
+
import * as path21 from "path";
|
|
19153
19181
|
function getTrackedFiles(cwd = process.cwd()) {
|
|
19154
19182
|
try {
|
|
19155
19183
|
let dir = cwd;
|
|
19156
19184
|
let onlyFile = "";
|
|
19157
19185
|
try {
|
|
19158
|
-
if (
|
|
19159
|
-
dir =
|
|
19160
|
-
onlyFile =
|
|
19186
|
+
if (fs24.statSync(cwd).isFile()) {
|
|
19187
|
+
dir = path21.dirname(cwd);
|
|
19188
|
+
onlyFile = path21.basename(cwd);
|
|
19161
19189
|
}
|
|
19162
19190
|
} catch {
|
|
19163
19191
|
}
|
|
@@ -19166,7 +19194,7 @@ function getTrackedFiles(cwd = process.cwd()) {
|
|
|
19166
19194
|
if (result.exitCode !== 0 || !result.stdout) {
|
|
19167
19195
|
return [];
|
|
19168
19196
|
}
|
|
19169
|
-
return result.stdout.split("\n").filter((line) => line.trim().length > 0).map((rel) =>
|
|
19197
|
+
return result.stdout.split("\n").filter((line) => line.trim().length > 0).map((rel) => path21.join(dir, rel));
|
|
19170
19198
|
} catch {
|
|
19171
19199
|
return [];
|
|
19172
19200
|
}
|
|
@@ -19406,6 +19434,8 @@ export {
|
|
|
19406
19434
|
unzipBounded,
|
|
19407
19435
|
docxOutline,
|
|
19408
19436
|
docxText,
|
|
19437
|
+
docxTables,
|
|
19438
|
+
formatDocxTables,
|
|
19409
19439
|
pptxOutline,
|
|
19410
19440
|
pptxSlideText,
|
|
19411
19441
|
pptxNotesText,
|
|
@@ -19420,6 +19450,8 @@ export {
|
|
|
19420
19450
|
rangeSheet,
|
|
19421
19451
|
formatXlsxRange,
|
|
19422
19452
|
querySheet,
|
|
19453
|
+
xlsxColumns,
|
|
19454
|
+
formatXlsxColumns,
|
|
19423
19455
|
isEmbeddableDocument,
|
|
19424
19456
|
pathEqClause,
|
|
19425
19457
|
detectWalkMode,
|
|
@@ -19485,13 +19517,6 @@ export {
|
|
|
19485
19517
|
getFileServedOutputs,
|
|
19486
19518
|
markFileTruncated,
|
|
19487
19519
|
getSessionId,
|
|
19488
|
-
DEFAULT_MAX_COUNT,
|
|
19489
|
-
DEFAULT_MAX_AGE_MS,
|
|
19490
|
-
isBlobStale,
|
|
19491
|
-
storeBlob,
|
|
19492
|
-
loadBlob,
|
|
19493
|
-
listBlobs,
|
|
19494
|
-
pruneBlobs,
|
|
19495
19520
|
isBuildCommand,
|
|
19496
19521
|
isTestRunnerCommand,
|
|
19497
19522
|
getMonitoringRecallHint,
|
|
@@ -19550,10 +19575,6 @@ export {
|
|
|
19550
19575
|
getSkillFilePath,
|
|
19551
19576
|
installedSkillPath,
|
|
19552
19577
|
pruneSkillOutputs,
|
|
19553
|
-
ocrIntegrityFailed,
|
|
19554
|
-
isOcrEngineAvailable,
|
|
19555
|
-
ocrImage,
|
|
19556
|
-
isTextHeavy,
|
|
19557
19578
|
visionTokens,
|
|
19558
19579
|
visionTokensSaved,
|
|
19559
19580
|
visionTokensSavedByText,
|