token-goat 2.6.36 → 2.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -8
- package/SECURITY.md +46 -17
- package/dist/{token-goat-chunk-UFOVM7ZN.mjs → token-goat-chunk-2G6RAB4G.mjs} +1391 -378
- package/dist/token-goat-chunk-324QOJYZ.mjs +91 -0
- package/dist/token-goat-chunk-5CVKO3DA.mjs +185 -0
- package/dist/{token-goat-chunk-65BKISIS.mjs → token-goat-chunk-73MF6YWW.mjs} +4 -4
- package/dist/{token-goat-chunk-MGOUYAA2.mjs → token-goat-chunk-7JDXDERZ.mjs} +1 -1
- package/dist/{token-goat-hook-chunk-BDR6C6IE.mjs → token-goat-chunk-AOBZUFNJ.mjs} +318 -51
- package/dist/{token-goat-chunk-V465YKOR.mjs → token-goat-chunk-ELDJRLHZ.mjs} +492 -28
- package/dist/{token-goat-chunk-CNDOJ3ZP.mjs → token-goat-chunk-I6TOUPLP.mjs} +2 -2
- package/dist/token-goat-chunk-R4SR7MQY.mjs +486 -0
- package/dist/{token-goat-chunk-IYTVE6KN.mjs → token-goat-chunk-RMDQFTQD.mjs} +226 -150
- package/dist/{token-goat-chunk-KYFJC37X.mjs → token-goat-chunk-VBXBLGTO.mjs} +792 -144
- package/dist/{token-goat-chunk-DG53MVNJ.mjs → token-goat-chunk-WN5T5EW5.mjs} +212 -212
- package/dist/{token-goat-chunk-VYMGEVZS.mjs → token-goat-chunk-XEDQH5DA.mjs} +317 -11
- package/dist/{token-goat-chunk-LN6OUHTV.mjs → token-goat-chunk-Y2SYNH3S.mjs} +5 -5
- package/dist/token-goat-chunk-YUGNM3KL.mjs +23 -0
- package/dist/token-goat-hook.mjs +7 -7
- package/dist/token-goat.core.mjs +5 -5
- package/package.json +12 -8
- package/dist/token-goat-chunk-FRTBMRP7.mjs +0 -10048
- package/dist/token-goat-hook-chunk-3QYSN4QV.mjs +0 -14764
- package/dist/token-goat-hook-chunk-3ZDBWJDF.mjs +0 -13659
- package/dist/token-goat-hook-chunk-5UH54CW6.mjs +0 -912
- package/dist/token-goat-hook-chunk-6ODM3MP7.mjs +0 -706
- package/dist/token-goat-hook-chunk-A77A26A7.mjs +0 -18184
- package/dist/token-goat-hook-chunk-BUOCULAM.mjs +0 -29
- package/dist/token-goat-hook-chunk-C6GIABOX.mjs +0 -15971
- package/dist/token-goat-hook-chunk-E257IGSN.mjs +0 -153
- package/dist/token-goat-hook-chunk-MW5HPEGD.mjs +0 -10411
- package/dist/token-goat-hook-chunk-QSCYNJ2B.mjs +0 -23
- package/dist/token-goat-hook-chunk-RFRLWOQH.mjs +0 -11
- package/dist/token-goat-hook-chunk-XUMIVYEN.mjs +0 -109
- package/dist/token-goat-hook-chunk-Y2WHX2P3.mjs +0 -6463
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
detectLanguage,
|
|
20
20
|
displaySafePath,
|
|
21
21
|
displaySafeText,
|
|
22
|
+
ensureDataDirPrivate,
|
|
22
23
|
ensureDirSync,
|
|
23
24
|
escapeRegExp,
|
|
24
25
|
extractEnv,
|
|
@@ -58,6 +59,7 @@ import {
|
|
|
58
59
|
propagateEndLinesToSymbols,
|
|
59
60
|
pushAll,
|
|
60
61
|
recordStat,
|
|
62
|
+
recordUnmappedTool,
|
|
61
63
|
redactIfDotenv,
|
|
62
64
|
redactSecrets,
|
|
63
65
|
resolveIndexPath,
|
|
@@ -89,7 +91,7 @@ import {
|
|
|
89
91
|
withFileLock,
|
|
90
92
|
writeIfDifferent,
|
|
91
93
|
writeJsonSettings
|
|
92
|
-
} from "./token-goat-chunk-
|
|
94
|
+
} from "./token-goat-chunk-ELDJRLHZ.mjs";
|
|
93
95
|
import {
|
|
94
96
|
registerReset
|
|
95
97
|
} from "./token-goat-chunk-AO2QD2AG.mjs";
|
|
@@ -113,15 +115,221 @@ function createLazyModuleLoader(load3, errorLabel) {
|
|
|
113
115
|
};
|
|
114
116
|
}
|
|
115
117
|
|
|
118
|
+
// src/xml_parser.ts
|
|
119
|
+
var MAX_XML_DEPTH = 512;
|
|
120
|
+
var NAMED_ENTITIES = {
|
|
121
|
+
lt: "<",
|
|
122
|
+
gt: ">",
|
|
123
|
+
amp: "&",
|
|
124
|
+
quot: '"',
|
|
125
|
+
apos: "'"
|
|
126
|
+
};
|
|
127
|
+
function decodeXmlEntities(text) {
|
|
128
|
+
if (!text.includes("&")) return text;
|
|
129
|
+
return text.replace(/&(#[xX][0-9a-fA-F]+|#[0-9]+|[a-zA-Z_][\w.:-]*);/g, (whole, body) => {
|
|
130
|
+
if (body.charCodeAt(0) === 35) {
|
|
131
|
+
const isHex = body.charCodeAt(1) === 120 || body.charCodeAt(1) === 88;
|
|
132
|
+
const digits = isHex ? body.slice(2) : body.slice(1);
|
|
133
|
+
const code = parseInt(digits, isHex ? 16 : 10);
|
|
134
|
+
if (!Number.isFinite(code) || code < 0 || code > 1114111) return whole;
|
|
135
|
+
if (code >= 55296 && code <= 57343) return whole;
|
|
136
|
+
return String.fromCodePoint(code);
|
|
137
|
+
}
|
|
138
|
+
const named = NAMED_ENTITIES[body];
|
|
139
|
+
return named ?? whole;
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
function newFrame(name) {
|
|
143
|
+
return { name, children: /* @__PURE__ */ new Map(), text: [], attrs: [] };
|
|
144
|
+
}
|
|
145
|
+
function finishFrame(frame) {
|
|
146
|
+
const text = frame.text.join("");
|
|
147
|
+
if (frame.children.size === 0 && frame.attrs.length === 0) return text;
|
|
148
|
+
const obj = {};
|
|
149
|
+
for (const [key, value] of frame.children) obj[key] = value;
|
|
150
|
+
if (text.length > 0) obj["#text"] = text;
|
|
151
|
+
for (const [key, value] of frame.attrs) obj[`@_${key}`] = value;
|
|
152
|
+
return obj;
|
|
153
|
+
}
|
|
154
|
+
function addChild(parent, name, value) {
|
|
155
|
+
const existing = parent.children.get(name);
|
|
156
|
+
if (existing === void 0 && !parent.children.has(name)) {
|
|
157
|
+
parent.children.set(name, value);
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
if (Array.isArray(existing)) {
|
|
161
|
+
existing.push(value);
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
parent.children.set(name, [existing, value]);
|
|
165
|
+
}
|
|
166
|
+
var WHITESPACE = /* @__PURE__ */ new Set([" ", " ", "\n", "\r"]);
|
|
167
|
+
function findTagEnd(src, from) {
|
|
168
|
+
let quote = "";
|
|
169
|
+
for (let i = from; i < src.length; i++) {
|
|
170
|
+
const ch = src[i];
|
|
171
|
+
if (quote !== "") {
|
|
172
|
+
if (ch === quote) quote = "";
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
if (ch === '"' || ch === "'") {
|
|
176
|
+
quote = ch;
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
if (ch === ">") return i;
|
|
180
|
+
}
|
|
181
|
+
return -1;
|
|
182
|
+
}
|
|
183
|
+
function skipDeclaration(src, from) {
|
|
184
|
+
let quote = "";
|
|
185
|
+
let inSubset = false;
|
|
186
|
+
for (let i = from + 2; i < src.length; i++) {
|
|
187
|
+
const ch = src[i];
|
|
188
|
+
if (quote !== "") {
|
|
189
|
+
if (ch === quote) quote = "";
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
if (ch === '"' || ch === "'") {
|
|
193
|
+
quote = ch;
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
if (ch === "[") inSubset = true;
|
|
197
|
+
else if (ch === "]") inSubset = false;
|
|
198
|
+
else if (ch === ">" && !inSubset) return i + 1;
|
|
199
|
+
}
|
|
200
|
+
return src.length;
|
|
201
|
+
}
|
|
202
|
+
function parseTagBody(body) {
|
|
203
|
+
let i = 0;
|
|
204
|
+
while (i < body.length && !WHITESPACE.has(body[i])) i++;
|
|
205
|
+
const name = body.slice(0, i);
|
|
206
|
+
const attrs = [];
|
|
207
|
+
while (i < body.length) {
|
|
208
|
+
while (i < body.length && WHITESPACE.has(body[i])) i++;
|
|
209
|
+
if (i >= body.length) break;
|
|
210
|
+
const nameStart = i;
|
|
211
|
+
while (i < body.length && !WHITESPACE.has(body[i]) && body[i] !== "=") i++;
|
|
212
|
+
const attrName = body.slice(nameStart, i);
|
|
213
|
+
if (attrName.length === 0) {
|
|
214
|
+
i++;
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
while (i < body.length && WHITESPACE.has(body[i])) i++;
|
|
218
|
+
if (body[i] !== "=") {
|
|
219
|
+
attrs.push([attrName, ""]);
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
i++;
|
|
223
|
+
while (i < body.length && WHITESPACE.has(body[i])) i++;
|
|
224
|
+
const quote = body[i];
|
|
225
|
+
if (quote === '"' || quote === "'") {
|
|
226
|
+
const valueStart = i + 1;
|
|
227
|
+
const valueEnd = body.indexOf(quote, valueStart);
|
|
228
|
+
const end = valueEnd === -1 ? body.length : valueEnd;
|
|
229
|
+
attrs.push([attrName, decodeXmlEntities(body.slice(valueStart, end))]);
|
|
230
|
+
i = end + 1;
|
|
231
|
+
} else {
|
|
232
|
+
const valueStart = i;
|
|
233
|
+
while (i < body.length && !WHITESPACE.has(body[i])) i++;
|
|
234
|
+
attrs.push([attrName, decodeXmlEntities(body.slice(valueStart, i))]);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
return { name, attrs };
|
|
238
|
+
}
|
|
239
|
+
function parseXml(xml) {
|
|
240
|
+
const src = xml.charCodeAt(0) === 65279 ? xml.slice(1) : xml;
|
|
241
|
+
const root = newFrame("");
|
|
242
|
+
const stack = [root];
|
|
243
|
+
const len = src.length;
|
|
244
|
+
let i = 0;
|
|
245
|
+
const top = () => stack[stack.length - 1];
|
|
246
|
+
const pushText = (from, to) => {
|
|
247
|
+
if (to > from) top().text.push(decodeXmlEntities(src.slice(from, to)));
|
|
248
|
+
};
|
|
249
|
+
const closeElement = (name) => {
|
|
250
|
+
let target = -1;
|
|
251
|
+
for (let k = stack.length - 1; k >= 1; k--) {
|
|
252
|
+
if (stack[k].name === name) {
|
|
253
|
+
target = k;
|
|
254
|
+
break;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
if (target === -1) return;
|
|
258
|
+
while (stack.length > target) {
|
|
259
|
+
const frame = stack.pop();
|
|
260
|
+
addChild(top(), frame.name, finishFrame(frame));
|
|
261
|
+
}
|
|
262
|
+
};
|
|
263
|
+
while (i < len) {
|
|
264
|
+
const lt = src.indexOf("<", i);
|
|
265
|
+
if (lt === -1) {
|
|
266
|
+
pushText(i, len);
|
|
267
|
+
break;
|
|
268
|
+
}
|
|
269
|
+
pushText(i, lt);
|
|
270
|
+
if (src.startsWith("<!--", lt)) {
|
|
271
|
+
const end2 = src.indexOf("-->", lt + 4);
|
|
272
|
+
i = end2 === -1 ? len : end2 + 3;
|
|
273
|
+
continue;
|
|
274
|
+
}
|
|
275
|
+
if (src.startsWith("<![CDATA[", lt)) {
|
|
276
|
+
const end2 = src.indexOf("]]>", lt + 9);
|
|
277
|
+
top().text.push(src.slice(lt + 9, end2 === -1 ? len : end2));
|
|
278
|
+
i = end2 === -1 ? len : end2 + 3;
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
if (src.startsWith("<!", lt)) {
|
|
282
|
+
i = skipDeclaration(src, lt);
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
if (src.startsWith("<?", lt)) {
|
|
286
|
+
const end2 = src.indexOf("?>", lt + 2);
|
|
287
|
+
const { name: name2, attrs: attrs2 } = parseTagBody(src.slice(lt + 2, end2 === -1 ? len : end2));
|
|
288
|
+
if (name2.length > 0) {
|
|
289
|
+
const frame2 = newFrame(`?${name2}`);
|
|
290
|
+
frame2.attrs = attrs2;
|
|
291
|
+
addChild(top(), frame2.name, finishFrame(frame2));
|
|
292
|
+
}
|
|
293
|
+
i = end2 === -1 ? len : end2 + 2;
|
|
294
|
+
continue;
|
|
295
|
+
}
|
|
296
|
+
if (src.startsWith("</", lt)) {
|
|
297
|
+
const end2 = src.indexOf(">", lt + 2);
|
|
298
|
+
closeElement(src.slice(lt + 2, end2 === -1 ? len : end2).trim());
|
|
299
|
+
i = end2 === -1 ? len : end2 + 1;
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
const end = findTagEnd(src, lt + 1);
|
|
303
|
+
const tagEnd = end === -1 ? len : end;
|
|
304
|
+
let body = src.slice(lt + 1, tagEnd);
|
|
305
|
+
const selfClosing = body.endsWith("/");
|
|
306
|
+
if (selfClosing) body = body.slice(0, -1);
|
|
307
|
+
const { name, attrs } = parseTagBody(body);
|
|
308
|
+
i = end === -1 ? len : end + 1;
|
|
309
|
+
if (name.length === 0) continue;
|
|
310
|
+
const frame = newFrame(name);
|
|
311
|
+
frame.attrs = attrs;
|
|
312
|
+
if (selfClosing) {
|
|
313
|
+
addChild(top(), name, finishFrame(frame));
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
if (stack.length > MAX_XML_DEPTH) {
|
|
317
|
+
throw new Error(`XML nesting deeper than ${MAX_XML_DEPTH} elements; refusing to parse (this file is not something any office application produces)`);
|
|
318
|
+
}
|
|
319
|
+
stack.push(frame);
|
|
320
|
+
}
|
|
321
|
+
while (stack.length > 1) {
|
|
322
|
+
const frame = stack.pop();
|
|
323
|
+
addChild(top(), frame.name, finishFrame(frame));
|
|
324
|
+
}
|
|
325
|
+
return Object.fromEntries(root.children);
|
|
326
|
+
}
|
|
327
|
+
|
|
116
328
|
// src/ooxml_extract.ts
|
|
117
329
|
var loadFflate = createLazyModuleLoader(
|
|
118
330
|
async () => await import("fflate"),
|
|
119
331
|
"office-file reading disabled (fflate unavailable)"
|
|
120
332
|
);
|
|
121
|
-
var loadXmlParser = createLazyModuleLoader(
|
|
122
|
-
async () => await import("fast-xml-parser"),
|
|
123
|
-
"office-file reading disabled (fast-xml-parser unavailable)"
|
|
124
|
-
);
|
|
125
333
|
var MAX_OOXML_INPUT_BYTES = 50 * 1024 * 1024;
|
|
126
334
|
function accessFailureMessage(err, filePath) {
|
|
127
335
|
const code = err?.code;
|
|
@@ -159,15 +367,7 @@ function decodeZipEntry(entries, entryPath) {
|
|
|
159
367
|
return new TextDecoder("utf-8").decode(bytes);
|
|
160
368
|
}
|
|
161
369
|
async function parseOoxmlPart(xmlText2) {
|
|
162
|
-
|
|
163
|
-
if (!fxp) throw new Error("fast-xml-parser is not installed; run `npm install fast-xml-parser` to enable this command");
|
|
164
|
-
const parser = new fxp.XMLParser({
|
|
165
|
-
ignoreAttributes: false,
|
|
166
|
-
preserveOrder: false,
|
|
167
|
-
trimValues: false,
|
|
168
|
-
parseTagValue: false
|
|
169
|
-
});
|
|
170
|
-
return parser.parse(xmlText2);
|
|
370
|
+
return parseXml(xmlText2);
|
|
171
371
|
}
|
|
172
372
|
function pushTextValue(runs, val) {
|
|
173
373
|
if (Array.isArray(val)) {
|
|
@@ -345,9 +545,9 @@ async function notesPathFor(entries, slidePath) {
|
|
|
345
545
|
}
|
|
346
546
|
return null;
|
|
347
547
|
}
|
|
348
|
-
async function parseSlide(entries,
|
|
349
|
-
const xml = decodeZipEntry(entries,
|
|
350
|
-
if (xml === null) throw new Error(`missing part: ${
|
|
548
|
+
async function parseSlide(entries, path31) {
|
|
549
|
+
const xml = decodeZipEntry(entries, path31);
|
|
550
|
+
if (xml === null) throw new Error(`missing part: ${path31}`);
|
|
351
551
|
return parseOoxmlPart(xml);
|
|
352
552
|
}
|
|
353
553
|
async function notesTextFor(entries, notesPath) {
|
|
@@ -363,8 +563,8 @@ async function pptxOutline(filePath) {
|
|
|
363
563
|
const { entries, slidePaths } = await listSlideParts(filePath);
|
|
364
564
|
const out = [];
|
|
365
565
|
for (let i = 0; i < slidePaths.length; i++) {
|
|
366
|
-
const
|
|
367
|
-
const parsed = await parseSlide(entries,
|
|
566
|
+
const path31 = slidePaths[i];
|
|
567
|
+
const parsed = await parseSlide(entries, path31);
|
|
368
568
|
const shapes = slideShapes(parsed);
|
|
369
569
|
const titleShape = shapes.find((s) => {
|
|
370
570
|
const t = shapePlaceholderType(s);
|
|
@@ -373,7 +573,7 @@ async function pptxOutline(filePath) {
|
|
|
373
573
|
const title = titleShape !== void 0 ? shapeText(titleShape) : "";
|
|
374
574
|
const allText = collectTextRuns(parsed, "a:t").join(" ");
|
|
375
575
|
const bodyChars = Math.max(0, allText.length - title.length);
|
|
376
|
-
const hasNotes = (await notesTextFor(entries, await notesPathFor(entries,
|
|
576
|
+
const hasNotes = (await notesTextFor(entries, await notesPathFor(entries, path31))).length > 0;
|
|
377
577
|
out.push({ slide: i + 1, title, bodyChars, hasNotes });
|
|
378
578
|
}
|
|
379
579
|
return out;
|
|
@@ -383,13 +583,13 @@ async function pptxSlideText(filePath, slideNumber, includeNotes) {
|
|
|
383
583
|
if (slideNumber < 1 || slideNumber > slidePaths.length) {
|
|
384
584
|
throw new Error(`slide ${slideNumber} out of range (this deck has ${slidePaths.length} slides)`);
|
|
385
585
|
}
|
|
386
|
-
const
|
|
387
|
-
const parsed = await parseSlide(entries,
|
|
586
|
+
const path31 = slidePaths[slideNumber - 1];
|
|
587
|
+
const parsed = await parseSlide(entries, path31);
|
|
388
588
|
const shapes = slideShapes(parsed);
|
|
389
589
|
const blocks = [...shapes.map(shapeText).filter((t) => t.length > 0), ...tableRowBlocks(parsed)];
|
|
390
590
|
const lines2 = [`# Slide ${slideNumber}`, ...blocks];
|
|
391
591
|
if (includeNotes) {
|
|
392
|
-
const notes = await notesTextFor(entries, await notesPathFor(entries,
|
|
592
|
+
const notes = await notesTextFor(entries, await notesPathFor(entries, path31));
|
|
393
593
|
if (notes.length > 0) lines2.push("", "## Speaker notes", notes);
|
|
394
594
|
}
|
|
395
595
|
return lines2.join("\n\n");
|
|
@@ -2132,10 +2332,18 @@ var parse2 = function(data, opts = {}) {
|
|
|
2132
2332
|
function parseRecords(content, opts) {
|
|
2133
2333
|
const delimiter = opts.delimiter ?? ",";
|
|
2134
2334
|
if (opts.noHeader === true) {
|
|
2135
|
-
const rows = parse2(content, { columns: false, skip_empty_lines: true, trim: true, delimiter, bom: true });
|
|
2335
|
+
const rows = parse2(content, { columns: false, skip_empty_lines: true, trim: true, delimiter, bom: true, relax_column_count: true });
|
|
2136
2336
|
return rows.map((row) => Object.fromEntries(row.map((cell, i) => [`col${i + 1}`, cell])));
|
|
2137
2337
|
}
|
|
2138
|
-
|
|
2338
|
+
const header = csvHeader(content, opts);
|
|
2339
|
+
const dupes = header.filter((name, i) => name !== "" && header.indexOf(name) !== i);
|
|
2340
|
+
if (dupes.length > 0) {
|
|
2341
|
+
const unique = [...new Set(dupes)];
|
|
2342
|
+
throw new Error(
|
|
2343
|
+
`duplicate column ${unique.length === 1 ? "name" : "names"} in header: ${unique.join(", ")} \u2014 rename the duplicates or pass --no-header to address columns positionally as col1, col2, \u2026`
|
|
2344
|
+
);
|
|
2345
|
+
}
|
|
2346
|
+
return parse2(content, { columns: true, skip_empty_lines: true, trim: true, delimiter, bom: true, relax_column_count: true });
|
|
2139
2347
|
}
|
|
2140
2348
|
function csvHeader(content, opts) {
|
|
2141
2349
|
if (opts.noHeader === true) return [];
|
|
@@ -2629,8 +2837,10 @@ function usedRange(ws) {
|
|
|
2629
2837
|
}
|
|
2630
2838
|
const rows = rowCount;
|
|
2631
2839
|
const cols = maxCol;
|
|
2632
|
-
|
|
2633
|
-
|
|
2840
|
+
if (rows === 0 || cols === 0) {
|
|
2841
|
+
return { ref: "(empty)", rows: 0, cols: 0 };
|
|
2842
|
+
}
|
|
2843
|
+
return { ref: `A1:${indexToColLetters(cols)}${rows}`, rows, cols };
|
|
2634
2844
|
}
|
|
2635
2845
|
async function listSheets(filePath) {
|
|
2636
2846
|
const wb = await loadWorkbook(filePath);
|
|
@@ -2794,6 +3004,40 @@ async function extractPdfText(data, pagesSpec, layout = false) {
|
|
|
2794
3004
|
return { text: pages.join("\n\n"), pageCount: doc.numPages, pagesExtracted: end - start + 1 };
|
|
2795
3005
|
});
|
|
2796
3006
|
}
|
|
3007
|
+
async function locatePdfPages(data, pattern, opts) {
|
|
3008
|
+
let re;
|
|
3009
|
+
try {
|
|
3010
|
+
re = new RegExp(pattern, opts.ignoreCase === true ? "i" : "");
|
|
3011
|
+
} catch (e) {
|
|
3012
|
+
throw new Error(`invalid regex pattern: ${pattern} (${e instanceof Error ? e.message : String(e)})`, { cause: e });
|
|
3013
|
+
}
|
|
3014
|
+
const pdfjs = await loadPdfjs();
|
|
3015
|
+
if (!pdfjs) throw new Error("pdfjs-dist is not installed; run `npm install pdfjs-dist` to enable pdf-extract");
|
|
3016
|
+
const maxMatches = opts.maxMatches ?? 50;
|
|
3017
|
+
const context = opts.context ?? 80;
|
|
3018
|
+
return withPdfDocument(pdfjs, data, async (doc) => {
|
|
3019
|
+
const range = parsePageRange(opts.pages, doc.numPages);
|
|
3020
|
+
const start = range ? range.start : 1;
|
|
3021
|
+
const end = range ? range.end : doc.numPages;
|
|
3022
|
+
const matches = [];
|
|
3023
|
+
for (let i = start; i <= end && matches.length < maxMatches; i++) {
|
|
3024
|
+
const page = await doc.getPage(i);
|
|
3025
|
+
const content = await page.getTextContent();
|
|
3026
|
+
const textItems = content.items.filter((item) => "str" in item);
|
|
3027
|
+
const pageText = textItems.map((item) => item.str).join(" ");
|
|
3028
|
+
const m = re.exec(pageText);
|
|
3029
|
+
if (m === null) continue;
|
|
3030
|
+
matches.push({ page: i, snippet: locateSnippet(pageText, m.index, m[0].length, context) });
|
|
3031
|
+
}
|
|
3032
|
+
return matches;
|
|
3033
|
+
});
|
|
3034
|
+
}
|
|
3035
|
+
function locateSnippet(text, index, matchLen, context) {
|
|
3036
|
+
const pad = Math.max(0, context - matchLen);
|
|
3037
|
+
const from = Math.max(0, index - Math.floor(pad / 2));
|
|
3038
|
+
const to = Math.min(text.length, index + matchLen + Math.ceil(pad / 2));
|
|
3039
|
+
return text.slice(from, to).replace(/\s+/g, " ").trim();
|
|
3040
|
+
}
|
|
2797
3041
|
async function resolveDestPage(doc, dest) {
|
|
2798
3042
|
let explicitDest = dest;
|
|
2799
3043
|
if (typeof explicitDest === "string") {
|
|
@@ -2991,9 +3235,27 @@ function toolMatcherFor(eventName) {
|
|
|
2991
3235
|
if (parts.length === 0) return null;
|
|
2992
3236
|
return parts.join("|");
|
|
2993
3237
|
}
|
|
3238
|
+
function foldToolName(name) {
|
|
3239
|
+
return name.toLowerCase().replace(/[_-]/g, "");
|
|
3240
|
+
}
|
|
3241
|
+
function noteUnrecognizedTool(event, list) {
|
|
3242
|
+
const toolName = event.toolName;
|
|
3243
|
+
if (typeof toolName !== "string" || toolName === "") return;
|
|
3244
|
+
const named = [];
|
|
3245
|
+
for (const { toolName: want } of list) {
|
|
3246
|
+
if (want === void 0) continue;
|
|
3247
|
+
if (want === toolName) return;
|
|
3248
|
+
named.push(want);
|
|
3249
|
+
}
|
|
3250
|
+
if (named.length === 0) return;
|
|
3251
|
+
const folded = foldToolName(toolName);
|
|
3252
|
+
const nearMiss = named.find((n) => foldToolName(n) === folded) ?? null;
|
|
3253
|
+
recordUnmappedTool(toolName, event.eventName, nearMiss);
|
|
3254
|
+
}
|
|
2994
3255
|
async function runHook(event) {
|
|
2995
3256
|
const list = _handlers.get(event.eventName);
|
|
2996
3257
|
if (list === void 0) return { hookType: "pass" };
|
|
3258
|
+
noteUnrecognizedTool(event, list);
|
|
2997
3259
|
let advisoryResult;
|
|
2998
3260
|
for (const { handler, toolName, advisory } of list) {
|
|
2999
3261
|
if (toolName !== void 0 && toolName !== event.toolName) continue;
|
|
@@ -3023,19 +3285,30 @@ var CLAUDE_CODE_EVENT_NAMES = {
|
|
|
3023
3285
|
notification: "Notification",
|
|
3024
3286
|
stop: "Stop",
|
|
3025
3287
|
pre_compact: "PreCompact",
|
|
3288
|
+
post_compact: "PostCompact",
|
|
3026
3289
|
user_prompt_submit: "UserPromptSubmit",
|
|
3027
3290
|
subagent_stop: "SubagentStop",
|
|
3028
|
-
session_start: "SessionStart"
|
|
3291
|
+
session_start: "SessionStart",
|
|
3292
|
+
// Claude Code has no separate failure event -- a failed tool arrives on PostToolUse there,
|
|
3293
|
+
// and only Copilot splits it out. This entry exists because the map is exhaustive over
|
|
3294
|
+
// HookEventName, and it is a spelling for the response envelope rather than a claim that
|
|
3295
|
+
// Claude Code will ever send this event. Nothing iterates this map to build install config,
|
|
3296
|
+
// so naming an event Claude Code does not have cannot register one.
|
|
3297
|
+
post_tool_use_failure: "PostToolUseFailure"
|
|
3029
3298
|
};
|
|
3030
3299
|
var EVENTS_WITHOUT_ADDITIONAL_CONTEXT = /* @__PURE__ */ new Set([
|
|
3031
3300
|
"notification",
|
|
3032
3301
|
"pre_compact"
|
|
3033
3302
|
]);
|
|
3034
|
-
|
|
3303
|
+
var EVENTS_WITH_RAW_STDOUT_CONTEXT = /* @__PURE__ */ new Set(["pre_compact"]);
|
|
3304
|
+
function serializeOutput(output, eventName, harness) {
|
|
3035
3305
|
switch (output.hookType) {
|
|
3036
3306
|
case "deny":
|
|
3037
3307
|
return JSON.stringify({ decision: "block", reason: output.message });
|
|
3038
3308
|
case "context":
|
|
3309
|
+
if (EVENTS_WITH_RAW_STDOUT_CONTEXT.has(eventName) && harness === "claudecode") {
|
|
3310
|
+
return output.context;
|
|
3311
|
+
}
|
|
3039
3312
|
if (EVENTS_WITHOUT_ADDITIONAL_CONTEXT.has(eventName)) {
|
|
3040
3313
|
return JSON.stringify({ systemMessage: output.context });
|
|
3041
3314
|
}
|
|
@@ -3499,6 +3772,7 @@ var _bashOutputs = /* @__PURE__ */ new Map();
|
|
|
3499
3772
|
var _grepQueries = /* @__PURE__ */ new Map();
|
|
3500
3773
|
var _globQueries = /* @__PURE__ */ new Map();
|
|
3501
3774
|
var _lastTabContext = null;
|
|
3775
|
+
var _seenImageHashes = [];
|
|
3502
3776
|
var _outstandingAgentSpawns = [];
|
|
3503
3777
|
var _outstandingAgentSpawnsAtLoad = [];
|
|
3504
3778
|
var _bashReruns = /* @__PURE__ */ new Set();
|
|
@@ -3698,6 +3972,18 @@ function setLastTabContext(text) {
|
|
|
3698
3972
|
function getLastTabContext() {
|
|
3699
3973
|
return _lastTabContext;
|
|
3700
3974
|
}
|
|
3975
|
+
var MAX_SEEN_IMAGE_HASHES = 16;
|
|
3976
|
+
function hasSeenImage(hash2) {
|
|
3977
|
+
return _seenImageHashes.includes(hash2);
|
|
3978
|
+
}
|
|
3979
|
+
function recordSeenImage(hash2) {
|
|
3980
|
+
const at = _seenImageHashes.indexOf(hash2);
|
|
3981
|
+
if (at !== -1) _seenImageHashes.splice(at, 1);
|
|
3982
|
+
_seenImageHashes.push(hash2);
|
|
3983
|
+
if (_seenImageHashes.length > MAX_SEEN_IMAGE_HASHES) {
|
|
3984
|
+
_seenImageHashes.splice(0, _seenImageHashes.length - MAX_SEEN_IMAGE_HASHES);
|
|
3985
|
+
}
|
|
3986
|
+
}
|
|
3701
3987
|
var MAX_OUTSTANDING_AGENT_SPAWNS = 30;
|
|
3702
3988
|
function recordOutstandingAgentSpawn(prompt) {
|
|
3703
3989
|
_outstandingAgentSpawns.push({ prompt, ts: Date.now() });
|
|
@@ -3811,6 +4097,7 @@ function exportSessionState() {
|
|
|
3811
4097
|
globQueries: Array.from(_globQueries.entries()),
|
|
3812
4098
|
outstandingAgentSpawns: _outstandingAgentSpawns.map((e) => [e.prompt, e.ts]),
|
|
3813
4099
|
..._lastTabContext !== null ? { lastTabContext: _lastTabContext } : {},
|
|
4100
|
+
..._seenImageHashes.length > 0 ? { seenImageHashes: [..._seenImageHashes] } : {},
|
|
3814
4101
|
..._compactedAt > 0 ? { compactedAt: _compactedAt } : {}
|
|
3815
4102
|
};
|
|
3816
4103
|
}
|
|
@@ -3836,6 +4123,7 @@ function importSessionState(s) {
|
|
|
3836
4123
|
_outstandingAgentSpawns = (s.outstandingAgentSpawns ?? []).map(([prompt, ts]) => ({ prompt, ts }));
|
|
3837
4124
|
_outstandingAgentSpawnsAtLoad = [..._outstandingAgentSpawns];
|
|
3838
4125
|
_lastTabContext = s.lastTabContext ?? null;
|
|
4126
|
+
_seenImageHashes = [...s.seenImageHashes ?? []];
|
|
3839
4127
|
_compactedAt = s.compactedAt ?? 0;
|
|
3840
4128
|
}
|
|
3841
4129
|
registerReset(() => {
|
|
@@ -3857,6 +4145,7 @@ registerReset(() => {
|
|
|
3857
4145
|
_outstandingAgentSpawns = [];
|
|
3858
4146
|
_outstandingAgentSpawnsAtLoad = [];
|
|
3859
4147
|
_lastTabContext = null;
|
|
4148
|
+
_seenImageHashes = [];
|
|
3860
4149
|
_compactedAt = 0;
|
|
3861
4150
|
_sessionId = null;
|
|
3862
4151
|
});
|
|
@@ -3879,9 +4168,11 @@ const VALID_HOOK_EVENTS = new Set([
|
|
|
3879
4168
|
'notification',
|
|
3880
4169
|
'stop',
|
|
3881
4170
|
'pre_compact',
|
|
4171
|
+
'post_compact',
|
|
3882
4172
|
'user_prompt_submit',
|
|
3883
4173
|
'subagent_stop',
|
|
3884
4174
|
'session_start',
|
|
4175
|
+
'post_tool_use_failure',
|
|
3885
4176
|
])`;
|
|
3886
4177
|
var SHIM_TRY_IN_PROCESS = `// Attempts the in-process hook call: import()s dist/token-goat-hook.mjs (a sibling of
|
|
3887
4178
|
// the baked token-goat entry path, built with zero load-time side effects -- unlike
|
|
@@ -4009,9 +4300,9 @@ function buildGuidanceBody(fallbackToolClause, opts = {}) {
|
|
|
4009
4300
|
"- pulling one value or subtree out of a JSON/YAML/XML file (manifest, lockfile, spec, config) \u2192 `json-query file 'a.b.c'` / `yaml-query file 'a.b.c'` / `xml-query file 'a.b.c'`",
|
|
4010
4301
|
"- opening an image to check its dimensions, format, or size \u2192 `image-meta file`",
|
|
4011
4302
|
"- opening a screenshot, diagram, or scan to read the text in it \u2192 `image-text file`",
|
|
4012
|
-
"- opening a PDF or Office document \u2192 inspect its format first, then read a narrow slice: PDF `pdf-meta`/`pdf-outline` then `pdf-extract
|
|
4303
|
+
"- opening a PDF or Office document \u2192 inspect its format first, then read a narrow slice: PDF `pdf-meta`/`pdf-outline` then `pdf-locate` to find the pages and `pdf-extract` only those; Word `docx-outline` then `docx-text`; PowerPoint `pptx-outline` then `pptx-slide`/`pptx-notes`; Excel `xlsx-sheets` then `xlsx-head`/`xlsx-range`/`xlsx-query`",
|
|
4013
4304
|
"",
|
|
4014
|
-
'Commands: `symbol NAME`, `read "file::symbol"`, `brief "file::symbol"`, `section "file::Heading"`, `semantic "description"`, `outline file`/`skeleton file`, `map --compact`, `refs file::symbol --callers`, `changed --symbol`, `config-get file KEY`, `json-query file \'a.b.c\'`/`yaml-query`/`xml-query`, `json-outline file`/`yaml-outline`/`xml-outline`, `bash-output`/`web-output`/`mcp-output`, ' + (gdrive ? "`gdrive-sections <file-id>`, " : "") + "`image-meta file`/`image-text file`, `pdf-meta`/`pdf-outline`/`pdf-extract`, `docx-outline`/`docx-text`, `pptx-outline`/`pptx-slide`/`pptx-notes`/`pptx-text`, `xlsx-sheets`/`xlsx-head`/`xlsx-range`/`xlsx-query`.",
|
|
4305
|
+
'Commands: `symbol NAME`, `read "file::symbol"`, `brief "file::symbol"`, `section "file::Heading"`, `semantic "description"`, `outline file`/`skeleton file`, `map --compact`, `refs file::symbol --callers`, `changed --symbol`, `config-get file KEY`, `json-query file \'a.b.c\'`/`yaml-query`/`xml-query`, `json-outline file`/`yaml-outline`/`xml-outline`, `bash-output`/`web-output`/`mcp-output`, ' + (gdrive ? "`gdrive-sections <file-id>`, " : "") + "`image-meta file`/`image-text file`, `pdf-meta`/`pdf-outline`/`pdf-locate`/`pdf-extract`, `docx-outline`/`docx-text`, `pptx-outline`/`pptx-slide`/`pptx-notes`/`pptx-text`, `xlsx-sheets`/`xlsx-head`/`xlsx-range`/`xlsx-query`.",
|
|
4015
4306
|
"",
|
|
4016
4307
|
"Sub-agent briefs must carry this gate verbatim: a sub-agent inherits none of this context and its reads spend the same token budget.",
|
|
4017
4308
|
"",
|
|
@@ -4030,6 +4321,7 @@ var HOOK_EVENT_MAP = [
|
|
|
4030
4321
|
["PreToolUse", "pre_tool_use"],
|
|
4031
4322
|
["PostToolUse", "post_tool_use"],
|
|
4032
4323
|
["PreCompact", "pre_compact"],
|
|
4324
|
+
["PostCompact", "post_compact"],
|
|
4033
4325
|
["UserPromptSubmit", "user_prompt_submit"],
|
|
4034
4326
|
["SubagentStop", "subagent_stop"],
|
|
4035
4327
|
["SessionStart", "session_start"]
|
|
@@ -4346,9 +4638,11 @@ const HOOK_EVENT_NAME_MAP = {
|
|
|
4346
4638
|
notification: 'Notification',
|
|
4347
4639
|
stop: 'Stop',
|
|
4348
4640
|
pre_compact: 'PreCompact',
|
|
4641
|
+
post_compact: 'PostCompact',
|
|
4349
4642
|
user_prompt_submit: 'UserPromptSubmit',
|
|
4350
4643
|
subagent_stop: 'SubagentStop',
|
|
4351
4644
|
session_start: 'SessionStart',
|
|
4645
|
+
post_tool_use_failure: 'PostToolUseFailure',
|
|
4352
4646
|
}
|
|
4353
4647
|
|
|
4354
4648
|
function stripTg(value) {
|
|
@@ -4729,10 +5023,19 @@ const path = require('node:path')
|
|
|
4729
5023
|
const { pathToFileURL } = require('node:url')
|
|
4730
5024
|
|
|
4731
5025
|
// Copilot event name -> token-goat internal HookEventName (src/types.ts's
|
|
4732
|
-
// HOOK_EVENTS). Only these
|
|
4733
|
-
// Copilot event (sessionEnd,
|
|
4734
|
-
//
|
|
5026
|
+
// HOOK_EVENTS). Only these eight have a token-goat handler; every other real
|
|
5027
|
+
// Copilot event (sessionEnd, subagentStart, errorOccurred, notification,
|
|
5028
|
+
// permissionRequest) is left unimplemented
|
|
4735
5029
|
// rather than guessed at, and falls through to the default no-op below.
|
|
5030
|
+
// postToolUseFailure is the newest of the eight and the only one whose channel
|
|
5031
|
+
// costs tokens instead of saving them: it fires instead of postToolUse when a
|
|
5032
|
+
// tool result is a failure, and accepts only additionalContext back --
|
|
5033
|
+
// modifiedResult and suppressOutput are not honored there, so a failed result
|
|
5034
|
+
// still cannot be fenced, compressed or shrunk. It is wired anyway because
|
|
5035
|
+
// additionalContext demonstrably reaches the model (see the postToolUseFailure
|
|
5036
|
+
// branch in translate() for the bundle offset), and hooks_tool_failure.ts
|
|
5037
|
+
// spends that channel only on an exact repeat failure, where staying silent
|
|
5038
|
+
// costs a whole wasted retry.
|
|
4736
5039
|
// 'sessionStart' was previously a permanent no-op on the stated grounds that
|
|
4737
5040
|
// token-goat has no internal session_start handler. That was simply wrong --
|
|
4738
5041
|
// hooks_session_start.ts has long emitted the command-routing reminder that
|
|
@@ -4752,6 +5055,7 @@ const COPILOT_TO_TG_EVENT = {
|
|
|
4752
5055
|
agentStop: 'stop',
|
|
4753
5056
|
subagentStop: 'subagent_stop',
|
|
4754
5057
|
userPromptSubmitted: 'user_prompt_submit',
|
|
5058
|
+
postToolUseFailure: 'post_tool_use_failure',
|
|
4755
5059
|
}
|
|
4756
5060
|
|
|
4757
5061
|
// Copilot built-in tool name -> token-goat internal tool name. Confirmed via
|
|
@@ -4766,9 +5070,47 @@ const COPILOT_TO_TG_EVENT = {
|
|
|
4766
5070
|
// MCP-server tool invocations (<server-name>-<tool-name>) have no
|
|
4767
5071
|
// token-goat equivalent and are passed through unmapped (safe no-op for
|
|
4768
5072
|
// handlers that don't recognize the name).
|
|
5073
|
+
// 'read_bash'/'read_powershell' are the background-shell output pollers, and
|
|
5074
|
+
// they are the exact shape of Claude Code's BashOutput: Copilot's shell tool
|
|
5075
|
+
// is async, so a long-running command is started once and then re-read over
|
|
5076
|
+
// and over within one turn, each read returning the accumulated output again.
|
|
5077
|
+
// hooks_bashoutput.ts already collapses that into a delta (or a short
|
|
5078
|
+
// "unchanged" marker) for Claude Code, and now does the same here. Both the
|
|
5079
|
+
// names and the argument key were read out of the shipping 1.0.80 bundle
|
|
5080
|
+
// rather than guessed: the builtin tool-name table in
|
|
5081
|
+
// prebuilds/win32-x64/runtime.node lists read_bash/stop_bash/list_bash and
|
|
5082
|
+
// read_powershell/stop_powershell/list_powershell, and the poller's own input
|
|
5083
|
+
// schema in app.js is {shellId, delay} -- so the shell id needs the
|
|
5084
|
+
// POLL_ID_ARG_KEY remap below to reach postBashOutputHandler, which reads
|
|
5085
|
+
// 'bash_id'. read_shell/stop_shell/list_shells, which an earlier static sweep
|
|
5086
|
+
// of the same binary suggested were the real names, are internal Rust
|
|
5087
|
+
// identifiers (tool_read_shell_prepare_input, tool_list_shells_descriptor,
|
|
5088
|
+
// PreparedStopShellInput, and the serde field names of the shell config
|
|
5089
|
+
// struct), never wire tool names, so they are deliberately absent here.
|
|
5090
|
+
// stop_bash/list_bash and their powershell twins stay unmapped because
|
|
5091
|
+
// token-goat has no KillShell-equivalent handler for them to reach; a mapping
|
|
5092
|
+
// would be pure decoration.
|
|
5093
|
+
// 'bash'/'powershell' stay exactly as they were. The same sweep suggested the
|
|
5094
|
+
// executor had been renamed to write_bash/write_powershell and that this
|
|
5095
|
+
// mapping was dead, and that is not what the bundle says: app.js resolves the
|
|
5096
|
+
// executor as shellConfig?.shellToolName ?? "bash" (with "powershell" as the
|
|
5097
|
+
// Windows default of the same config), and its command lives under 'command'
|
|
5098
|
+
// exactly as hooks_bash.ts expects. write_bash/write_powershell are real tool
|
|
5099
|
+
// names, but they are a different tool -- their schema is {shellId, input,
|
|
5100
|
+
// delay} and the bundle files them under the subtype "write_shell", i.e. send
|
|
5101
|
+
// stdin to an already-running shell, not run a command. Mapping them to Bash
|
|
5102
|
+
// would label a stdin write as a shell execution, so they are left unmapped.
|
|
5103
|
+
// 'task', 'read_agent' and 'memory'-family tools are likewise left alone:
|
|
5104
|
+
// task's result is assembled in the native addon and its shape is unknown,
|
|
5105
|
+
// read_agent is an incremental poll that postAgentHandler is not written for,
|
|
5106
|
+
// and the real tool names behind memory were never confirmed. Each would put
|
|
5107
|
+
// a handler that rewrites model-visible output in front of a payload shape
|
|
5108
|
+
// nobody has seen, which is worse than leaving the compression on the table.
|
|
4769
5109
|
const TOOL_TO_TG = {
|
|
4770
5110
|
bash: 'Bash',
|
|
4771
5111
|
powershell: 'Bash',
|
|
5112
|
+
read_bash: 'BashOutput',
|
|
5113
|
+
read_powershell: 'BashOutput',
|
|
4772
5114
|
view: 'Read',
|
|
4773
5115
|
create: 'Write',
|
|
4774
5116
|
edit: 'Edit',
|
|
@@ -4809,6 +5151,17 @@ const FILE_PATH_ARG_KEY = {
|
|
|
4809
5151
|
create: 'path',
|
|
4810
5152
|
}
|
|
4811
5153
|
|
|
5154
|
+
// read_bash/read_powershell send the background shell's id under 'shellId' (confirmed
|
|
5155
|
+
// against the poller's input schema in the shipping 1.0.80 app.js: {shellId, delay});
|
|
5156
|
+
// postBashOutputHandler reads 'bash_id'. Without this the tool-name mapping above would
|
|
5157
|
+
// be inert -- the handler bails on a missing bash_id and every poll would keep costing
|
|
5158
|
+
// the full accumulated output. Same keying convention as FILE_PATH_ARG_KEY: the ORIGINAL
|
|
5159
|
+
// Copilot tool name, since that is what the argument shape belongs to.
|
|
5160
|
+
const POLL_ID_ARG_KEY = {
|
|
5161
|
+
read_bash: 'shellId',
|
|
5162
|
+
read_powershell: 'shellId',
|
|
5163
|
+
}
|
|
5164
|
+
|
|
4812
5165
|
// Copilot spawns a brand-new process for every single hook invocation (no long-lived plugin
|
|
4813
5166
|
// process the way OpenClaw's is -- OPENCLAW_HOOK_SCRIPT's own \`copilot-\${process.pid}-\${Date.now()}\`
|
|
4814
5167
|
// fallback is safe there specifically because that process lives for the whole session, so the
|
|
@@ -4817,7 +5170,14 @@ const FILE_PATH_ARG_KEY = {
|
|
|
4817
5170
|
// the same session, since process.pid varies per invocation -- breaking token-goat's
|
|
4818
5171
|
// session-based dedup/state ledger, which never accumulates across calls as a result. Derive a
|
|
4819
5172
|
// stable id instead from the one thing that's actually constant across calls for the same
|
|
4820
|
-
// session: the working directory Copilot reports
|
|
5173
|
+
// session: the working directory Copilot reports. That field is \`workingDirectory\`, declared
|
|
5174
|
+
// required on BaseHookInput in copilot-sdk/types.d.ts since 1.0.76, so it is present on EVERY
|
|
5175
|
+
// hook event. This previously read \`payload.cwd\`, which Copilot has never sent under any name in
|
|
5176
|
+
// any version -- the key simply did not exist, so this derived every fallback id from
|
|
5177
|
+
// process.cwd() instead and \`canonical.cwd\` below was undefined on every single call. It went
|
|
5178
|
+
// unnoticed because process.cwd() happens to be the project directory Copilot spawns the hook in,
|
|
5179
|
+
// so the fallback was accidentally right; nothing about that was by design. \`cwd\` is still read
|
|
5180
|
+
// as a secondary in case a future version adds it under the shorter name.
|
|
4821
5181
|
function stableFallbackSessionId(cwd) {
|
|
4822
5182
|
const key = typeof cwd === 'string' && cwd ? cwd : process.cwd()
|
|
4823
5183
|
const hash = require('node:crypto').createHash('sha256').update(key).digest('hex').slice(0, 16)
|
|
@@ -4825,13 +5185,19 @@ function stableFallbackSessionId(cwd) {
|
|
|
4825
5185
|
}
|
|
4826
5186
|
|
|
4827
5187
|
function remapToolInput(copilotToolName, input) {
|
|
5188
|
+
if (!input || typeof input !== 'object') return input
|
|
5189
|
+
let out = input
|
|
5190
|
+
// Add the canonical key alongside the original rather than renaming it, so nothing that
|
|
5191
|
+
// might read the original 'path'/'shellId' key elsewhere (e.g. a future handler) loses it.
|
|
4828
5192
|
const pathKey = FILE_PATH_ARG_KEY[copilotToolName]
|
|
4829
|
-
if (pathKey
|
|
4830
|
-
|
|
5193
|
+
if (pathKey !== undefined && pathKey in out) {
|
|
5194
|
+
out = Object.assign({}, out, { file_path: out[pathKey] })
|
|
4831
5195
|
}
|
|
4832
|
-
|
|
4833
|
-
|
|
4834
|
-
|
|
5196
|
+
const idKey = POLL_ID_ARG_KEY[copilotToolName]
|
|
5197
|
+
if (idKey !== undefined && idKey in out) {
|
|
5198
|
+
out = Object.assign({}, out, { bash_id: out[idKey] })
|
|
5199
|
+
}
|
|
5200
|
+
return out
|
|
4835
5201
|
}
|
|
4836
5202
|
|
|
4837
5203
|
// Attempts the in-process hook call: import()s dist/token-goat-hook.mjs (a sibling of
|
|
@@ -4881,8 +5247,19 @@ async function main() {
|
|
|
4881
5247
|
|
|
4882
5248
|
const toolName = payload && payload.toolName
|
|
4883
5249
|
const canonical = {
|
|
4884
|
-
session_id:
|
|
4885
|
-
|
|
5250
|
+
session_id:
|
|
5251
|
+
(payload && payload.sessionId) ||
|
|
5252
|
+
stableFallbackSessionId(payload && (payload.workingDirectory || payload.cwd)),
|
|
5253
|
+
cwd: payload && (payload.workingDirectory || payload.cwd),
|
|
5254
|
+
}
|
|
5255
|
+
|
|
5256
|
+
// userPromptSubmitted only: Copilot declares \`prompt\` required on UserPromptSubmittedHookInput.
|
|
5257
|
+
// hooks_session.ts's userPromptSubmitHandler reads it as \`event.raw['prompt']\` and gates every
|
|
5258
|
+
// branch it has on the text, so without this it saw '' on every Copilot prompt and the
|
|
5259
|
+
// embedded-skill dedup hint could never fire. Same shape as the postToolUseFailure \`error\`
|
|
5260
|
+
// drop: a required field the canonical builder simply did not list.
|
|
5261
|
+
if (typeof (payload && payload.prompt) === 'string' && payload.prompt !== '') {
|
|
5262
|
+
canonical.prompt = payload.prompt
|
|
4886
5263
|
}
|
|
4887
5264
|
if (toolName) {
|
|
4888
5265
|
canonical.tool_name = TOOL_TO_TG[toolName] || toolName
|
|
@@ -4897,6 +5274,16 @@ async function main() {
|
|
|
4897
5274
|
// post-read/post-bash stats stay empty no matter how many tool calls happen. Extract the
|
|
4898
5275
|
// LLM-facing text directly rather than forwarding the raw object, since textResultForLlm
|
|
4899
5276
|
// isn't one of those recognized object keys.
|
|
5277
|
+
// postToolUseFailure only: Copilot's PostToolUseFailureHookInput (copilot-sdk/types.d.ts:1042)
|
|
5278
|
+
// carries {toolName, toolArgs, error} and no toolResult at all -- the failure text lives in
|
|
5279
|
+
// \`error\`, a plain string. Without forwarding it, hooks_tool_failure.ts's extractFailureText
|
|
5280
|
+
// finds nothing to key on and the repeat-failure brake returns pass on every single call: wired,
|
|
5281
|
+
// green, and doing nothing. Found by driving the installed shim rather than by a test, because
|
|
5282
|
+
// the handler's own tests hand it a raw payload that already has the field.
|
|
5283
|
+
if (typeof (payload && payload.error) === 'string' && payload.error !== '') {
|
|
5284
|
+
canonical.error = payload.error
|
|
5285
|
+
}
|
|
5286
|
+
|
|
4900
5287
|
const rawResult = payload && payload.toolResult
|
|
4901
5288
|
if (rawResult && typeof rawResult === 'object') {
|
|
4902
5289
|
const tr = rawResult
|
|
@@ -4983,7 +5370,46 @@ function translate(copilotEvent, resp) {
|
|
|
4983
5370
|
}
|
|
4984
5371
|
|
|
4985
5372
|
if (copilotEvent === 'postToolUse') {
|
|
4986
|
-
//
|
|
5373
|
+
// Verified against the shipping @github/copilot 1.0.80 bundle, not against the docs page.
|
|
5374
|
+
// NativeHookPipelineProcessor.postToolExecution (app.js offset 2043150) gates on
|
|
5375
|
+
// toolResult.resultType === "success", then does n.toolResultJson && CSr(e.toolResult,
|
|
5376
|
+
// n.toolResultJson), where CSr (offset 2032350) is an in-place Object.assign; the mutated
|
|
5377
|
+
// object is re-serialized back to native in the postTool callback at offset 1793926. So
|
|
5378
|
+
// modifiedResult IS honored on this event, and token-goat's rewriteOutput producers
|
|
5379
|
+
// (compression, injection fencing, image shrink) really do reach the model. resultType is
|
|
5380
|
+
// hardcoded 'success' because success is the only branch this event ever runs on.
|
|
5381
|
+
//
|
|
5382
|
+
// additionalContext on THIS event is dropped on the JS path. grep -abo "onAdditionalContext:"
|
|
5383
|
+
// app.js returns nothing, so the callback is never supplied, and the two "onAdditionalContext?"
|
|
5384
|
+
// call sites (offsets 2041896 and 2043300) are both no-ops. preToolsExecution (offset 2041832)
|
|
5385
|
+
// additionally pushes each context into an array that IS drained into the native return
|
|
5386
|
+
// payload (additional_contexts, offset 1791950); postToolExecution has no such push and its
|
|
5387
|
+
// native return payload (offset 1793926) has no additional_contexts key. Residual, stated
|
|
5388
|
+
// honestly: the native hookProcessorPostToolUse might fold additionalContext into the
|
|
5389
|
+
// toolResultJson it returns, and that was NOT verified. The evidence leans against it, since
|
|
5390
|
+
// the failure sibling path appends its context explicitly in JS via
|
|
5391
|
+
// hookAppendPostToolUseFailureContext and there is no success-path counterpart. The
|
|
5392
|
+
// out.additionalContext below is kept as cheap best-effort, not as a channel anything should
|
|
5393
|
+
// depend on -- see src/pending_context.ts, whose whole design depended on it.
|
|
5394
|
+
//
|
|
5395
|
+
// Failed tool calls never reach this event. postToolUse and postToolUseFailure are two
|
|
5396
|
+
// distinct hook events (both listed in the runtime.node hook-event enum at offset 101618150),
|
|
5397
|
+
// and the shipped copilot-sdk/types.d.ts says onPostToolUse "does not fire for non-success
|
|
5398
|
+
// results". The failure event cannot carry a fence either: PostToolUseFailureHookInput carries
|
|
5399
|
+
// only a stringified error message, not the tool result, and PostToolUseFailureHookOutput
|
|
5400
|
+
// consumes only additionalContext -- "modifiedResult or suppressOutput are not honored for
|
|
5401
|
+
// failure hooks". rejected/denied/timeout results trigger no post hook at all. So on Copilot,
|
|
5402
|
+
// the output of a failed tool call reaches the model unfenced, uncompressed and unshrunk, and
|
|
5403
|
+
// no response shape this shim could emit changes that. That gap is real and still open. What
|
|
5404
|
+
// is now wired is the narrower thing that IS possible there: the postToolUseFailure branch
|
|
5405
|
+
// below carries advisory text alongside the failure, and never rewrites it.
|
|
5406
|
+
//
|
|
5407
|
+
// Emitted camelCase-only. The inbound side around line 247 also tolerates a snake_case "VS
|
|
5408
|
+
// Code compatible" shape, and this comment used to justify the camelCase choice by claiming
|
|
5409
|
+
// PascalCase event registration selects the snake_case response format. That reasoning is
|
|
5410
|
+
// UNVERIFIED in 1.0.80: every hook-response field in the native string tables is camelCase,
|
|
5411
|
+
// and the snake_case hits in the bundle are unrelated internal Rust identifiers. camelCase
|
|
5412
|
+
// stays because it is what the bundle reads; the old justification is no longer asserted.
|
|
4987
5413
|
const hso = resp && resp.hookSpecificOutput
|
|
4988
5414
|
const updatedToolOutput = hso && hso.updatedToolOutput
|
|
4989
5415
|
const context = extractContext(resp)
|
|
@@ -4995,6 +5421,25 @@ function translate(copilotEvent, resp) {
|
|
|
4995
5421
|
return out
|
|
4996
5422
|
}
|
|
4997
5423
|
|
|
5424
|
+
if (copilotEvent === 'postToolUseFailure') {
|
|
5425
|
+
// The failed-tool twin of postToolUse, and the only response field it accepts is
|
|
5426
|
+
// additionalContext: the shipped copilot-sdk/types.d.ts says "modifiedResult or suppressOutput
|
|
5427
|
+
// are not honored for failure hooks", so nothing here can fence, compress or shrink the failed
|
|
5428
|
+
// output -- that gap is real and stays open. What is NOT open, and was the reason this event
|
|
5429
|
+
// went unwired for so long, is whether additionalContext reaches the model at all. It does:
|
|
5430
|
+
// app.js 1.0.80 at offset 2043380 either folds it into textResultForLlm (when
|
|
5431
|
+
// appendFailureContextToToolResult is set) or has the native
|
|
5432
|
+
// hookAppendPostToolUseFailureContext push {content, source:'system'} onto
|
|
5433
|
+
// toolResult.newMessages. That is the exact opposite of postToolUse, whose additionalContext
|
|
5434
|
+
// the JS side drops on the floor, so neither event's behaviour generalises to the other.
|
|
5435
|
+
//
|
|
5436
|
+
// Because this channel spends tokens rather than saving them, the handler behind it
|
|
5437
|
+
// (hooks_tool_failure.ts) is silent on a first failure and speaks only on an exact repeat.
|
|
5438
|
+
const context = extractContext(resp)
|
|
5439
|
+
if (context) return { additionalContext: context }
|
|
5440
|
+
return {}
|
|
5441
|
+
}
|
|
5442
|
+
|
|
4998
5443
|
if (copilotEvent === 'sessionStart') {
|
|
4999
5444
|
// sessionStart has no tool result to modify -- only additionalContext applies, and it's the
|
|
5000
5445
|
// one channel that reaches the model before it picks its first read tool, so this is where
|
|
@@ -5018,12 +5463,47 @@ function translate(copilotEvent, resp) {
|
|
|
5018
5463
|
return { decision: 'allow' }
|
|
5019
5464
|
}
|
|
5020
5465
|
|
|
5021
|
-
|
|
5022
|
-
|
|
5023
|
-
|
|
5024
|
-
|
|
5025
|
-
|
|
5026
|
-
|
|
5466
|
+
if (copilotEvent === 'userPromptSubmitted') {
|
|
5467
|
+
// Copilot's own hooks reference says command-hook output here "is dropped, including
|
|
5468
|
+
// modifiedPrompt", and this branch used to believe it and return nothing. That is wrong for
|
|
5469
|
+
// additionalContext on 1.0.80, established by experiment rather than by reading: a config-file
|
|
5470
|
+
// command hook (at <cwd>/.github/hooks/, the project scope -- the user scope ~/.copilot/hooks/
|
|
5471
|
+
// was never exercised) returned {"additionalContext":"<marker>"} and the marker turned up
|
|
5472
|
+
// verbatim inside the session's user.message.transformedContent, wrapped in a
|
|
5473
|
+
// <system_reminder> block. What settles that it reached the model rather than only the on-disk
|
|
5474
|
+
// record is the billing: the provider's returned usage charged ~140 input tokens for a turn
|
|
5475
|
+
// whose raw content field is 35 bytes, so the marker's bytes were paid for whichever field
|
|
5476
|
+
// carried them. (The weaker argument first offered for this -- that transformedContent carries
|
|
5477
|
+
// an envelope the content field lacks, 29 B vs 195 B -- was measured on the control turn that
|
|
5478
|
+
// had no marker in it, and proves nothing about the marker.) That the model sees it is the
|
|
5479
|
+
// whole point: hook.start/hook.end records also persist and reach nothing.
|
|
5480
|
+
//
|
|
5481
|
+
// Scope of the finding, stated honestly: demonstrated ONCE on 1.0.80, not shown to be
|
|
5482
|
+
// reliable. Of two turns in that experiment, one delivered the marker and one fired a
|
|
5483
|
+
// userPromptSubmitted hook that produced no output and never ran the script; no explanation
|
|
5484
|
+
// was established and the rate is unknown. A hint that silently fails to arrive costs nothing
|
|
5485
|
+
// and breaks nothing here, which is why the direct return is still the right default.
|
|
5486
|
+
//
|
|
5487
|
+
// modifiedPrompt is NOT claimed to work and is not wanted: rewriting a user's prompt is far
|
|
5488
|
+
// more invasive than anything token-goat does, so only additionalContext is forwarded.
|
|
5489
|
+
//
|
|
5490
|
+
// This is also the write end of any post-compaction channel. Copilot has no postCompact hook.
|
|
5491
|
+
// Whether the summary is recoverable from events.jsonl is NOT settled -- the emit()/
|
|
5492
|
+
// emitEphemeral() distinction does not gate the writer, and the real decision is in native
|
|
5493
|
+
// code; see COPILOT_NO_POST_COMPACT_REASON in ../bridges_status.ts for what was and was not
|
|
5494
|
+
// established. The read end that IS confirmed is preCompact, which fires as a notification, so
|
|
5495
|
+
// a manifest can be built there and drained here without reading the event log at all.
|
|
5496
|
+
const context = extractContext(resp)
|
|
5497
|
+
if (context) return { additionalContext: context }
|
|
5498
|
+
return {}
|
|
5499
|
+
}
|
|
5500
|
+
|
|
5501
|
+
// preCompact is the genuine notification-only case, and the contrast with userPromptSubmitted
|
|
5502
|
+
// above is why this fallthrough is worth a comment at all. The hooks reference marks it
|
|
5503
|
+
// "No -- notification only" for output processing, and unlike the additionalContext claim that
|
|
5504
|
+
// turned out to be false, this one is confirmed in the shipping bundle: both preCompact call
|
|
5505
|
+
// sites in app.js (1.0.79 and re-checked in 1.0.80) await the hook and never assign its result.
|
|
5506
|
+
// There is no field to aim at here, so nothing to reconsider on the next version bump.
|
|
5027
5507
|
return {}
|
|
5028
5508
|
}
|
|
5029
5509
|
|
|
@@ -5065,7 +5545,8 @@ var COPILOT_CLI_HOOK_EVENTS = [
|
|
|
5065
5545
|
"preCompact",
|
|
5066
5546
|
"agentStop",
|
|
5067
5547
|
"subagentStop",
|
|
5068
|
-
"userPromptSubmitted"
|
|
5548
|
+
"userPromptSubmitted",
|
|
5549
|
+
"postToolUseFailure"
|
|
5069
5550
|
];
|
|
5070
5551
|
function copilotCliUserRoot() {
|
|
5071
5552
|
const override = process.env["COPILOT_HOME"];
|
|
@@ -5078,6 +5559,23 @@ function copilotCliUserHooksDir() {
|
|
|
5078
5559
|
function copilotCliProjectHooksDir() {
|
|
5079
5560
|
return path5.join(process.cwd(), ".github", "hooks");
|
|
5080
5561
|
}
|
|
5562
|
+
function copilotCliCacheRoot() {
|
|
5563
|
+
const override = process.env["COPILOT_CACHE_HOME"];
|
|
5564
|
+
if (override !== void 0 && override.trim() !== "") return path5.resolve(override);
|
|
5565
|
+
const home = os3.homedir();
|
|
5566
|
+
if (process.platform === "darwin") return path5.join(home, "Library", "Caches", "copilot");
|
|
5567
|
+
if (process.platform === "win32") {
|
|
5568
|
+
const local = process.env["LOCALAPPDATA"];
|
|
5569
|
+
const base2 = local !== void 0 && local.trim() !== "" ? local : path5.join(home, ".cache");
|
|
5570
|
+
return path5.join(base2, "copilot");
|
|
5571
|
+
}
|
|
5572
|
+
const xdg = process.env["XDG_CACHE_HOME"];
|
|
5573
|
+
const base = xdg !== void 0 && xdg.trim() !== "" ? xdg : path5.join(home, ".cache");
|
|
5574
|
+
return path5.join(base, "copilot");
|
|
5575
|
+
}
|
|
5576
|
+
function copilotCliMcpToolsDir() {
|
|
5577
|
+
return path5.join(copilotCliCacheRoot(), "mcp-tools");
|
|
5578
|
+
}
|
|
5081
5579
|
function copilotCliHooksDir(opts = {}) {
|
|
5082
5580
|
return opts.local === true ? copilotCliProjectHooksDir() : copilotCliUserHooksDir();
|
|
5083
5581
|
}
|
|
@@ -6006,11 +6504,15 @@ function sessionFileStem(sessionId) {
|
|
|
6006
6504
|
return `${saltedStemPrefix(sessionId.slice(0, sep))}${digest}`;
|
|
6007
6505
|
}
|
|
6008
6506
|
function sessionPath(sessionId) {
|
|
6507
|
+
return sessionSidecarPath(sessionId, ".json");
|
|
6508
|
+
}
|
|
6509
|
+
function sessionSidecarPath(sessionId, suffix) {
|
|
6009
6510
|
if (!sessionId) return null;
|
|
6010
6511
|
const safe = sessionFileStem(sessionId);
|
|
6011
6512
|
if (!safe) return null;
|
|
6513
|
+
if (suffix.includes("/") || suffix.includes("\\") || suffix.includes("..")) return null;
|
|
6012
6514
|
const dir = path10.join(tokenGoatHome(), SESSIONS_SUBDIR);
|
|
6013
|
-
const candidate = path10.join(dir, `${safe}
|
|
6515
|
+
const candidate = path10.join(dir, `${safe}${suffix}`);
|
|
6014
6516
|
try {
|
|
6015
6517
|
const rel = path10.relative(dir, candidate);
|
|
6016
6518
|
if (rel.startsWith("..")) return null;
|
|
@@ -6099,6 +6601,7 @@ function coerce(raw) {
|
|
|
6099
6601
|
) : [];
|
|
6100
6602
|
const cliReads = Array.isArray(o["cliReads"]) ? o["cliReads"].filter((h) => typeof h === "string") : [];
|
|
6101
6603
|
const bashReruns = Array.isArray(o["bashReruns"]) ? o["bashReruns"].filter((h) => typeof h === "string") : [];
|
|
6604
|
+
const seenImageHashes = Array.isArray(o["seenImageHashes"]) ? o["seenImageHashes"].filter((h) => typeof h === "string") : [];
|
|
6102
6605
|
const pendingLargeFileHints = Array.isArray(o["pendingLargeFileHints"]) ? o["pendingLargeFileHints"].filter(
|
|
6103
6606
|
(p) => Array.isArray(p) && p.length === 2 && typeof p[0] === "string" && typeof p[1] === "number"
|
|
6104
6607
|
) : [];
|
|
@@ -6126,6 +6629,7 @@ function coerce(raw) {
|
|
|
6126
6629
|
globQueries,
|
|
6127
6630
|
outstandingAgentSpawns,
|
|
6128
6631
|
...typeof o["lastTabContext"] === "string" ? { lastTabContext: o["lastTabContext"] } : {},
|
|
6632
|
+
...seenImageHashes.length > 0 ? { seenImageHashes } : {},
|
|
6129
6633
|
...typeof o["compactedAt"] === "number" ? { compactedAt: o["compactedAt"] } : {},
|
|
6130
6634
|
...typeof o["created_ts"] === "number" ? { created_ts: o["created_ts"] } : {}
|
|
6131
6635
|
};
|
|
@@ -6233,12 +6737,21 @@ function mergeSessionState(disk, mem) {
|
|
|
6233
6737
|
grepQueries: mergePairs(disk.grepQueries ?? [], mem.grepQueries ?? []),
|
|
6234
6738
|
globQueries: mergePairs(disk.globQueries ?? [], mem.globQueries ?? []),
|
|
6235
6739
|
outstandingAgentSpawns: mergeOutstandingAgentSpawns(disk.outstandingAgentSpawns ?? [], mem.outstandingAgentSpawns ?? []),
|
|
6740
|
+
// An accumulating collection, so union rather than pick a winner: two hook processes can
|
|
6741
|
+
// each see a screenshot the other never did. Disk first, then mem, so the order stays
|
|
6742
|
+
// oldest-to-newest and the cap evicts the oldest -- the same policy recordSeenImage applies
|
|
6743
|
+
// in memory, applied again here because a union of two capped lists can exceed the cap.
|
|
6744
|
+
seenImageHashes: mergeSeenImageHashes(disk.seenImageHashes ?? [], mem.seenImageHashes ?? []),
|
|
6236
6745
|
// Last-seen scalar, not an accumulating collection: prefer mem's value (this process's freshest observation) over disk's, since a newer write always supersedes an older one.
|
|
6237
6746
|
...mem.lastTabContext !== void 0 ? { lastTabContext: mem.lastTabContext } : disk.lastTabContext !== void 0 ? { lastTabContext: disk.lastTabContext } : {},
|
|
6238
6747
|
// Prefer the value already on disk: it marks the original creation time, and must never be bumped forward to the merge's "now". `mem` never carries one (it is not tracked in memory), so this is really "keep whatever disk has".
|
|
6239
6748
|
...disk.created_ts !== void 0 ? { created_ts: disk.created_ts } : mem.created_ts !== void 0 ? { created_ts: mem.created_ts } : {}
|
|
6240
6749
|
};
|
|
6241
6750
|
}
|
|
6751
|
+
function mergeSeenImageHashes(disk, mem) {
|
|
6752
|
+
const merged = Array.from(/* @__PURE__ */ new Set([...disk, ...mem]));
|
|
6753
|
+
return merged.length > MAX_SEEN_IMAGE_HASHES ? merged.slice(merged.length - MAX_SEEN_IMAGE_HASHES) : merged;
|
|
6754
|
+
}
|
|
6242
6755
|
function capFiles(s, max) {
|
|
6243
6756
|
if (s.files.length <= max) return s;
|
|
6244
6757
|
const kept = [...s.files].sort((a, b) => b.lastReadAt - a.lastReadAt).slice(0, max);
|
|
@@ -6951,6 +7464,9 @@ function resolveTesseractEntry() {
|
|
|
6951
7464
|
}
|
|
6952
7465
|
return _tesseractEntryPath;
|
|
6953
7466
|
}
|
|
7467
|
+
function isOcrEngineAvailable() {
|
|
7468
|
+
return !_ocrUnavailableThisProcess && resolveTesseractEntry() !== null;
|
|
7469
|
+
}
|
|
6954
7470
|
var _ocrUnavailableThisProcess = false;
|
|
6955
7471
|
function buildChildScript(entryPath, cacheDir) {
|
|
6956
7472
|
return [
|
|
@@ -7065,14 +7581,29 @@ var loadSharp = createLazyModuleLoader(async () => {
|
|
|
7065
7581
|
function isImagePath(p) {
|
|
7066
7582
|
return IMAGE_EXTENSIONS.has(path12.extname(p).toLowerCase());
|
|
7067
7583
|
}
|
|
7584
|
+
var ImageDecodeError = class extends Error {
|
|
7585
|
+
};
|
|
7068
7586
|
async function probeImageMeta(input) {
|
|
7069
7587
|
const sharp = await loadSharp();
|
|
7070
7588
|
if (sharp === null) return null;
|
|
7071
7589
|
try {
|
|
7072
7590
|
const meta = await sharp(input, { limitInputPixels: false }).metadata();
|
|
7073
7591
|
return { width: meta.width ?? 0, height: meta.height ?? 0, format: meta.format ?? null, pages: meta.pages ?? 1 };
|
|
7592
|
+
} catch (e) {
|
|
7593
|
+
throw new ImageDecodeError(e?.message ?? "image could not be decoded");
|
|
7594
|
+
}
|
|
7595
|
+
}
|
|
7596
|
+
async function imageQualifiesForShrink(input) {
|
|
7597
|
+
if (input.length >= DEFAULT_SIZE_THRESHOLD_BYTES) return true;
|
|
7598
|
+
const sharp = await loadSharp();
|
|
7599
|
+
if (sharp === null) return false;
|
|
7600
|
+
try {
|
|
7601
|
+
const cfg = loadConfig().image_shrink;
|
|
7602
|
+
const limitInputPixels = cfg.max_image_pixels > 0 ? cfg.max_image_pixels : false;
|
|
7603
|
+
const meta = await sharp(input, { limitInputPixels }).metadata();
|
|
7604
|
+
return Math.max(meta.width ?? 0, meta.height ?? 0) > DEFAULT_MAX_DIMENSION;
|
|
7074
7605
|
} catch {
|
|
7075
|
-
return
|
|
7606
|
+
return false;
|
|
7076
7607
|
}
|
|
7077
7608
|
}
|
|
7078
7609
|
async function shrinkImage(input, opts) {
|
|
@@ -7196,7 +7727,6 @@ async function preReadImageHandler(event) {
|
|
|
7196
7727
|
pruneShrinkCache();
|
|
7197
7728
|
const stat2 = statInfo(filePath);
|
|
7198
7729
|
if (stat2 === null) return passOutput();
|
|
7199
|
-
const size = stat2.size;
|
|
7200
7730
|
const quality = loadConfig().image_shrink.jpeg_quality;
|
|
7201
7731
|
const cached = findCachedShrink(filePath, stat2.size, stat2.mtimeMs, quality);
|
|
7202
7732
|
if (cached !== null) {
|
|
@@ -7206,7 +7736,14 @@ async function preReadImageHandler(event) {
|
|
|
7206
7736
|
} catch {
|
|
7207
7737
|
cachedData = null;
|
|
7208
7738
|
}
|
|
7209
|
-
|
|
7739
|
+
let meta = null;
|
|
7740
|
+
if (cachedData !== null) {
|
|
7741
|
+
try {
|
|
7742
|
+
meta = await probeImageMeta(cachedData);
|
|
7743
|
+
} catch {
|
|
7744
|
+
meta = null;
|
|
7745
|
+
}
|
|
7746
|
+
}
|
|
7210
7747
|
if (cachedData !== null && meta !== null) {
|
|
7211
7748
|
const result2 = {
|
|
7212
7749
|
data: cachedData,
|
|
@@ -7223,34 +7760,13 @@ async function preReadImageHandler(event) {
|
|
|
7223
7760
|
} catch {
|
|
7224
7761
|
}
|
|
7225
7762
|
}
|
|
7226
|
-
let input
|
|
7227
|
-
|
|
7228
|
-
|
|
7229
|
-
|
|
7230
|
-
|
|
7231
|
-
} catch {
|
|
7232
|
-
return passOutput();
|
|
7233
|
-
}
|
|
7234
|
-
const sharp = await loadSharp();
|
|
7235
|
-
if (sharp === null) return passOutput();
|
|
7236
|
-
try {
|
|
7237
|
-
const cfg = loadConfig().image_shrink;
|
|
7238
|
-
const limitInputPixels = cfg.max_image_pixels > 0 ? cfg.max_image_pixels : false;
|
|
7239
|
-
const meta = await sharp(input, { limitInputPixels }).metadata();
|
|
7240
|
-
const longestEdge = Math.max(meta.width ?? 0, meta.height ?? 0);
|
|
7241
|
-
qualifies = longestEdge > DEFAULT_MAX_DIMENSION;
|
|
7242
|
-
} catch {
|
|
7243
|
-
return passOutput();
|
|
7244
|
-
}
|
|
7245
|
-
}
|
|
7246
|
-
if (!qualifies) return passOutput();
|
|
7247
|
-
if (input === null) {
|
|
7248
|
-
try {
|
|
7249
|
-
input = fs16.readFileSync(filePath);
|
|
7250
|
-
} catch {
|
|
7251
|
-
return passOutput();
|
|
7252
|
-
}
|
|
7763
|
+
let input;
|
|
7764
|
+
try {
|
|
7765
|
+
input = fs16.readFileSync(filePath);
|
|
7766
|
+
} catch {
|
|
7767
|
+
return passOutput();
|
|
7253
7768
|
}
|
|
7769
|
+
if (!await imageQualifiesForShrink(input)) return passOutput();
|
|
7254
7770
|
const result = await shrinkImage(input, { quality, sizeThresholdBytes: 0 });
|
|
7255
7771
|
if (result === null) {
|
|
7256
7772
|
recordStat("image_shrink_skipped");
|
|
@@ -7261,30 +7777,432 @@ async function preReadImageHandler(event) {
|
|
|
7261
7777
|
}
|
|
7262
7778
|
registerHook("pre_tool_use", preReadImageHandler, { toolName: "Read" });
|
|
7263
7779
|
|
|
7264
|
-
// src/
|
|
7780
|
+
// src/embed_model.ts
|
|
7781
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
7782
|
+
import * as fs17 from "node:fs";
|
|
7265
7783
|
import { createRequire as createRequire3 } from "node:module";
|
|
7266
|
-
|
|
7267
|
-
|
|
7268
|
-
|
|
7269
|
-
|
|
7270
|
-
|
|
7271
|
-
|
|
7272
|
-
|
|
7273
|
-
|
|
7274
|
-
|
|
7275
|
-
|
|
7276
|
-
|
|
7277
|
-
|
|
7784
|
+
import * as path13 from "node:path";
|
|
7785
|
+
import { pipeline } from "node:stream/promises";
|
|
7786
|
+
|
|
7787
|
+
// src/embed_tokenizer.ts
|
|
7788
|
+
var MAX_SEQUENCE_TOKENS = 512;
|
|
7789
|
+
var UnsupportedTokenizerError = class extends Error {
|
|
7790
|
+
constructor(message) {
|
|
7791
|
+
super(message);
|
|
7792
|
+
this.name = "UnsupportedTokenizerError";
|
|
7793
|
+
}
|
|
7794
|
+
};
|
|
7795
|
+
function fail(message) {
|
|
7796
|
+
throw new UnsupportedTokenizerError(message);
|
|
7797
|
+
}
|
|
7798
|
+
function asRecord(value, path31) {
|
|
7799
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${path31} is not an object`);
|
|
7800
|
+
return value;
|
|
7801
|
+
}
|
|
7802
|
+
function asString(value, path31) {
|
|
7803
|
+
if (typeof value !== "string") fail(`${path31} is not a string`);
|
|
7804
|
+
return value;
|
|
7805
|
+
}
|
|
7806
|
+
function asArray2(value, path31) {
|
|
7807
|
+
if (!Array.isArray(value)) fail(`${path31} is not an array`);
|
|
7808
|
+
return value;
|
|
7809
|
+
}
|
|
7810
|
+
function expectValue(actual, wanted, path31) {
|
|
7811
|
+
if (actual !== wanted) fail(`${path31} is ${JSON.stringify(actual)}, expected ${JSON.stringify(wanted)}`);
|
|
7812
|
+
}
|
|
7813
|
+
function isChinese(cp) {
|
|
7814
|
+
return cp >= 19968 && cp <= 40959 || cp >= 13312 && cp <= 19903 || cp >= 131072 && cp <= 173791 || cp >= 173824 && cp <= 177983 || cp >= 177984 && cp <= 178207 || cp >= 178208 && cp <= 183983 || cp >= 63744 && cp <= 64255 || cp >= 194560 && cp <= 195103;
|
|
7815
|
+
}
|
|
7816
|
+
var PUNCTUATION = new RegExp("\\p{P}", "u");
|
|
7817
|
+
function isPunctuation(ch) {
|
|
7818
|
+
const cp = ch.codePointAt(0) ?? 0;
|
|
7819
|
+
if (cp >= 33 && cp <= 47 || cp >= 58 && cp <= 64 || cp >= 91 && cp <= 96 || cp >= 123 && cp <= 126) {
|
|
7820
|
+
return true;
|
|
7821
|
+
}
|
|
7822
|
+
return PUNCTUATION.test(ch);
|
|
7823
|
+
}
|
|
7824
|
+
var CONTROL = new RegExp("\\p{Cc}|\\p{Cf}|\\p{Co}|\\p{Cs}", "u");
|
|
7825
|
+
function isControl(ch) {
|
|
7826
|
+
if (ch === " " || ch === "\n" || ch === "\r") return false;
|
|
7827
|
+
return CONTROL.test(ch);
|
|
7828
|
+
}
|
|
7829
|
+
var SPACE_SEPARATOR = new RegExp("\\p{Zs}", "u");
|
|
7830
|
+
function isWhitespace(ch) {
|
|
7831
|
+
return ch === " " || ch === " " || ch === "\n" || ch === "\r" || SPACE_SEPARATOR.test(ch);
|
|
7832
|
+
}
|
|
7833
|
+
var COMBINING_MARK = /[\u0300-\u036f]/gu;
|
|
7834
|
+
function normalize(text) {
|
|
7835
|
+
let out = "";
|
|
7836
|
+
for (const ch of text) {
|
|
7837
|
+
const cp = ch.codePointAt(0) ?? 0;
|
|
7838
|
+
if (cp === 0 || cp === 65533 || isControl(ch)) continue;
|
|
7839
|
+
if (isWhitespace(ch)) {
|
|
7840
|
+
out += " ";
|
|
7841
|
+
continue;
|
|
7278
7842
|
}
|
|
7279
|
-
|
|
7280
|
-
|
|
7843
|
+
if (isChinese(cp)) {
|
|
7844
|
+
out += ` ${ch} `;
|
|
7845
|
+
continue;
|
|
7846
|
+
}
|
|
7847
|
+
out += ch;
|
|
7281
7848
|
}
|
|
7849
|
+
return out.normalize("NFD").replace(COMBINING_MARK, "").toLowerCase();
|
|
7850
|
+
}
|
|
7851
|
+
function preTokenize(text) {
|
|
7852
|
+
const words = [];
|
|
7853
|
+
for (const piece of text.split(/\s+/)) {
|
|
7854
|
+
if (!piece) continue;
|
|
7855
|
+
let buf = "";
|
|
7856
|
+
for (const ch of piece) {
|
|
7857
|
+
if (isPunctuation(ch)) {
|
|
7858
|
+
if (buf) {
|
|
7859
|
+
words.push(buf);
|
|
7860
|
+
buf = "";
|
|
7861
|
+
}
|
|
7862
|
+
words.push(ch);
|
|
7863
|
+
} else {
|
|
7864
|
+
buf += ch;
|
|
7865
|
+
}
|
|
7866
|
+
}
|
|
7867
|
+
if (buf) words.push(buf);
|
|
7868
|
+
}
|
|
7869
|
+
return words;
|
|
7870
|
+
}
|
|
7871
|
+
function readSpec(raw) {
|
|
7872
|
+
const spec = asRecord(raw, "tokenizer.json");
|
|
7873
|
+
const normalizer = asRecord(spec["normalizer"], "normalizer");
|
|
7874
|
+
expectValue(normalizer["type"], "BertNormalizer", "normalizer.type");
|
|
7875
|
+
expectValue(normalizer["clean_text"], true, "normalizer.clean_text");
|
|
7876
|
+
expectValue(normalizer["handle_chinese_chars"], true, "normalizer.handle_chinese_chars");
|
|
7877
|
+
expectValue(normalizer["lowercase"], true, "normalizer.lowercase");
|
|
7878
|
+
const stripAccents = normalizer["strip_accents"];
|
|
7879
|
+
if (stripAccents !== null && stripAccents !== true) {
|
|
7880
|
+
fail(`normalizer.strip_accents is ${JSON.stringify(stripAccents)}, expected null or true`);
|
|
7881
|
+
}
|
|
7882
|
+
expectValue(asRecord(spec["pre_tokenizer"], "pre_tokenizer")["type"], "BertPreTokenizer", "pre_tokenizer.type");
|
|
7883
|
+
const model = asRecord(spec["model"], "model");
|
|
7884
|
+
expectValue(model["type"], "WordPiece", "model.type");
|
|
7885
|
+
expectValue(model["continuing_subword_prefix"], "##", "model.continuing_subword_prefix");
|
|
7886
|
+
const post = asRecord(spec["post_processor"], "post_processor");
|
|
7887
|
+
expectValue(post["type"], "TemplateProcessing", "post_processor.type");
|
|
7888
|
+
const template = asArray2(post["single"], "post_processor.single").map((part) => {
|
|
7889
|
+
const special = asRecord(part, "post_processor.single[]")["SpecialToken"];
|
|
7890
|
+
return special === void 0 ? "A" : asString(asRecord(special, "SpecialToken")["id"], "SpecialToken.id");
|
|
7891
|
+
}).join(" ");
|
|
7892
|
+
if (template !== "[CLS] A [SEP]") fail(`post_processor.single is "${template}", expected "[CLS] A [SEP]"`);
|
|
7893
|
+
const vocab = /* @__PURE__ */ new Map();
|
|
7894
|
+
for (const [token, id] of Object.entries(asRecord(model["vocab"], "model.vocab"))) {
|
|
7895
|
+
if (typeof id !== "number" || !Number.isInteger(id)) fail(`model.vocab["${token}"] is not an integer id`);
|
|
7896
|
+
vocab.set(token, id);
|
|
7897
|
+
}
|
|
7898
|
+
const maxChars = model["max_input_chars_per_word"];
|
|
7899
|
+
if (maxChars !== void 0 && (typeof maxChars !== "number" || !Number.isInteger(maxChars) || maxChars < 1)) {
|
|
7900
|
+
fail("model.max_input_chars_per_word is not a positive integer");
|
|
7901
|
+
}
|
|
7902
|
+
return {
|
|
7903
|
+
vocab,
|
|
7904
|
+
unkToken: asString(model["unk_token"], "model.unk_token"),
|
|
7905
|
+
maxInputCharsPerWord: typeof maxChars === "number" ? maxChars : 100
|
|
7906
|
+
};
|
|
7282
7907
|
}
|
|
7908
|
+
var BertWordPiece = class _BertWordPiece {
|
|
7909
|
+
vocab;
|
|
7910
|
+
maxInputCharsPerWord;
|
|
7911
|
+
clsId;
|
|
7912
|
+
sepId;
|
|
7913
|
+
padId;
|
|
7914
|
+
unkId;
|
|
7915
|
+
/** @param raw the parsed contents of a tokenizer.json. */
|
|
7916
|
+
constructor(raw) {
|
|
7917
|
+
const spec = readSpec(raw);
|
|
7918
|
+
this.vocab = spec.vocab;
|
|
7919
|
+
this.maxInputCharsPerWord = spec.maxInputCharsPerWord;
|
|
7920
|
+
this.clsId = this.requireToken("[CLS]");
|
|
7921
|
+
this.sepId = this.requireToken("[SEP]");
|
|
7922
|
+
this.padId = this.requireToken("[PAD]");
|
|
7923
|
+
this.unkId = this.requireToken(spec.unkToken);
|
|
7924
|
+
}
|
|
7925
|
+
/** Parse and validate in one step, for the common case of reading the file off disk. */
|
|
7926
|
+
static fromJson(json) {
|
|
7927
|
+
let parsed;
|
|
7928
|
+
try {
|
|
7929
|
+
parsed = JSON.parse(json);
|
|
7930
|
+
} catch (err) {
|
|
7931
|
+
throw new UnsupportedTokenizerError(`tokenizer.json is not valid JSON: ${err.message}`);
|
|
7932
|
+
}
|
|
7933
|
+
return new _BertWordPiece(parsed);
|
|
7934
|
+
}
|
|
7935
|
+
requireToken(token) {
|
|
7936
|
+
const id = this.vocab.get(token);
|
|
7937
|
+
if (id === void 0) fail(`model.vocab is missing the ${token} token`);
|
|
7938
|
+
return id;
|
|
7939
|
+
}
|
|
7940
|
+
/** Greedy longest-match-first over one whitespace- and punctuation-free word. */
|
|
7941
|
+
wordToIds(word, into) {
|
|
7942
|
+
if (word.length > this.maxInputCharsPerWord) {
|
|
7943
|
+
into.push(this.unkId);
|
|
7944
|
+
return;
|
|
7945
|
+
}
|
|
7946
|
+
const pieces = [];
|
|
7947
|
+
let start = 0;
|
|
7948
|
+
while (start < word.length) {
|
|
7949
|
+
let end = word.length;
|
|
7950
|
+
let found = -1;
|
|
7951
|
+
while (start < end) {
|
|
7952
|
+
const sub = start === 0 ? word.slice(start, end) : `##${word.slice(start, end)}`;
|
|
7953
|
+
const id = this.vocab.get(sub);
|
|
7954
|
+
if (id !== void 0) {
|
|
7955
|
+
found = id;
|
|
7956
|
+
break;
|
|
7957
|
+
}
|
|
7958
|
+
end--;
|
|
7959
|
+
}
|
|
7960
|
+
if (found === -1) {
|
|
7961
|
+
into.push(this.unkId);
|
|
7962
|
+
return;
|
|
7963
|
+
}
|
|
7964
|
+
pieces.push(found);
|
|
7965
|
+
start = end;
|
|
7966
|
+
}
|
|
7967
|
+
for (const id of pieces) into.push(id);
|
|
7968
|
+
}
|
|
7969
|
+
/**
|
|
7970
|
+
* `[CLS] ... [SEP]`, cut to `maxLength` tokens.
|
|
7971
|
+
*
|
|
7972
|
+
* The cut is taken after the markers are added, not before, which means a sequence long enough to
|
|
7973
|
+
* be truncated ends on an ordinary token and has no [SEP] at all. That is what the reference does
|
|
7974
|
+
* and therefore what the model has been fed all along, so it is deliberate rather than an
|
|
7975
|
+
* oversight: keeping the [SEP] would be the more defensible sequence and a different one, and a
|
|
7976
|
+
* tokenizer whose whole justification is producing identical ids does not get to improve on them.
|
|
7977
|
+
* The oracle carries two cases that reach the limit; both end mid-text.
|
|
7978
|
+
*/
|
|
7979
|
+
encode(text, maxLength = MAX_SEQUENCE_TOKENS) {
|
|
7980
|
+
if (!Number.isInteger(maxLength) || maxLength < 2) {
|
|
7981
|
+
throw new RangeError(`maxLength must be an integer >= 2 (both markers), got ${maxLength}`);
|
|
7982
|
+
}
|
|
7983
|
+
const ids = [this.clsId];
|
|
7984
|
+
for (const word of preTokenize(normalize(text))) {
|
|
7985
|
+
this.wordToIds(word, ids);
|
|
7986
|
+
if (ids.length >= maxLength) return ids.slice(0, maxLength);
|
|
7987
|
+
}
|
|
7988
|
+
ids.push(this.sepId);
|
|
7989
|
+
return ids.length > maxLength ? ids.slice(0, maxLength) : ids;
|
|
7990
|
+
}
|
|
7991
|
+
};
|
|
7992
|
+
|
|
7993
|
+
// src/embed_model.ts
|
|
7994
|
+
var _require2 = createRequire3(import.meta.url);
|
|
7283
7995
|
var DEFAULT_MODEL = "Xenova/bge-small-en-v1.5";
|
|
7284
7996
|
var DEFAULT_DIM = 384;
|
|
7285
7997
|
var PINNED_MODEL_REVISION = "ea104dacec62c0de699686887e3f920caeb4f3e3";
|
|
7998
|
+
var MODEL_FILES = [
|
|
7999
|
+
{
|
|
8000
|
+
name: "tokenizer.json",
|
|
8001
|
+
sha256: "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66",
|
|
8002
|
+
bytes: 711396
|
|
8003
|
+
},
|
|
8004
|
+
{
|
|
8005
|
+
name: "onnx/model_quantized.onnx",
|
|
8006
|
+
sha256: "6c9c6101a956d62dfb5e7190c538226c0c5bb9cb27b651234b6df063ee7dbfe4",
|
|
8007
|
+
bytes: 34014426
|
|
8008
|
+
}
|
|
8009
|
+
];
|
|
8010
|
+
function modelDir() {
|
|
8011
|
+
return path13.join(dataDir(), "models", ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
8012
|
+
}
|
|
8013
|
+
function downloadUrl(file) {
|
|
8014
|
+
return `https://huggingface.co/${DEFAULT_MODEL}/resolve/${PINNED_MODEL_REVISION}/${file.name}`;
|
|
8015
|
+
}
|
|
8016
|
+
var _ort = null;
|
|
8017
|
+
var _ortError = null;
|
|
8018
|
+
var _ortLoadAttempted = false;
|
|
8019
|
+
function ensureRuntimeLoaded() {
|
|
8020
|
+
if (_ortLoadAttempted) return;
|
|
8021
|
+
_ortLoadAttempted = true;
|
|
8022
|
+
try {
|
|
8023
|
+
_ort = _require2("onnxruntime-node");
|
|
8024
|
+
} catch (e) {
|
|
8025
|
+
_ortError = e instanceof Error ? e : new Error(String(e));
|
|
8026
|
+
}
|
|
8027
|
+
}
|
|
8028
|
+
function isRuntimeAvailable() {
|
|
8029
|
+
ensureRuntimeLoaded();
|
|
8030
|
+
return _ort !== null && _ortError === null;
|
|
8031
|
+
}
|
|
8032
|
+
function runtimeLoadError() {
|
|
8033
|
+
ensureRuntimeLoaded();
|
|
8034
|
+
return _ortError;
|
|
8035
|
+
}
|
|
8036
|
+
function runtimeVersion() {
|
|
8037
|
+
ensureRuntimeLoaded();
|
|
8038
|
+
if (_ort === null) return "unknown";
|
|
8039
|
+
try {
|
|
8040
|
+
let dir = path13.dirname(_require2.resolve("onnxruntime-node"));
|
|
8041
|
+
for (let depth = 0; depth < 6; depth++) {
|
|
8042
|
+
const manifest = path13.join(dir, "package.json");
|
|
8043
|
+
if (fs17.existsSync(manifest)) {
|
|
8044
|
+
const parsed = JSON.parse(fs17.readFileSync(manifest, "utf8"));
|
|
8045
|
+
if (parsed.name === "onnxruntime-node" && typeof parsed.version === "string") return parsed.version;
|
|
8046
|
+
}
|
|
8047
|
+
const up = path13.dirname(dir);
|
|
8048
|
+
if (up === dir) break;
|
|
8049
|
+
dir = up;
|
|
8050
|
+
}
|
|
8051
|
+
} catch {
|
|
8052
|
+
}
|
|
8053
|
+
return "unknown";
|
|
8054
|
+
}
|
|
8055
|
+
function sha256Of(filePath) {
|
|
8056
|
+
return new Promise((resolve10, reject) => {
|
|
8057
|
+
const hash2 = createHash4("sha256");
|
|
8058
|
+
const stream = fs17.createReadStream(filePath);
|
|
8059
|
+
stream.on("error", reject);
|
|
8060
|
+
stream.on("data", (chunk) => hash2.update(chunk));
|
|
8061
|
+
stream.on("end", () => resolve10(hash2.digest("hex")));
|
|
8062
|
+
});
|
|
8063
|
+
}
|
|
8064
|
+
async function download(file, target) {
|
|
8065
|
+
const url = downloadUrl(file);
|
|
8066
|
+
const response = await fetch(url, { redirect: "follow" });
|
|
8067
|
+
if (!response.ok) throw new Error(`GET ${url} returned ${response.status} ${response.statusText}`);
|
|
8068
|
+
if (!response.body) throw new Error(`GET ${url} returned no body`);
|
|
8069
|
+
const temp = `${target}.${process.pid}.partial`;
|
|
8070
|
+
const hash2 = createHash4("sha256");
|
|
8071
|
+
let written = 0;
|
|
8072
|
+
const out = fs17.createWriteStream(temp);
|
|
8073
|
+
try {
|
|
8074
|
+
await pipeline(async function* () {
|
|
8075
|
+
for await (const chunk of response.body) {
|
|
8076
|
+
written += chunk.byteLength;
|
|
8077
|
+
if (written > file.bytes) throw new Error(`${file.name} is longer than the pinned ${file.bytes} bytes`);
|
|
8078
|
+
hash2.update(chunk);
|
|
8079
|
+
yield chunk;
|
|
8080
|
+
}
|
|
8081
|
+
}, out);
|
|
8082
|
+
if (written !== file.bytes) {
|
|
8083
|
+
throw new Error(`${file.name} is ${written} bytes, expected the pinned ${file.bytes}`);
|
|
8084
|
+
}
|
|
8085
|
+
const digest = hash2.digest("hex");
|
|
8086
|
+
if (digest !== file.sha256) {
|
|
8087
|
+
throw new Error(`${file.name} has sha256 ${digest}, expected the pinned ${file.sha256}`);
|
|
8088
|
+
}
|
|
8089
|
+
fs17.renameSync(temp, target);
|
|
8090
|
+
} catch (e) {
|
|
8091
|
+
await new Promise((resolve10) => {
|
|
8092
|
+
if (out.closed) resolve10();
|
|
8093
|
+
else out.once("close", () => resolve10());
|
|
8094
|
+
});
|
|
8095
|
+
try {
|
|
8096
|
+
fs17.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
8097
|
+
} catch {
|
|
8098
|
+
}
|
|
8099
|
+
throw e;
|
|
8100
|
+
}
|
|
8101
|
+
}
|
|
8102
|
+
async function ensureModelFiles(modelName = DEFAULT_MODEL) {
|
|
8103
|
+
if (modelName !== DEFAULT_MODEL) {
|
|
8104
|
+
throw new Error(
|
|
8105
|
+
`Only ${DEFAULT_MODEL} is supported: its files are pinned to a revision and to a sha256 each, and "${modelName}" has neither, so there would be nothing to check the download against.`
|
|
8106
|
+
);
|
|
8107
|
+
}
|
|
8108
|
+
ensureDataDirPrivate();
|
|
8109
|
+
const dir = modelDir();
|
|
8110
|
+
const offline = loadConfig().network.offline;
|
|
8111
|
+
for (const file of MODEL_FILES) {
|
|
8112
|
+
const target = path13.join(dir, file.name);
|
|
8113
|
+
if (fs17.existsSync(target)) {
|
|
8114
|
+
const digest = await sha256Of(target);
|
|
8115
|
+
if (digest === file.sha256) continue;
|
|
8116
|
+
fs17.rmSync(target, { force: true });
|
|
8117
|
+
}
|
|
8118
|
+
if (offline) {
|
|
8119
|
+
throw new Error(
|
|
8120
|
+
`Offline mode is on (network.offline): refusing to download ${file.name} for the embedding model. Copy the pinned files into ${dir} on a connected machine to use semantic search here.`
|
|
8121
|
+
);
|
|
8122
|
+
}
|
|
8123
|
+
fs17.mkdirSync(path13.dirname(target), { recursive: true });
|
|
8124
|
+
console.warn(
|
|
8125
|
+
`Downloading the embedding model, once (${file.name}, ${Math.round(file.bytes / 1024 / 1024)} MB) into ${dir}`
|
|
8126
|
+
);
|
|
8127
|
+
await download(file, target);
|
|
8128
|
+
}
|
|
8129
|
+
return dir;
|
|
8130
|
+
}
|
|
8131
|
+
function poolAndNormalize(hidden, seq, dim) {
|
|
8132
|
+
const pooled = new Float64Array(dim);
|
|
8133
|
+
for (let t = 0; t < seq; t++) {
|
|
8134
|
+
const base = t * dim;
|
|
8135
|
+
for (let d = 0; d < dim; d++) pooled[d] = (pooled[d] ?? 0) + (hidden[base + d] ?? 0);
|
|
8136
|
+
}
|
|
8137
|
+
let sumOfSquares = 0;
|
|
8138
|
+
for (let d = 0; d < dim; d++) {
|
|
8139
|
+
const mean = (pooled[d] ?? 0) / seq;
|
|
8140
|
+
pooled[d] = mean;
|
|
8141
|
+
sumOfSquares += mean * mean;
|
|
8142
|
+
}
|
|
8143
|
+
const norm = Math.sqrt(sumOfSquares);
|
|
8144
|
+
const out = new Float32Array(dim);
|
|
8145
|
+
if (norm === 0 || !Number.isFinite(norm)) return out;
|
|
8146
|
+
for (let d = 0; d < dim; d++) out[d] = (pooled[d] ?? 0) / norm;
|
|
8147
|
+
return out;
|
|
8148
|
+
}
|
|
8149
|
+
var EmbeddingModel = class _EmbeddingModel {
|
|
8150
|
+
constructor(tokenizer, session, tensorFactory) {
|
|
8151
|
+
this.tokenizer = tokenizer;
|
|
8152
|
+
this.session = session;
|
|
8153
|
+
this.tensorFactory = tensorFactory;
|
|
8154
|
+
}
|
|
8155
|
+
tokenizer;
|
|
8156
|
+
session;
|
|
8157
|
+
tensorFactory;
|
|
8158
|
+
static async load(modelName = DEFAULT_MODEL) {
|
|
8159
|
+
if (!isRuntimeAvailable()) {
|
|
8160
|
+
throw new Error(`onnxruntime-node is not available: ${_ortError?.message ?? "unknown error"}`);
|
|
8161
|
+
}
|
|
8162
|
+
const dir = await ensureModelFiles(modelName);
|
|
8163
|
+
const tokenizer = BertWordPiece.fromJson(fs17.readFileSync(path13.join(dir, "tokenizer.json"), "utf8"));
|
|
8164
|
+
const ort = _ort;
|
|
8165
|
+
const session = await ort.InferenceSession.create(path13.join(dir, "onnx", "model_quantized.onnx"));
|
|
8166
|
+
return new _EmbeddingModel(tokenizer, session, ort.Tensor);
|
|
8167
|
+
}
|
|
8168
|
+
/** Embed one text. Sequences are run singly, so there is no padding and no mask to get wrong. */
|
|
8169
|
+
async embed(text) {
|
|
8170
|
+
const ids = this.tokenizer.encode(text);
|
|
8171
|
+
const length = ids.length;
|
|
8172
|
+
const feeds = {
|
|
8173
|
+
input_ids: new this.tensorFactory("int64", BigInt64Array.from(ids, BigInt), [1, length]),
|
|
8174
|
+
attention_mask: new this.tensorFactory("int64", new BigInt64Array(length).fill(1n), [1, length])
|
|
8175
|
+
};
|
|
8176
|
+
if (this.session.inputNames.includes("token_type_ids")) {
|
|
8177
|
+
feeds["token_type_ids"] = new this.tensorFactory("int64", new BigInt64Array(length), [1, length]);
|
|
8178
|
+
}
|
|
8179
|
+
const outputName = this.session.outputNames[0];
|
|
8180
|
+
if (outputName === void 0) throw new Error("the model declares no outputs");
|
|
8181
|
+
const output = (await this.session.run(feeds))[outputName];
|
|
8182
|
+
if (!output) throw new Error(`the model produced no ${outputName}`);
|
|
8183
|
+
const [, seq, dim] = output.dims;
|
|
8184
|
+
if (seq === void 0 || dim === void 0) {
|
|
8185
|
+
throw new Error(`expected a [batch, sequence, dimension] output, got [${output.dims.join(", ")}]`);
|
|
8186
|
+
}
|
|
8187
|
+
if (dim !== DEFAULT_DIM) {
|
|
8188
|
+
throw new Error(`the model produced ${dim}-dimension vectors, expected ${DEFAULT_DIM}`);
|
|
8189
|
+
}
|
|
8190
|
+
return poolAndNormalize(output.data, seq, dim);
|
|
8191
|
+
}
|
|
8192
|
+
};
|
|
8193
|
+
registerReset(() => {
|
|
8194
|
+
_ort = null;
|
|
8195
|
+
_ortError = null;
|
|
8196
|
+
_ortLoadAttempted = false;
|
|
8197
|
+
});
|
|
8198
|
+
|
|
8199
|
+
// src/embeddings.ts
|
|
7286
8200
|
var QUERY_INSTRUCTION_PREFIX = "Represent this sentence for searching relevant passages: ";
|
|
7287
8201
|
var _extractorCache = /* @__PURE__ */ new Map();
|
|
8202
|
+
var inHousePipelineFn = async (_task, modelName) => {
|
|
8203
|
+
const model = await EmbeddingModel.load(modelName);
|
|
8204
|
+
return async (text) => ({ data: await model.embed(text) });
|
|
8205
|
+
};
|
|
7288
8206
|
var _pipelineFnOverride = null;
|
|
7289
8207
|
registerReset(() => {
|
|
7290
8208
|
_extractorCache.clear();
|
|
@@ -7303,8 +8221,7 @@ async function buildExtractorWithRetry(pipelineFn, modelName) {
|
|
|
7303
8221
|
let lastError;
|
|
7304
8222
|
for (let attempt = 1; attempt <= PIPELINE_RETRY_ATTEMPTS; attempt++) {
|
|
7305
8223
|
try {
|
|
7306
|
-
|
|
7307
|
-
return await pipelineFn("feature-extraction", modelName, pipelineOptions);
|
|
8224
|
+
return await pipelineFn("feature-extraction", modelName);
|
|
7308
8225
|
} catch (e) {
|
|
7309
8226
|
lastError = e;
|
|
7310
8227
|
if (attempt < PIPELINE_RETRY_ATTEMPTS) await sleep(PIPELINE_RETRY_DELAY_MS * attempt);
|
|
@@ -7356,25 +8273,23 @@ var _MIN_TOKEN_LEN = 3;
|
|
|
7356
8273
|
var OVER_FETCH_FACTOR = 4;
|
|
7357
8274
|
var MAX_OVER_FETCH = 100;
|
|
7358
8275
|
function isAvailable() {
|
|
7359
|
-
|
|
7360
|
-
|
|
8276
|
+
return isRuntimeAvailable();
|
|
8277
|
+
}
|
|
8278
|
+
function embeddingBackendLoadError() {
|
|
8279
|
+
return runtimeLoadError();
|
|
7361
8280
|
}
|
|
7362
8281
|
async function embedTexts(texts, modelName = DEFAULT_MODEL) {
|
|
7363
8282
|
if (!isAvailable()) {
|
|
7364
8283
|
throw new Error(
|
|
7365
|
-
`
|
|
8284
|
+
`Embedding backend not available: ${runtimeLoadError()?.message ?? "unknown error"}`
|
|
7366
8285
|
);
|
|
7367
8286
|
}
|
|
7368
8287
|
if (texts.length === 0) {
|
|
7369
8288
|
return [];
|
|
7370
8289
|
}
|
|
7371
|
-
if (!_transformer || typeof _transformer !== "object") {
|
|
7372
|
-
throw new Error("Transformer module is unavailable");
|
|
7373
|
-
}
|
|
7374
8290
|
let extractorPromise = _extractorCache.get(modelName);
|
|
7375
8291
|
if (!extractorPromise) {
|
|
7376
|
-
const
|
|
7377
|
-
const pipelineFn = _pipelineFnOverride ?? transformerObj["pipeline"];
|
|
8292
|
+
const pipelineFn = _pipelineFnOverride ?? inHousePipelineFn;
|
|
7378
8293
|
extractorPromise = buildExtractorWithRetry(pipelineFn, modelName);
|
|
7379
8294
|
_extractorCache.set(modelName, extractorPromise);
|
|
7380
8295
|
extractorPromise.catch(() => {
|
|
@@ -7586,6 +8501,7 @@ async function upsertChunks(db, chunks) {
|
|
|
7586
8501
|
deleteFileEmbeddings(db, filePath);
|
|
7587
8502
|
return "unavailable";
|
|
7588
8503
|
}
|
|
8504
|
+
ensureEmbeddingProvenance(db);
|
|
7589
8505
|
const texts = chunks.map((c) => c.text);
|
|
7590
8506
|
const embeddings = await embedTexts(texts);
|
|
7591
8507
|
const chunkInsertStmt = db.prepare(`
|
|
@@ -7656,7 +8572,6 @@ function fetchScopedHits(db, queryVec, k, maxDistance, rootDir) {
|
|
|
7656
8572
|
}
|
|
7657
8573
|
async function searchSemantic(db, query, topK = 8, modelName = DEFAULT_MODEL, maxDistance = DEFAULT_DISTANCE_THRESHOLD, rootDir) {
|
|
7658
8574
|
if (!isAvailable()) {
|
|
7659
|
-
console.warn("Embeddings not available; semantic search disabled");
|
|
7660
8575
|
return [];
|
|
7661
8576
|
}
|
|
7662
8577
|
if (query.trim().length === 0) {
|
|
@@ -7665,6 +8580,7 @@ async function searchSemantic(db, query, topK = 8, modelName = DEFAULT_MODEL, ma
|
|
|
7665
8580
|
if (!chunkVectorsTableExists(db)) {
|
|
7666
8581
|
return [];
|
|
7667
8582
|
}
|
|
8583
|
+
ensureEmbeddingProvenance(db, modelName);
|
|
7668
8584
|
const queryEmbeddings = await embedTexts([`${QUERY_INSTRUCTION_PREFIX}${query}`], modelName);
|
|
7669
8585
|
if (queryEmbeddings.length === 0) {
|
|
7670
8586
|
return [];
|
|
@@ -7813,6 +8729,44 @@ function deleteFileEmbeddings(db, filePath) {
|
|
|
7813
8729
|
}
|
|
7814
8730
|
db.prepare(`DELETE FROM chunks WHERE ${pathEqClause("file_path")}`).run(folded);
|
|
7815
8731
|
}
|
|
8732
|
+
function resetAllEmbeddings(db) {
|
|
8733
|
+
const paths = db.prepare("SELECT DISTINCT file_path FROM chunks").pluck().all();
|
|
8734
|
+
const clearEmbedSha = db.prepare(`UPDATE files SET embed_sha = NULL WHERE ${pathEqClause("path")}`);
|
|
8735
|
+
const tx = db.transaction(() => {
|
|
8736
|
+
for (const p of paths) deleteFileEmbeddings(db, p);
|
|
8737
|
+
for (const p of paths) clearEmbedSha.run(foldPath(p));
|
|
8738
|
+
});
|
|
8739
|
+
tx.immediate();
|
|
8740
|
+
return paths.length;
|
|
8741
|
+
}
|
|
8742
|
+
function embeddingProvenance(modelName = DEFAULT_MODEL) {
|
|
8743
|
+
const revision = modelName === DEFAULT_MODEL ? PINNED_MODEL_REVISION.slice(0, 12) : "unpinned";
|
|
8744
|
+
return `${modelName}@${revision}/${backendId()}`;
|
|
8745
|
+
}
|
|
8746
|
+
function backendId() {
|
|
8747
|
+
return `onnxruntime-node@${majorMinor(runtimeVersion())}`;
|
|
8748
|
+
}
|
|
8749
|
+
function majorMinor(version) {
|
|
8750
|
+
const parts = version.split(".");
|
|
8751
|
+
return parts.length >= 2 ? `${parts[0]}.${parts[1]}` : version;
|
|
8752
|
+
}
|
|
8753
|
+
var _provenanceChecked = /* @__PURE__ */ new WeakSet();
|
|
8754
|
+
function ensureEmbeddingProvenance(db, modelName = DEFAULT_MODEL) {
|
|
8755
|
+
if (_provenanceChecked.has(db)) return;
|
|
8756
|
+
_provenanceChecked.add(db);
|
|
8757
|
+
const current = embeddingProvenance(modelName);
|
|
8758
|
+
const stored = db.prepare("SELECT provenance FROM embedding_provenance WHERE id = 1").pluck().get();
|
|
8759
|
+
if (stored === current) return;
|
|
8760
|
+
const cleared = resetAllEmbeddings(db);
|
|
8761
|
+
db.prepare(
|
|
8762
|
+
"INSERT INTO embedding_provenance (id, provenance) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET provenance = excluded.provenance"
|
|
8763
|
+
).run(current);
|
|
8764
|
+
if (cleared > 0) {
|
|
8765
|
+
console.warn(
|
|
8766
|
+
`Embedding stack changed (${stored ?? "unrecorded"} -> ${current}); discarded ${cleared} file${cleared === 1 ? "" : "s"} worth of vectors because old and new ones are not comparable. Run \`token-goat index\` to rebuild them.`
|
|
8767
|
+
);
|
|
8768
|
+
}
|
|
8769
|
+
}
|
|
7816
8770
|
function _extractQueryTokens(query) {
|
|
7817
8771
|
const tokens = /* @__PURE__ */ new Set();
|
|
7818
8772
|
const matches = query.matchAll(_TOKEN_RE);
|
|
@@ -7849,17 +8803,19 @@ function _pathPriorityPenalty(filePath) {
|
|
|
7849
8803
|
}
|
|
7850
8804
|
|
|
7851
8805
|
// src/hooks_read.ts
|
|
7852
|
-
import * as
|
|
7853
|
-
import * as
|
|
8806
|
+
import * as fs23 from "node:fs";
|
|
8807
|
+
import * as path19 from "node:path";
|
|
7854
8808
|
|
|
7855
8809
|
// src/compact.ts
|
|
7856
|
-
import * as
|
|
7857
|
-
import * as
|
|
8810
|
+
import * as fs18 from "node:fs";
|
|
8811
|
+
import * as path14 from "node:path";
|
|
7858
8812
|
|
|
7859
8813
|
// src/overflow_guard.ts
|
|
8814
|
+
function estimateTokensFromLength(length) {
|
|
8815
|
+
return Math.max(1, Math.floor(Math.max(0, length) / 3) + 1);
|
|
8816
|
+
}
|
|
7860
8817
|
function estimateTokens(text) {
|
|
7861
|
-
|
|
7862
|
-
return Math.max(1, Math.floor(stripped.length / 3) + 1);
|
|
8818
|
+
return estimateTokensFromLength(stripAnsiCodes(text).length);
|
|
7863
8819
|
}
|
|
7864
8820
|
function trimToBudget(text, budgetTokens, command) {
|
|
7865
8821
|
const markerMarginTokens = 64;
|
|
@@ -8116,7 +9072,7 @@ function inferSessionGoal(cache, maxTokens = 80) {
|
|
|
8116
9072
|
const dirCounts = new Counter();
|
|
8117
9073
|
for (const fpath of editedPaths) {
|
|
8118
9074
|
try {
|
|
8119
|
-
let parent =
|
|
9075
|
+
let parent = path14.dirname(fpath);
|
|
8120
9076
|
if (parent === ".") {
|
|
8121
9077
|
parent = "root";
|
|
8122
9078
|
} else if (parent.startsWith("./")) {
|
|
@@ -8192,11 +9148,11 @@ function isNoisePath(inputPath) {
|
|
|
8192
9148
|
}
|
|
8193
9149
|
function findLatestSessionId() {
|
|
8194
9150
|
try {
|
|
8195
|
-
const sessionsDir =
|
|
8196
|
-
if (!
|
|
9151
|
+
const sessionsDir = path14.join(tokenGoatHome(), "sessions");
|
|
9152
|
+
if (!fs18.existsSync(sessionsDir)) {
|
|
8197
9153
|
return null;
|
|
8198
9154
|
}
|
|
8199
|
-
const files =
|
|
9155
|
+
const files = fs18.readdirSync(sessionsDir);
|
|
8200
9156
|
const jsonFiles = files.filter((f) => f.endsWith(".json") && !f.includes(AGENT_SALT_MARKER));
|
|
8201
9157
|
if (jsonFiles.length === 0) {
|
|
8202
9158
|
return null;
|
|
@@ -8206,9 +9162,9 @@ function findLatestSessionId() {
|
|
|
8206
9162
|
return null;
|
|
8207
9163
|
}
|
|
8208
9164
|
let latestFile = firstFile;
|
|
8209
|
-
let latestMtime =
|
|
9165
|
+
let latestMtime = fs18.statSync(path14.join(sessionsDir, firstFile)).mtimeMs;
|
|
8210
9166
|
for (const file of jsonFiles) {
|
|
8211
|
-
const mtime =
|
|
9167
|
+
const mtime = fs18.statSync(path14.join(sessionsDir, file)).mtimeMs;
|
|
8212
9168
|
if (mtime > latestMtime) {
|
|
8213
9169
|
latestFile = file;
|
|
8214
9170
|
latestMtime = mtime;
|
|
@@ -8230,37 +9186,37 @@ function eventCount(cache) {
|
|
|
8230
9186
|
function writeSessionManifest(projectHash, sessionId, manifestJson) {
|
|
8231
9187
|
const safeSessionId = sanitizeIdForFilename(sessionId, 64);
|
|
8232
9188
|
if (!safeSessionId) return;
|
|
8233
|
-
const sessionsDir =
|
|
8234
|
-
if (!
|
|
9189
|
+
const sessionsDir = path14.join(dataDir(), "projects", projectHash, "sessions");
|
|
9190
|
+
if (!fs18.existsSync(sessionsDir)) {
|
|
8235
9191
|
ensureDirSync(sessionsDir);
|
|
8236
9192
|
}
|
|
8237
|
-
const dest =
|
|
9193
|
+
const dest = path14.join(sessionsDir, `${safeSessionId}.json`);
|
|
8238
9194
|
atomicWriteText(dest, JSON.stringify(manifestJson));
|
|
8239
9195
|
}
|
|
8240
9196
|
function readAllSessionManifests(projectHash, maxAgeSecs = 3600) {
|
|
8241
|
-
const sessionsDir =
|
|
8242
|
-
if (!
|
|
9197
|
+
const sessionsDir = path14.join(dataDir(), "projects", projectHash, "sessions");
|
|
9198
|
+
if (!fs18.existsSync(sessionsDir)) {
|
|
8243
9199
|
return [];
|
|
8244
9200
|
}
|
|
8245
9201
|
const now = Date.now() / 1e3;
|
|
8246
9202
|
const results = [];
|
|
8247
9203
|
try {
|
|
8248
|
-
const files =
|
|
9204
|
+
const files = fs18.readdirSync(sessionsDir);
|
|
8249
9205
|
for (const file of files) {
|
|
8250
9206
|
if (!file.endsWith(".json")) {
|
|
8251
9207
|
continue;
|
|
8252
9208
|
}
|
|
8253
9209
|
try {
|
|
8254
|
-
const fullPath =
|
|
8255
|
-
const stat2 =
|
|
9210
|
+
const fullPath = path14.join(sessionsDir, file);
|
|
9211
|
+
const stat2 = fs18.statSync(fullPath);
|
|
8256
9212
|
if (now - stat2.mtimeMs / 1e3 > maxAgeSecs) {
|
|
8257
9213
|
try {
|
|
8258
|
-
|
|
9214
|
+
fs18.unlinkSync(fullPath);
|
|
8259
9215
|
} catch {
|
|
8260
9216
|
}
|
|
8261
9217
|
continue;
|
|
8262
9218
|
}
|
|
8263
|
-
const text =
|
|
9219
|
+
const text = fs18.readFileSync(fullPath, "utf8");
|
|
8264
9220
|
const data = JSON.parse(text);
|
|
8265
9221
|
if (typeof data === "object" && data !== null && "files" in data) {
|
|
8266
9222
|
results.push(data);
|
|
@@ -8431,8 +9387,8 @@ function buildManifestWithCount(sessionId, opts) {
|
|
|
8431
9387
|
}
|
|
8432
9388
|
|
|
8433
9389
|
// src/snapshots.ts
|
|
8434
|
-
import * as
|
|
8435
|
-
import * as
|
|
9390
|
+
import * as fs19 from "node:fs";
|
|
9391
|
+
import * as path15 from "node:path";
|
|
8436
9392
|
var MAX_SNAPSHOTS_PER_SESSION = 150;
|
|
8437
9393
|
var MAX_SNAPSHOT_BYTES = 256 * 1024;
|
|
8438
9394
|
var SNAPSHOT_TRUNCATE_BYTES = 50 * 1024;
|
|
@@ -8442,10 +9398,10 @@ var VALID_KINDS = /* @__PURE__ */ new Set([KIND_READ, KIND_PREDICTIVE]);
|
|
|
8442
9398
|
function sessionDir(sessionId) {
|
|
8443
9399
|
if (!sessionId) return null;
|
|
8444
9400
|
const safe = sanitizeIdForFilename(sessionId, 64, "anon");
|
|
8445
|
-
const base =
|
|
8446
|
-
const candidate =
|
|
9401
|
+
const base = path15.join(tokenGoatHome(), "session_snapshots");
|
|
9402
|
+
const candidate = path15.join(base, safe);
|
|
8447
9403
|
try {
|
|
8448
|
-
const rel =
|
|
9404
|
+
const rel = path15.relative(base, candidate);
|
|
8449
9405
|
if (rel.startsWith("..")) return null;
|
|
8450
9406
|
} catch {
|
|
8451
9407
|
return null;
|
|
@@ -8458,7 +9414,7 @@ function pathKey(filePath) {
|
|
|
8458
9414
|
function snapshot_path(sessionId, filePath) {
|
|
8459
9415
|
const d = sessionDir(sessionId);
|
|
8460
9416
|
if (!d) return null;
|
|
8461
|
-
return
|
|
9417
|
+
return path15.join(d, `${pathKey(filePath)}.bin`);
|
|
8462
9418
|
}
|
|
8463
9419
|
function kindSidecarPath(snapshotPath) {
|
|
8464
9420
|
return snapshotPath + ".kind";
|
|
@@ -8466,11 +9422,11 @@ function kindSidecarPath(snapshotPath) {
|
|
|
8466
9422
|
function writeSnapshotKind(sidecarPath, kind) {
|
|
8467
9423
|
try {
|
|
8468
9424
|
const safeKind = VALID_KINDS.has(kind) ? kind : KIND_READ;
|
|
8469
|
-
const dir =
|
|
8470
|
-
if (!
|
|
9425
|
+
const dir = path15.dirname(sidecarPath);
|
|
9426
|
+
if (!fs19.existsSync(dir)) {
|
|
8471
9427
|
ensureDirSync(dir);
|
|
8472
9428
|
}
|
|
8473
|
-
|
|
9429
|
+
fs19.writeFileSync(sidecarPath, safeKind, "utf8");
|
|
8474
9430
|
return true;
|
|
8475
9431
|
} catch {
|
|
8476
9432
|
return false;
|
|
@@ -8479,12 +9435,12 @@ function writeSnapshotKind(sidecarPath, kind) {
|
|
|
8479
9435
|
function evictOldest(d, maxCount) {
|
|
8480
9436
|
try {
|
|
8481
9437
|
const entries = [];
|
|
8482
|
-
const files =
|
|
9438
|
+
const files = fs19.readdirSync(d);
|
|
8483
9439
|
for (const file of files) {
|
|
8484
|
-
const fullPath =
|
|
9440
|
+
const fullPath = path15.join(d, file);
|
|
8485
9441
|
if (!file.endsWith(".bin")) continue;
|
|
8486
9442
|
try {
|
|
8487
|
-
const stat2 =
|
|
9443
|
+
const stat2 = fs19.statSync(fullPath);
|
|
8488
9444
|
entries.push([fullPath, stat2.mtimeMs]);
|
|
8489
9445
|
} catch {
|
|
8490
9446
|
continue;
|
|
@@ -8496,10 +9452,10 @@ function evictOldest(d, maxCount) {
|
|
|
8496
9452
|
const over = entries.length - maxCount;
|
|
8497
9453
|
for (const [p] of entries.slice(0, over)) {
|
|
8498
9454
|
try {
|
|
8499
|
-
|
|
9455
|
+
fs19.unlinkSync(p);
|
|
8500
9456
|
removed++;
|
|
8501
9457
|
try {
|
|
8502
|
-
|
|
9458
|
+
fs19.unlinkSync(kindSidecarPath(p));
|
|
8503
9459
|
} catch {
|
|
8504
9460
|
}
|
|
8505
9461
|
} catch {
|
|
@@ -8528,10 +9484,10 @@ function store(sessionId, filePath, content, opts = {}) {
|
|
|
8528
9484
|
if (!p) return null;
|
|
8529
9485
|
const sha = fingerprintContent(stored);
|
|
8530
9486
|
try {
|
|
8531
|
-
const isNewEntry = !
|
|
9487
|
+
const isNewEntry = !fs19.existsSync(p);
|
|
8532
9488
|
if (!isNewEntry) {
|
|
8533
9489
|
try {
|
|
8534
|
-
const existing =
|
|
9490
|
+
const existing = fs19.readFileSync(p);
|
|
8535
9491
|
if (Buffer.from(existing).equals(stored)) {
|
|
8536
9492
|
return {
|
|
8537
9493
|
path: p,
|
|
@@ -8542,8 +9498,8 @@ function store(sessionId, filePath, content, opts = {}) {
|
|
|
8542
9498
|
} catch {
|
|
8543
9499
|
}
|
|
8544
9500
|
}
|
|
8545
|
-
const dir =
|
|
8546
|
-
if (!
|
|
9501
|
+
const dir = path15.dirname(p);
|
|
9502
|
+
if (!fs19.existsSync(dir)) {
|
|
8547
9503
|
ensureDirSync(dir);
|
|
8548
9504
|
}
|
|
8549
9505
|
if (isNewEntry) {
|
|
@@ -8563,9 +9519,9 @@ function store(sessionId, filePath, content, opts = {}) {
|
|
|
8563
9519
|
}
|
|
8564
9520
|
function load(sessionId, filePath, opts = {}) {
|
|
8565
9521
|
const p = snapshot_path(sessionId, filePath);
|
|
8566
|
-
if (!p || !
|
|
9522
|
+
if (!p || !fs19.existsSync(p)) return null;
|
|
8567
9523
|
try {
|
|
8568
|
-
const stat2 =
|
|
9524
|
+
const stat2 = fs19.statSync(p);
|
|
8569
9525
|
if (stat2.size > MAX_SNAPSHOT_BYTES) {
|
|
8570
9526
|
return null;
|
|
8571
9527
|
}
|
|
@@ -8573,7 +9529,7 @@ function load(sessionId, filePath, opts = {}) {
|
|
|
8573
9529
|
return null;
|
|
8574
9530
|
}
|
|
8575
9531
|
try {
|
|
8576
|
-
const data =
|
|
9532
|
+
const data = fs19.readFileSync(p);
|
|
8577
9533
|
if (opts.expected_sha) {
|
|
8578
9534
|
const actualSha = fingerprintContent(data);
|
|
8579
9535
|
if (actualSha.toLowerCase() !== opts.expected_sha.toLowerCase()) {
|
|
@@ -8586,31 +9542,31 @@ function load(sessionId, filePath, opts = {}) {
|
|
|
8586
9542
|
}
|
|
8587
9543
|
}
|
|
8588
9544
|
function removeEligibleSnapshotFile(fullPath, file, cutoff) {
|
|
8589
|
-
const stat2 =
|
|
9545
|
+
const stat2 = fs19.lstatSync(fullPath);
|
|
8590
9546
|
if ((stat2.mode & 61440) === 40960) return false;
|
|
8591
9547
|
if (cutoff !== void 0 && stat2.mtimeMs >= cutoff) return false;
|
|
8592
|
-
|
|
9548
|
+
fs19.unlinkSync(fullPath);
|
|
8593
9549
|
return file.endsWith(".bin");
|
|
8594
9550
|
}
|
|
8595
9551
|
function cleanup_stale(maxAgeHours = 24) {
|
|
8596
|
-
const base =
|
|
8597
|
-
if (!
|
|
9552
|
+
const base = path15.join(tokenGoatHome(), "session_snapshots");
|
|
9553
|
+
if (!fs19.existsSync(base)) return 0;
|
|
8598
9554
|
const cutoff = Date.now() - maxAgeHours * 3600 * 1e3;
|
|
8599
9555
|
let removed = 0;
|
|
8600
9556
|
try {
|
|
8601
|
-
const sessionDirs =
|
|
9557
|
+
const sessionDirs = fs19.readdirSync(base);
|
|
8602
9558
|
for (const sessionDir2 of sessionDirs) {
|
|
8603
|
-
const sessionPath2 =
|
|
9559
|
+
const sessionPath2 = path15.join(base, sessionDir2);
|
|
8604
9560
|
try {
|
|
8605
|
-
const stat2 =
|
|
9561
|
+
const stat2 = fs19.statSync(sessionPath2);
|
|
8606
9562
|
if (!stat2.isDirectory()) continue;
|
|
8607
9563
|
} catch {
|
|
8608
9564
|
continue;
|
|
8609
9565
|
}
|
|
8610
9566
|
try {
|
|
8611
|
-
const files =
|
|
9567
|
+
const files = fs19.readdirSync(sessionPath2);
|
|
8612
9568
|
for (const file of files) {
|
|
8613
|
-
const fullPath =
|
|
9569
|
+
const fullPath = path15.join(sessionPath2, file);
|
|
8614
9570
|
try {
|
|
8615
9571
|
if (removeEligibleSnapshotFile(fullPath, file, cutoff)) removed++;
|
|
8616
9572
|
} catch {
|
|
@@ -8618,7 +9574,7 @@ function cleanup_stale(maxAgeHours = 24) {
|
|
|
8618
9574
|
}
|
|
8619
9575
|
}
|
|
8620
9576
|
try {
|
|
8621
|
-
|
|
9577
|
+
fs19.rmdirSync(sessionPath2);
|
|
8622
9578
|
} catch {
|
|
8623
9579
|
}
|
|
8624
9580
|
} catch {
|
|
@@ -8654,11 +9610,11 @@ function buildPackageManifestHint(options) {
|
|
|
8654
9610
|
return null;
|
|
8655
9611
|
}
|
|
8656
9612
|
}
|
|
8657
|
-
function _sanitizeHintPath(
|
|
8658
|
-
if (typeof
|
|
9613
|
+
function _sanitizeHintPath(path31) {
|
|
9614
|
+
if (typeof path31 !== "string") {
|
|
8659
9615
|
return "???";
|
|
8660
9616
|
}
|
|
8661
|
-
return
|
|
9617
|
+
return path31.replace(/[\x00]/g, "").slice(0, 200);
|
|
8662
9618
|
}
|
|
8663
9619
|
|
|
8664
9620
|
// src/hints/lang_patterns.ts
|
|
@@ -9286,33 +10242,33 @@ function dispatchFileTypeHandler(filePath, content, contentLengthHint) {
|
|
|
9286
10242
|
}
|
|
9287
10243
|
|
|
9288
10244
|
// src/doc_compact.ts
|
|
9289
|
-
import * as
|
|
9290
|
-
import * as
|
|
10245
|
+
import * as fs20 from "fs";
|
|
10246
|
+
import * as path16 from "path";
|
|
9291
10247
|
var defaultSentencesPerSection = 2;
|
|
9292
10248
|
var headerPrefix = "<!-- token-goat doc-compact source-hash:";
|
|
9293
10249
|
var headerRegex = /^<!-- token-goat doc-compact source-hash:(\S+) source:(.+?) -->\r?$/;
|
|
9294
10250
|
var compactSubdir = "doc_compacts";
|
|
9295
10251
|
function sourceHash(filePath) {
|
|
9296
10252
|
try {
|
|
9297
|
-
return fingerprintContent(
|
|
10253
|
+
return fingerprintContent(fs20.readFileSync(filePath));
|
|
9298
10254
|
} catch {
|
|
9299
10255
|
return "";
|
|
9300
10256
|
}
|
|
9301
10257
|
}
|
|
9302
10258
|
function _compactSlug(absPathStr) {
|
|
9303
10259
|
const h = fingerprintContent(foldPath(absPathStr)).slice(0, 12);
|
|
9304
|
-
const ext =
|
|
9305
|
-
const stem =
|
|
10260
|
+
const ext = path16.extname(absPathStr);
|
|
10261
|
+
const stem = path16.basename(absPathStr, ext);
|
|
9306
10262
|
const safeStem = sanitizeIdForFilename(stem, 32);
|
|
9307
10263
|
return `${h}_${safeStem}`;
|
|
9308
10264
|
}
|
|
9309
10265
|
function compactPathFor(sourcePath) {
|
|
9310
10266
|
const abs = resolveIndexPath(sourcePath);
|
|
9311
|
-
return
|
|
10267
|
+
return path16.join(dataDir(), compactSubdir, `${_compactSlug(abs)}.md`);
|
|
9312
10268
|
}
|
|
9313
10269
|
function readCompactHeader(compactPath) {
|
|
9314
10270
|
try {
|
|
9315
|
-
const text =
|
|
10271
|
+
const text = fs20.readFileSync(compactPath, "utf-8");
|
|
9316
10272
|
const firstLine = text.split("\n")[0] || "";
|
|
9317
10273
|
const m = firstLine.match(headerRegex);
|
|
9318
10274
|
if (!m || !m[1] || !m[2]) return null;
|
|
@@ -9331,8 +10287,8 @@ function isCompactFresh(compactPath, sourcePath) {
|
|
|
9331
10287
|
}
|
|
9332
10288
|
function markCompactStale(compactPath) {
|
|
9333
10289
|
try {
|
|
9334
|
-
if (!
|
|
9335
|
-
const text =
|
|
10290
|
+
if (!fs20.existsSync(compactPath)) return false;
|
|
10291
|
+
const text = fs20.readFileSync(compactPath, "utf-8");
|
|
9336
10292
|
const lines2 = text.split("\n");
|
|
9337
10293
|
if (!lines2[0]) return false;
|
|
9338
10294
|
const m = lines2[0].match(headerRegex);
|
|
@@ -9347,7 +10303,7 @@ function markCompactStale(compactPath) {
|
|
|
9347
10303
|
}
|
|
9348
10304
|
function readCompactBody(compactPath) {
|
|
9349
10305
|
try {
|
|
9350
|
-
const text =
|
|
10306
|
+
const text = fs20.readFileSync(compactPath, "utf-8");
|
|
9351
10307
|
const lines2 = text.split("\n");
|
|
9352
10308
|
if (lines2.length < 2) return null;
|
|
9353
10309
|
const body = lines2.slice(1).join("\n").trimStart();
|
|
@@ -9357,14 +10313,14 @@ function readCompactBody(compactPath) {
|
|
|
9357
10313
|
}
|
|
9358
10314
|
}
|
|
9359
10315
|
function writeCompact(compactPath, sourcePath, compactBody, sourceRel) {
|
|
9360
|
-
const srcPath =
|
|
10316
|
+
const srcPath = path16.resolve(sourcePath);
|
|
9361
10317
|
const sha = sourceHash(srcPath);
|
|
9362
|
-
const displayRel = sourceRel ||
|
|
10318
|
+
const displayRel = sourceRel || path16.basename(srcPath);
|
|
9363
10319
|
const header = `${headerPrefix}${sha} source:${displayRel} -->
|
|
9364
10320
|
`;
|
|
9365
10321
|
const fullText = header + compactBody.trimStart();
|
|
9366
|
-
const dir =
|
|
9367
|
-
if (!
|
|
10322
|
+
const dir = path16.dirname(compactPath);
|
|
10323
|
+
if (!fs20.existsSync(dir)) {
|
|
9368
10324
|
ensureDirSync(dir);
|
|
9369
10325
|
}
|
|
9370
10326
|
atomicWriteText(compactPath, fullText);
|
|
@@ -9488,7 +10444,7 @@ function extractDocCompact(body, heading) {
|
|
|
9488
10444
|
}
|
|
9489
10445
|
function compactDoc(filePath, heading) {
|
|
9490
10446
|
try {
|
|
9491
|
-
const body =
|
|
10447
|
+
const body = fs20.readFileSync(filePath, "utf-8");
|
|
9492
10448
|
const compact = extractDocCompact(body, heading);
|
|
9493
10449
|
return compact || null;
|
|
9494
10450
|
} catch {
|
|
@@ -9498,21 +10454,21 @@ function compactDoc(filePath, heading) {
|
|
|
9498
10454
|
|
|
9499
10455
|
// src/evidence_cache.ts
|
|
9500
10456
|
import crypto2 from "node:crypto";
|
|
9501
|
-
import
|
|
9502
|
-
import
|
|
10457
|
+
import fs21 from "node:fs";
|
|
10458
|
+
import path17 from "node:path";
|
|
9503
10459
|
var MAX_ENTRIES = 500;
|
|
9504
10460
|
var MAX_TEXT_BYTES = 128 * 1024;
|
|
9505
10461
|
var MAX_SEMANTIC_CANDIDATES = 100;
|
|
9506
10462
|
var CACHE_FILE = "workspace-evidence.json";
|
|
9507
10463
|
function cachePath() {
|
|
9508
|
-
return
|
|
10464
|
+
return path17.join(dataDir(), CACHE_FILE);
|
|
9509
10465
|
}
|
|
9510
10466
|
function hash(text) {
|
|
9511
10467
|
return crypto2.createHash("sha256").update(text).digest("hex");
|
|
9512
10468
|
}
|
|
9513
10469
|
function load2() {
|
|
9514
10470
|
try {
|
|
9515
|
-
const parsed = JSON.parse(
|
|
10471
|
+
const parsed = JSON.parse(fs21.readFileSync(cachePath(), "utf8"));
|
|
9516
10472
|
if (!Array.isArray(parsed)) return [];
|
|
9517
10473
|
return parsed.filter(
|
|
9518
10474
|
(entry) => typeof entry === "object" && entry !== null && typeof entry.id === "string" && typeof entry.projectRoot === "string" && typeof entry.source === "string" && (entry.representation === "file" || entry.representation === "tool-output") && typeof entry.contentHash === "string" && typeof entry.text === "string" && typeof entry.createdAt === "number" && (entry.embedding === void 0 || typeof entry.embedding === "string")
|
|
@@ -9524,7 +10480,7 @@ function load2() {
|
|
|
9524
10480
|
function save(entries) {
|
|
9525
10481
|
try {
|
|
9526
10482
|
ensureDirSync(dataDir());
|
|
9527
|
-
|
|
10483
|
+
fs21.writeFileSync(cachePath(), JSON.stringify(entries.slice(0, MAX_ENTRIES)), "utf8");
|
|
9528
10484
|
} catch {
|
|
9529
10485
|
}
|
|
9530
10486
|
}
|
|
@@ -9615,7 +10571,7 @@ function buildDeltaCapsule(projectRoot, limit = 8) {
|
|
|
9615
10571
|
const root = normalizePath(projectRoot);
|
|
9616
10572
|
const changed = load2().filter((entry) => entry.projectRoot === root && entry.representation === "file").filter((entry) => {
|
|
9617
10573
|
try {
|
|
9618
|
-
return hash(
|
|
10574
|
+
return hash(fs21.readFileSync(entry.source, "utf8")) !== entry.contentHash;
|
|
9619
10575
|
} catch {
|
|
9620
10576
|
return true;
|
|
9621
10577
|
}
|
|
@@ -9626,8 +10582,8 @@ ${changed.map((entry) => `- ${entry.source} (use a fresh surgical read)`).join("
|
|
|
9626
10582
|
}
|
|
9627
10583
|
|
|
9628
10584
|
// src/notebook_compact.ts
|
|
9629
|
-
import * as
|
|
9630
|
-
import * as
|
|
10585
|
+
import * as fs22 from "node:fs";
|
|
10586
|
+
import * as path18 from "node:path";
|
|
9631
10587
|
var NB_STRIP_MIN_SAVINGS = 4096;
|
|
9632
10588
|
function stripNotebook(nbDict) {
|
|
9633
10589
|
const cells = [];
|
|
@@ -9647,23 +10603,23 @@ function stripNotebook(nbDict) {
|
|
|
9647
10603
|
var SIDECAR_DEFAULT_MAX_COUNT = 200;
|
|
9648
10604
|
var SIDECAR_DEFAULT_MAX_AGE_MS = 24 * 3600 * 1e3;
|
|
9649
10605
|
function pruneSidecars(cacheRoot, maxCount = SIDECAR_DEFAULT_MAX_COUNT, maxAgeMs = SIDECAR_DEFAULT_MAX_AGE_MS) {
|
|
9650
|
-
const nbStripDir =
|
|
10606
|
+
const nbStripDir = path18.join(cacheRoot, "nb_strip");
|
|
9651
10607
|
let removed = 0;
|
|
9652
10608
|
try {
|
|
9653
|
-
if (!
|
|
10609
|
+
if (!fs22.existsSync(nbStripDir)) return 0;
|
|
9654
10610
|
const cutoff = Date.now() - maxAgeMs;
|
|
9655
10611
|
const kept = [];
|
|
9656
|
-
for (const entry of
|
|
9657
|
-
const dir =
|
|
10612
|
+
for (const entry of fs22.readdirSync(nbStripDir)) {
|
|
10613
|
+
const dir = path18.join(nbStripDir, entry);
|
|
9658
10614
|
let mtime;
|
|
9659
10615
|
try {
|
|
9660
|
-
mtime =
|
|
10616
|
+
mtime = fs22.statSync(dir).mtimeMs;
|
|
9661
10617
|
} catch {
|
|
9662
10618
|
continue;
|
|
9663
10619
|
}
|
|
9664
10620
|
if (mtime < cutoff) {
|
|
9665
10621
|
try {
|
|
9666
|
-
|
|
10622
|
+
fs22.rmSync(dir, { recursive: true, force: true });
|
|
9667
10623
|
removed++;
|
|
9668
10624
|
} catch {
|
|
9669
10625
|
continue;
|
|
@@ -9676,7 +10632,7 @@ function pruneSidecars(cacheRoot, maxCount = SIDECAR_DEFAULT_MAX_COUNT, maxAgeMs
|
|
|
9676
10632
|
kept.sort((a, b) => a[1] - b[1]);
|
|
9677
10633
|
for (const [dir] of kept.slice(0, kept.length - maxCount)) {
|
|
9678
10634
|
try {
|
|
9679
|
-
|
|
10635
|
+
fs22.rmSync(dir, { recursive: true, force: true });
|
|
9680
10636
|
removed++;
|
|
9681
10637
|
} catch {
|
|
9682
10638
|
continue;
|
|
@@ -9690,9 +10646,9 @@ function pruneSidecars(cacheRoot, maxCount = SIDECAR_DEFAULT_MAX_COUNT, maxAgeMs
|
|
|
9690
10646
|
}
|
|
9691
10647
|
function getOrCreateSidecar(rawBytes, cacheRoot, opts = {}) {
|
|
9692
10648
|
const sha = fingerprintContent(rawBytes);
|
|
9693
|
-
const sidecarDir =
|
|
9694
|
-
const sidecarPath =
|
|
9695
|
-
if (
|
|
10649
|
+
const sidecarDir = path18.join(cacheRoot, "nb_strip", sha);
|
|
10650
|
+
const sidecarPath = path18.join(sidecarDir, "stripped.ipynb");
|
|
10651
|
+
if (fs22.existsSync(sidecarPath)) {
|
|
9696
10652
|
return [sidecarPath, false];
|
|
9697
10653
|
}
|
|
9698
10654
|
let nb;
|
|
@@ -9709,7 +10665,7 @@ function getOrCreateSidecar(rawBytes, cacheRoot, opts = {}) {
|
|
|
9709
10665
|
try {
|
|
9710
10666
|
ensureDirSync(sidecarDir);
|
|
9711
10667
|
} catch (err) {
|
|
9712
|
-
if (!
|
|
10668
|
+
if (!fs22.existsSync(sidecarDir)) {
|
|
9713
10669
|
throw err;
|
|
9714
10670
|
}
|
|
9715
10671
|
}
|
|
@@ -9742,8 +10698,8 @@ function isNodeModulesPath(p) {
|
|
|
9742
10698
|
return check.includes("/node_modules/") || check.includes("\\node_modules\\");
|
|
9743
10699
|
}
|
|
9744
10700
|
function relPathWithinRoot(root, target) {
|
|
9745
|
-
const rel =
|
|
9746
|
-
if (rel.startsWith("..") ||
|
|
10701
|
+
const rel = path19.relative(root, target).replace(/\\/g, "/");
|
|
10702
|
+
if (rel.startsWith("..") || path19.isAbsolute(rel)) return null;
|
|
9747
10703
|
return rel;
|
|
9748
10704
|
}
|
|
9749
10705
|
function _isDocFile(filePath) {
|
|
@@ -9774,7 +10730,7 @@ function scanRequestedSlice(absPath, offset, limit) {
|
|
|
9774
10730
|
const windowEnd = offset + limit;
|
|
9775
10731
|
let fd;
|
|
9776
10732
|
try {
|
|
9777
|
-
fd =
|
|
10733
|
+
fd = fs23.openSync(absPath, "r");
|
|
9778
10734
|
} catch {
|
|
9779
10735
|
return null;
|
|
9780
10736
|
}
|
|
@@ -9788,7 +10744,7 @@ function scanRequestedSlice(absPath, offset, limit) {
|
|
|
9788
10744
|
const nearSingleLine = lineNumber < NEAR_SINGLE_LINE_SCAN_THRESHOLD;
|
|
9789
10745
|
return { bytes: sliceBytes, trustworthy: nearSingleLine, nearSingleLine };
|
|
9790
10746
|
}
|
|
9791
|
-
const bytesRead =
|
|
10747
|
+
const bytesRead = fs23.readSync(fd, buf, 0, buf.length, null);
|
|
9792
10748
|
if (bytesRead === 0) {
|
|
9793
10749
|
return {
|
|
9794
10750
|
bytes: sliceBytes,
|
|
@@ -9807,7 +10763,7 @@ function scanRequestedSlice(absPath, offset, limit) {
|
|
|
9807
10763
|
}
|
|
9808
10764
|
} finally {
|
|
9809
10765
|
try {
|
|
9810
|
-
|
|
10766
|
+
fs23.closeSync(fd);
|
|
9811
10767
|
} catch {
|
|
9812
10768
|
}
|
|
9813
10769
|
}
|
|
@@ -9844,7 +10800,7 @@ var BINARY_FILE_TYPE_EXTS = /* @__PURE__ */ new Set(["pdf", "docx", "xlsx", "ppt
|
|
|
9844
10800
|
var TEXT_FILE_TYPE_EXTS = /* @__PURE__ */ new Set(["html", "htm", "xhtml", "txt", "log", "out", "err", "trace", "csv", "tsv", "vtt", "srt"]);
|
|
9845
10801
|
var DISPATCHED_FILE_TYPE_EXTS = /* @__PURE__ */ new Set([...BINARY_FILE_TYPE_EXTS, ...TEXT_FILE_TYPE_EXTS]);
|
|
9846
10802
|
function isDispatchedFileType(basename12) {
|
|
9847
|
-
return DISPATCHED_FILE_TYPE_EXTS.has(
|
|
10803
|
+
return DISPATCHED_FILE_TYPE_EXTS.has(path19.extname(basename12).slice(1).toLowerCase());
|
|
9848
10804
|
}
|
|
9849
10805
|
function surgicalHint(filePath, basename12, lineCount) {
|
|
9850
10806
|
if (lineCount < loadConfig().hints.min_file_lines_for_hint) return "";
|
|
@@ -9907,7 +10863,7 @@ function loadSnapshotDiff(sessionId, normalized, basename12) {
|
|
|
9907
10863
|
try {
|
|
9908
10864
|
const sz = statSize(normalized);
|
|
9909
10865
|
if (sz === null || sz > 256 * 1024) return { kind: "none" };
|
|
9910
|
-
const currentContent =
|
|
10866
|
+
const currentContent = fs23.readFileSync(normalized, "utf8");
|
|
9911
10867
|
const TRUNC_MARKER = "\n<snapshot truncated at ";
|
|
9912
10868
|
const oldRaw = oldSnap.toString("utf8");
|
|
9913
10869
|
const truncIdx = oldRaw.indexOf(TRUNC_MARKER);
|
|
@@ -9993,7 +10949,7 @@ function preReadHandlerInner(event) {
|
|
|
9993
10949
|
try {
|
|
9994
10950
|
const cwd = getCwd(event) ?? process.cwd();
|
|
9995
10951
|
const project = findProject(cwd) ?? makeProjectAt(cwd);
|
|
9996
|
-
const current =
|
|
10952
|
+
const current = fs23.readFileSync(normalized, "utf8");
|
|
9997
10953
|
const evidence = findVerifiedFileEvidence(project.root, normalized, current);
|
|
9998
10954
|
if (evidence !== null) {
|
|
9999
10955
|
recordStat("evidence_cache_hit", 0);
|
|
@@ -10004,7 +10960,7 @@ function preReadHandlerInner(event) {
|
|
|
10004
10960
|
} catch {
|
|
10005
10961
|
}
|
|
10006
10962
|
}
|
|
10007
|
-
const basename12 =
|
|
10963
|
+
const basename12 = path19.basename(normalized);
|
|
10008
10964
|
if (isLockFile(basename12)) {
|
|
10009
10965
|
return denyOutput(
|
|
10010
10966
|
'Lock files are rarely useful to read in full. Use `token-goat section "' + shown + '::<section>"` to extract a specific dependency, or read the relevant manifest instead.'
|
|
@@ -10043,7 +10999,7 @@ function preReadHandlerInner(event) {
|
|
|
10043
10999
|
const skillName = detectSkillFile(normalized);
|
|
10044
11000
|
if (skillName && basename12 === "SKILL.md") {
|
|
10045
11001
|
try {
|
|
10046
|
-
const body =
|
|
11002
|
+
const body = fs23.readFileSync(normalized, "utf-8");
|
|
10047
11003
|
const bodySha = contentHash(body);
|
|
10048
11004
|
const compact = getCompactAnySessionSync(skillName);
|
|
10049
11005
|
const stale = isCompactStale(compact, skillName, bodySha);
|
|
@@ -10074,9 +11030,9 @@ function preReadHandlerInner(event) {
|
|
|
10074
11030
|
const isNotebook = /\.ipynb$/i.test(basename12);
|
|
10075
11031
|
if (event.toolName !== "Grep" && isNotebook) {
|
|
10076
11032
|
try {
|
|
10077
|
-
const rawBytes =
|
|
11033
|
+
const rawBytes = fs23.readFileSync(normalized);
|
|
10078
11034
|
const [sidecarPath] = getOrCreateSidecar(rawBytes, dataDir());
|
|
10079
|
-
const sidecarContent =
|
|
11035
|
+
const sidecarContent = fs23.readFileSync(sidecarPath, "utf-8");
|
|
10080
11036
|
const savedBytes = rawBytes.length - sidecarContent.length;
|
|
10081
11037
|
if (savedBytes >= NB_STRIP_MIN_SAVINGS) {
|
|
10082
11038
|
recordActualRead(event, normalized);
|
|
@@ -10096,7 +11052,7 @@ function preReadHandlerInner(event) {
|
|
|
10096
11052
|
const sz = statSize(normalized);
|
|
10097
11053
|
if (sz !== null && sz >= MARKDOWN_SIZE_THRESHOLD) {
|
|
10098
11054
|
markdownSize = sz;
|
|
10099
|
-
fileContent =
|
|
11055
|
+
fileContent = fs23.readFileSync(normalized, "utf8");
|
|
10100
11056
|
}
|
|
10101
11057
|
} catch {
|
|
10102
11058
|
}
|
|
@@ -10325,7 +11281,7 @@ function preReadHandlerInner(event) {
|
|
|
10325
11281
|
"Note: " + shown + " is large (" + kb + "KB). " + hint + contextPressureAdvisorySuffix()
|
|
10326
11282
|
);
|
|
10327
11283
|
}
|
|
10328
|
-
const fileTypeExt =
|
|
11284
|
+
const fileTypeExt = path19.extname(normalized).slice(1).toLowerCase();
|
|
10329
11285
|
const fileStatSize = size ?? statSize(normalized) ?? 0;
|
|
10330
11286
|
const isKnownFileType = DISPATCHED_FILE_TYPE_EXTS.has(fileTypeExt);
|
|
10331
11287
|
if (event.toolName !== "Grep" && !isImagePath(normalized) && (isKnownFileType || fileStatSize >= FILE_TYPE_THRESHOLDS.generic)) {
|
|
@@ -10334,7 +11290,7 @@ function preReadHandlerInner(event) {
|
|
|
10334
11290
|
let ftContent = "";
|
|
10335
11291
|
if (!BINARY_FILE_TYPE_EXTS.has(fileTypeExt) && fileStatSize <= SLICE_ESTIMATE_SCAN_CAP_BYTES) {
|
|
10336
11292
|
try {
|
|
10337
|
-
ftContent =
|
|
11293
|
+
ftContent = fs23.readFileSync(normalized, "utf8");
|
|
10338
11294
|
} catch {
|
|
10339
11295
|
}
|
|
10340
11296
|
}
|
|
@@ -10364,7 +11320,7 @@ function estimateTruncatedLineCount(normalized) {
|
|
|
10364
11320
|
try {
|
|
10365
11321
|
const sz = statSize(normalized);
|
|
10366
11322
|
if (sz !== null && sz <= SLICE_ESTIMATE_SCAN_CAP_BYTES) {
|
|
10367
|
-
return countTextLines(
|
|
11323
|
+
return countTextLines(fs23.readFileSync(normalized, "utf8"));
|
|
10368
11324
|
}
|
|
10369
11325
|
} catch {
|
|
10370
11326
|
}
|
|
@@ -10387,13 +11343,13 @@ function postReadHandlerInner(event) {
|
|
|
10387
11343
|
if (respText.includes("[Truncated:") || respText.includes("Truncated: PARTIAL view")) {
|
|
10388
11344
|
markFileTruncated(normalized);
|
|
10389
11345
|
}
|
|
10390
|
-
const postBasename =
|
|
11346
|
+
const postBasename = path19.basename(normalized);
|
|
10391
11347
|
const diffSourcesEnabled = loadConfig().hints.serve_diff_on_reread;
|
|
10392
11348
|
if (/\.(md|mdx|markdown|rst|txt)$/i.test(postBasename) || isSessionArtifactFile(normalized) || diffSourcesEnabled && DIFFABLE_SOURCE_RE.test(postBasename)) {
|
|
10393
11349
|
try {
|
|
10394
11350
|
const sz = statSize(normalized);
|
|
10395
11351
|
if (sz !== null && sz <= 256 * 1024) {
|
|
10396
|
-
const content =
|
|
11352
|
+
const content = fs23.readFileSync(normalized);
|
|
10397
11353
|
store(getSessionId(), normalized, content);
|
|
10398
11354
|
}
|
|
10399
11355
|
} catch {
|
|
@@ -10403,7 +11359,7 @@ function postReadHandlerInner(event) {
|
|
|
10403
11359
|
try {
|
|
10404
11360
|
const cwd = getCwd(event) ?? process.cwd();
|
|
10405
11361
|
const project = findProject(cwd) ?? makeProjectAt(cwd);
|
|
10406
|
-
const source = decodeSource(
|
|
11362
|
+
const source = decodeSource(fs23.readFileSync(normalized));
|
|
10407
11363
|
recordEvidence({ projectRoot: project.root, source: normalized, representation: "file", text: source });
|
|
10408
11364
|
} catch {
|
|
10409
11365
|
}
|
|
@@ -10434,7 +11390,7 @@ function postReadHandlerInner(event) {
|
|
|
10434
11390
|
try {
|
|
10435
11391
|
const sz = statSize(normalized);
|
|
10436
11392
|
if (sz !== null && sz <= SLICE_ESTIMATE_SCAN_CAP_BYTES) {
|
|
10437
|
-
const lineCount = countTextLines(
|
|
11393
|
+
const lineCount = countTextLines(fs23.readFileSync(normalized, "utf8"));
|
|
10438
11394
|
const minLines = loadConfig().post_read_code_compress.min_lines;
|
|
10439
11395
|
if (lineCount >= minLines && meetsSavingsFloor(sz)) {
|
|
10440
11396
|
recordStat("session_hint", 0, 0);
|
|
@@ -10454,9 +11410,9 @@ function postReadHandler(event) {
|
|
|
10454
11410
|
registerHook("post_tool_use", postReadHandler, { toolName: "Read" });
|
|
10455
11411
|
|
|
10456
11412
|
// src/cli_context_stats.ts
|
|
10457
|
-
import * as
|
|
11413
|
+
import * as fs25 from "node:fs";
|
|
10458
11414
|
import * as os8 from "node:os";
|
|
10459
|
-
import * as
|
|
11415
|
+
import * as path21 from "node:path";
|
|
10460
11416
|
|
|
10461
11417
|
// src/confirm_apply.ts
|
|
10462
11418
|
import * as readline from "node:readline";
|
|
@@ -10510,8 +11466,8 @@ ${diff}
|
|
|
10510
11466
|
}
|
|
10511
11467
|
|
|
10512
11468
|
// src/memory_prune.ts
|
|
10513
|
-
import * as
|
|
10514
|
-
import * as
|
|
11469
|
+
import * as fs24 from "node:fs";
|
|
11470
|
+
import * as path20 from "node:path";
|
|
10515
11471
|
var ENTRY_RE = /^\s*-\s*\[(?<title>[^\]]+)\]\((?<target>[^)]+?\.md)\)/;
|
|
10516
11472
|
var URL_SCHEME_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
10517
11473
|
function parseIndex(text) {
|
|
@@ -10556,10 +11512,10 @@ function pruneIndex(memoryDir, opts) {
|
|
|
10556
11512
|
changed: false,
|
|
10557
11513
|
tokensSaved: 0
|
|
10558
11514
|
};
|
|
10559
|
-
const memoryMd =
|
|
11515
|
+
const memoryMd = path20.join(memoryDir, "MEMORY.md");
|
|
10560
11516
|
let text;
|
|
10561
11517
|
try {
|
|
10562
|
-
text =
|
|
11518
|
+
text = fs24.readFileSync(memoryMd, "utf-8");
|
|
10563
11519
|
} catch {
|
|
10564
11520
|
return result;
|
|
10565
11521
|
}
|
|
@@ -10570,7 +11526,7 @@ function pruneIndex(memoryDir, opts) {
|
|
|
10570
11526
|
const dups = [];
|
|
10571
11527
|
for (const entry of entries) {
|
|
10572
11528
|
const isUrl = URL_SCHEME_RE.test(entry.target);
|
|
10573
|
-
const targetExists = isUrl ? true :
|
|
11529
|
+
const targetExists = isUrl ? true : path20.isAbsolute(entry.target) ? fs24.existsSync(entry.target) : fs24.existsSync(path20.join(memoryDir, entry.target));
|
|
10574
11530
|
const foldedTarget = foldPath(entry.target);
|
|
10575
11531
|
if (!targetExists) {
|
|
10576
11532
|
dead.push(entry);
|
|
@@ -10624,7 +11580,7 @@ function jaccard(a, b) {
|
|
|
10624
11580
|
function siblingSnippet(filePath) {
|
|
10625
11581
|
let text;
|
|
10626
11582
|
try {
|
|
10627
|
-
text =
|
|
11583
|
+
text = fs24.readFileSync(filePath, { encoding: "utf-8" });
|
|
10628
11584
|
} catch {
|
|
10629
11585
|
return "";
|
|
10630
11586
|
}
|
|
@@ -10714,7 +11670,7 @@ async function tryEmbeddingClusters(siblings, snippets, threshold) {
|
|
|
10714
11670
|
}
|
|
10715
11671
|
async function findContentDuplicates(memoryDir, _opts) {
|
|
10716
11672
|
const threshold = _opts?.threshold ?? 0.92;
|
|
10717
|
-
const siblings =
|
|
11673
|
+
const siblings = fs24.readdirSync(memoryDir).filter((name) => name.toLowerCase().endsWith(".md") && name.toLowerCase() !== "memory.md").map((name) => path20.join(memoryDir, name)).sort();
|
|
10718
11674
|
if (siblings.length < 2) {
|
|
10719
11675
|
return [];
|
|
10720
11676
|
}
|
|
@@ -10738,7 +11694,7 @@ function auditClaudeMd(files) {
|
|
|
10738
11694
|
for (const filePath of files) {
|
|
10739
11695
|
let text;
|
|
10740
11696
|
try {
|
|
10741
|
-
text =
|
|
11697
|
+
text = fs24.readFileSync(filePath, { encoding: "utf-8" });
|
|
10742
11698
|
} catch {
|
|
10743
11699
|
continue;
|
|
10744
11700
|
}
|
|
@@ -10798,7 +11754,7 @@ function auditClaudeMd(files) {
|
|
|
10798
11754
|
const overlaps = [];
|
|
10799
11755
|
for (const [stripped, filesSet] of lineToFiles) {
|
|
10800
11756
|
if (filesSet.has(report.path) && filesSet.size > 1) {
|
|
10801
|
-
const others = Array.from(filesSet).filter((p) => p !== report.path).map((p) =>
|
|
11757
|
+
const others = Array.from(filesSet).filter((p) => p !== report.path).map((p) => path20.basename(p));
|
|
10802
11758
|
if (others.length > 0) {
|
|
10803
11759
|
if (stripped.length > 60) {
|
|
10804
11760
|
overlaps.push(
|
|
@@ -10818,7 +11774,7 @@ function auditClaudeMd(files) {
|
|
|
10818
11774
|
// src/cli_context_stats.ts
|
|
10819
11775
|
function tok(filePath) {
|
|
10820
11776
|
try {
|
|
10821
|
-
const size =
|
|
11777
|
+
const size = fs25.statSync(filePath).size;
|
|
10822
11778
|
return Math.floor(size / 4);
|
|
10823
11779
|
} catch {
|
|
10824
11780
|
return 0;
|
|
@@ -10827,42 +11783,42 @@ function tok(filePath) {
|
|
|
10827
11783
|
function findClaudeMdFiles(projectRoot, homeDir = os8.homedir()) {
|
|
10828
11784
|
const found = [];
|
|
10829
11785
|
const seen = /* @__PURE__ */ new Set();
|
|
10830
|
-
let current =
|
|
11786
|
+
let current = path21.resolve(projectRoot);
|
|
10831
11787
|
while (true) {
|
|
10832
|
-
const candidate =
|
|
10833
|
-
if (!seen.has(candidate) &&
|
|
11788
|
+
const candidate = path21.join(current, "CLAUDE.md");
|
|
11789
|
+
if (!seen.has(candidate) && fs25.existsSync(candidate)) {
|
|
10834
11790
|
found.push(candidate);
|
|
10835
11791
|
seen.add(candidate);
|
|
10836
11792
|
}
|
|
10837
|
-
const parent =
|
|
11793
|
+
const parent = path21.dirname(current);
|
|
10838
11794
|
if (parent === current) break;
|
|
10839
11795
|
current = parent;
|
|
10840
11796
|
}
|
|
10841
|
-
const globalMd =
|
|
10842
|
-
if (!seen.has(globalMd) &&
|
|
11797
|
+
const globalMd = path21.join(homeDir, ".claude", "CLAUDE.md");
|
|
11798
|
+
if (!seen.has(globalMd) && fs25.existsSync(globalMd)) {
|
|
10843
11799
|
found.push(globalMd);
|
|
10844
11800
|
}
|
|
10845
11801
|
return found;
|
|
10846
11802
|
}
|
|
10847
11803
|
function findMemoryMd(projectRoot, homeDir = os8.homedir(), alternateRoots = []) {
|
|
10848
11804
|
try {
|
|
10849
|
-
const projectsDir =
|
|
10850
|
-
if (!
|
|
10851
|
-
const rootStr =
|
|
11805
|
+
const projectsDir = path21.join(homeDir, ".claude", "projects");
|
|
11806
|
+
if (!fs25.existsSync(projectsDir)) return null;
|
|
11807
|
+
const rootStr = path21.resolve(projectRoot);
|
|
10852
11808
|
const candidateRoots = [rootStr];
|
|
10853
11809
|
try {
|
|
10854
|
-
const realRoot =
|
|
11810
|
+
const realRoot = fs25.realpathSync.native(rootStr);
|
|
10855
11811
|
if (realRoot !== rootStr) candidateRoots.push(realRoot);
|
|
10856
11812
|
} catch {
|
|
10857
11813
|
}
|
|
10858
11814
|
for (const alternate of alternateRoots) {
|
|
10859
|
-
const resolved =
|
|
11815
|
+
const resolved = path21.resolve(alternate);
|
|
10860
11816
|
if (!candidateRoots.includes(resolved)) candidateRoots.push(resolved);
|
|
10861
11817
|
}
|
|
10862
11818
|
for (const root of candidateRoots) {
|
|
10863
11819
|
const expectedSlug = root.replace(/[^A-Za-z0-9]/g, "-");
|
|
10864
|
-
const candidate =
|
|
10865
|
-
if (
|
|
11820
|
+
const candidate = path21.join(projectsDir, expectedSlug, "memory", "MEMORY.md");
|
|
11821
|
+
if (fs25.existsSync(candidate)) return candidate;
|
|
10866
11822
|
}
|
|
10867
11823
|
return null;
|
|
10868
11824
|
} catch {
|
|
@@ -10876,10 +11832,10 @@ function buildStats(projectRoot, homeDir = os8.homedir(), alternateRoots = []) {
|
|
|
10876
11832
|
for (const p of claudeMds) {
|
|
10877
11833
|
const t = tok(p);
|
|
10878
11834
|
claudeMdTotal += t;
|
|
10879
|
-
const parentDir =
|
|
11835
|
+
const parentDir = path21.basename(path21.dirname(p));
|
|
10880
11836
|
const label = parentDir === ".claude" ? "~/.claude/CLAUDE.md" : (() => {
|
|
10881
11837
|
try {
|
|
10882
|
-
return
|
|
11838
|
+
return path21.relative(projectRoot, p);
|
|
10883
11839
|
} catch {
|
|
10884
11840
|
return p;
|
|
10885
11841
|
}
|
|
@@ -10943,11 +11899,11 @@ async function runContextStats(opts = {}) {
|
|
|
10943
11899
|
process.stdout.write("[--fix] No MEMORY.md found; nothing to prune.\n");
|
|
10944
11900
|
} else {
|
|
10945
11901
|
const memPath = result.memory_md_path;
|
|
10946
|
-
const pruneResult = pruneIndex(
|
|
11902
|
+
const pruneResult = pruneIndex(path21.dirname(memPath), { dryRun: true });
|
|
10947
11903
|
if (!pruneResult.changed || pruneResult.after === void 0) {
|
|
10948
11904
|
process.stdout.write("[--fix] MEMORY.md already clean; nothing to prune.\n");
|
|
10949
11905
|
} else {
|
|
10950
|
-
const before =
|
|
11906
|
+
const before = fs25.readFileSync(memPath, "utf-8");
|
|
10951
11907
|
const applyResult = await confirmAndApply(
|
|
10952
11908
|
[{ path: memPath, before, after: pruneResult.after, label: "MEMORY.md" }],
|
|
10953
11909
|
opts.yes === true ? { yes: true } : {}
|
|
@@ -10974,8 +11930,8 @@ async function runContextStats(opts = {}) {
|
|
|
10974
11930
|
}
|
|
10975
11931
|
|
|
10976
11932
|
// src/baseline.ts
|
|
10977
|
-
import * as
|
|
10978
|
-
import * as
|
|
11933
|
+
import * as fs26 from "node:fs";
|
|
11934
|
+
import * as path22 from "node:path";
|
|
10979
11935
|
var SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
10980
11936
|
"node_modules",
|
|
10981
11937
|
".git",
|
|
@@ -11013,12 +11969,12 @@ function walkProject(rootDir, opts = {}) {
|
|
|
11013
11969
|
if (dir === void 0) break;
|
|
11014
11970
|
let entries;
|
|
11015
11971
|
try {
|
|
11016
|
-
entries =
|
|
11972
|
+
entries = fs26.readdirSync(dir, { withFileTypes: true });
|
|
11017
11973
|
} catch {
|
|
11018
11974
|
continue;
|
|
11019
11975
|
}
|
|
11020
11976
|
for (const entry of entries) {
|
|
11021
|
-
const full =
|
|
11977
|
+
const full = path22.join(dir, entry.name);
|
|
11022
11978
|
if (entry.isDirectory()) {
|
|
11023
11979
|
if (SKIP_DIRS.has(entry.name) || extraSkipDirs.includes(entry.name)) continue;
|
|
11024
11980
|
if (entry.name.startsWith(".") && entry.name !== ".") {
|
|
@@ -11093,7 +12049,7 @@ function fetchTopSymbols(limit, dbPath, rootDir) {
|
|
|
11093
12049
|
}
|
|
11094
12050
|
}
|
|
11095
12051
|
function buildProjectMap(rootDir = process.cwd(), opts = {}) {
|
|
11096
|
-
const root =
|
|
12052
|
+
const root = path22.resolve(rootDir);
|
|
11097
12053
|
const config = loadConfig();
|
|
11098
12054
|
const { files, languages } = walkProject(root, { excludeTests: config.repomap.exclude_tests });
|
|
11099
12055
|
const compact = opts.compact === true || files.length > config.repomap.compact_file_threshold;
|
|
@@ -11102,12 +12058,12 @@ function buildProjectMap(rootDir = process.cwd(), opts = {}) {
|
|
|
11102
12058
|
const recentFiles = files.map((f) => {
|
|
11103
12059
|
let mtime;
|
|
11104
12060
|
try {
|
|
11105
|
-
mtime =
|
|
12061
|
+
mtime = fs26.statSync(f).mtimeMs;
|
|
11106
12062
|
} catch {
|
|
11107
12063
|
mtime = 0;
|
|
11108
12064
|
}
|
|
11109
12065
|
return { f, mtime };
|
|
11110
|
-
}).sort((a, b) => b.mtime - a.mtime).slice(0, compact ? 5 : 15).map((x) =>
|
|
12066
|
+
}).sort((a, b) => b.mtime - a.mtime).slice(0, compact ? 5 : 15).map((x) => path22.relative(root, x.f));
|
|
11111
12067
|
return {
|
|
11112
12068
|
rootDir: root,
|
|
11113
12069
|
fileCount: files.length,
|
|
@@ -11119,7 +12075,7 @@ function buildProjectMap(rootDir = process.cwd(), opts = {}) {
|
|
|
11119
12075
|
}
|
|
11120
12076
|
function formatProjectMap(map, compact = false) {
|
|
11121
12077
|
const lines2 = [];
|
|
11122
|
-
const rel =
|
|
12078
|
+
const rel = path22.basename(map.rootDir);
|
|
11123
12079
|
lines2.push(`# Project map: ${rel}`);
|
|
11124
12080
|
lines2.push(`Files: ${map.fileCount}`);
|
|
11125
12081
|
const langPairs = Object.entries(map.languages).sort((a, b) => b[1] - a[1]);
|
|
@@ -11147,13 +12103,13 @@ function formatProjectMap(map, compact = false) {
|
|
|
11147
12103
|
}
|
|
11148
12104
|
function mapLookupBytesSaved(map, emittedText) {
|
|
11149
12105
|
const referencedFiles = /* @__PURE__ */ new Set([
|
|
11150
|
-
...map.recentFiles.map((f) => normalizePath(
|
|
12106
|
+
...map.recentFiles.map((f) => normalizePath(path22.resolve(map.rootDir, f))),
|
|
11151
12107
|
...map.topSymbols.map((s) => normalizePath(s.filePath))
|
|
11152
12108
|
]);
|
|
11153
12109
|
let fullSourceBytes = 0;
|
|
11154
12110
|
for (const fp of referencedFiles) {
|
|
11155
12111
|
try {
|
|
11156
|
-
fullSourceBytes +=
|
|
12112
|
+
fullSourceBytes += fs26.statSync(fp).size;
|
|
11157
12113
|
} catch {
|
|
11158
12114
|
}
|
|
11159
12115
|
}
|
|
@@ -11167,14 +12123,14 @@ function findMemSuggestionCandidates(projectRoot) {
|
|
|
11167
12123
|
const claudeMdFiles = findClaudeMdFiles(projectRoot);
|
|
11168
12124
|
const candidateFiles = new Set(claudeMdFiles);
|
|
11169
12125
|
for (const claudeMd of claudeMdFiles) {
|
|
11170
|
-
const agentsMd =
|
|
11171
|
-
if (
|
|
12126
|
+
const agentsMd = path22.join(path22.dirname(claudeMd), "AGENTS.md");
|
|
12127
|
+
if (fs26.existsSync(agentsMd)) candidateFiles.add(agentsMd);
|
|
11172
12128
|
}
|
|
11173
12129
|
const suggestions = [];
|
|
11174
12130
|
for (const filePath of candidateFiles) {
|
|
11175
12131
|
let text;
|
|
11176
12132
|
try {
|
|
11177
|
-
text =
|
|
12133
|
+
text = fs26.readFileSync(filePath, { encoding: "utf-8" });
|
|
11178
12134
|
} catch {
|
|
11179
12135
|
continue;
|
|
11180
12136
|
}
|
|
@@ -11203,7 +12159,7 @@ function formatMemSuggestions(projectRoot) {
|
|
|
11203
12159
|
if (suggestions.length === 0) return "";
|
|
11204
12160
|
const lines2 = ["", "## mem suggestions"];
|
|
11205
12161
|
for (const s of suggestions) {
|
|
11206
|
-
const basename12 =
|
|
12162
|
+
const basename12 = path22.basename(s.path);
|
|
11207
12163
|
lines2.push(
|
|
11208
12164
|
"Consider: mem import --from-md " + s.path + " # migrates " + s.count + " preference-shaped lines from " + basename12 + " as pending facts for review"
|
|
11209
12165
|
);
|
|
@@ -11343,9 +12299,9 @@ function getFileEntry(filePath, dbPath = globalDbPath()) {
|
|
|
11343
12299
|
embedSha: row.embed_sha ?? ""
|
|
11344
12300
|
};
|
|
11345
12301
|
}
|
|
11346
|
-
function sanitizeFtsQuery(query,
|
|
12302
|
+
function sanitizeFtsQuery(query, join23 = "AND") {
|
|
11347
12303
|
const terms = query.split(/\s+/).filter((t) => t.length > 0).map((t) => '"' + t.replace(/"/g, '""') + '"');
|
|
11348
|
-
return terms.join(
|
|
12304
|
+
return terms.join(join23 === "OR" ? " OR " : " ");
|
|
11349
12305
|
}
|
|
11350
12306
|
function runFtsQuery(db, match, limit, scope, rootDir) {
|
|
11351
12307
|
const sql = `SELECT s.file_path, s.name, s.kind, s.line_start, s.line_end, s.body, s.docstring, s.parent FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid WHERE symbols_fts MATCH ?${scope !== void 0 ? ` AND ${scope.clause}` : ""} ORDER BY bm25(symbols_fts) LIMIT ?`;
|
|
@@ -11374,9 +12330,9 @@ function searchSymbolsFts(query, limit = 50, dbPath = globalDbPath(), rootDir) {
|
|
|
11374
12330
|
}
|
|
11375
12331
|
|
|
11376
12332
|
// src/parser.ts
|
|
11377
|
-
import * as
|
|
12333
|
+
import * as fs27 from "node:fs";
|
|
11378
12334
|
import { createRequire as createRequire4 } from "node:module";
|
|
11379
|
-
import * as
|
|
12335
|
+
import * as path27 from "node:path";
|
|
11380
12336
|
|
|
11381
12337
|
// src/languages/csharp.ts
|
|
11382
12338
|
var USING_RE = /^(?:global\s+)?using\s+(?:static\s+)?([A-Za-z_][A-Za-z0-9_.]*)\s*(?:=\s*([A-Za-z_][A-Za-z0-9_.<>,\s]*))?\s*;/;
|
|
@@ -11809,7 +12765,7 @@ function extractHtml(content, filePath) {
|
|
|
11809
12765
|
}
|
|
11810
12766
|
|
|
11811
12767
|
// src/languages/liquid.ts
|
|
11812
|
-
import * as
|
|
12768
|
+
import * as path23 from "node:path";
|
|
11813
12769
|
var INCLUDE_RE = /{%-?\s*include\s+(['"])((?:(?!\1)[\s\S])+?)\1/gi;
|
|
11814
12770
|
var SECTION_RE = /{%-?\s*section\s+(['"])((?:(?!\1)[\s\S])+?)\1/gi;
|
|
11815
12771
|
var RENDER_RE = /{%-?\s*render\s+(['"])((?:(?!\1)[\s\S])+?)\1/gi;
|
|
@@ -11852,7 +12808,7 @@ function extractLiquid(content, filePath, relPath) {
|
|
|
11852
12808
|
const resolvedRel = relPath ?? filePath;
|
|
11853
12809
|
const relPosix = resolvedRel.replace(/\\/g, "/");
|
|
11854
12810
|
if (relPosix.startsWith("sections/") || relPosix.includes("/sections/")) {
|
|
11855
|
-
const stem =
|
|
12811
|
+
const stem = path23.basename(resolvedRel, path23.extname(resolvedRel));
|
|
11856
12812
|
symbols.push({ filePath, name: stem, kind: "liquid_section_file", lineStart: 1, lineEnd: 1, body: "", docstring: "", parent: "" });
|
|
11857
12813
|
}
|
|
11858
12814
|
const totalLines = countContentLines(content);
|
|
@@ -14155,7 +15111,7 @@ function extractApex(content, filePath) {
|
|
|
14155
15111
|
}
|
|
14156
15112
|
|
|
14157
15113
|
// src/languages/salesforce_metadata.ts
|
|
14158
|
-
import * as
|
|
15114
|
+
import * as path24 from "node:path";
|
|
14159
15115
|
var MAX_SYMBOLS10 = 1e3;
|
|
14160
15116
|
var MAX_REFS = 1e3;
|
|
14161
15117
|
var FLOW_TAG_KIND = {
|
|
@@ -14213,7 +15169,7 @@ function normalizedPath(filePath) {
|
|
|
14213
15169
|
return filePath.replace(/\\/g, "/");
|
|
14214
15170
|
}
|
|
14215
15171
|
function basenameWithout(filePath, suffix) {
|
|
14216
|
-
const base =
|
|
15172
|
+
const base = path24.basename(filePath);
|
|
14217
15173
|
return base.toLowerCase().endsWith(suffix.toLowerCase()) ? base.slice(0, base.length - suffix.length) : base;
|
|
14218
15174
|
}
|
|
14219
15175
|
function objectNameFromPath(filePath) {
|
|
@@ -14252,12 +15208,12 @@ function snakeCase(value) {
|
|
|
14252
15208
|
return value.replace(/([a-z0-9])([A-Z])/g, "$1_$2").replace(/([A-Z]+)([A-Z][a-z])/g, "$1_$2").replace(/[^A-Za-z0-9]+/g, "_").replace(/^_+|_+$/g, "").toLowerCase();
|
|
14253
15209
|
}
|
|
14254
15210
|
function companionName(filePath) {
|
|
14255
|
-
const base =
|
|
15211
|
+
const base = path24.basename(filePath);
|
|
14256
15212
|
const match = /^(.+)\.(?:cls|trigger|page|component|cmp|app|evt|intf|design|auradoc|tokens|js)-meta\.xml$/i.exec(base);
|
|
14257
15213
|
return match?.[1] === void 0 ? null : `${match[1]}.metadata`;
|
|
14258
15214
|
}
|
|
14259
15215
|
function metadataArtifactName(filePath) {
|
|
14260
|
-
const base =
|
|
15216
|
+
const base = path24.basename(filePath);
|
|
14261
15217
|
const match = /^(.+)\.[^.]+-meta\.xml$/i.exec(base);
|
|
14262
15218
|
return match?.[1] ?? basenameWithout(filePath, "-meta.xml");
|
|
14263
15219
|
}
|
|
@@ -14348,7 +15304,7 @@ function extractSalesforceMetadata(rawContent, filePath) {
|
|
|
14348
15304
|
const seen = /* @__PURE__ */ new Set();
|
|
14349
15305
|
const refs = [];
|
|
14350
15306
|
const seenRefs = /* @__PURE__ */ new Set();
|
|
14351
|
-
const base =
|
|
15307
|
+
const base = path24.basename(filePath).toLowerCase();
|
|
14352
15308
|
const whole = wholeFileSpan(content);
|
|
14353
15309
|
const root = rootElement(content);
|
|
14354
15310
|
if (root === null) return { symbols, refs };
|
|
@@ -14492,14 +15448,14 @@ function extractSalesforceMetadata(rawContent, filePath) {
|
|
|
14492
15448
|
}
|
|
14493
15449
|
|
|
14494
15450
|
// src/languages/salesforce_frontend.ts
|
|
14495
|
-
import * as
|
|
15451
|
+
import * as path25 from "node:path";
|
|
14496
15452
|
function lines(content) {
|
|
14497
15453
|
return content.split("\n");
|
|
14498
15454
|
}
|
|
14499
15455
|
function bundleName(filePath) {
|
|
14500
15456
|
const normalized = filePath.replaceAll("\\", "/");
|
|
14501
|
-
const parent =
|
|
14502
|
-
const base =
|
|
15457
|
+
const parent = path25.posix.basename(path25.posix.dirname(normalized));
|
|
15458
|
+
const base = path25.posix.basename(normalized).replace(/\.[^.]+$/, "");
|
|
14503
15459
|
return parent === "lwc" || parent === "aura" ? base : parent;
|
|
14504
15460
|
}
|
|
14505
15461
|
function lwcTagAlias(name) {
|
|
@@ -14603,7 +15559,7 @@ var MARKUP_KIND = {
|
|
|
14603
15559
|
};
|
|
14604
15560
|
function markupArtifactName(filePath, extension) {
|
|
14605
15561
|
if (extension === ".page" || extension === ".component" || extension === ".email") {
|
|
14606
|
-
return
|
|
15562
|
+
return path25.posix.basename(filePath.replaceAll("\\", "/")).replace(new RegExp(`${extension.replace(".", "\\.")}$`, "i"), "");
|
|
14607
15563
|
}
|
|
14608
15564
|
return bundleName(filePath);
|
|
14609
15565
|
}
|
|
@@ -14623,7 +15579,7 @@ function attributeRefs(refs, content, filePath, attribute, split = false) {
|
|
|
14623
15579
|
}
|
|
14624
15580
|
function extractSalesforceMarkup(content, filePath) {
|
|
14625
15581
|
const normalized = filePath.replaceAll("\\", "/");
|
|
14626
|
-
const extension =
|
|
15582
|
+
const extension = path25.posix.extname(normalized).toLowerCase();
|
|
14627
15583
|
const kind = MARKUP_KIND[extension] ?? "salesforce_markup";
|
|
14628
15584
|
const symbols = [
|
|
14629
15585
|
symbol(filePath, markupArtifactName(normalized, extension), kind, 1, countContentLines(content))
|
|
@@ -14655,7 +15611,7 @@ function extractSalesforceMarkup(content, filePath) {
|
|
|
14655
15611
|
}
|
|
14656
15612
|
|
|
14657
15613
|
// src/languages/sfc_idx.ts
|
|
14658
|
-
import * as
|
|
15614
|
+
import * as path26 from "node:path";
|
|
14659
15615
|
var MAX_SYMBOLS11 = 500;
|
|
14660
15616
|
function dedupe2(values, key) {
|
|
14661
15617
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -14674,7 +15630,7 @@ function finalize(symbols, refs) {
|
|
|
14674
15630
|
}
|
|
14675
15631
|
function componentName(filePath) {
|
|
14676
15632
|
const normalized = filePath.replaceAll("\\", "/");
|
|
14677
|
-
return
|
|
15633
|
+
return path26.posix.basename(normalized).replace(/\.[^.]+$/, "");
|
|
14678
15634
|
}
|
|
14679
15635
|
function matchLine2(content, offset) {
|
|
14680
15636
|
return content.slice(0, offset).split("\n").length;
|
|
@@ -14986,8 +15942,8 @@ function countNewlines(s) {
|
|
|
14986
15942
|
return n;
|
|
14987
15943
|
}
|
|
14988
15944
|
function loadGrammar(lang, filePath, content) {
|
|
14989
|
-
const useTsx = lang === "typescript" && filePath !== void 0 &&
|
|
14990
|
-
const useCppHeader = lang === "c" && filePath !== void 0 &&
|
|
15945
|
+
const useTsx = lang === "typescript" && filePath !== void 0 && path27.extname(filePath).toLowerCase() === ".tsx";
|
|
15946
|
+
const useCppHeader = lang === "c" && filePath !== void 0 && path27.extname(filePath).toLowerCase() === ".h" && content !== void 0 && CPP_HEADER_SNIFF_RE.test(content);
|
|
14991
15947
|
const cacheKey = useTsx ? "typescript:tsx" : useCppHeader ? "c:cpp-header" : lang;
|
|
14992
15948
|
const cached = _grammarCache.get(cacheKey);
|
|
14993
15949
|
if (cached !== void 0) return cached;
|
|
@@ -15135,6 +16091,9 @@ function makeSymbol(filePath, name, kind, node, lines2, style) {
|
|
|
15135
16091
|
parent: ""
|
|
15136
16092
|
};
|
|
15137
16093
|
}
|
|
16094
|
+
function fanOutElidesBodies(nameCount, declarationChars) {
|
|
16095
|
+
return nameCount > 1 && nameCount * declarationChars > MAX_SYMBOL_BODY_CHARS;
|
|
16096
|
+
}
|
|
15138
16097
|
function collectPatternBindings(node) {
|
|
15139
16098
|
const names = [];
|
|
15140
16099
|
const walk = (n) => {
|
|
@@ -15142,6 +16101,16 @@ function collectPatternBindings(node) {
|
|
|
15142
16101
|
if (n.text !== "") names.push(n.text);
|
|
15143
16102
|
return;
|
|
15144
16103
|
}
|
|
16104
|
+
if (n.type === "assignment_pattern" || n.type === "object_assignment_pattern") {
|
|
16105
|
+
const left = n.childForFieldName("left");
|
|
16106
|
+
if (left !== null) walk(left);
|
|
16107
|
+
return;
|
|
16108
|
+
}
|
|
16109
|
+
if (n.type === "pair_pattern") {
|
|
16110
|
+
const value = n.childForFieldName("value");
|
|
16111
|
+
if (value !== null) walk(value);
|
|
16112
|
+
return;
|
|
16113
|
+
}
|
|
15145
16114
|
for (const child of n.namedChildren) walk(child);
|
|
15146
16115
|
};
|
|
15147
16116
|
walk(node);
|
|
@@ -15203,7 +16172,7 @@ function extractTsJsSymbols(root, filePath, lines2) {
|
|
|
15203
16172
|
out.push(makeSymbol(filePath, name.text, isFn ? "function" : "variable", child, lines2, "c"));
|
|
15204
16173
|
} else {
|
|
15205
16174
|
const bindings = collectPatternBindings(name);
|
|
15206
|
-
const elideBodies = bindings.length
|
|
16175
|
+
const elideBodies = fanOutElidesBodies(bindings.length, child.text.length);
|
|
15207
16176
|
for (const bound of bindings) {
|
|
15208
16177
|
const sym = makeSymbol(filePath, bound, "variable", child, lines2, "c");
|
|
15209
16178
|
out.push(elideBodies ? { ...sym, body: "" } : sym);
|
|
@@ -15321,14 +16290,26 @@ var GO_LOCAL_KINDS = /* @__PURE__ */ new Set([
|
|
|
15321
16290
|
// the interface type declaring them does not.
|
|
15322
16291
|
"method_elem"
|
|
15323
16292
|
]);
|
|
16293
|
+
var GO_MULTI_NAME_SPECS = /* @__PURE__ */ new Set(["var_spec", "const_spec"]);
|
|
15324
16294
|
function extractGoSymbols(root, filePath, lines2) {
|
|
15325
16295
|
const out = [];
|
|
15326
16296
|
const visit = (node, insideFunction) => {
|
|
15327
16297
|
const kind = GO_KIND_BY_TYPE.get(node.type);
|
|
15328
16298
|
if (kind !== void 0 && !(insideFunction && GO_LOCAL_KINDS.has(node.type))) {
|
|
15329
|
-
|
|
15330
|
-
|
|
15331
|
-
|
|
16299
|
+
if (GO_MULTI_NAME_SPECS.has(node.type)) {
|
|
16300
|
+
const declared = node.namedChildren.filter(
|
|
16301
|
+
(c) => c.type === "identifier" && c.text !== "" && c.text !== "_"
|
|
16302
|
+
);
|
|
16303
|
+
const elideBodies = fanOutElidesBodies(declared.length, node.text.length);
|
|
16304
|
+
for (const child of declared) {
|
|
16305
|
+
const sym = makeSymbol(filePath, child.text, kind, node, lines2, "c");
|
|
16306
|
+
out.push(elideBodies ? { ...sym, body: "" } : sym);
|
|
16307
|
+
}
|
|
16308
|
+
} else {
|
|
16309
|
+
const name = nodeName(node);
|
|
16310
|
+
if (name !== null && name !== "") {
|
|
16311
|
+
out.push(makeSymbol(filePath, name, kind, node, lines2, "c"));
|
|
16312
|
+
}
|
|
15332
16313
|
}
|
|
15333
16314
|
}
|
|
15334
16315
|
const childInside = insideFunction || GO_FN_SCOPE_TYPES.has(node.type);
|
|
@@ -16654,9 +17635,9 @@ function isUnderSkipDir(filePath, skipDirs) {
|
|
|
16654
17635
|
}
|
|
16655
17636
|
function isParseSkipEligible(filePath, cfg) {
|
|
16656
17637
|
if (isUnderSkipDir(filePath, cfg.skip_dirs)) return true;
|
|
16657
|
-
if (cfg.skip_files.includes(
|
|
17638
|
+
if (cfg.skip_files.includes(path27.basename(filePath))) return true;
|
|
16658
17639
|
try {
|
|
16659
|
-
const stat2 =
|
|
17640
|
+
const stat2 = fs27.statSync(filePath);
|
|
16660
17641
|
if (stat2.size > cfg.large_file_skip_kb * 1024) return true;
|
|
16661
17642
|
} catch {
|
|
16662
17643
|
}
|
|
@@ -16698,7 +17679,8 @@ function writeParseResult(filePath, content, result, dbPath) {
|
|
|
16698
17679
|
});
|
|
16699
17680
|
writeAll.immediate();
|
|
16700
17681
|
}
|
|
16701
|
-
function indexFileSync(
|
|
17682
|
+
function indexFileSync(rawPath, dbPath = globalDbPath(), preReadBytes) {
|
|
17683
|
+
const filePath = canonicalizeIndexPath(rawPath);
|
|
16702
17684
|
const ixCfg = loadConfig().indexing;
|
|
16703
17685
|
if (ixCfg !== void 0 && isParseSkipEligible(filePath, ixCfg)) {
|
|
16704
17686
|
const db = getDb(dbPath);
|
|
@@ -16712,7 +17694,7 @@ function indexFileSync(filePath, dbPath = globalDbPath(), preReadBytes) {
|
|
|
16712
17694
|
raw = preReadBytes;
|
|
16713
17695
|
} else {
|
|
16714
17696
|
try {
|
|
16715
|
-
raw =
|
|
17697
|
+
raw = fs27.readFileSync(filePath);
|
|
16716
17698
|
} catch (err) {
|
|
16717
17699
|
if (err.code === "ENOENT") return;
|
|
16718
17700
|
throw err;
|
|
@@ -16743,14 +17725,32 @@ var UNAVAILABLE_EMBED_SHA_PREFIX = "unavailable:";
|
|
|
16743
17725
|
function unavailableEmbedSha(sha) {
|
|
16744
17726
|
return UNAVAILABLE_EMBED_SHA_PREFIX + sha;
|
|
16745
17727
|
}
|
|
17728
|
+
function canonicalizeIndexPath(absPath) {
|
|
17729
|
+
if (!isCaseInsensitiveFs()) return absPath;
|
|
17730
|
+
let real;
|
|
17731
|
+
try {
|
|
17732
|
+
real = fs27.realpathSync.native(absPath);
|
|
17733
|
+
} catch {
|
|
17734
|
+
return absPath;
|
|
17735
|
+
}
|
|
17736
|
+
const cut = Math.max(absPath.lastIndexOf("/"), absPath.lastIndexOf("\\"));
|
|
17737
|
+
const base = absPath.slice(cut + 1);
|
|
17738
|
+
const realNorm = normalizePath(real);
|
|
17739
|
+
const realBase = path27.basename(realNorm);
|
|
17740
|
+
if (base === realBase) return absPath;
|
|
17741
|
+
if (foldPath(base) !== foldPath(realBase)) return absPath;
|
|
17742
|
+
const callerDir = cut < 0 ? normalizePath(path27.resolve(".")) : normalizePath(absPath.slice(0, cut));
|
|
17743
|
+
if (foldPath(callerDir) !== foldPath(path27.dirname(realNorm))) return absPath;
|
|
17744
|
+
return absPath.slice(0, cut + 1) + realBase;
|
|
17745
|
+
}
|
|
16746
17746
|
function indexedPathSpellingIsStale(storedPath, absPath) {
|
|
16747
17747
|
if (!isCaseInsensitiveFs()) return false;
|
|
16748
17748
|
const stored = normalizePath(storedPath);
|
|
16749
|
-
const candidate = normalizePath(
|
|
17749
|
+
const candidate = normalizePath(path27.resolve(absPath));
|
|
16750
17750
|
if (foldPath(stored) !== foldPath(candidate)) return false;
|
|
16751
17751
|
let real;
|
|
16752
17752
|
try {
|
|
16753
|
-
real = normalizePath(
|
|
17753
|
+
real = normalizePath(fs27.realpathSync.native(absPath));
|
|
16754
17754
|
} catch {
|
|
16755
17755
|
return false;
|
|
16756
17756
|
}
|
|
@@ -16779,7 +17779,8 @@ function isEmbedFresh(storedEmbedSha, sha, embeddingsEnabled, depsAvailable) {
|
|
|
16779
17779
|
if (!depsAvailable && storedEmbedSha === unavailableEmbedSha(sha)) return true;
|
|
16780
17780
|
return false;
|
|
16781
17781
|
}
|
|
16782
|
-
async function indexFileEmbeddings(
|
|
17782
|
+
async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onError) {
|
|
17783
|
+
const filePath = canonicalizeIndexPath(rawPath);
|
|
16783
17784
|
const ixCfg = loadConfig().indexing;
|
|
16784
17785
|
if (!ixCfg.embeddings_enabled) {
|
|
16785
17786
|
stampEmbedSha(getDb(dbPath), filePath, sha, disabledEmbedSha);
|
|
@@ -16816,7 +17817,7 @@ async function indexFileEmbeddings(filePath, dbPath = globalDbPath(), sha, onErr
|
|
|
16816
17817
|
}
|
|
16817
17818
|
let content;
|
|
16818
17819
|
try {
|
|
16819
|
-
content = decodeSource(await
|
|
17820
|
+
content = decodeSource(await fs27.promises.readFile(filePath));
|
|
16820
17821
|
} catch {
|
|
16821
17822
|
return;
|
|
16822
17823
|
}
|
|
@@ -16862,15 +17863,15 @@ function stampEmbedSha(db, filePath, sha, makeValue) {
|
|
|
16862
17863
|
}
|
|
16863
17864
|
function safeMtime(filePath) {
|
|
16864
17865
|
try {
|
|
16865
|
-
return
|
|
17866
|
+
return fs27.statSync(filePath).mtimeMs / 1e3;
|
|
16866
17867
|
} catch {
|
|
16867
17868
|
return 0;
|
|
16868
17869
|
}
|
|
16869
17870
|
}
|
|
16870
17871
|
|
|
16871
17872
|
// src/index_prune.ts
|
|
16872
|
-
import * as
|
|
16873
|
-
import * as
|
|
17873
|
+
import * as fs28 from "node:fs";
|
|
17874
|
+
import * as path28 from "node:path";
|
|
16874
17875
|
function removeFileFromIndex(db, filePath) {
|
|
16875
17876
|
const tx = db.transaction(() => {
|
|
16876
17877
|
deleteFileRows(db, filePath);
|
|
@@ -16907,7 +17908,7 @@ function findDeletablePaths(rootPrefix, dbPath) {
|
|
|
16907
17908
|
for (const p of foldedPathsUnderRoot(rootPrefix, dbPath)) {
|
|
16908
17909
|
let stillExists;
|
|
16909
17910
|
try {
|
|
16910
|
-
const st =
|
|
17911
|
+
const st = fs28.statSync(p, { throwIfNoEntry: false });
|
|
16911
17912
|
stillExists = st !== void 0 && st.isFile();
|
|
16912
17913
|
} catch {
|
|
16913
17914
|
continue;
|
|
@@ -16935,7 +17936,7 @@ function removeDeletedFilesBestEffort(db, paths) {
|
|
|
16935
17936
|
for (const p of paths) {
|
|
16936
17937
|
let gone;
|
|
16937
17938
|
try {
|
|
16938
|
-
const st =
|
|
17939
|
+
const st = fs28.statSync(p, { throwIfNoEntry: false });
|
|
16939
17940
|
gone = st === void 0 || !st.isFile();
|
|
16940
17941
|
} catch {
|
|
16941
17942
|
continue;
|
|
@@ -17002,7 +18003,7 @@ function pruneOrphanedChunks(dbPath = globalDbPath()) {
|
|
|
17002
18003
|
return removed;
|
|
17003
18004
|
}
|
|
17004
18005
|
function recordKnownRoot(filePath, dbPath = globalDbPath()) {
|
|
17005
|
-
const project = findProject(
|
|
18006
|
+
const project = findProject(path28.dirname(filePath));
|
|
17006
18007
|
if (project === null || isTooShallowToPrune(project.root)) return;
|
|
17007
18008
|
const db = getDb(dbPath);
|
|
17008
18009
|
db.prepare(
|
|
@@ -17025,7 +18026,7 @@ function sweepKnownRoots(dbPath = globalDbPath(), opts) {
|
|
|
17025
18026
|
if (isTooShallowToPrune(root)) continue;
|
|
17026
18027
|
let reachable;
|
|
17027
18028
|
try {
|
|
17028
|
-
reachable =
|
|
18029
|
+
reachable = fs28.statSync(root, { throwIfNoEntry: false })?.isDirectory() === true;
|
|
17029
18030
|
} catch {
|
|
17030
18031
|
reachable = false;
|
|
17031
18032
|
}
|
|
@@ -17060,19 +18061,19 @@ function sweepKnownRoots(dbPath = globalDbPath(), opts) {
|
|
|
17060
18061
|
}
|
|
17061
18062
|
var KNOWN_ROOT_RECORD_MIN_INTERVAL_MS = 60 * 60 * 1e3;
|
|
17062
18063
|
function knownRootRecordMarkerPath(dir, filePath) {
|
|
17063
|
-
return
|
|
18064
|
+
return path28.join(dir, `known-root-record-${shortFingerprint(path28.dirname(filePath))}.marker`);
|
|
17064
18065
|
}
|
|
17065
18066
|
var KNOWN_ROOT_MARKER_PREFIX = "known-root-record-";
|
|
17066
18067
|
function sweepExpiredKnownRootMarkers(dir = dataDir()) {
|
|
17067
18068
|
let removed = 0;
|
|
17068
18069
|
try {
|
|
17069
18070
|
const cutoff = Date.now() - KNOWN_ROOT_RECORD_MIN_INTERVAL_MS;
|
|
17070
|
-
for (const file of
|
|
18071
|
+
for (const file of fs28.readdirSync(dir)) {
|
|
17071
18072
|
if (!file.startsWith(KNOWN_ROOT_MARKER_PREFIX) || !file.endsWith(".marker")) continue;
|
|
17072
|
-
const full =
|
|
18073
|
+
const full = path28.join(dir, file);
|
|
17073
18074
|
try {
|
|
17074
|
-
if (
|
|
17075
|
-
|
|
18075
|
+
if (fs28.statSync(full).mtimeMs < cutoff) {
|
|
18076
|
+
fs28.unlinkSync(full);
|
|
17076
18077
|
removed += 1;
|
|
17077
18078
|
}
|
|
17078
18079
|
} catch {
|
|
@@ -17085,13 +18086,13 @@ function sweepExpiredKnownRootMarkers(dir = dataDir()) {
|
|
|
17085
18086
|
function recordKnownRootThrottled(filePath, dir = dataDir(), dbPath = globalDbPath()) {
|
|
17086
18087
|
const markerPath = knownRootRecordMarkerPath(dir, filePath);
|
|
17087
18088
|
try {
|
|
17088
|
-
const stat2 =
|
|
18089
|
+
const stat2 = fs28.statSync(markerPath);
|
|
17089
18090
|
if (Date.now() - stat2.mtimeMs < KNOWN_ROOT_RECORD_MIN_INTERVAL_MS) return;
|
|
17090
18091
|
} catch {
|
|
17091
18092
|
}
|
|
17092
18093
|
try {
|
|
17093
18094
|
ensureDirSync(dir);
|
|
17094
|
-
|
|
18095
|
+
fs28.writeFileSync(markerPath, "");
|
|
17095
18096
|
} catch {
|
|
17096
18097
|
}
|
|
17097
18098
|
recordKnownRoot(filePath, dbPath);
|
|
@@ -17099,8 +18100,8 @@ function recordKnownRootThrottled(filePath, dir = dataDir(), dbPath = globalDbPa
|
|
|
17099
18100
|
|
|
17100
18101
|
// src/worker.ts
|
|
17101
18102
|
import { spawn as spawn2 } from "node:child_process";
|
|
17102
|
-
import * as
|
|
17103
|
-
import * as
|
|
18103
|
+
import * as fs29 from "node:fs";
|
|
18104
|
+
import * as path29 from "node:path";
|
|
17104
18105
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
17105
18106
|
var DEFAULT_POLL_INTERVAL_MS = 2e3;
|
|
17106
18107
|
function resolvePollIntervalMs(explicit) {
|
|
@@ -17120,7 +18121,7 @@ var unclearedDrainingSnapshots = /* @__PURE__ */ new Map();
|
|
|
17120
18121
|
function drainingSnapshotStamp(file, content) {
|
|
17121
18122
|
let identity = "unknown";
|
|
17122
18123
|
try {
|
|
17123
|
-
const stat2 =
|
|
18124
|
+
const stat2 = fs29.statSync(file);
|
|
17124
18125
|
identity = `${stat2.mtimeMs}:${stat2.size}:${stat2.ino}`;
|
|
17125
18126
|
} catch {
|
|
17126
18127
|
}
|
|
@@ -17129,15 +18130,15 @@ function drainingSnapshotStamp(file, content) {
|
|
|
17129
18130
|
var DRAINING_READ_ATTEMPTS = 5;
|
|
17130
18131
|
var DRAINING_READ_RETRY_DELAY_MS = 50;
|
|
17131
18132
|
function listDrainingFiles(queuePath) {
|
|
17132
|
-
const dir =
|
|
17133
|
-
const base = `${
|
|
18133
|
+
const dir = path29.dirname(queuePath);
|
|
18134
|
+
const base = `${path29.basename(queuePath)}.draining`;
|
|
17134
18135
|
let entries;
|
|
17135
18136
|
try {
|
|
17136
|
-
entries =
|
|
18137
|
+
entries = fs29.readdirSync(dir);
|
|
17137
18138
|
} catch {
|
|
17138
18139
|
return [];
|
|
17139
18140
|
}
|
|
17140
|
-
return entries.filter((name) => name === base || name.startsWith(`${base}.alt-`) && !name.includes(".corrupt-")).sort().map((name) =>
|
|
18141
|
+
return entries.filter((name) => name === base || name.startsWith(`${base}.alt-`) && !name.includes(".corrupt-")).sort().map((name) => path29.join(dir, name));
|
|
17141
18142
|
}
|
|
17142
18143
|
var MAX_TRANSIENT_RETRIES = 5;
|
|
17143
18144
|
function bumpRetryCount(dbPath, absPath) {
|
|
@@ -17165,17 +18166,17 @@ function clearRetryCount(dbPath, absPath) {
|
|
|
17165
18166
|
}
|
|
17166
18167
|
}
|
|
17167
18168
|
function dirtyQueuePathFor(dir) {
|
|
17168
|
-
return
|
|
18169
|
+
return path29.join(dir, "queue", "dirty.txt");
|
|
17169
18170
|
}
|
|
17170
18171
|
function drainHeartbeatPathFor(dir) {
|
|
17171
|
-
return
|
|
18172
|
+
return path29.join(dir, "queue", "drain-heartbeat");
|
|
17172
18173
|
}
|
|
17173
18174
|
function writeDrainHeartbeat(dir, force = false) {
|
|
17174
18175
|
const now = Date.now();
|
|
17175
18176
|
if (!force && now - (heartbeatWriteTimes.get(dir) ?? 0) < WORKER_HEARTBEAT_REFRESH_MS) return;
|
|
17176
18177
|
try {
|
|
17177
|
-
ensureDirSync(
|
|
17178
|
-
|
|
18178
|
+
ensureDirSync(path29.dirname(drainHeartbeatPathFor(dir)));
|
|
18179
|
+
fs29.writeFileSync(drainHeartbeatPathFor(dir), `${process.pid}
|
|
17179
18180
|
`);
|
|
17180
18181
|
heartbeatWriteTimes.set(dir, now);
|
|
17181
18182
|
} catch {
|
|
@@ -17184,15 +18185,15 @@ function writeDrainHeartbeat(dir, force = false) {
|
|
|
17184
18185
|
function hasFreshWorkerHeartbeat(dir, pid) {
|
|
17185
18186
|
try {
|
|
17186
18187
|
const heartbeatPath = drainHeartbeatPathFor(dir);
|
|
17187
|
-
if (Date.now() -
|
|
17188
|
-
return
|
|
18188
|
+
if (Date.now() - fs29.statSync(heartbeatPath).mtimeMs > WORKER_HEARTBEAT_STALE_MS) return false;
|
|
18189
|
+
return fs29.readFileSync(heartbeatPath, "utf8").trim() === String(pid);
|
|
17189
18190
|
} catch {
|
|
17190
18191
|
return false;
|
|
17191
18192
|
}
|
|
17192
18193
|
}
|
|
17193
18194
|
function pidFileIsWithinStartupGrace(dir) {
|
|
17194
18195
|
try {
|
|
17195
|
-
return Date.now() -
|
|
18196
|
+
return Date.now() - fs29.statSync(workerPidPath(dir)).mtimeMs < WORKER_STARTUP_GRACE_MS;
|
|
17196
18197
|
} catch {
|
|
17197
18198
|
return false;
|
|
17198
18199
|
}
|
|
@@ -17226,28 +18227,28 @@ function parseDirtyQueueLines(raw) {
|
|
|
17226
18227
|
return out;
|
|
17227
18228
|
}
|
|
17228
18229
|
function workerPidPath(dir = dataDir()) {
|
|
17229
|
-
return
|
|
18230
|
+
return path29.join(dir, "worker.pid");
|
|
17230
18231
|
}
|
|
17231
18232
|
function getDirtyPathsFor(dir) {
|
|
17232
18233
|
let raw;
|
|
17233
18234
|
try {
|
|
17234
|
-
raw =
|
|
18235
|
+
raw = fs29.readFileSync(dirtyQueuePathFor(dir), "utf8");
|
|
17235
18236
|
} catch {
|
|
17236
18237
|
return [];
|
|
17237
18238
|
}
|
|
17238
18239
|
return parseDirtyQueueLines(raw);
|
|
17239
18240
|
}
|
|
17240
18241
|
function workerErrorLogPath(dir) {
|
|
17241
|
-
return
|
|
18242
|
+
return path29.join(dir, "worker-errors.log");
|
|
17242
18243
|
}
|
|
17243
18244
|
var WORKER_ERROR_LOG_MAX_BYTES = 5 * 1024 * 1024;
|
|
17244
18245
|
var CORRUPT_QUARANTINE_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1e3;
|
|
17245
18246
|
function cleanupWorkerStateFiles(dir) {
|
|
17246
18247
|
try {
|
|
17247
18248
|
const logPath = workerErrorLogPath(dir);
|
|
17248
|
-
const stat2 =
|
|
18249
|
+
const stat2 = fs29.statSync(logPath);
|
|
17249
18250
|
if (stat2.size > WORKER_ERROR_LOG_MAX_BYTES) {
|
|
17250
|
-
|
|
18251
|
+
fs29.writeFileSync(
|
|
17251
18252
|
logPath,
|
|
17252
18253
|
`${(/* @__PURE__ */ new Date()).toISOString()} worker-errors.log rotated (exceeded ${WORKER_ERROR_LOG_MAX_BYTES} bytes)
|
|
17253
18254
|
`
|
|
@@ -17256,13 +18257,13 @@ function cleanupWorkerStateFiles(dir) {
|
|
|
17256
18257
|
} catch {
|
|
17257
18258
|
}
|
|
17258
18259
|
try {
|
|
17259
|
-
const queueDir =
|
|
18260
|
+
const queueDir = path29.dirname(dirtyQueuePathFor(dir));
|
|
17260
18261
|
const cutoff = Date.now() - CORRUPT_QUARANTINE_MAX_AGE_MS;
|
|
17261
|
-
for (const file of
|
|
18262
|
+
for (const file of fs29.readdirSync(queueDir)) {
|
|
17262
18263
|
if (!file.includes(".corrupt-")) continue;
|
|
17263
|
-
const full =
|
|
18264
|
+
const full = path29.join(queueDir, file);
|
|
17264
18265
|
try {
|
|
17265
|
-
if (
|
|
18266
|
+
if (fs29.statSync(full).mtimeMs < cutoff) fs29.unlinkSync(full);
|
|
17266
18267
|
} catch {
|
|
17267
18268
|
}
|
|
17268
18269
|
}
|
|
@@ -17293,7 +18294,7 @@ registerReset(() => {
|
|
|
17293
18294
|
function embedFileSerialized(absPath, dbPath, sha) {
|
|
17294
18295
|
const key = foldPath(absPath);
|
|
17295
18296
|
const prior = inFlightEmbeddings.get(key);
|
|
17296
|
-
const dir =
|
|
18297
|
+
const dir = path29.dirname(dbPath);
|
|
17297
18298
|
const onEmbedError = (err) => {
|
|
17298
18299
|
const message = extractErrorMessage(err);
|
|
17299
18300
|
appendWorkerErrorLog(dir, `${(/* @__PURE__ */ new Date()).toISOString()} indexFileEmbeddings failed for ${absPath}: ${message}
|
|
@@ -17327,7 +18328,7 @@ function embedFileSerialized(absPath, dbPath, sha) {
|
|
|
17327
18328
|
}
|
|
17328
18329
|
function appendWorkerErrorLog(dir, line) {
|
|
17329
18330
|
try {
|
|
17330
|
-
|
|
18331
|
+
fs29.appendFileSync(workerErrorLogPath(dir), line);
|
|
17331
18332
|
} catch {
|
|
17332
18333
|
}
|
|
17333
18334
|
}
|
|
@@ -17344,7 +18345,7 @@ function logTransientReadFailure(dir, absPath) {
|
|
|
17344
18345
|
);
|
|
17345
18346
|
}
|
|
17346
18347
|
function bumpAndCheckRetry(dir, absPath) {
|
|
17347
|
-
const dbPath =
|
|
18348
|
+
const dbPath = path29.join(dir, "global.db");
|
|
17348
18349
|
const attempts = bumpRetryCount(dbPath, absPath);
|
|
17349
18350
|
if (attempts > MAX_TRANSIENT_RETRIES) {
|
|
17350
18351
|
if (attempts === MAX_TRANSIENT_RETRIES + 1) {
|
|
@@ -17361,14 +18362,14 @@ function bumpAndCheckRetry(dir, absPath) {
|
|
|
17361
18362
|
function appendToDirtyQueue(dir, absPath) {
|
|
17362
18363
|
const queuePath = dirtyQueuePathFor(dir);
|
|
17363
18364
|
try {
|
|
17364
|
-
ensureDirSync(
|
|
18365
|
+
ensureDirSync(path29.dirname(queuePath));
|
|
17365
18366
|
let leadingNewline = "";
|
|
17366
18367
|
try {
|
|
17367
|
-
const existing =
|
|
18368
|
+
const existing = fs29.readFileSync(queuePath, "utf8");
|
|
17368
18369
|
if (existing.length > 0 && !existing.endsWith("\n")) leadingNewline = "\n";
|
|
17369
18370
|
} catch {
|
|
17370
18371
|
}
|
|
17371
|
-
|
|
18372
|
+
fs29.appendFileSync(queuePath, `${leadingNewline}${encodeDirtyQueueLine(absPath)}
|
|
17372
18373
|
`);
|
|
17373
18374
|
} catch {
|
|
17374
18375
|
}
|
|
@@ -17377,7 +18378,7 @@ function requeueDirtyPath(dir, absPath) {
|
|
|
17377
18378
|
if (bumpAndCheckRetry(dir, absPath)) appendToDirtyQueue(dir, absPath);
|
|
17378
18379
|
}
|
|
17379
18380
|
function makeIndexer(dbPath) {
|
|
17380
|
-
const dir =
|
|
18381
|
+
const dir = path29.dirname(dbPath);
|
|
17381
18382
|
return (absPath, sha) => {
|
|
17382
18383
|
try {
|
|
17383
18384
|
const ixCfgForSkip = loadConfig().indexing;
|
|
@@ -17429,16 +18430,16 @@ function processDirtyBatch(paths, index = makeIndexer(globalDbPath()), remove =
|
|
|
17429
18430
|
requeue(dir, p);
|
|
17430
18431
|
continue;
|
|
17431
18432
|
}
|
|
17432
|
-
clearRetryCount(
|
|
18433
|
+
clearRetryCount(path29.join(dir, "global.db"), p);
|
|
17433
18434
|
try {
|
|
17434
|
-
const
|
|
18435
|
+
const dirname19 = path29.dirname(p);
|
|
17435
18436
|
let root;
|
|
17436
|
-
if (projectRootCache.has(
|
|
17437
|
-
root = projectRootCache.get(
|
|
18437
|
+
if (projectRootCache.has(dirname19)) {
|
|
18438
|
+
root = projectRootCache.get(dirname19) ?? null;
|
|
17438
18439
|
} else {
|
|
17439
|
-
const project = findProject(
|
|
18440
|
+
const project = findProject(dirname19);
|
|
17440
18441
|
root = project?.root ?? null;
|
|
17441
|
-
projectRootCache.set(
|
|
18442
|
+
projectRootCache.set(dirname19, root);
|
|
17442
18443
|
}
|
|
17443
18444
|
if (root) lastKnownProjectRoots.set(dir, root);
|
|
17444
18445
|
} catch {
|
|
@@ -17457,7 +18458,7 @@ function sleepSyncMs(ms) {
|
|
|
17457
18458
|
function drainOnce(dir, index, remove) {
|
|
17458
18459
|
const queuePath = dirtyQueuePathFor(dir);
|
|
17459
18460
|
const draining = `${queuePath}.draining`;
|
|
17460
|
-
const dbPath =
|
|
18461
|
+
const dbPath = path29.join(dir, "global.db");
|
|
17461
18462
|
const indexFn = index ?? makeIndexer(dbPath);
|
|
17462
18463
|
const removeFn = remove ?? makeRemover(dbPath);
|
|
17463
18464
|
let processed = 0;
|
|
@@ -17474,7 +18475,7 @@ function drainOnce(dir, index, remove) {
|
|
|
17474
18475
|
let drainingContent = null;
|
|
17475
18476
|
for (let attempt = 0; attempt < DRAINING_READ_ATTEMPTS; attempt++) {
|
|
17476
18477
|
try {
|
|
17477
|
-
drainingContent =
|
|
18478
|
+
drainingContent = fs29.readFileSync(drainingFile, "utf8");
|
|
17478
18479
|
break;
|
|
17479
18480
|
} catch {
|
|
17480
18481
|
if (attempt < DRAINING_READ_ATTEMPTS - 1) sleepSyncMs(DRAINING_READ_RETRY_DELAY_MS);
|
|
@@ -17482,7 +18483,7 @@ function drainOnce(dir, index, remove) {
|
|
|
17482
18483
|
}
|
|
17483
18484
|
if (drainingContent === null) {
|
|
17484
18485
|
try {
|
|
17485
|
-
|
|
18486
|
+
fs29.renameSync(drainingFile, `${drainingFile}.corrupt-${Date.now()}`);
|
|
17486
18487
|
unclearedDrainingSnapshots.delete(drainingFile);
|
|
17487
18488
|
} catch {
|
|
17488
18489
|
}
|
|
@@ -17492,23 +18493,23 @@ function drainOnce(dir, index, remove) {
|
|
|
17492
18493
|
processed += processDirtyBatch(parseDirtyQueueLines(drainingContent), indexFn, removeFn, dir, requeueFn);
|
|
17493
18494
|
}
|
|
17494
18495
|
try {
|
|
17495
|
-
|
|
18496
|
+
fs29.rmSync(drainingFile, { force: true });
|
|
17496
18497
|
unclearedDrainingSnapshots.delete(drainingFile);
|
|
17497
18498
|
} catch {
|
|
17498
18499
|
try {
|
|
17499
|
-
|
|
18500
|
+
fs29.renameSync(drainingFile, `${drainingFile}.corrupt-${Date.now()}`);
|
|
17500
18501
|
unclearedDrainingSnapshots.delete(drainingFile);
|
|
17501
18502
|
} catch {
|
|
17502
18503
|
unclearedDrainingSnapshots.set(drainingFile, drainingSnapshotStamp(drainingFile, drainingContent));
|
|
17503
18504
|
}
|
|
17504
18505
|
}
|
|
17505
18506
|
}
|
|
17506
|
-
if (
|
|
17507
|
-
const claimTarget =
|
|
18507
|
+
if (fs29.existsSync(queuePath)) {
|
|
18508
|
+
const claimTarget = fs29.existsSync(draining) ? `${draining}.alt-${Date.now()}` : draining;
|
|
17508
18509
|
let claimed = false;
|
|
17509
18510
|
for (let attempt = 0; attempt < 5; attempt++) {
|
|
17510
18511
|
try {
|
|
17511
|
-
|
|
18512
|
+
fs29.renameSync(queuePath, claimTarget);
|
|
17512
18513
|
claimed = true;
|
|
17513
18514
|
break;
|
|
17514
18515
|
} catch {
|
|
@@ -17519,14 +18520,14 @@ function drainOnce(dir, index, remove) {
|
|
|
17519
18520
|
let claimedContent = "";
|
|
17520
18521
|
let readOk = false;
|
|
17521
18522
|
try {
|
|
17522
|
-
claimedContent =
|
|
18523
|
+
claimedContent = fs29.readFileSync(claimTarget, "utf8");
|
|
17523
18524
|
readOk = true;
|
|
17524
18525
|
} catch {
|
|
17525
18526
|
}
|
|
17526
18527
|
if (readOk) {
|
|
17527
18528
|
processed += processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn);
|
|
17528
18529
|
try {
|
|
17529
|
-
const recheck =
|
|
18530
|
+
const recheck = fs29.readFileSync(claimTarget, "utf8");
|
|
17530
18531
|
if (recheck !== claimedContent) {
|
|
17531
18532
|
const extra = recheck.startsWith(claimedContent) ? recheck.slice(claimedContent.length) : recheck;
|
|
17532
18533
|
for (const p of parseDirtyQueueLines(extra)) appendToDirtyQueue(dir, p);
|
|
@@ -17534,10 +18535,10 @@ function drainOnce(dir, index, remove) {
|
|
|
17534
18535
|
} catch {
|
|
17535
18536
|
}
|
|
17536
18537
|
try {
|
|
17537
|
-
|
|
18538
|
+
fs29.rmSync(claimTarget, { force: true });
|
|
17538
18539
|
} catch {
|
|
17539
18540
|
try {
|
|
17540
|
-
|
|
18541
|
+
fs29.renameSync(claimTarget, `${claimTarget}.corrupt-${Date.now()}`);
|
|
17541
18542
|
} catch {
|
|
17542
18543
|
unclearedDrainingSnapshots.set(claimTarget, drainingSnapshotStamp(claimTarget, claimedContent));
|
|
17543
18544
|
}
|
|
@@ -17570,7 +18571,7 @@ function pidAlive(pid) {
|
|
|
17570
18571
|
}
|
|
17571
18572
|
function readPidFile(dir) {
|
|
17572
18573
|
try {
|
|
17573
|
-
const raw =
|
|
18574
|
+
const raw = fs29.readFileSync(workerPidPath(dir), "utf8").trim();
|
|
17574
18575
|
if (!/^\d+$/.test(raw)) return null;
|
|
17575
18576
|
return parseInt(raw, 10);
|
|
17576
18577
|
} catch {
|
|
@@ -17583,20 +18584,20 @@ function isWorkerRunning(dir = dataDir()) {
|
|
|
17583
18584
|
return pidAlive(pid) && hasFreshWorkerHeartbeat(dir, pid);
|
|
17584
18585
|
}
|
|
17585
18586
|
function workerHealthCheckMarkerPath(dir) {
|
|
17586
|
-
return
|
|
18587
|
+
return path29.join(dir, "worker-healthcheck.marker");
|
|
17587
18588
|
}
|
|
17588
18589
|
var WORKER_HEALTHCHECK_MIN_INTERVAL_MS = 5 * 60 * 1e3;
|
|
17589
18590
|
function ensureWorkerAlive(dir = dataDir()) {
|
|
17590
18591
|
if (process.env["TOKEN_GOAT_NO_WORKER_SPAWN"] === "1") return;
|
|
17591
18592
|
const markerPath = workerHealthCheckMarkerPath(dir);
|
|
17592
18593
|
try {
|
|
17593
|
-
const stat2 =
|
|
18594
|
+
const stat2 = fs29.statSync(markerPath);
|
|
17594
18595
|
if (Date.now() - stat2.mtimeMs < WORKER_HEALTHCHECK_MIN_INTERVAL_MS) return;
|
|
17595
18596
|
} catch {
|
|
17596
18597
|
}
|
|
17597
18598
|
try {
|
|
17598
18599
|
ensureDirSync(dir);
|
|
17599
|
-
|
|
18600
|
+
fs29.writeFileSync(markerPath, "");
|
|
17600
18601
|
} catch {
|
|
17601
18602
|
}
|
|
17602
18603
|
if (isWorkerRunning(dir)) return;
|
|
@@ -17626,7 +18627,7 @@ function stopWorker(dir = dataDir()) {
|
|
|
17626
18627
|
}
|
|
17627
18628
|
if (readPidFile(dir) === pid) {
|
|
17628
18629
|
try {
|
|
17629
|
-
|
|
18630
|
+
fs29.rmSync(workerPidPath(dir), { force: true });
|
|
17630
18631
|
} catch {
|
|
17631
18632
|
}
|
|
17632
18633
|
}
|
|
@@ -17641,7 +18642,7 @@ var WorkerAlreadyRunningError = class extends Error {
|
|
|
17641
18642
|
function claimWorkerPidFile(dir, pid) {
|
|
17642
18643
|
const pidPath = workerPidPath(dir);
|
|
17643
18644
|
try {
|
|
17644
|
-
|
|
18645
|
+
fs29.writeFileSync(pidPath, `${pid}
|
|
17645
18646
|
`, { flag: "wx" });
|
|
17646
18647
|
return true;
|
|
17647
18648
|
} catch (e) {
|
|
@@ -17658,11 +18659,11 @@ function claimWorkerPidFile(dir, pid) {
|
|
|
17658
18659
|
}
|
|
17659
18660
|
}
|
|
17660
18661
|
try {
|
|
17661
|
-
|
|
18662
|
+
fs29.rmSync(pidPath, { force: true });
|
|
17662
18663
|
} catch {
|
|
17663
18664
|
}
|
|
17664
18665
|
try {
|
|
17665
|
-
|
|
18666
|
+
fs29.writeFileSync(pidPath, `${pid}
|
|
17666
18667
|
`, { flag: "wx" });
|
|
17667
18668
|
return true;
|
|
17668
18669
|
} catch (e2) {
|
|
@@ -17672,9 +18673,9 @@ function claimWorkerPidFile(dir, pid) {
|
|
|
17672
18673
|
}
|
|
17673
18674
|
function daemonEntryScript() {
|
|
17674
18675
|
const self = fileURLToPath2(import.meta.url);
|
|
17675
|
-
const launcher =
|
|
18676
|
+
const launcher = path29.join(path29.dirname(self), "token-goat.mjs");
|
|
17676
18677
|
try {
|
|
17677
|
-
if (
|
|
18678
|
+
if (fs29.existsSync(launcher)) return launcher;
|
|
17678
18679
|
} catch {
|
|
17679
18680
|
}
|
|
17680
18681
|
return self;
|
|
@@ -17685,7 +18686,7 @@ function startDetachedWorker(opts) {
|
|
|
17685
18686
|
try {
|
|
17686
18687
|
ensureDirSync(dir);
|
|
17687
18688
|
} catch (e) {
|
|
17688
|
-
if (e.code !== "EEXIST" || !
|
|
18689
|
+
if (e.code !== "EEXIST" || !fs29.existsSync(dir)) throw e;
|
|
17689
18690
|
}
|
|
17690
18691
|
const child = spawn2(
|
|
17691
18692
|
process.execPath,
|
|
@@ -17722,7 +18723,7 @@ async function runWorkerLoop(dir, pollIntervalMs, shouldStop = () => false) {
|
|
|
17722
18723
|
let lastKnownRootsSweepMs = 0;
|
|
17723
18724
|
let ownedPidFile = false;
|
|
17724
18725
|
while (!shouldStop()) {
|
|
17725
|
-
if (!
|
|
18726
|
+
if (!fs29.existsSync(dir)) break;
|
|
17726
18727
|
const pidOwner = readPidFile(dir);
|
|
17727
18728
|
if (pidOwner === process.pid) ownedPidFile = true;
|
|
17728
18729
|
else if (ownedPidFile && pidOwner !== null) break;
|
|
@@ -17747,7 +18748,7 @@ async function runWorkerLoop(dir, pollIntervalMs, shouldStop = () => false) {
|
|
|
17747
18748
|
}
|
|
17748
18749
|
if (Date.now() - lastKnownRootsSweepMs >= KNOWN_ROOTS_SWEEP_INTERVAL_MS) {
|
|
17749
18750
|
try {
|
|
17750
|
-
const result = sweepKnownRoots(
|
|
18751
|
+
const result = sweepKnownRoots(path29.join(dir, "global.db"));
|
|
17751
18752
|
if (result.flaggedRoots.length > 0) {
|
|
17752
18753
|
appendWorkerErrorLog(
|
|
17753
18754
|
dir,
|
|
@@ -17770,7 +18771,7 @@ function runDetachedWorkerDaemon() {
|
|
|
17770
18771
|
process.on("exit", () => {
|
|
17771
18772
|
if (readPidFile(dir) === process.pid) {
|
|
17772
18773
|
try {
|
|
17773
|
-
|
|
18774
|
+
fs29.rmSync(workerPidPath(dir), { force: true });
|
|
17774
18775
|
} catch {
|
|
17775
18776
|
}
|
|
17776
18777
|
}
|
|
@@ -17780,26 +18781,26 @@ function runDetachedWorkerDaemon() {
|
|
|
17780
18781
|
}
|
|
17781
18782
|
|
|
17782
18783
|
// src/hooks_index.ts
|
|
17783
|
-
import * as
|
|
17784
|
-
import * as
|
|
18784
|
+
import * as fs30 from "node:fs";
|
|
18785
|
+
import * as path30 from "node:path";
|
|
17785
18786
|
function dirtyQueuePath() {
|
|
17786
|
-
return
|
|
18787
|
+
return path30.join(dataDir(), "queue", "dirty.txt");
|
|
17787
18788
|
}
|
|
17788
18789
|
function appendDirtyPath(normalizedPath2) {
|
|
17789
18790
|
const queuePath = dirtyQueuePath();
|
|
17790
|
-
const dir =
|
|
18791
|
+
const dir = path30.dirname(queuePath);
|
|
17791
18792
|
try {
|
|
17792
18793
|
ensureDirSync(dir);
|
|
17793
18794
|
} catch (e) {
|
|
17794
|
-
if (e.code !== "EEXIST" || !
|
|
18795
|
+
if (e.code !== "EEXIST" || !fs30.existsSync(dir)) throw e;
|
|
17795
18796
|
}
|
|
17796
18797
|
let leadingNewline = "";
|
|
17797
18798
|
try {
|
|
17798
|
-
const existing =
|
|
18799
|
+
const existing = fs30.readFileSync(queuePath, "utf8");
|
|
17799
18800
|
if (existing.length > 0 && !existing.endsWith("\n")) leadingNewline = "\n";
|
|
17800
18801
|
} catch {
|
|
17801
18802
|
}
|
|
17802
|
-
|
|
18803
|
+
fs30.appendFileSync(queuePath, `${leadingNewline}${encodeDirtyQueueLine(normalizedPath2)}
|
|
17803
18804
|
`);
|
|
17804
18805
|
}
|
|
17805
18806
|
function enqueueDirtyPathSafe(filePath, opts) {
|
|
@@ -17812,7 +18813,7 @@ function getDirtyPaths() {
|
|
|
17812
18813
|
const queuePath = dirtyQueuePath();
|
|
17813
18814
|
let raw;
|
|
17814
18815
|
try {
|
|
17815
|
-
raw =
|
|
18816
|
+
raw = fs30.readFileSync(queuePath, "utf8");
|
|
17816
18817
|
} catch {
|
|
17817
18818
|
return [];
|
|
17818
18819
|
}
|
|
@@ -17821,9 +18822,9 @@ function getDirtyPaths() {
|
|
|
17821
18822
|
function preCompactIndexHandler(_event) {
|
|
17822
18823
|
const paths = getDirtyPaths();
|
|
17823
18824
|
if (paths.length > 0) {
|
|
17824
|
-
const sidecar =
|
|
18825
|
+
const sidecar = path30.join(dataDir(), "queue", "pending.txt");
|
|
17825
18826
|
try {
|
|
17826
|
-
ensureDirSync(
|
|
18827
|
+
ensureDirSync(path30.dirname(sidecar));
|
|
17827
18828
|
atomicWriteBytes(sidecar, Buffer.from(`${paths.join("\n")}
|
|
17828
18829
|
`, "utf8"));
|
|
17829
18830
|
} catch {
|
|
@@ -17941,6 +18942,7 @@ function urlPolicyDenialReason(url, policy) {
|
|
|
17941
18942
|
export {
|
|
17942
18943
|
createLazyModuleLoader,
|
|
17943
18944
|
extractPdfText,
|
|
18945
|
+
locatePdfPages,
|
|
17944
18946
|
extractPdfOutline,
|
|
17945
18947
|
extractPdfMeta,
|
|
17946
18948
|
docxOutline,
|
|
@@ -18017,6 +19019,8 @@ export {
|
|
|
18017
19019
|
getGlobMatchCount,
|
|
18018
19020
|
setLastTabContext,
|
|
18019
19021
|
getLastTabContext,
|
|
19022
|
+
hasSeenImage,
|
|
19023
|
+
recordSeenImage,
|
|
18020
19024
|
recordOutstandingAgentSpawn,
|
|
18021
19025
|
getOutstandingAgentSpawns,
|
|
18022
19026
|
removeOutstandingAgentSpawn,
|
|
@@ -18040,6 +19044,8 @@ export {
|
|
|
18040
19044
|
installCodex,
|
|
18041
19045
|
uninstallCodex,
|
|
18042
19046
|
isCodexInstalled,
|
|
19047
|
+
copilotCliUserRoot,
|
|
19048
|
+
copilotCliMcpToolsDir,
|
|
18043
19049
|
copilotCliConfigPath,
|
|
18044
19050
|
copilotCliScriptPath,
|
|
18045
19051
|
installCopilotCli,
|
|
@@ -18063,11 +19069,13 @@ export {
|
|
|
18063
19069
|
loadBlob,
|
|
18064
19070
|
listBlobs,
|
|
18065
19071
|
pruneBlobs,
|
|
19072
|
+
estimateTokensFromLength,
|
|
18066
19073
|
estimateTokens,
|
|
18067
19074
|
trimToBudget,
|
|
18068
19075
|
capJsonRows,
|
|
18069
19076
|
SESSIONS_SUBDIR,
|
|
18070
19077
|
AGENT_SALT_MARKER,
|
|
19078
|
+
sessionSidecarPath,
|
|
18071
19079
|
listSiblingSessionStates,
|
|
18072
19080
|
loadSessionState,
|
|
18073
19081
|
saveSessionState,
|
|
@@ -18101,11 +19109,14 @@ export {
|
|
|
18101
19109
|
getSkillFilePath,
|
|
18102
19110
|
installedSkillPath,
|
|
18103
19111
|
pruneSkillOutputs,
|
|
19112
|
+
isOcrEngineAvailable,
|
|
18104
19113
|
ocrImage,
|
|
18105
19114
|
isTextHeavy,
|
|
18106
19115
|
formatShrinkSummary,
|
|
18107
19116
|
isImagePath,
|
|
19117
|
+
ImageDecodeError,
|
|
18108
19118
|
probeImageMeta,
|
|
19119
|
+
imageQualifiesForShrink,
|
|
18109
19120
|
shrinkImage,
|
|
18110
19121
|
compactPathFor,
|
|
18111
19122
|
isCompactFresh,
|
|
@@ -18116,6 +19127,8 @@ export {
|
|
|
18116
19127
|
compactDoc,
|
|
18117
19128
|
OVER_FETCH_FACTOR,
|
|
18118
19129
|
MAX_OVER_FETCH,
|
|
19130
|
+
isAvailable,
|
|
19131
|
+
embeddingBackendLoadError,
|
|
18119
19132
|
searchSemantic,
|
|
18120
19133
|
mergeNearbyHits,
|
|
18121
19134
|
embeddingsDepsAvailable,
|