token-goat 2.6.35 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -7
- package/SECURITY.md +47 -16
- package/dist/{token-goat-chunk-I2KCEGIP.mjs → token-goat-chunk-2F6TFBZE.mjs} +1355 -408
- package/dist/token-goat-chunk-324QOJYZ.mjs +91 -0
- package/dist/{token-goat-chunk-K7W3P2TJ.mjs → token-goat-chunk-44Y77VHR.mjs} +425 -39
- package/dist/{token-goat-chunk-UIZVX3HN.mjs → token-goat-chunk-4KZILRZN.mjs} +2 -2
- package/dist/token-goat-chunk-5CVKO3DA.mjs +185 -0
- package/dist/token-goat-chunk-A62K4XW2.mjs +23 -0
- package/dist/{token-goat-chunk-SIHYQCTM.mjs → token-goat-chunk-AO6MFFTW.mjs} +331 -13
- package/dist/{token-goat-chunk-DRAIYVUI.mjs → token-goat-chunk-FBGBTICM.mjs} +4 -4
- package/dist/{token-goat-chunk-YF67EWRH.mjs → token-goat-chunk-J35AKWEQ.mjs} +5 -5
- package/dist/{token-goat-hook-chunk-4POU6OHK.mjs → token-goat-chunk-LVCBDJVE.mjs} +318 -51
- package/dist/token-goat-chunk-R4SR7MQY.mjs +486 -0
- package/dist/{token-goat-chunk-D5IZEGN2.mjs → token-goat-chunk-SRAR6DOK.mjs} +144 -147
- package/dist/{token-goat-hook-chunk-NIPXIAPA.mjs → token-goat-chunk-TUPJRK7R.mjs} +1 -1
- package/dist/{token-goat-chunk-HW4VUKJ5.mjs → token-goat-chunk-VXSYZGBA.mjs} +589 -141
- package/dist/{token-goat-chunk-DG53MVNJ.mjs → token-goat-chunk-WN5T5EW5.mjs} +212 -212
- package/dist/token-goat-hook.mjs +7 -7
- package/dist/token-goat.core.mjs +5 -5
- package/package.json +18 -15
- package/scripts/install-git-hooks.mjs +55 -0
- package/dist/token-goat-chunk-DQIT5JIF.mjs +0 -13659
- package/dist/token-goat-chunk-Y2NH6DGZ.mjs +0 -10048
- package/dist/token-goat-hook-chunk-2O7P4Z6Q.mjs +0 -6419
- package/dist/token-goat-hook-chunk-3AAH72SO.mjs +0 -15970
- package/dist/token-goat-hook-chunk-3QYSN4QV.mjs +0 -14764
- package/dist/token-goat-hook-chunk-BUOCULAM.mjs +0 -29
- package/dist/token-goat-hook-chunk-CRBMPT74.mjs +0 -900
- package/dist/token-goat-hook-chunk-D6UJEFHX.mjs +0 -153
- package/dist/token-goat-hook-chunk-J4HKWUWZ.mjs +0 -23
- package/dist/token-goat-hook-chunk-NXVT4F3S.mjs +0 -10406
- package/dist/token-goat-hook-chunk-RFRLWOQH.mjs +0 -11
- package/dist/token-goat-hook-chunk-VU6VGZBM.mjs +0 -706
- package/dist/token-goat-hook-chunk-XM4W3WCO.mjs +0 -18144
- package/dist/token-goat-hook-chunk-XUMIVYEN.mjs +0 -109
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
detectLanguage,
|
|
20
20
|
displaySafePath,
|
|
21
21
|
displaySafeText,
|
|
22
|
+
ensureDataDirPrivate,
|
|
22
23
|
ensureDirSync,
|
|
23
24
|
escapeRegExp,
|
|
24
25
|
extractEnv,
|
|
@@ -56,6 +57,7 @@ import {
|
|
|
56
57
|
parse,
|
|
57
58
|
precedingDocComment,
|
|
58
59
|
propagateEndLinesToSymbols,
|
|
60
|
+
pushAll,
|
|
59
61
|
recordStat,
|
|
60
62
|
redactIfDotenv,
|
|
61
63
|
redactSecrets,
|
|
@@ -88,7 +90,7 @@ import {
|
|
|
88
90
|
withFileLock,
|
|
89
91
|
writeIfDifferent,
|
|
90
92
|
writeJsonSettings
|
|
91
|
-
} from "./token-goat-chunk-
|
|
93
|
+
} from "./token-goat-chunk-44Y77VHR.mjs";
|
|
92
94
|
import {
|
|
93
95
|
registerReset
|
|
94
96
|
} from "./token-goat-chunk-AO2QD2AG.mjs";
|
|
@@ -112,15 +114,221 @@ function createLazyModuleLoader(load3, errorLabel) {
|
|
|
112
114
|
};
|
|
113
115
|
}
|
|
114
116
|
|
|
117
|
+
// src/xml_parser.ts
|
|
118
|
+
var MAX_XML_DEPTH = 512;
|
|
119
|
+
var NAMED_ENTITIES = {
|
|
120
|
+
lt: "<",
|
|
121
|
+
gt: ">",
|
|
122
|
+
amp: "&",
|
|
123
|
+
quot: '"',
|
|
124
|
+
apos: "'"
|
|
125
|
+
};
|
|
126
|
+
function decodeXmlEntities(text) {
|
|
127
|
+
if (!text.includes("&")) return text;
|
|
128
|
+
return text.replace(/&(#[xX][0-9a-fA-F]+|#[0-9]+|[a-zA-Z_][\w.:-]*);/g, (whole, body) => {
|
|
129
|
+
if (body.charCodeAt(0) === 35) {
|
|
130
|
+
const isHex = body.charCodeAt(1) === 120 || body.charCodeAt(1) === 88;
|
|
131
|
+
const digits = isHex ? body.slice(2) : body.slice(1);
|
|
132
|
+
const code = parseInt(digits, isHex ? 16 : 10);
|
|
133
|
+
if (!Number.isFinite(code) || code < 0 || code > 1114111) return whole;
|
|
134
|
+
if (code >= 55296 && code <= 57343) return whole;
|
|
135
|
+
return String.fromCodePoint(code);
|
|
136
|
+
}
|
|
137
|
+
const named = NAMED_ENTITIES[body];
|
|
138
|
+
return named ?? whole;
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
function newFrame(name) {
|
|
142
|
+
return { name, children: /* @__PURE__ */ new Map(), text: [], attrs: [] };
|
|
143
|
+
}
|
|
144
|
+
function finishFrame(frame) {
|
|
145
|
+
const text = frame.text.join("");
|
|
146
|
+
if (frame.children.size === 0 && frame.attrs.length === 0) return text;
|
|
147
|
+
const obj = {};
|
|
148
|
+
for (const [key, value] of frame.children) obj[key] = value;
|
|
149
|
+
if (text.length > 0) obj["#text"] = text;
|
|
150
|
+
for (const [key, value] of frame.attrs) obj[`@_${key}`] = value;
|
|
151
|
+
return obj;
|
|
152
|
+
}
|
|
153
|
+
function addChild(parent, name, value) {
|
|
154
|
+
const existing = parent.children.get(name);
|
|
155
|
+
if (existing === void 0 && !parent.children.has(name)) {
|
|
156
|
+
parent.children.set(name, value);
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
if (Array.isArray(existing)) {
|
|
160
|
+
existing.push(value);
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
parent.children.set(name, [existing, value]);
|
|
164
|
+
}
|
|
165
|
+
var WHITESPACE = /* @__PURE__ */ new Set([" ", " ", "\n", "\r"]);
|
|
166
|
+
function findTagEnd(src, from) {
|
|
167
|
+
let quote = "";
|
|
168
|
+
for (let i = from; i < src.length; i++) {
|
|
169
|
+
const ch = src[i];
|
|
170
|
+
if (quote !== "") {
|
|
171
|
+
if (ch === quote) quote = "";
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
if (ch === '"' || ch === "'") {
|
|
175
|
+
quote = ch;
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
if (ch === ">") return i;
|
|
179
|
+
}
|
|
180
|
+
return -1;
|
|
181
|
+
}
|
|
182
|
+
function skipDeclaration(src, from) {
|
|
183
|
+
let quote = "";
|
|
184
|
+
let inSubset = false;
|
|
185
|
+
for (let i = from + 2; i < src.length; i++) {
|
|
186
|
+
const ch = src[i];
|
|
187
|
+
if (quote !== "") {
|
|
188
|
+
if (ch === quote) quote = "";
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
if (ch === '"' || ch === "'") {
|
|
192
|
+
quote = ch;
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
if (ch === "[") inSubset = true;
|
|
196
|
+
else if (ch === "]") inSubset = false;
|
|
197
|
+
else if (ch === ">" && !inSubset) return i + 1;
|
|
198
|
+
}
|
|
199
|
+
return src.length;
|
|
200
|
+
}
|
|
201
|
+
function parseTagBody(body) {
|
|
202
|
+
let i = 0;
|
|
203
|
+
while (i < body.length && !WHITESPACE.has(body[i])) i++;
|
|
204
|
+
const name = body.slice(0, i);
|
|
205
|
+
const attrs = [];
|
|
206
|
+
while (i < body.length) {
|
|
207
|
+
while (i < body.length && WHITESPACE.has(body[i])) i++;
|
|
208
|
+
if (i >= body.length) break;
|
|
209
|
+
const nameStart = i;
|
|
210
|
+
while (i < body.length && !WHITESPACE.has(body[i]) && body[i] !== "=") i++;
|
|
211
|
+
const attrName = body.slice(nameStart, i);
|
|
212
|
+
if (attrName.length === 0) {
|
|
213
|
+
i++;
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
while (i < body.length && WHITESPACE.has(body[i])) i++;
|
|
217
|
+
if (body[i] !== "=") {
|
|
218
|
+
attrs.push([attrName, ""]);
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
i++;
|
|
222
|
+
while (i < body.length && WHITESPACE.has(body[i])) i++;
|
|
223
|
+
const quote = body[i];
|
|
224
|
+
if (quote === '"' || quote === "'") {
|
|
225
|
+
const valueStart = i + 1;
|
|
226
|
+
const valueEnd = body.indexOf(quote, valueStart);
|
|
227
|
+
const end = valueEnd === -1 ? body.length : valueEnd;
|
|
228
|
+
attrs.push([attrName, decodeXmlEntities(body.slice(valueStart, end))]);
|
|
229
|
+
i = end + 1;
|
|
230
|
+
} else {
|
|
231
|
+
const valueStart = i;
|
|
232
|
+
while (i < body.length && !WHITESPACE.has(body[i])) i++;
|
|
233
|
+
attrs.push([attrName, decodeXmlEntities(body.slice(valueStart, i))]);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
return { name, attrs };
|
|
237
|
+
}
|
|
238
|
+
function parseXml(xml) {
|
|
239
|
+
const src = xml.charCodeAt(0) === 65279 ? xml.slice(1) : xml;
|
|
240
|
+
const root = newFrame("");
|
|
241
|
+
const stack = [root];
|
|
242
|
+
const len = src.length;
|
|
243
|
+
let i = 0;
|
|
244
|
+
const top = () => stack[stack.length - 1];
|
|
245
|
+
const pushText = (from, to) => {
|
|
246
|
+
if (to > from) top().text.push(decodeXmlEntities(src.slice(from, to)));
|
|
247
|
+
};
|
|
248
|
+
const closeElement = (name) => {
|
|
249
|
+
let target = -1;
|
|
250
|
+
for (let k = stack.length - 1; k >= 1; k--) {
|
|
251
|
+
if (stack[k].name === name) {
|
|
252
|
+
target = k;
|
|
253
|
+
break;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
if (target === -1) return;
|
|
257
|
+
while (stack.length > target) {
|
|
258
|
+
const frame = stack.pop();
|
|
259
|
+
addChild(top(), frame.name, finishFrame(frame));
|
|
260
|
+
}
|
|
261
|
+
};
|
|
262
|
+
while (i < len) {
|
|
263
|
+
const lt = src.indexOf("<", i);
|
|
264
|
+
if (lt === -1) {
|
|
265
|
+
pushText(i, len);
|
|
266
|
+
break;
|
|
267
|
+
}
|
|
268
|
+
pushText(i, lt);
|
|
269
|
+
if (src.startsWith("<!--", lt)) {
|
|
270
|
+
const end2 = src.indexOf("-->", lt + 4);
|
|
271
|
+
i = end2 === -1 ? len : end2 + 3;
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
274
|
+
if (src.startsWith("<![CDATA[", lt)) {
|
|
275
|
+
const end2 = src.indexOf("]]>", lt + 9);
|
|
276
|
+
top().text.push(src.slice(lt + 9, end2 === -1 ? len : end2));
|
|
277
|
+
i = end2 === -1 ? len : end2 + 3;
|
|
278
|
+
continue;
|
|
279
|
+
}
|
|
280
|
+
if (src.startsWith("<!", lt)) {
|
|
281
|
+
i = skipDeclaration(src, lt);
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
if (src.startsWith("<?", lt)) {
|
|
285
|
+
const end2 = src.indexOf("?>", lt + 2);
|
|
286
|
+
const { name: name2, attrs: attrs2 } = parseTagBody(src.slice(lt + 2, end2 === -1 ? len : end2));
|
|
287
|
+
if (name2.length > 0) {
|
|
288
|
+
const frame2 = newFrame(`?${name2}`);
|
|
289
|
+
frame2.attrs = attrs2;
|
|
290
|
+
addChild(top(), frame2.name, finishFrame(frame2));
|
|
291
|
+
}
|
|
292
|
+
i = end2 === -1 ? len : end2 + 2;
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
if (src.startsWith("</", lt)) {
|
|
296
|
+
const end2 = src.indexOf(">", lt + 2);
|
|
297
|
+
closeElement(src.slice(lt + 2, end2 === -1 ? len : end2).trim());
|
|
298
|
+
i = end2 === -1 ? len : end2 + 1;
|
|
299
|
+
continue;
|
|
300
|
+
}
|
|
301
|
+
const end = findTagEnd(src, lt + 1);
|
|
302
|
+
const tagEnd = end === -1 ? len : end;
|
|
303
|
+
let body = src.slice(lt + 1, tagEnd);
|
|
304
|
+
const selfClosing = body.endsWith("/");
|
|
305
|
+
if (selfClosing) body = body.slice(0, -1);
|
|
306
|
+
const { name, attrs } = parseTagBody(body);
|
|
307
|
+
i = end === -1 ? len : end + 1;
|
|
308
|
+
if (name.length === 0) continue;
|
|
309
|
+
const frame = newFrame(name);
|
|
310
|
+
frame.attrs = attrs;
|
|
311
|
+
if (selfClosing) {
|
|
312
|
+
addChild(top(), name, finishFrame(frame));
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
if (stack.length > MAX_XML_DEPTH) {
|
|
316
|
+
throw new Error(`XML nesting deeper than ${MAX_XML_DEPTH} elements; refusing to parse (this file is not something any office application produces)`);
|
|
317
|
+
}
|
|
318
|
+
stack.push(frame);
|
|
319
|
+
}
|
|
320
|
+
while (stack.length > 1) {
|
|
321
|
+
const frame = stack.pop();
|
|
322
|
+
addChild(top(), frame.name, finishFrame(frame));
|
|
323
|
+
}
|
|
324
|
+
return Object.fromEntries(root.children);
|
|
325
|
+
}
|
|
326
|
+
|
|
115
327
|
// src/ooxml_extract.ts
|
|
116
328
|
var loadFflate = createLazyModuleLoader(
|
|
117
329
|
async () => await import("fflate"),
|
|
118
330
|
"office-file reading disabled (fflate unavailable)"
|
|
119
331
|
);
|
|
120
|
-
var loadXmlParser = createLazyModuleLoader(
|
|
121
|
-
async () => await import("fast-xml-parser"),
|
|
122
|
-
"office-file reading disabled (fast-xml-parser unavailable)"
|
|
123
|
-
);
|
|
124
332
|
var MAX_OOXML_INPUT_BYTES = 50 * 1024 * 1024;
|
|
125
333
|
function accessFailureMessage(err, filePath) {
|
|
126
334
|
const code = err?.code;
|
|
@@ -158,15 +366,7 @@ function decodeZipEntry(entries, entryPath) {
|
|
|
158
366
|
return new TextDecoder("utf-8").decode(bytes);
|
|
159
367
|
}
|
|
160
368
|
async function parseOoxmlPart(xmlText2) {
|
|
161
|
-
|
|
162
|
-
if (!fxp) throw new Error("fast-xml-parser is not installed; run `npm install fast-xml-parser` to enable this command");
|
|
163
|
-
const parser = new fxp.XMLParser({
|
|
164
|
-
ignoreAttributes: false,
|
|
165
|
-
preserveOrder: false,
|
|
166
|
-
trimValues: false,
|
|
167
|
-
parseTagValue: false
|
|
168
|
-
});
|
|
169
|
-
return parser.parse(xmlText2);
|
|
369
|
+
return parseXml(xmlText2);
|
|
170
370
|
}
|
|
171
371
|
function pushTextValue(runs, val) {
|
|
172
372
|
if (Array.isArray(val)) {
|
|
@@ -211,7 +411,7 @@ function collectElements(node, tag) {
|
|
|
211
411
|
const obj = n;
|
|
212
412
|
for (const [key, val] of Object.entries(obj)) {
|
|
213
413
|
if (key === tag) {
|
|
214
|
-
if (Array.isArray(val)) out
|
|
414
|
+
if (Array.isArray(val)) pushAll(out, val);
|
|
215
415
|
else out.push(val);
|
|
216
416
|
} else if (val !== null && typeof val === "object") {
|
|
217
417
|
walk(val);
|
|
@@ -344,9 +544,9 @@ async function notesPathFor(entries, slidePath) {
|
|
|
344
544
|
}
|
|
345
545
|
return null;
|
|
346
546
|
}
|
|
347
|
-
async function parseSlide(entries,
|
|
348
|
-
const xml = decodeZipEntry(entries,
|
|
349
|
-
if (xml === null) throw new Error(`missing part: ${
|
|
547
|
+
async function parseSlide(entries, path31) {
|
|
548
|
+
const xml = decodeZipEntry(entries, path31);
|
|
549
|
+
if (xml === null) throw new Error(`missing part: ${path31}`);
|
|
350
550
|
return parseOoxmlPart(xml);
|
|
351
551
|
}
|
|
352
552
|
async function notesTextFor(entries, notesPath) {
|
|
@@ -362,8 +562,8 @@ async function pptxOutline(filePath) {
|
|
|
362
562
|
const { entries, slidePaths } = await listSlideParts(filePath);
|
|
363
563
|
const out = [];
|
|
364
564
|
for (let i = 0; i < slidePaths.length; i++) {
|
|
365
|
-
const
|
|
366
|
-
const parsed = await parseSlide(entries,
|
|
565
|
+
const path31 = slidePaths[i];
|
|
566
|
+
const parsed = await parseSlide(entries, path31);
|
|
367
567
|
const shapes = slideShapes(parsed);
|
|
368
568
|
const titleShape = shapes.find((s) => {
|
|
369
569
|
const t = shapePlaceholderType(s);
|
|
@@ -372,7 +572,7 @@ async function pptxOutline(filePath) {
|
|
|
372
572
|
const title = titleShape !== void 0 ? shapeText(titleShape) : "";
|
|
373
573
|
const allText = collectTextRuns(parsed, "a:t").join(" ");
|
|
374
574
|
const bodyChars = Math.max(0, allText.length - title.length);
|
|
375
|
-
const hasNotes = (await notesTextFor(entries, await notesPathFor(entries,
|
|
575
|
+
const hasNotes = (await notesTextFor(entries, await notesPathFor(entries, path31))).length > 0;
|
|
376
576
|
out.push({ slide: i + 1, title, bodyChars, hasNotes });
|
|
377
577
|
}
|
|
378
578
|
return out;
|
|
@@ -382,13 +582,13 @@ async function pptxSlideText(filePath, slideNumber, includeNotes) {
|
|
|
382
582
|
if (slideNumber < 1 || slideNumber > slidePaths.length) {
|
|
383
583
|
throw new Error(`slide ${slideNumber} out of range (this deck has ${slidePaths.length} slides)`);
|
|
384
584
|
}
|
|
385
|
-
const
|
|
386
|
-
const parsed = await parseSlide(entries,
|
|
585
|
+
const path31 = slidePaths[slideNumber - 1];
|
|
586
|
+
const parsed = await parseSlide(entries, path31);
|
|
387
587
|
const shapes = slideShapes(parsed);
|
|
388
588
|
const blocks = [...shapes.map(shapeText).filter((t) => t.length > 0), ...tableRowBlocks(parsed)];
|
|
389
589
|
const lines2 = [`# Slide ${slideNumber}`, ...blocks];
|
|
390
590
|
if (includeNotes) {
|
|
391
|
-
const notes = await notesTextFor(entries, await notesPathFor(entries,
|
|
591
|
+
const notes = await notesTextFor(entries, await notesPathFor(entries, path31));
|
|
392
592
|
if (notes.length > 0) lines2.push("", "## Speaker notes", notes);
|
|
393
593
|
}
|
|
394
594
|
return lines2.join("\n\n");
|
|
@@ -2488,7 +2688,10 @@ function makeWorksheet(name, data) {
|
|
|
2488
2688
|
},
|
|
2489
2689
|
eachCell(opts, cb) {
|
|
2490
2690
|
if (opts.includeEmpty) {
|
|
2491
|
-
|
|
2691
|
+
let maxCol = 0;
|
|
2692
|
+
if (rowCells !== void 0) {
|
|
2693
|
+
for (const c of rowCells.keys()) if (c > maxCol) maxCol = c;
|
|
2694
|
+
}
|
|
2492
2695
|
for (let c = 1; c <= maxCol; c++) cb(rowCells?.get(c) ?? EMPTY_CELL, c);
|
|
2493
2696
|
return;
|
|
2494
2697
|
}
|
|
@@ -3019,19 +3222,30 @@ var CLAUDE_CODE_EVENT_NAMES = {
|
|
|
3019
3222
|
notification: "Notification",
|
|
3020
3223
|
stop: "Stop",
|
|
3021
3224
|
pre_compact: "PreCompact",
|
|
3225
|
+
post_compact: "PostCompact",
|
|
3022
3226
|
user_prompt_submit: "UserPromptSubmit",
|
|
3023
3227
|
subagent_stop: "SubagentStop",
|
|
3024
|
-
session_start: "SessionStart"
|
|
3228
|
+
session_start: "SessionStart",
|
|
3229
|
+
// Claude Code has no separate failure event -- a failed tool arrives on PostToolUse there,
|
|
3230
|
+
// and only Copilot splits it out. This entry exists because the map is exhaustive over
|
|
3231
|
+
// HookEventName, and it is a spelling for the response envelope rather than a claim that
|
|
3232
|
+
// Claude Code will ever send this event. Nothing iterates this map to build install config,
|
|
3233
|
+
// so naming an event Claude Code does not have cannot register one.
|
|
3234
|
+
post_tool_use_failure: "PostToolUseFailure"
|
|
3025
3235
|
};
|
|
3026
3236
|
var EVENTS_WITHOUT_ADDITIONAL_CONTEXT = /* @__PURE__ */ new Set([
|
|
3027
3237
|
"notification",
|
|
3028
3238
|
"pre_compact"
|
|
3029
3239
|
]);
|
|
3030
|
-
|
|
3240
|
+
var EVENTS_WITH_RAW_STDOUT_CONTEXT = /* @__PURE__ */ new Set(["pre_compact"]);
|
|
3241
|
+
function serializeOutput(output, eventName, harness) {
|
|
3031
3242
|
switch (output.hookType) {
|
|
3032
3243
|
case "deny":
|
|
3033
3244
|
return JSON.stringify({ decision: "block", reason: output.message });
|
|
3034
3245
|
case "context":
|
|
3246
|
+
if (EVENTS_WITH_RAW_STDOUT_CONTEXT.has(eventName) && harness === "claudecode") {
|
|
3247
|
+
return output.context;
|
|
3248
|
+
}
|
|
3035
3249
|
if (EVENTS_WITHOUT_ADDITIONAL_CONTEXT.has(eventName)) {
|
|
3036
3250
|
return JSON.stringify({ systemMessage: output.context });
|
|
3037
3251
|
}
|
|
@@ -3495,6 +3709,7 @@ var _bashOutputs = /* @__PURE__ */ new Map();
|
|
|
3495
3709
|
var _grepQueries = /* @__PURE__ */ new Map();
|
|
3496
3710
|
var _globQueries = /* @__PURE__ */ new Map();
|
|
3497
3711
|
var _lastTabContext = null;
|
|
3712
|
+
var _seenImageHashes = [];
|
|
3498
3713
|
var _outstandingAgentSpawns = [];
|
|
3499
3714
|
var _outstandingAgentSpawnsAtLoad = [];
|
|
3500
3715
|
var _bashReruns = /* @__PURE__ */ new Set();
|
|
@@ -3694,6 +3909,18 @@ function setLastTabContext(text) {
|
|
|
3694
3909
|
function getLastTabContext() {
|
|
3695
3910
|
return _lastTabContext;
|
|
3696
3911
|
}
|
|
3912
|
+
var MAX_SEEN_IMAGE_HASHES = 16;
|
|
3913
|
+
function hasSeenImage(hash2) {
|
|
3914
|
+
return _seenImageHashes.includes(hash2);
|
|
3915
|
+
}
|
|
3916
|
+
function recordSeenImage(hash2) {
|
|
3917
|
+
const at = _seenImageHashes.indexOf(hash2);
|
|
3918
|
+
if (at !== -1) _seenImageHashes.splice(at, 1);
|
|
3919
|
+
_seenImageHashes.push(hash2);
|
|
3920
|
+
if (_seenImageHashes.length > MAX_SEEN_IMAGE_HASHES) {
|
|
3921
|
+
_seenImageHashes.splice(0, _seenImageHashes.length - MAX_SEEN_IMAGE_HASHES);
|
|
3922
|
+
}
|
|
3923
|
+
}
|
|
3697
3924
|
var MAX_OUTSTANDING_AGENT_SPAWNS = 30;
|
|
3698
3925
|
function recordOutstandingAgentSpawn(prompt) {
|
|
3699
3926
|
_outstandingAgentSpawns.push({ prompt, ts: Date.now() });
|
|
@@ -3807,6 +4034,7 @@ function exportSessionState() {
|
|
|
3807
4034
|
globQueries: Array.from(_globQueries.entries()),
|
|
3808
4035
|
outstandingAgentSpawns: _outstandingAgentSpawns.map((e) => [e.prompt, e.ts]),
|
|
3809
4036
|
..._lastTabContext !== null ? { lastTabContext: _lastTabContext } : {},
|
|
4037
|
+
..._seenImageHashes.length > 0 ? { seenImageHashes: [..._seenImageHashes] } : {},
|
|
3810
4038
|
..._compactedAt > 0 ? { compactedAt: _compactedAt } : {}
|
|
3811
4039
|
};
|
|
3812
4040
|
}
|
|
@@ -3832,6 +4060,7 @@ function importSessionState(s) {
|
|
|
3832
4060
|
_outstandingAgentSpawns = (s.outstandingAgentSpawns ?? []).map(([prompt, ts]) => ({ prompt, ts }));
|
|
3833
4061
|
_outstandingAgentSpawnsAtLoad = [..._outstandingAgentSpawns];
|
|
3834
4062
|
_lastTabContext = s.lastTabContext ?? null;
|
|
4063
|
+
_seenImageHashes = [...s.seenImageHashes ?? []];
|
|
3835
4064
|
_compactedAt = s.compactedAt ?? 0;
|
|
3836
4065
|
}
|
|
3837
4066
|
registerReset(() => {
|
|
@@ -3853,6 +4082,7 @@ registerReset(() => {
|
|
|
3853
4082
|
_outstandingAgentSpawns = [];
|
|
3854
4083
|
_outstandingAgentSpawnsAtLoad = [];
|
|
3855
4084
|
_lastTabContext = null;
|
|
4085
|
+
_seenImageHashes = [];
|
|
3856
4086
|
_compactedAt = 0;
|
|
3857
4087
|
_sessionId = null;
|
|
3858
4088
|
});
|
|
@@ -3875,9 +4105,11 @@ const VALID_HOOK_EVENTS = new Set([
|
|
|
3875
4105
|
'notification',
|
|
3876
4106
|
'stop',
|
|
3877
4107
|
'pre_compact',
|
|
4108
|
+
'post_compact',
|
|
3878
4109
|
'user_prompt_submit',
|
|
3879
4110
|
'subagent_stop',
|
|
3880
4111
|
'session_start',
|
|
4112
|
+
'post_tool_use_failure',
|
|
3881
4113
|
])`;
|
|
3882
4114
|
var SHIM_TRY_IN_PROCESS = `// Attempts the in-process hook call: import()s dist/token-goat-hook.mjs (a sibling of
|
|
3883
4115
|
// the baked token-goat entry path, built with zero load-time side effects -- unlike
|
|
@@ -4026,6 +4258,7 @@ var HOOK_EVENT_MAP = [
|
|
|
4026
4258
|
["PreToolUse", "pre_tool_use"],
|
|
4027
4259
|
["PostToolUse", "post_tool_use"],
|
|
4028
4260
|
["PreCompact", "pre_compact"],
|
|
4261
|
+
["PostCompact", "post_compact"],
|
|
4029
4262
|
["UserPromptSubmit", "user_prompt_submit"],
|
|
4030
4263
|
["SubagentStop", "subagent_stop"],
|
|
4031
4264
|
["SessionStart", "session_start"]
|
|
@@ -4342,9 +4575,11 @@ const HOOK_EVENT_NAME_MAP = {
|
|
|
4342
4575
|
notification: 'Notification',
|
|
4343
4576
|
stop: 'Stop',
|
|
4344
4577
|
pre_compact: 'PreCompact',
|
|
4578
|
+
post_compact: 'PostCompact',
|
|
4345
4579
|
user_prompt_submit: 'UserPromptSubmit',
|
|
4346
4580
|
subagent_stop: 'SubagentStop',
|
|
4347
4581
|
session_start: 'SessionStart',
|
|
4582
|
+
post_tool_use_failure: 'PostToolUseFailure',
|
|
4348
4583
|
}
|
|
4349
4584
|
|
|
4350
4585
|
function stripTg(value) {
|
|
@@ -4725,10 +4960,19 @@ const path = require('node:path')
|
|
|
4725
4960
|
const { pathToFileURL } = require('node:url')
|
|
4726
4961
|
|
|
4727
4962
|
// Copilot event name -> token-goat internal HookEventName (src/types.ts's
|
|
4728
|
-
// HOOK_EVENTS). Only these
|
|
4729
|
-
// Copilot event (sessionEnd,
|
|
4730
|
-
//
|
|
4963
|
+
// HOOK_EVENTS). Only these eight have a token-goat handler; every other real
|
|
4964
|
+
// Copilot event (sessionEnd, subagentStart, errorOccurred, notification,
|
|
4965
|
+
// permissionRequest) is left unimplemented
|
|
4731
4966
|
// rather than guessed at, and falls through to the default no-op below.
|
|
4967
|
+
// postToolUseFailure is the newest of the eight and the only one whose channel
|
|
4968
|
+
// costs tokens instead of saving them: it fires instead of postToolUse when a
|
|
4969
|
+
// tool result is a failure, and accepts only additionalContext back --
|
|
4970
|
+
// modifiedResult and suppressOutput are not honored there, so a failed result
|
|
4971
|
+
// still cannot be fenced, compressed or shrunk. It is wired anyway because
|
|
4972
|
+
// additionalContext demonstrably reaches the model (see the postToolUseFailure
|
|
4973
|
+
// branch in translate() for the bundle offset), and hooks_tool_failure.ts
|
|
4974
|
+
// spends that channel only on an exact repeat failure, where staying silent
|
|
4975
|
+
// costs a whole wasted retry.
|
|
4732
4976
|
// 'sessionStart' was previously a permanent no-op on the stated grounds that
|
|
4733
4977
|
// token-goat has no internal session_start handler. That was simply wrong --
|
|
4734
4978
|
// hooks_session_start.ts has long emitted the command-routing reminder that
|
|
@@ -4748,6 +4992,7 @@ const COPILOT_TO_TG_EVENT = {
|
|
|
4748
4992
|
agentStop: 'stop',
|
|
4749
4993
|
subagentStop: 'subagent_stop',
|
|
4750
4994
|
userPromptSubmitted: 'user_prompt_submit',
|
|
4995
|
+
postToolUseFailure: 'post_tool_use_failure',
|
|
4751
4996
|
}
|
|
4752
4997
|
|
|
4753
4998
|
// Copilot built-in tool name -> token-goat internal tool name. Confirmed via
|
|
@@ -4762,9 +5007,47 @@ const COPILOT_TO_TG_EVENT = {
|
|
|
4762
5007
|
// MCP-server tool invocations (<server-name>-<tool-name>) have no
|
|
4763
5008
|
// token-goat equivalent and are passed through unmapped (safe no-op for
|
|
4764
5009
|
// handlers that don't recognize the name).
|
|
5010
|
+
// 'read_bash'/'read_powershell' are the background-shell output pollers, and
|
|
5011
|
+
// they are the exact shape of Claude Code's BashOutput: Copilot's shell tool
|
|
5012
|
+
// is async, so a long-running command is started once and then re-read over
|
|
5013
|
+
// and over within one turn, each read returning the accumulated output again.
|
|
5014
|
+
// hooks_bashoutput.ts already collapses that into a delta (or a short
|
|
5015
|
+
// "unchanged" marker) for Claude Code, and now does the same here. Both the
|
|
5016
|
+
// names and the argument key were read out of the shipping 1.0.80 bundle
|
|
5017
|
+
// rather than guessed: the builtin tool-name table in
|
|
5018
|
+
// prebuilds/win32-x64/runtime.node lists read_bash/stop_bash/list_bash and
|
|
5019
|
+
// read_powershell/stop_powershell/list_powershell, and the poller's own input
|
|
5020
|
+
// schema in app.js is {shellId, delay} -- so the shell id needs the
|
|
5021
|
+
// POLL_ID_ARG_KEY remap below to reach postBashOutputHandler, which reads
|
|
5022
|
+
// 'bash_id'. read_shell/stop_shell/list_shells, which an earlier static sweep
|
|
5023
|
+
// of the same binary suggested were the real names, are internal Rust
|
|
5024
|
+
// identifiers (tool_read_shell_prepare_input, tool_list_shells_descriptor,
|
|
5025
|
+
// PreparedStopShellInput, and the serde field names of the shell config
|
|
5026
|
+
// struct), never wire tool names, so they are deliberately absent here.
|
|
5027
|
+
// stop_bash/list_bash and their powershell twins stay unmapped because
|
|
5028
|
+
// token-goat has no KillShell-equivalent handler for them to reach; a mapping
|
|
5029
|
+
// would be pure decoration.
|
|
5030
|
+
// 'bash'/'powershell' stay exactly as they were. The same sweep suggested the
|
|
5031
|
+
// executor had been renamed to write_bash/write_powershell and that this
|
|
5032
|
+
// mapping was dead, and that is not what the bundle says: app.js resolves the
|
|
5033
|
+
// executor as shellConfig?.shellToolName ?? "bash" (with "powershell" as the
|
|
5034
|
+
// Windows default of the same config), and its command lives under 'command'
|
|
5035
|
+
// exactly as hooks_bash.ts expects. write_bash/write_powershell are real tool
|
|
5036
|
+
// names, but they are a different tool -- their schema is {shellId, input,
|
|
5037
|
+
// delay} and the bundle files them under the subtype "write_shell", i.e. send
|
|
5038
|
+
// stdin to an already-running shell, not run a command. Mapping them to Bash
|
|
5039
|
+
// would label a stdin write as a shell execution, so they are left unmapped.
|
|
5040
|
+
// 'task', 'read_agent' and 'memory'-family tools are likewise left alone:
|
|
5041
|
+
// task's result is assembled in the native addon and its shape is unknown,
|
|
5042
|
+
// read_agent is an incremental poll that postAgentHandler is not written for,
|
|
5043
|
+
// and the real tool names behind memory were never confirmed. Each would put
|
|
5044
|
+
// a handler that rewrites model-visible output in front of a payload shape
|
|
5045
|
+
// nobody has seen, which is worse than leaving the compression on the table.
|
|
4765
5046
|
const TOOL_TO_TG = {
|
|
4766
5047
|
bash: 'Bash',
|
|
4767
5048
|
powershell: 'Bash',
|
|
5049
|
+
read_bash: 'BashOutput',
|
|
5050
|
+
read_powershell: 'BashOutput',
|
|
4768
5051
|
view: 'Read',
|
|
4769
5052
|
create: 'Write',
|
|
4770
5053
|
edit: 'Edit',
|
|
@@ -4805,6 +5088,17 @@ const FILE_PATH_ARG_KEY = {
|
|
|
4805
5088
|
create: 'path',
|
|
4806
5089
|
}
|
|
4807
5090
|
|
|
5091
|
+
// read_bash/read_powershell send the background shell's id under 'shellId' (confirmed
|
|
5092
|
+
// against the poller's input schema in the shipping 1.0.80 app.js: {shellId, delay});
|
|
5093
|
+
// postBashOutputHandler reads 'bash_id'. Without this the tool-name mapping above would
|
|
5094
|
+
// be inert -- the handler bails on a missing bash_id and every poll would keep costing
|
|
5095
|
+
// the full accumulated output. Same keying convention as FILE_PATH_ARG_KEY: the ORIGINAL
|
|
5096
|
+
// Copilot tool name, since that is what the argument shape belongs to.
|
|
5097
|
+
const POLL_ID_ARG_KEY = {
|
|
5098
|
+
read_bash: 'shellId',
|
|
5099
|
+
read_powershell: 'shellId',
|
|
5100
|
+
}
|
|
5101
|
+
|
|
4808
5102
|
// Copilot spawns a brand-new process for every single hook invocation (no long-lived plugin
|
|
4809
5103
|
// process the way OpenClaw's is -- OPENCLAW_HOOK_SCRIPT's own \`copilot-\${process.pid}-\${Date.now()}\`
|
|
4810
5104
|
// fallback is safe there specifically because that process lives for the whole session, so the
|
|
@@ -4821,13 +5115,19 @@ function stableFallbackSessionId(cwd) {
|
|
|
4821
5115
|
}
|
|
4822
5116
|
|
|
4823
5117
|
function remapToolInput(copilotToolName, input) {
|
|
5118
|
+
if (!input || typeof input !== 'object') return input
|
|
5119
|
+
let out = input
|
|
5120
|
+
// Add the canonical key alongside the original rather than renaming it, so nothing that
|
|
5121
|
+
// might read the original 'path'/'shellId' key elsewhere (e.g. a future handler) loses it.
|
|
4824
5122
|
const pathKey = FILE_PATH_ARG_KEY[copilotToolName]
|
|
4825
|
-
if (pathKey
|
|
4826
|
-
|
|
5123
|
+
if (pathKey !== undefined && pathKey in out) {
|
|
5124
|
+
out = Object.assign({}, out, { file_path: out[pathKey] })
|
|
4827
5125
|
}
|
|
4828
|
-
|
|
4829
|
-
|
|
4830
|
-
|
|
5126
|
+
const idKey = POLL_ID_ARG_KEY[copilotToolName]
|
|
5127
|
+
if (idKey !== undefined && idKey in out) {
|
|
5128
|
+
out = Object.assign({}, out, { bash_id: out[idKey] })
|
|
5129
|
+
}
|
|
5130
|
+
return out
|
|
4831
5131
|
}
|
|
4832
5132
|
|
|
4833
5133
|
// Attempts the in-process hook call: import()s dist/token-goat-hook.mjs (a sibling of
|
|
@@ -4979,7 +5279,46 @@ function translate(copilotEvent, resp) {
|
|
|
4979
5279
|
}
|
|
4980
5280
|
|
|
4981
5281
|
if (copilotEvent === 'postToolUse') {
|
|
4982
|
-
//
|
|
5282
|
+
// Verified against the shipping @github/copilot 1.0.80 bundle, not against the docs page.
|
|
5283
|
+
// NativeHookPipelineProcessor.postToolExecution (app.js offset 2043150) gates on
|
|
5284
|
+
// toolResult.resultType === "success", then does n.toolResultJson && CSr(e.toolResult,
|
|
5285
|
+
// n.toolResultJson), where CSr (offset 2032350) is an in-place Object.assign; the mutated
|
|
5286
|
+
// object is re-serialized back to native in the postTool callback at offset 1793926. So
|
|
5287
|
+
// modifiedResult IS honored on this event, and token-goat's rewriteOutput producers
|
|
5288
|
+
// (compression, injection fencing, image shrink) really do reach the model. resultType is
|
|
5289
|
+
// hardcoded 'success' because success is the only branch this event ever runs on.
|
|
5290
|
+
//
|
|
5291
|
+
// additionalContext on THIS event is dropped on the JS path. grep -abo "onAdditionalContext:"
|
|
5292
|
+
// app.js returns nothing, so the callback is never supplied, and the two "onAdditionalContext?"
|
|
5293
|
+
// call sites (offsets 2041896 and 2043300) are both no-ops. preToolsExecution (offset 2041832)
|
|
5294
|
+
// additionally pushes each context into an array that IS drained into the native return
|
|
5295
|
+
// payload (additional_contexts, offset 1791950); postToolExecution has no such push and its
|
|
5296
|
+
// native return payload (offset 1793926) has no additional_contexts key. Residual, stated
|
|
5297
|
+
// honestly: the native hookProcessorPostToolUse might fold additionalContext into the
|
|
5298
|
+
// toolResultJson it returns, and that was NOT verified. The evidence leans against it, since
|
|
5299
|
+
// the failure sibling path appends its context explicitly in JS via
|
|
5300
|
+
// hookAppendPostToolUseFailureContext and there is no success-path counterpart. The
|
|
5301
|
+
// out.additionalContext below is kept as cheap best-effort, not as a channel anything should
|
|
5302
|
+
// depend on -- see src/pending_context.ts, whose whole design depended on it.
|
|
5303
|
+
//
|
|
5304
|
+
// Failed tool calls never reach this event. postToolUse and postToolUseFailure are two
|
|
5305
|
+
// distinct hook events (both listed in the runtime.node hook-event enum at offset 101618150),
|
|
5306
|
+
// and the shipped copilot-sdk/types.d.ts says onPostToolUse "does not fire for non-success
|
|
5307
|
+
// results". The failure event cannot carry a fence either: PostToolUseFailureHookInput carries
|
|
5308
|
+
// only a stringified error message, not the tool result, and PostToolUseFailureHookOutput
|
|
5309
|
+
// consumes only additionalContext -- "modifiedResult or suppressOutput are not honored for
|
|
5310
|
+
// failure hooks". rejected/denied/timeout results trigger no post hook at all. So on Copilot,
|
|
5311
|
+
// the output of a failed tool call reaches the model unfenced, uncompressed and unshrunk, and
|
|
5312
|
+
// no response shape this shim could emit changes that. That gap is real and still open. What
|
|
5313
|
+
// is now wired is the narrower thing that IS possible there: the postToolUseFailure branch
|
|
5314
|
+
// below carries advisory text alongside the failure, and never rewrites it.
|
|
5315
|
+
//
|
|
5316
|
+
// Emitted camelCase-only. The inbound side around line 247 also tolerates a snake_case "VS
|
|
5317
|
+
// Code compatible" shape, and this comment used to justify the camelCase choice by claiming
|
|
5318
|
+
// PascalCase event registration selects the snake_case response format. That reasoning is
|
|
5319
|
+
// UNVERIFIED in 1.0.80: every hook-response field in the native string tables is camelCase,
|
|
5320
|
+
// and the snake_case hits in the bundle are unrelated internal Rust identifiers. camelCase
|
|
5321
|
+
// stays because it is what the bundle reads; the old justification is no longer asserted.
|
|
4983
5322
|
const hso = resp && resp.hookSpecificOutput
|
|
4984
5323
|
const updatedToolOutput = hso && hso.updatedToolOutput
|
|
4985
5324
|
const context = extractContext(resp)
|
|
@@ -4991,6 +5330,25 @@ function translate(copilotEvent, resp) {
|
|
|
4991
5330
|
return out
|
|
4992
5331
|
}
|
|
4993
5332
|
|
|
5333
|
+
if (copilotEvent === 'postToolUseFailure') {
|
|
5334
|
+
// The failed-tool twin of postToolUse, and the only response field it accepts is
|
|
5335
|
+
// additionalContext: the shipped copilot-sdk/types.d.ts says "modifiedResult or suppressOutput
|
|
5336
|
+
// are not honored for failure hooks", so nothing here can fence, compress or shrink the failed
|
|
5337
|
+
// output -- that gap is real and stays open. What is NOT open, and was the reason this event
|
|
5338
|
+
// went unwired for so long, is whether additionalContext reaches the model at all. It does:
|
|
5339
|
+
// app.js 1.0.80 at offset 2043380 either folds it into textResultForLlm (when
|
|
5340
|
+
// appendFailureContextToToolResult is set) or has the native
|
|
5341
|
+
// hookAppendPostToolUseFailureContext push {content, source:'system'} onto
|
|
5342
|
+
// toolResult.newMessages. That is the exact opposite of postToolUse, whose additionalContext
|
|
5343
|
+
// the JS side drops on the floor, so neither event's behaviour generalises to the other.
|
|
5344
|
+
//
|
|
5345
|
+
// Because this channel spends tokens rather than saving them, the handler behind it
|
|
5346
|
+
// (hooks_tool_failure.ts) is silent on a first failure and speaks only on an exact repeat.
|
|
5347
|
+
const context = extractContext(resp)
|
|
5348
|
+
if (context) return { additionalContext: context }
|
|
5349
|
+
return {}
|
|
5350
|
+
}
|
|
5351
|
+
|
|
4994
5352
|
if (copilotEvent === 'sessionStart') {
|
|
4995
5353
|
// sessionStart has no tool result to modify -- only additionalContext applies, and it's the
|
|
4996
5354
|
// one channel that reaches the model before it picks its first read tool, so this is where
|
|
@@ -5014,12 +5372,47 @@ function translate(copilotEvent, resp) {
|
|
|
5014
5372
|
return { decision: 'allow' }
|
|
5015
5373
|
}
|
|
5016
5374
|
|
|
5017
|
-
|
|
5018
|
-
|
|
5019
|
-
|
|
5020
|
-
|
|
5021
|
-
|
|
5022
|
-
|
|
5375
|
+
if (copilotEvent === 'userPromptSubmitted') {
|
|
5376
|
+
// Copilot's own hooks reference says command-hook output here "is dropped, including
|
|
5377
|
+
// modifiedPrompt", and this branch used to believe it and return nothing. That is wrong for
|
|
5378
|
+
// additionalContext on 1.0.80, established by experiment rather than by reading: a config-file
|
|
5379
|
+
// command hook (at <cwd>/.github/hooks/, the project scope -- the user scope ~/.copilot/hooks/
|
|
5380
|
+
// was never exercised) returned {"additionalContext":"<marker>"} and the marker turned up
|
|
5381
|
+
// verbatim inside the session's user.message.transformedContent, wrapped in a
|
|
5382
|
+
// <system_reminder> block. What settles that it reached the model rather than only the on-disk
|
|
5383
|
+
// record is the billing: the provider's returned usage charged ~140 input tokens for a turn
|
|
5384
|
+
// whose raw content field is 35 bytes, so the marker's bytes were paid for whichever field
|
|
5385
|
+
// carried them. (The weaker argument first offered for this -- that transformedContent carries
|
|
5386
|
+
// an envelope the content field lacks, 29 B vs 195 B -- was measured on the control turn that
|
|
5387
|
+
// had no marker in it, and proves nothing about the marker.) That the model sees it is the
|
|
5388
|
+
// whole point: hook.start/hook.end records also persist and reach nothing.
|
|
5389
|
+
//
|
|
5390
|
+
// Scope of the finding, stated honestly: demonstrated ONCE on 1.0.80, not shown to be
|
|
5391
|
+
// reliable. Of two turns in that experiment, one delivered the marker and one fired a
|
|
5392
|
+
// userPromptSubmitted hook that produced no output and never ran the script; no explanation
|
|
5393
|
+
// was established and the rate is unknown. A hint that silently fails to arrive costs nothing
|
|
5394
|
+
// and breaks nothing here, which is why the direct return is still the right default.
|
|
5395
|
+
//
|
|
5396
|
+
// modifiedPrompt is NOT claimed to work and is not wanted: rewriting a user's prompt is far
|
|
5397
|
+
// more invasive than anything token-goat does, so only additionalContext is forwarded.
|
|
5398
|
+
//
|
|
5399
|
+
// This is also the write end of any post-compaction channel. Copilot has no postCompact hook.
|
|
5400
|
+
// Whether the summary is recoverable from events.jsonl is NOT settled -- the emit()/
|
|
5401
|
+
// emitEphemeral() distinction does not gate the writer, and the real decision is in native
|
|
5402
|
+
// code; see COPILOT_NO_POST_COMPACT_REASON in ../bridges_status.ts for what was and was not
|
|
5403
|
+
// established. The read end that IS confirmed is preCompact, which fires as a notification, so
|
|
5404
|
+
// a manifest can be built there and drained here without reading the event log at all.
|
|
5405
|
+
const context = extractContext(resp)
|
|
5406
|
+
if (context) return { additionalContext: context }
|
|
5407
|
+
return {}
|
|
5408
|
+
}
|
|
5409
|
+
|
|
5410
|
+
// preCompact is the genuine notification-only case, and the contrast with userPromptSubmitted
|
|
5411
|
+
// above is why this fallthrough is worth a comment at all. The hooks reference marks it
|
|
5412
|
+
// "No -- notification only" for output processing, and unlike the additionalContext claim that
|
|
5413
|
+
// turned out to be false, this one is confirmed in the shipping bundle: both preCompact call
|
|
5414
|
+
// sites in app.js (1.0.79 and re-checked in 1.0.80) await the hook and never assign its result.
|
|
5415
|
+
// There is no field to aim at here, so nothing to reconsider on the next version bump.
|
|
5023
5416
|
return {}
|
|
5024
5417
|
}
|
|
5025
5418
|
|
|
@@ -5061,7 +5454,8 @@ var COPILOT_CLI_HOOK_EVENTS = [
|
|
|
5061
5454
|
"preCompact",
|
|
5062
5455
|
"agentStop",
|
|
5063
5456
|
"subagentStop",
|
|
5064
|
-
"userPromptSubmitted"
|
|
5457
|
+
"userPromptSubmitted",
|
|
5458
|
+
"postToolUseFailure"
|
|
5065
5459
|
];
|
|
5066
5460
|
function copilotCliUserRoot() {
|
|
5067
5461
|
const override = process.env["COPILOT_HOME"];
|
|
@@ -5074,6 +5468,23 @@ function copilotCliUserHooksDir() {
|
|
|
5074
5468
|
function copilotCliProjectHooksDir() {
|
|
5075
5469
|
return path5.join(process.cwd(), ".github", "hooks");
|
|
5076
5470
|
}
|
|
5471
|
+
function copilotCliCacheRoot() {
|
|
5472
|
+
const override = process.env["COPILOT_CACHE_HOME"];
|
|
5473
|
+
if (override !== void 0 && override.trim() !== "") return path5.resolve(override);
|
|
5474
|
+
const home = os3.homedir();
|
|
5475
|
+
if (process.platform === "darwin") return path5.join(home, "Library", "Caches", "copilot");
|
|
5476
|
+
if (process.platform === "win32") {
|
|
5477
|
+
const local = process.env["LOCALAPPDATA"];
|
|
5478
|
+
const base2 = local !== void 0 && local.trim() !== "" ? local : path5.join(home, ".cache");
|
|
5479
|
+
return path5.join(base2, "copilot");
|
|
5480
|
+
}
|
|
5481
|
+
const xdg = process.env["XDG_CACHE_HOME"];
|
|
5482
|
+
const base = xdg !== void 0 && xdg.trim() !== "" ? xdg : path5.join(home, ".cache");
|
|
5483
|
+
return path5.join(base, "copilot");
|
|
5484
|
+
}
|
|
5485
|
+
function copilotCliMcpToolsDir() {
|
|
5486
|
+
return path5.join(copilotCliCacheRoot(), "mcp-tools");
|
|
5487
|
+
}
|
|
5077
5488
|
function copilotCliHooksDir(opts = {}) {
|
|
5078
5489
|
return opts.local === true ? copilotCliProjectHooksDir() : copilotCliUserHooksDir();
|
|
5079
5490
|
}
|
|
@@ -5761,6 +6172,7 @@ function uninstallVscode(opts = {}) {
|
|
|
5761
6172
|
}
|
|
5762
6173
|
|
|
5763
6174
|
// src/session_store.ts
|
|
6175
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
5764
6176
|
import * as fs13 from "node:fs";
|
|
5765
6177
|
import * as path10 from "node:path";
|
|
5766
6178
|
|
|
@@ -5987,12 +6399,29 @@ function sweepCacheRoots(extraRoots = []) {
|
|
|
5987
6399
|
var MAX_FILES = 500;
|
|
5988
6400
|
var SESSIONS_SUBDIR = "sessions";
|
|
5989
6401
|
var AGENT_SALT_MARKER = sanitizeIdForFilename(":agent:");
|
|
6402
|
+
var AGENT_SALT_SEPARATOR = ":agent:";
|
|
6403
|
+
var AGENT_DIGEST_CHARS = 12;
|
|
6404
|
+
var SALTED_SESSION_MAX = 64 - AGENT_SALT_MARKER.length - AGENT_DIGEST_CHARS;
|
|
6405
|
+
function saltedStemPrefix(sessionId) {
|
|
6406
|
+
return `${sanitizeIdForFilename(sessionId, SALTED_SESSION_MAX)}${AGENT_SALT_MARKER}`;
|
|
6407
|
+
}
|
|
6408
|
+
function sessionFileStem(sessionId) {
|
|
6409
|
+
const sep = sessionId.indexOf(AGENT_SALT_SEPARATOR);
|
|
6410
|
+
if (sep < 0) return sanitizeIdForFilename(sessionId, 64);
|
|
6411
|
+
const agentId = sessionId.slice(sep + AGENT_SALT_SEPARATOR.length);
|
|
6412
|
+
const digest = createHash2("sha256").update(agentId).digest("hex").slice(0, AGENT_DIGEST_CHARS);
|
|
6413
|
+
return `${saltedStemPrefix(sessionId.slice(0, sep))}${digest}`;
|
|
6414
|
+
}
|
|
5990
6415
|
function sessionPath(sessionId) {
|
|
6416
|
+
return sessionSidecarPath(sessionId, ".json");
|
|
6417
|
+
}
|
|
6418
|
+
function sessionSidecarPath(sessionId, suffix) {
|
|
5991
6419
|
if (!sessionId) return null;
|
|
5992
|
-
const safe =
|
|
6420
|
+
const safe = sessionFileStem(sessionId);
|
|
5993
6421
|
if (!safe) return null;
|
|
6422
|
+
if (suffix.includes("/") || suffix.includes("\\") || suffix.includes("..")) return null;
|
|
5994
6423
|
const dir = path10.join(tokenGoatHome(), SESSIONS_SUBDIR);
|
|
5995
|
-
const candidate = path10.join(dir, `${safe}
|
|
6424
|
+
const candidate = path10.join(dir, `${safe}${suffix}`);
|
|
5996
6425
|
try {
|
|
5997
6426
|
const rel = path10.relative(dir, candidate);
|
|
5998
6427
|
if (rel.startsWith("..")) return null;
|
|
@@ -6081,6 +6510,7 @@ function coerce(raw) {
|
|
|
6081
6510
|
) : [];
|
|
6082
6511
|
const cliReads = Array.isArray(o["cliReads"]) ? o["cliReads"].filter((h) => typeof h === "string") : [];
|
|
6083
6512
|
const bashReruns = Array.isArray(o["bashReruns"]) ? o["bashReruns"].filter((h) => typeof h === "string") : [];
|
|
6513
|
+
const seenImageHashes = Array.isArray(o["seenImageHashes"]) ? o["seenImageHashes"].filter((h) => typeof h === "string") : [];
|
|
6084
6514
|
const pendingLargeFileHints = Array.isArray(o["pendingLargeFileHints"]) ? o["pendingLargeFileHints"].filter(
|
|
6085
6515
|
(p) => Array.isArray(p) && p.length === 2 && typeof p[0] === "string" && typeof p[1] === "number"
|
|
6086
6516
|
) : [];
|
|
@@ -6108,6 +6538,7 @@ function coerce(raw) {
|
|
|
6108
6538
|
globQueries,
|
|
6109
6539
|
outstandingAgentSpawns,
|
|
6110
6540
|
...typeof o["lastTabContext"] === "string" ? { lastTabContext: o["lastTabContext"] } : {},
|
|
6541
|
+
...seenImageHashes.length > 0 ? { seenImageHashes } : {},
|
|
6111
6542
|
...typeof o["compactedAt"] === "number" ? { compactedAt: o["compactedAt"] } : {},
|
|
6112
6543
|
...typeof o["created_ts"] === "number" ? { created_ts: o["created_ts"] } : {}
|
|
6113
6544
|
};
|
|
@@ -6215,12 +6646,21 @@ function mergeSessionState(disk, mem) {
|
|
|
6215
6646
|
grepQueries: mergePairs(disk.grepQueries ?? [], mem.grepQueries ?? []),
|
|
6216
6647
|
globQueries: mergePairs(disk.globQueries ?? [], mem.globQueries ?? []),
|
|
6217
6648
|
outstandingAgentSpawns: mergeOutstandingAgentSpawns(disk.outstandingAgentSpawns ?? [], mem.outstandingAgentSpawns ?? []),
|
|
6649
|
+
// An accumulating collection, so union rather than pick a winner: two hook processes can
|
|
6650
|
+
// each see a screenshot the other never did. Disk first, then mem, so the order stays
|
|
6651
|
+
// oldest-to-newest and the cap evicts the oldest -- the same policy recordSeenImage applies
|
|
6652
|
+
// in memory, applied again here because a union of two capped lists can exceed the cap.
|
|
6653
|
+
seenImageHashes: mergeSeenImageHashes(disk.seenImageHashes ?? [], mem.seenImageHashes ?? []),
|
|
6218
6654
|
// Last-seen scalar, not an accumulating collection: prefer mem's value (this process's freshest observation) over disk's, since a newer write always supersedes an older one.
|
|
6219
6655
|
...mem.lastTabContext !== void 0 ? { lastTabContext: mem.lastTabContext } : disk.lastTabContext !== void 0 ? { lastTabContext: disk.lastTabContext } : {},
|
|
6220
6656
|
// Prefer the value already on disk: it marks the original creation time, and must never be bumped forward to the merge's "now". `mem` never carries one (it is not tracked in memory), so this is really "keep whatever disk has".
|
|
6221
6657
|
...disk.created_ts !== void 0 ? { created_ts: disk.created_ts } : mem.created_ts !== void 0 ? { created_ts: mem.created_ts } : {}
|
|
6222
6658
|
};
|
|
6223
6659
|
}
|
|
6660
|
+
function mergeSeenImageHashes(disk, mem) {
|
|
6661
|
+
const merged = Array.from(/* @__PURE__ */ new Set([...disk, ...mem]));
|
|
6662
|
+
return merged.length > MAX_SEEN_IMAGE_HASHES ? merged.slice(merged.length - MAX_SEEN_IMAGE_HASHES) : merged;
|
|
6663
|
+
}
|
|
6224
6664
|
function capFiles(s, max) {
|
|
6225
6665
|
if (s.files.length <= max) return s;
|
|
6226
6666
|
const kept = [...s.files].sort((a, b) => b.lastReadAt - a.lastReadAt).slice(0, max);
|
|
@@ -6241,9 +6681,8 @@ function readSessionStateFile(sessionId) {
|
|
|
6241
6681
|
}
|
|
6242
6682
|
function listSiblingSessionStates(sessionId) {
|
|
6243
6683
|
if (!sessionId) return [];
|
|
6244
|
-
const
|
|
6245
|
-
if (
|
|
6246
|
-
const prefix = `${safeSessionId}${AGENT_SALT_MARKER}`;
|
|
6684
|
+
const prefix = saltedStemPrefix(sessionId);
|
|
6685
|
+
if (prefix === AGENT_SALT_MARKER) return [];
|
|
6247
6686
|
const dir = path10.join(tokenGoatHome(), SESSIONS_SUBDIR);
|
|
6248
6687
|
const out = [];
|
|
6249
6688
|
try {
|
|
@@ -6307,7 +6746,7 @@ function scanForInjectionPatterns(text) {
|
|
|
6307
6746
|
var UNTRUSTED_WEB_TAG = "untrusted-web-content";
|
|
6308
6747
|
function neutralizeFenceMarkers(text, tag) {
|
|
6309
6748
|
const escapedTag = tag.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
6310
|
-
const marker = new RegExp(`<\\s*/?\\s*${escapedTag}(
|
|
6749
|
+
const marker = new RegExp(`<\\s*/?\\s*${escapedTag}(?=[\\s/>])[^>]*>`, "gi");
|
|
6311
6750
|
return text.replace(marker, (m) => m.replace(/</g, "<").replace(/>/g, ">"));
|
|
6312
6751
|
}
|
|
6313
6752
|
function fenceUntrustedContent(text, matchedPatternNames, tag = UNTRUSTED_WEB_TAG) {
|
|
@@ -6905,7 +7344,7 @@ function pruneSkillOutputs(maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
|
|
|
6905
7344
|
}
|
|
6906
7345
|
|
|
6907
7346
|
// src/image_shrink.ts
|
|
6908
|
-
import { createHash as
|
|
7347
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
6909
7348
|
import * as fs16 from "node:fs";
|
|
6910
7349
|
import * as path12 from "node:path";
|
|
6911
7350
|
|
|
@@ -7058,6 +7497,19 @@ async function probeImageMeta(input) {
|
|
|
7058
7497
|
return null;
|
|
7059
7498
|
}
|
|
7060
7499
|
}
|
|
7500
|
+
async function imageQualifiesForShrink(input) {
|
|
7501
|
+
if (input.length >= DEFAULT_SIZE_THRESHOLD_BYTES) return true;
|
|
7502
|
+
const sharp = await loadSharp();
|
|
7503
|
+
if (sharp === null) return false;
|
|
7504
|
+
try {
|
|
7505
|
+
const cfg = loadConfig().image_shrink;
|
|
7506
|
+
const limitInputPixels = cfg.max_image_pixels > 0 ? cfg.max_image_pixels : false;
|
|
7507
|
+
const meta = await sharp(input, { limitInputPixels }).metadata();
|
|
7508
|
+
return Math.max(meta.width ?? 0, meta.height ?? 0) > DEFAULT_MAX_DIMENSION;
|
|
7509
|
+
} catch {
|
|
7510
|
+
return false;
|
|
7511
|
+
}
|
|
7512
|
+
}
|
|
7061
7513
|
async function shrinkImage(input, opts) {
|
|
7062
7514
|
const cfg = loadConfig().image_shrink;
|
|
7063
7515
|
const maxDimension = opts?.maxDimension ?? DEFAULT_MAX_DIMENSION;
|
|
@@ -7107,7 +7559,7 @@ function imageShrinkCacheDir() {
|
|
|
7107
7559
|
return path12.join(tokenGoatHome(), "image_shrink_cache");
|
|
7108
7560
|
}
|
|
7109
7561
|
function shrinkCacheKey(originalPath, size, mtimeMs, quality) {
|
|
7110
|
-
return
|
|
7562
|
+
return createHash3("sha256").update(`${originalPath}:${size}:${mtimeMs}:${quality}`).digest("hex").slice(0, 16);
|
|
7111
7563
|
}
|
|
7112
7564
|
function findCachedShrink(originalPath, size, mtimeMs, quality) {
|
|
7113
7565
|
const key = shrinkCacheKey(originalPath, size, mtimeMs, quality);
|
|
@@ -7156,18 +7608,18 @@ function pruneShrinkCache() {
|
|
|
7156
7608
|
}
|
|
7157
7609
|
async function finalizeShrinkResult(result, filePath) {
|
|
7158
7610
|
const basename12 = path12.basename(filePath);
|
|
7611
|
+
const shrinkSaved = result.originalBytes - result.shrunkBytes;
|
|
7612
|
+
recordStat("image_shrink", shrinkSaved, Math.round(shrinkSaved / 4), void 0, basename12);
|
|
7159
7613
|
if (loadConfig().image_shrink.ocr_enabled) {
|
|
7160
7614
|
const ocr = await ocrImage(result.data);
|
|
7161
7615
|
if (ocr !== null && isTextHeavy(ocr, loadConfig().image_shrink.ocr_min_confidence)) {
|
|
7162
7616
|
const textBytes = Buffer.byteLength(ocr.text, "utf8");
|
|
7163
|
-
const
|
|
7164
|
-
recordStat("image_ocr",
|
|
7617
|
+
const saved = Math.max(0, result.shrunkBytes - textBytes);
|
|
7618
|
+
recordStat("image_ocr", saved, Math.round(saved / 4), void 0, basename12);
|
|
7165
7619
|
return contextOutput(formatOcrSummary(ocr, basename12, result.originalBytes));
|
|
7166
7620
|
}
|
|
7167
7621
|
}
|
|
7168
|
-
const saved = result.originalBytes - result.shrunkBytes;
|
|
7169
7622
|
const { summary, dataUrl } = formatShrinkSummary(result, basename12);
|
|
7170
|
-
recordStat("image_shrink", saved, Math.round(saved / 4), void 0, basename12);
|
|
7171
7623
|
return contextOutput(`${summary}
|
|
7172
7624
|
${dataUrl}`);
|
|
7173
7625
|
}
|
|
@@ -7179,7 +7631,6 @@ async function preReadImageHandler(event) {
|
|
|
7179
7631
|
pruneShrinkCache();
|
|
7180
7632
|
const stat2 = statInfo(filePath);
|
|
7181
7633
|
if (stat2 === null) return passOutput();
|
|
7182
|
-
const size = stat2.size;
|
|
7183
7634
|
const quality = loadConfig().image_shrink.jpeg_quality;
|
|
7184
7635
|
const cached = findCachedShrink(filePath, stat2.size, stat2.mtimeMs, quality);
|
|
7185
7636
|
if (cached !== null) {
|
|
@@ -7206,34 +7657,13 @@ async function preReadImageHandler(event) {
|
|
|
7206
7657
|
} catch {
|
|
7207
7658
|
}
|
|
7208
7659
|
}
|
|
7209
|
-
let input
|
|
7210
|
-
|
|
7211
|
-
|
|
7212
|
-
|
|
7213
|
-
|
|
7214
|
-
} catch {
|
|
7215
|
-
return passOutput();
|
|
7216
|
-
}
|
|
7217
|
-
const sharp = await loadSharp();
|
|
7218
|
-
if (sharp === null) return passOutput();
|
|
7219
|
-
try {
|
|
7220
|
-
const cfg = loadConfig().image_shrink;
|
|
7221
|
-
const limitInputPixels = cfg.max_image_pixels > 0 ? cfg.max_image_pixels : false;
|
|
7222
|
-
const meta = await sharp(input, { limitInputPixels }).metadata();
|
|
7223
|
-
const longestEdge = Math.max(meta.width ?? 0, meta.height ?? 0);
|
|
7224
|
-
qualifies = longestEdge > DEFAULT_MAX_DIMENSION;
|
|
7225
|
-
} catch {
|
|
7226
|
-
return passOutput();
|
|
7227
|
-
}
|
|
7228
|
-
}
|
|
7229
|
-
if (!qualifies) return passOutput();
|
|
7230
|
-
if (input === null) {
|
|
7231
|
-
try {
|
|
7232
|
-
input = fs16.readFileSync(filePath);
|
|
7233
|
-
} catch {
|
|
7234
|
-
return passOutput();
|
|
7235
|
-
}
|
|
7660
|
+
let input;
|
|
7661
|
+
try {
|
|
7662
|
+
input = fs16.readFileSync(filePath);
|
|
7663
|
+
} catch {
|
|
7664
|
+
return passOutput();
|
|
7236
7665
|
}
|
|
7666
|
+
if (!await imageQualifiesForShrink(input)) return passOutput();
|
|
7237
7667
|
const result = await shrinkImage(input, { quality, sizeThresholdBytes: 0 });
|
|
7238
7668
|
if (result === null) {
|
|
7239
7669
|
recordStat("image_shrink_skipped");
|
|
@@ -7244,30 +7674,432 @@ async function preReadImageHandler(event) {
|
|
|
7244
7674
|
}
|
|
7245
7675
|
registerHook("pre_tool_use", preReadImageHandler, { toolName: "Read" });
|
|
7246
7676
|
|
|
7247
|
-
// src/
|
|
7677
|
+
// src/embed_model.ts
|
|
7678
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
7679
|
+
import * as fs17 from "node:fs";
|
|
7248
7680
|
import { createRequire as createRequire3 } from "node:module";
|
|
7249
|
-
|
|
7250
|
-
|
|
7251
|
-
|
|
7252
|
-
|
|
7253
|
-
|
|
7254
|
-
|
|
7255
|
-
|
|
7256
|
-
|
|
7257
|
-
|
|
7258
|
-
|
|
7259
|
-
|
|
7260
|
-
|
|
7681
|
+
import * as path13 from "node:path";
|
|
7682
|
+
import { pipeline } from "node:stream/promises";
|
|
7683
|
+
|
|
7684
|
+
// src/embed_tokenizer.ts
|
|
7685
|
+
var MAX_SEQUENCE_TOKENS = 512;
|
|
7686
|
+
var UnsupportedTokenizerError = class extends Error {
|
|
7687
|
+
constructor(message) {
|
|
7688
|
+
super(message);
|
|
7689
|
+
this.name = "UnsupportedTokenizerError";
|
|
7690
|
+
}
|
|
7691
|
+
};
|
|
7692
|
+
function fail(message) {
|
|
7693
|
+
throw new UnsupportedTokenizerError(message);
|
|
7694
|
+
}
|
|
7695
|
+
function asRecord(value, path31) {
|
|
7696
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${path31} is not an object`);
|
|
7697
|
+
return value;
|
|
7698
|
+
}
|
|
7699
|
+
function asString(value, path31) {
|
|
7700
|
+
if (typeof value !== "string") fail(`${path31} is not a string`);
|
|
7701
|
+
return value;
|
|
7702
|
+
}
|
|
7703
|
+
function asArray2(value, path31) {
|
|
7704
|
+
if (!Array.isArray(value)) fail(`${path31} is not an array`);
|
|
7705
|
+
return value;
|
|
7706
|
+
}
|
|
7707
|
+
function expectValue(actual, wanted, path31) {
|
|
7708
|
+
if (actual !== wanted) fail(`${path31} is ${JSON.stringify(actual)}, expected ${JSON.stringify(wanted)}`);
|
|
7709
|
+
}
|
|
7710
|
+
function isChinese(cp) {
|
|
7711
|
+
return cp >= 19968 && cp <= 40959 || cp >= 13312 && cp <= 19903 || cp >= 131072 && cp <= 173791 || cp >= 173824 && cp <= 177983 || cp >= 177984 && cp <= 178207 || cp >= 178208 && cp <= 183983 || cp >= 63744 && cp <= 64255 || cp >= 194560 && cp <= 195103;
|
|
7712
|
+
}
|
|
7713
|
+
var PUNCTUATION = new RegExp("\\p{P}", "u");
|
|
7714
|
+
function isPunctuation(ch) {
|
|
7715
|
+
const cp = ch.codePointAt(0) ?? 0;
|
|
7716
|
+
if (cp >= 33 && cp <= 47 || cp >= 58 && cp <= 64 || cp >= 91 && cp <= 96 || cp >= 123 && cp <= 126) {
|
|
7717
|
+
return true;
|
|
7718
|
+
}
|
|
7719
|
+
return PUNCTUATION.test(ch);
|
|
7720
|
+
}
|
|
7721
|
+
var CONTROL = new RegExp("\\p{Cc}|\\p{Cf}|\\p{Co}|\\p{Cs}", "u");
|
|
7722
|
+
function isControl(ch) {
|
|
7723
|
+
if (ch === " " || ch === "\n" || ch === "\r") return false;
|
|
7724
|
+
return CONTROL.test(ch);
|
|
7725
|
+
}
|
|
7726
|
+
var SPACE_SEPARATOR = new RegExp("\\p{Zs}", "u");
|
|
7727
|
+
function isWhitespace(ch) {
|
|
7728
|
+
return ch === " " || ch === " " || ch === "\n" || ch === "\r" || SPACE_SEPARATOR.test(ch);
|
|
7729
|
+
}
|
|
7730
|
+
var COMBINING_MARK = /[\u0300-\u036f]/gu;
|
|
7731
|
+
function normalize(text) {
|
|
7732
|
+
let out = "";
|
|
7733
|
+
for (const ch of text) {
|
|
7734
|
+
const cp = ch.codePointAt(0) ?? 0;
|
|
7735
|
+
if (cp === 0 || cp === 65533 || isControl(ch)) continue;
|
|
7736
|
+
if (isWhitespace(ch)) {
|
|
7737
|
+
out += " ";
|
|
7738
|
+
continue;
|
|
7261
7739
|
}
|
|
7262
|
-
|
|
7263
|
-
|
|
7740
|
+
if (isChinese(cp)) {
|
|
7741
|
+
out += ` ${ch} `;
|
|
7742
|
+
continue;
|
|
7743
|
+
}
|
|
7744
|
+
out += ch;
|
|
7264
7745
|
}
|
|
7746
|
+
return out.normalize("NFD").replace(COMBINING_MARK, "").toLowerCase();
|
|
7747
|
+
}
|
|
7748
|
+
function preTokenize(text) {
|
|
7749
|
+
const words = [];
|
|
7750
|
+
for (const piece of text.split(/\s+/)) {
|
|
7751
|
+
if (!piece) continue;
|
|
7752
|
+
let buf = "";
|
|
7753
|
+
for (const ch of piece) {
|
|
7754
|
+
if (isPunctuation(ch)) {
|
|
7755
|
+
if (buf) {
|
|
7756
|
+
words.push(buf);
|
|
7757
|
+
buf = "";
|
|
7758
|
+
}
|
|
7759
|
+
words.push(ch);
|
|
7760
|
+
} else {
|
|
7761
|
+
buf += ch;
|
|
7762
|
+
}
|
|
7763
|
+
}
|
|
7764
|
+
if (buf) words.push(buf);
|
|
7765
|
+
}
|
|
7766
|
+
return words;
|
|
7767
|
+
}
|
|
7768
|
+
function readSpec(raw) {
|
|
7769
|
+
const spec = asRecord(raw, "tokenizer.json");
|
|
7770
|
+
const normalizer = asRecord(spec["normalizer"], "normalizer");
|
|
7771
|
+
expectValue(normalizer["type"], "BertNormalizer", "normalizer.type");
|
|
7772
|
+
expectValue(normalizer["clean_text"], true, "normalizer.clean_text");
|
|
7773
|
+
expectValue(normalizer["handle_chinese_chars"], true, "normalizer.handle_chinese_chars");
|
|
7774
|
+
expectValue(normalizer["lowercase"], true, "normalizer.lowercase");
|
|
7775
|
+
const stripAccents = normalizer["strip_accents"];
|
|
7776
|
+
if (stripAccents !== null && stripAccents !== true) {
|
|
7777
|
+
fail(`normalizer.strip_accents is ${JSON.stringify(stripAccents)}, expected null or true`);
|
|
7778
|
+
}
|
|
7779
|
+
expectValue(asRecord(spec["pre_tokenizer"], "pre_tokenizer")["type"], "BertPreTokenizer", "pre_tokenizer.type");
|
|
7780
|
+
const model = asRecord(spec["model"], "model");
|
|
7781
|
+
expectValue(model["type"], "WordPiece", "model.type");
|
|
7782
|
+
expectValue(model["continuing_subword_prefix"], "##", "model.continuing_subword_prefix");
|
|
7783
|
+
const post = asRecord(spec["post_processor"], "post_processor");
|
|
7784
|
+
expectValue(post["type"], "TemplateProcessing", "post_processor.type");
|
|
7785
|
+
const template = asArray2(post["single"], "post_processor.single").map((part) => {
|
|
7786
|
+
const special = asRecord(part, "post_processor.single[]")["SpecialToken"];
|
|
7787
|
+
return special === void 0 ? "A" : asString(asRecord(special, "SpecialToken")["id"], "SpecialToken.id");
|
|
7788
|
+
}).join(" ");
|
|
7789
|
+
if (template !== "[CLS] A [SEP]") fail(`post_processor.single is "${template}", expected "[CLS] A [SEP]"`);
|
|
7790
|
+
const vocab = /* @__PURE__ */ new Map();
|
|
7791
|
+
for (const [token, id] of Object.entries(asRecord(model["vocab"], "model.vocab"))) {
|
|
7792
|
+
if (typeof id !== "number" || !Number.isInteger(id)) fail(`model.vocab["${token}"] is not an integer id`);
|
|
7793
|
+
vocab.set(token, id);
|
|
7794
|
+
}
|
|
7795
|
+
const maxChars = model["max_input_chars_per_word"];
|
|
7796
|
+
if (maxChars !== void 0 && (typeof maxChars !== "number" || !Number.isInteger(maxChars) || maxChars < 1)) {
|
|
7797
|
+
fail("model.max_input_chars_per_word is not a positive integer");
|
|
7798
|
+
}
|
|
7799
|
+
return {
|
|
7800
|
+
vocab,
|
|
7801
|
+
unkToken: asString(model["unk_token"], "model.unk_token"),
|
|
7802
|
+
maxInputCharsPerWord: typeof maxChars === "number" ? maxChars : 100
|
|
7803
|
+
};
|
|
7265
7804
|
}
|
|
7805
|
+
var BertWordPiece = class _BertWordPiece {
|
|
7806
|
+
vocab;
|
|
7807
|
+
maxInputCharsPerWord;
|
|
7808
|
+
clsId;
|
|
7809
|
+
sepId;
|
|
7810
|
+
padId;
|
|
7811
|
+
unkId;
|
|
7812
|
+
/** @param raw the parsed contents of a tokenizer.json. */
|
|
7813
|
+
constructor(raw) {
|
|
7814
|
+
const spec = readSpec(raw);
|
|
7815
|
+
this.vocab = spec.vocab;
|
|
7816
|
+
this.maxInputCharsPerWord = spec.maxInputCharsPerWord;
|
|
7817
|
+
this.clsId = this.requireToken("[CLS]");
|
|
7818
|
+
this.sepId = this.requireToken("[SEP]");
|
|
7819
|
+
this.padId = this.requireToken("[PAD]");
|
|
7820
|
+
this.unkId = this.requireToken(spec.unkToken);
|
|
7821
|
+
}
|
|
7822
|
+
/** Parse and validate in one step, for the common case of reading the file off disk. */
|
|
7823
|
+
static fromJson(json) {
|
|
7824
|
+
let parsed;
|
|
7825
|
+
try {
|
|
7826
|
+
parsed = JSON.parse(json);
|
|
7827
|
+
} catch (err) {
|
|
7828
|
+
throw new UnsupportedTokenizerError(`tokenizer.json is not valid JSON: ${err.message}`);
|
|
7829
|
+
}
|
|
7830
|
+
return new _BertWordPiece(parsed);
|
|
7831
|
+
}
|
|
7832
|
+
requireToken(token) {
|
|
7833
|
+
const id = this.vocab.get(token);
|
|
7834
|
+
if (id === void 0) fail(`model.vocab is missing the ${token} token`);
|
|
7835
|
+
return id;
|
|
7836
|
+
}
|
|
7837
|
+
/** Greedy longest-match-first over one whitespace- and punctuation-free word. */
|
|
7838
|
+
wordToIds(word, into) {
|
|
7839
|
+
if (word.length > this.maxInputCharsPerWord) {
|
|
7840
|
+
into.push(this.unkId);
|
|
7841
|
+
return;
|
|
7842
|
+
}
|
|
7843
|
+
const pieces = [];
|
|
7844
|
+
let start = 0;
|
|
7845
|
+
while (start < word.length) {
|
|
7846
|
+
let end = word.length;
|
|
7847
|
+
let found = -1;
|
|
7848
|
+
while (start < end) {
|
|
7849
|
+
const sub = start === 0 ? word.slice(start, end) : `##${word.slice(start, end)}`;
|
|
7850
|
+
const id = this.vocab.get(sub);
|
|
7851
|
+
if (id !== void 0) {
|
|
7852
|
+
found = id;
|
|
7853
|
+
break;
|
|
7854
|
+
}
|
|
7855
|
+
end--;
|
|
7856
|
+
}
|
|
7857
|
+
if (found === -1) {
|
|
7858
|
+
into.push(this.unkId);
|
|
7859
|
+
return;
|
|
7860
|
+
}
|
|
7861
|
+
pieces.push(found);
|
|
7862
|
+
start = end;
|
|
7863
|
+
}
|
|
7864
|
+
for (const id of pieces) into.push(id);
|
|
7865
|
+
}
|
|
7866
|
+
/**
|
|
7867
|
+
* `[CLS] ... [SEP]`, cut to `maxLength` tokens.
|
|
7868
|
+
*
|
|
7869
|
+
* The cut is taken after the markers are added, not before, which means a sequence long enough to
|
|
7870
|
+
* be truncated ends on an ordinary token and has no [SEP] at all. That is what the reference does
|
|
7871
|
+
* and therefore what the model has been fed all along, so it is deliberate rather than an
|
|
7872
|
+
* oversight: keeping the [SEP] would be the more defensible sequence and a different one, and a
|
|
7873
|
+
* tokenizer whose whole justification is producing identical ids does not get to improve on them.
|
|
7874
|
+
* The oracle carries two cases that reach the limit; both end mid-text.
|
|
7875
|
+
*/
|
|
7876
|
+
encode(text, maxLength = MAX_SEQUENCE_TOKENS) {
|
|
7877
|
+
if (!Number.isInteger(maxLength) || maxLength < 2) {
|
|
7878
|
+
throw new RangeError(`maxLength must be an integer >= 2 (both markers), got ${maxLength}`);
|
|
7879
|
+
}
|
|
7880
|
+
const ids = [this.clsId];
|
|
7881
|
+
for (const word of preTokenize(normalize(text))) {
|
|
7882
|
+
this.wordToIds(word, ids);
|
|
7883
|
+
if (ids.length >= maxLength) return ids.slice(0, maxLength);
|
|
7884
|
+
}
|
|
7885
|
+
ids.push(this.sepId);
|
|
7886
|
+
return ids.length > maxLength ? ids.slice(0, maxLength) : ids;
|
|
7887
|
+
}
|
|
7888
|
+
};
|
|
7889
|
+
|
|
7890
|
+
// src/embed_model.ts
|
|
7891
|
+
var _require2 = createRequire3(import.meta.url);
|
|
7266
7892
|
var DEFAULT_MODEL = "Xenova/bge-small-en-v1.5";
|
|
7267
7893
|
var DEFAULT_DIM = 384;
|
|
7268
7894
|
var PINNED_MODEL_REVISION = "ea104dacec62c0de699686887e3f920caeb4f3e3";
|
|
7895
|
+
var MODEL_FILES = [
|
|
7896
|
+
{
|
|
7897
|
+
name: "tokenizer.json",
|
|
7898
|
+
sha256: "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66",
|
|
7899
|
+
bytes: 711396
|
|
7900
|
+
},
|
|
7901
|
+
{
|
|
7902
|
+
name: "onnx/model_quantized.onnx",
|
|
7903
|
+
sha256: "6c9c6101a956d62dfb5e7190c538226c0c5bb9cb27b651234b6df063ee7dbfe4",
|
|
7904
|
+
bytes: 34014426
|
|
7905
|
+
}
|
|
7906
|
+
];
|
|
7907
|
+
function modelDir() {
|
|
7908
|
+
return path13.join(dataDir(), "models", ...DEFAULT_MODEL.split("/"), PINNED_MODEL_REVISION);
|
|
7909
|
+
}
|
|
7910
|
+
function downloadUrl(file) {
|
|
7911
|
+
return `https://huggingface.co/${DEFAULT_MODEL}/resolve/${PINNED_MODEL_REVISION}/${file.name}`;
|
|
7912
|
+
}
|
|
7913
|
+
var _ort = null;
|
|
7914
|
+
var _ortError = null;
|
|
7915
|
+
var _ortLoadAttempted = false;
|
|
7916
|
+
function ensureRuntimeLoaded() {
|
|
7917
|
+
if (_ortLoadAttempted) return;
|
|
7918
|
+
_ortLoadAttempted = true;
|
|
7919
|
+
try {
|
|
7920
|
+
_ort = _require2("onnxruntime-node");
|
|
7921
|
+
} catch (e) {
|
|
7922
|
+
_ortError = e instanceof Error ? e : new Error(String(e));
|
|
7923
|
+
}
|
|
7924
|
+
}
|
|
7925
|
+
function isRuntimeAvailable() {
|
|
7926
|
+
ensureRuntimeLoaded();
|
|
7927
|
+
return _ort !== null && _ortError === null;
|
|
7928
|
+
}
|
|
7929
|
+
function runtimeLoadError() {
|
|
7930
|
+
ensureRuntimeLoaded();
|
|
7931
|
+
return _ortError;
|
|
7932
|
+
}
|
|
7933
|
+
function runtimeVersion() {
|
|
7934
|
+
ensureRuntimeLoaded();
|
|
7935
|
+
if (_ort === null) return "unknown";
|
|
7936
|
+
try {
|
|
7937
|
+
let dir = path13.dirname(_require2.resolve("onnxruntime-node"));
|
|
7938
|
+
for (let depth = 0; depth < 6; depth++) {
|
|
7939
|
+
const manifest = path13.join(dir, "package.json");
|
|
7940
|
+
if (fs17.existsSync(manifest)) {
|
|
7941
|
+
const parsed = JSON.parse(fs17.readFileSync(manifest, "utf8"));
|
|
7942
|
+
if (parsed.name === "onnxruntime-node" && typeof parsed.version === "string") return parsed.version;
|
|
7943
|
+
}
|
|
7944
|
+
const up = path13.dirname(dir);
|
|
7945
|
+
if (up === dir) break;
|
|
7946
|
+
dir = up;
|
|
7947
|
+
}
|
|
7948
|
+
} catch {
|
|
7949
|
+
}
|
|
7950
|
+
return "unknown";
|
|
7951
|
+
}
|
|
7952
|
+
function sha256Of(filePath) {
|
|
7953
|
+
return new Promise((resolve10, reject) => {
|
|
7954
|
+
const hash2 = createHash4("sha256");
|
|
7955
|
+
const stream = fs17.createReadStream(filePath);
|
|
7956
|
+
stream.on("error", reject);
|
|
7957
|
+
stream.on("data", (chunk) => hash2.update(chunk));
|
|
7958
|
+
stream.on("end", () => resolve10(hash2.digest("hex")));
|
|
7959
|
+
});
|
|
7960
|
+
}
|
|
7961
|
+
async function download(file, target) {
|
|
7962
|
+
const url = downloadUrl(file);
|
|
7963
|
+
const response = await fetch(url, { redirect: "follow" });
|
|
7964
|
+
if (!response.ok) throw new Error(`GET ${url} returned ${response.status} ${response.statusText}`);
|
|
7965
|
+
if (!response.body) throw new Error(`GET ${url} returned no body`);
|
|
7966
|
+
const temp = `${target}.${process.pid}.partial`;
|
|
7967
|
+
const hash2 = createHash4("sha256");
|
|
7968
|
+
let written = 0;
|
|
7969
|
+
const out = fs17.createWriteStream(temp);
|
|
7970
|
+
try {
|
|
7971
|
+
await pipeline(async function* () {
|
|
7972
|
+
for await (const chunk of response.body) {
|
|
7973
|
+
written += chunk.byteLength;
|
|
7974
|
+
if (written > file.bytes) throw new Error(`${file.name} is longer than the pinned ${file.bytes} bytes`);
|
|
7975
|
+
hash2.update(chunk);
|
|
7976
|
+
yield chunk;
|
|
7977
|
+
}
|
|
7978
|
+
}, out);
|
|
7979
|
+
if (written !== file.bytes) {
|
|
7980
|
+
throw new Error(`${file.name} is ${written} bytes, expected the pinned ${file.bytes}`);
|
|
7981
|
+
}
|
|
7982
|
+
const digest = hash2.digest("hex");
|
|
7983
|
+
if (digest !== file.sha256) {
|
|
7984
|
+
throw new Error(`${file.name} has sha256 ${digest}, expected the pinned ${file.sha256}`);
|
|
7985
|
+
}
|
|
7986
|
+
fs17.renameSync(temp, target);
|
|
7987
|
+
} catch (e) {
|
|
7988
|
+
await new Promise((resolve10) => {
|
|
7989
|
+
if (out.closed) resolve10();
|
|
7990
|
+
else out.once("close", () => resolve10());
|
|
7991
|
+
});
|
|
7992
|
+
try {
|
|
7993
|
+
fs17.rmSync(temp, { force: true, maxRetries: 20, retryDelay: 25 });
|
|
7994
|
+
} catch {
|
|
7995
|
+
}
|
|
7996
|
+
throw e;
|
|
7997
|
+
}
|
|
7998
|
+
}
|
|
7999
|
+
async function ensureModelFiles(modelName = DEFAULT_MODEL) {
|
|
8000
|
+
if (modelName !== DEFAULT_MODEL) {
|
|
8001
|
+
throw new Error(
|
|
8002
|
+
`Only ${DEFAULT_MODEL} is supported: its files are pinned to a revision and to a sha256 each, and "${modelName}" has neither, so there would be nothing to check the download against.`
|
|
8003
|
+
);
|
|
8004
|
+
}
|
|
8005
|
+
ensureDataDirPrivate();
|
|
8006
|
+
const dir = modelDir();
|
|
8007
|
+
const offline = loadConfig().network.offline;
|
|
8008
|
+
for (const file of MODEL_FILES) {
|
|
8009
|
+
const target = path13.join(dir, file.name);
|
|
8010
|
+
if (fs17.existsSync(target)) {
|
|
8011
|
+
const digest = await sha256Of(target);
|
|
8012
|
+
if (digest === file.sha256) continue;
|
|
8013
|
+
fs17.rmSync(target, { force: true });
|
|
8014
|
+
}
|
|
8015
|
+
if (offline) {
|
|
8016
|
+
throw new Error(
|
|
8017
|
+
`Offline mode is on (network.offline): refusing to download ${file.name} for the embedding model. Copy the pinned files into ${dir} on a connected machine to use semantic search here.`
|
|
8018
|
+
);
|
|
8019
|
+
}
|
|
8020
|
+
fs17.mkdirSync(path13.dirname(target), { recursive: true });
|
|
8021
|
+
console.warn(
|
|
8022
|
+
`Downloading the embedding model, once (${file.name}, ${Math.round(file.bytes / 1024 / 1024)} MB) into ${dir}`
|
|
8023
|
+
);
|
|
8024
|
+
await download(file, target);
|
|
8025
|
+
}
|
|
8026
|
+
return dir;
|
|
8027
|
+
}
|
|
8028
|
+
function poolAndNormalize(hidden, seq, dim) {
|
|
8029
|
+
const pooled = new Float64Array(dim);
|
|
8030
|
+
for (let t = 0; t < seq; t++) {
|
|
8031
|
+
const base = t * dim;
|
|
8032
|
+
for (let d = 0; d < dim; d++) pooled[d] = (pooled[d] ?? 0) + (hidden[base + d] ?? 0);
|
|
8033
|
+
}
|
|
8034
|
+
let sumOfSquares = 0;
|
|
8035
|
+
for (let d = 0; d < dim; d++) {
|
|
8036
|
+
const mean = (pooled[d] ?? 0) / seq;
|
|
8037
|
+
pooled[d] = mean;
|
|
8038
|
+
sumOfSquares += mean * mean;
|
|
8039
|
+
}
|
|
8040
|
+
const norm = Math.sqrt(sumOfSquares);
|
|
8041
|
+
const out = new Float32Array(dim);
|
|
8042
|
+
if (norm === 0 || !Number.isFinite(norm)) return out;
|
|
8043
|
+
for (let d = 0; d < dim; d++) out[d] = (pooled[d] ?? 0) / norm;
|
|
8044
|
+
return out;
|
|
8045
|
+
}
|
|
8046
|
+
var EmbeddingModel = class _EmbeddingModel {
|
|
8047
|
+
constructor(tokenizer, session, tensorFactory) {
|
|
8048
|
+
this.tokenizer = tokenizer;
|
|
8049
|
+
this.session = session;
|
|
8050
|
+
this.tensorFactory = tensorFactory;
|
|
8051
|
+
}
|
|
8052
|
+
tokenizer;
|
|
8053
|
+
session;
|
|
8054
|
+
tensorFactory;
|
|
8055
|
+
static async load(modelName = DEFAULT_MODEL) {
|
|
8056
|
+
if (!isRuntimeAvailable()) {
|
|
8057
|
+
throw new Error(`onnxruntime-node is not available: ${_ortError?.message ?? "unknown error"}`);
|
|
8058
|
+
}
|
|
8059
|
+
const dir = await ensureModelFiles(modelName);
|
|
8060
|
+
const tokenizer = BertWordPiece.fromJson(fs17.readFileSync(path13.join(dir, "tokenizer.json"), "utf8"));
|
|
8061
|
+
const ort = _ort;
|
|
8062
|
+
const session = await ort.InferenceSession.create(path13.join(dir, "onnx", "model_quantized.onnx"));
|
|
8063
|
+
return new _EmbeddingModel(tokenizer, session, ort.Tensor);
|
|
8064
|
+
}
|
|
8065
|
+
/** Embed one text. Sequences are run singly, so there is no padding and no mask to get wrong. */
|
|
8066
|
+
async embed(text) {
|
|
8067
|
+
const ids = this.tokenizer.encode(text);
|
|
8068
|
+
const length = ids.length;
|
|
8069
|
+
const feeds = {
|
|
8070
|
+
input_ids: new this.tensorFactory("int64", BigInt64Array.from(ids, BigInt), [1, length]),
|
|
8071
|
+
attention_mask: new this.tensorFactory("int64", new BigInt64Array(length).fill(1n), [1, length])
|
|
8072
|
+
};
|
|
8073
|
+
if (this.session.inputNames.includes("token_type_ids")) {
|
|
8074
|
+
feeds["token_type_ids"] = new this.tensorFactory("int64", new BigInt64Array(length), [1, length]);
|
|
8075
|
+
}
|
|
8076
|
+
const outputName = this.session.outputNames[0];
|
|
8077
|
+
if (outputName === void 0) throw new Error("the model declares no outputs");
|
|
8078
|
+
const output = (await this.session.run(feeds))[outputName];
|
|
8079
|
+
if (!output) throw new Error(`the model produced no ${outputName}`);
|
|
8080
|
+
const [, seq, dim] = output.dims;
|
|
8081
|
+
if (seq === void 0 || dim === void 0) {
|
|
8082
|
+
throw new Error(`expected a [batch, sequence, dimension] output, got [${output.dims.join(", ")}]`);
|
|
8083
|
+
}
|
|
8084
|
+
if (dim !== DEFAULT_DIM) {
|
|
8085
|
+
throw new Error(`the model produced ${dim}-dimension vectors, expected ${DEFAULT_DIM}`);
|
|
8086
|
+
}
|
|
8087
|
+
return poolAndNormalize(output.data, seq, dim);
|
|
8088
|
+
}
|
|
8089
|
+
};
|
|
8090
|
+
registerReset(() => {
|
|
8091
|
+
_ort = null;
|
|
8092
|
+
_ortError = null;
|
|
8093
|
+
_ortLoadAttempted = false;
|
|
8094
|
+
});
|
|
8095
|
+
|
|
8096
|
+
// src/embeddings.ts
|
|
7269
8097
|
var QUERY_INSTRUCTION_PREFIX = "Represent this sentence for searching relevant passages: ";
|
|
7270
8098
|
var _extractorCache = /* @__PURE__ */ new Map();
|
|
8099
|
+
var inHousePipelineFn = async (_task, modelName) => {
|
|
8100
|
+
const model = await EmbeddingModel.load(modelName);
|
|
8101
|
+
return async (text) => ({ data: await model.embed(text) });
|
|
8102
|
+
};
|
|
7271
8103
|
var _pipelineFnOverride = null;
|
|
7272
8104
|
registerReset(() => {
|
|
7273
8105
|
_extractorCache.clear();
|
|
@@ -7286,8 +8118,7 @@ async function buildExtractorWithRetry(pipelineFn, modelName) {
|
|
|
7286
8118
|
let lastError;
|
|
7287
8119
|
for (let attempt = 1; attempt <= PIPELINE_RETRY_ATTEMPTS; attempt++) {
|
|
7288
8120
|
try {
|
|
7289
|
-
|
|
7290
|
-
return await pipelineFn("feature-extraction", modelName, pipelineOptions);
|
|
8121
|
+
return await pipelineFn("feature-extraction", modelName);
|
|
7291
8122
|
} catch (e) {
|
|
7292
8123
|
lastError = e;
|
|
7293
8124
|
if (attempt < PIPELINE_RETRY_ATTEMPTS) await sleep(PIPELINE_RETRY_DELAY_MS * attempt);
|
|
@@ -7339,25 +8170,23 @@ var _MIN_TOKEN_LEN = 3;
|
|
|
7339
8170
|
var OVER_FETCH_FACTOR = 4;
|
|
7340
8171
|
var MAX_OVER_FETCH = 100;
|
|
7341
8172
|
function isAvailable() {
|
|
7342
|
-
|
|
7343
|
-
|
|
8173
|
+
return isRuntimeAvailable();
|
|
8174
|
+
}
|
|
8175
|
+
function embeddingBackendLoadError() {
|
|
8176
|
+
return runtimeLoadError();
|
|
7344
8177
|
}
|
|
7345
8178
|
async function embedTexts(texts, modelName = DEFAULT_MODEL) {
|
|
7346
8179
|
if (!isAvailable()) {
|
|
7347
8180
|
throw new Error(
|
|
7348
|
-
`
|
|
8181
|
+
`Embedding backend not available: ${runtimeLoadError()?.message ?? "unknown error"}`
|
|
7349
8182
|
);
|
|
7350
8183
|
}
|
|
7351
8184
|
if (texts.length === 0) {
|
|
7352
8185
|
return [];
|
|
7353
8186
|
}
|
|
7354
|
-
if (!_transformer || typeof _transformer !== "object") {
|
|
7355
|
-
throw new Error("Transformer module is unavailable");
|
|
7356
|
-
}
|
|
7357
8187
|
let extractorPromise = _extractorCache.get(modelName);
|
|
7358
8188
|
if (!extractorPromise) {
|
|
7359
|
-
const
|
|
7360
|
-
const pipelineFn = _pipelineFnOverride ?? transformerObj["pipeline"];
|
|
8189
|
+
const pipelineFn = _pipelineFnOverride ?? inHousePipelineFn;
|
|
7361
8190
|
extractorPromise = buildExtractorWithRetry(pipelineFn, modelName);
|
|
7362
8191
|
_extractorCache.set(modelName, extractorPromise);
|
|
7363
8192
|
extractorPromise.catch(() => {
|
|
@@ -7423,7 +8252,8 @@ function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize,
|
|
|
7423
8252
|
const line = lines2[lineNo - 1] ?? "";
|
|
7424
8253
|
const lineWithNewline = line + "\n";
|
|
7425
8254
|
if (currentChunk.length + lineWithNewline.length > chunkSize && currentChunk.length > 0) {
|
|
7426
|
-
const
|
|
8255
|
+
const trimmedLength = currentChunk.trim().length;
|
|
8256
|
+
const currentChunkTooSmall = trimmedLength < MIN_CHUNK_CHARS;
|
|
7427
8257
|
if (!currentChunkTooSmall) {
|
|
7428
8258
|
chunks.push({
|
|
7429
8259
|
filePath,
|
|
@@ -7433,12 +8263,21 @@ function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize,
|
|
|
7433
8263
|
kind
|
|
7434
8264
|
});
|
|
7435
8265
|
}
|
|
7436
|
-
|
|
7437
|
-
|
|
7438
|
-
|
|
7439
|
-
|
|
7440
|
-
|
|
7441
|
-
|
|
8266
|
+
let overlapChars = 0;
|
|
8267
|
+
let computedOverlapStart = currentLine;
|
|
8268
|
+
while (computedOverlapStart > rangeStart) {
|
|
8269
|
+
const candidateChars = (lines2[computedOverlapStart - 2] ?? "").length + 1;
|
|
8270
|
+
if (overlapChars + candidateChars > overlap) break;
|
|
8271
|
+
overlapChars += candidateChars;
|
|
8272
|
+
computedOverlapStart--;
|
|
8273
|
+
}
|
|
8274
|
+
const droppedChunkHadContent = currentChunkTooSmall && trimmedLength > 0;
|
|
8275
|
+
const overlapStart = droppedChunkHadContent ? Math.min(computedOverlapStart, startLine) : computedOverlapStart;
|
|
8276
|
+
if (overlapStart !== startLine) {
|
|
8277
|
+
const overlapText = lines2.slice(overlapStart - 1, currentLine - 1).join("\n");
|
|
8278
|
+
currentChunk = overlapText + "\n";
|
|
8279
|
+
startLine = overlapStart;
|
|
8280
|
+
}
|
|
7442
8281
|
}
|
|
7443
8282
|
currentChunk += lineWithNewline;
|
|
7444
8283
|
currentLine++;
|
|
@@ -7559,6 +8398,7 @@ async function upsertChunks(db, chunks) {
|
|
|
7559
8398
|
deleteFileEmbeddings(db, filePath);
|
|
7560
8399
|
return "unavailable";
|
|
7561
8400
|
}
|
|
8401
|
+
ensureEmbeddingProvenance(db);
|
|
7562
8402
|
const texts = chunks.map((c) => c.text);
|
|
7563
8403
|
const embeddings = await embedTexts(texts);
|
|
7564
8404
|
const chunkInsertStmt = db.prepare(`
|
|
@@ -7587,7 +8427,7 @@ async function upsertChunks(db, chunks) {
|
|
|
7587
8427
|
insertChunkVector(vectorInsertStmt, chunkResult.lastInsertRowid, embedding);
|
|
7588
8428
|
}
|
|
7589
8429
|
});
|
|
7590
|
-
tx();
|
|
8430
|
+
tx.immediate();
|
|
7591
8431
|
return "embedded";
|
|
7592
8432
|
}
|
|
7593
8433
|
var BACKFILL_MULTIPLIER = 3;
|
|
@@ -7629,7 +8469,6 @@ function fetchScopedHits(db, queryVec, k, maxDistance, rootDir) {
|
|
|
7629
8469
|
}
|
|
7630
8470
|
async function searchSemantic(db, query, topK = 8, modelName = DEFAULT_MODEL, maxDistance = DEFAULT_DISTANCE_THRESHOLD, rootDir) {
|
|
7631
8471
|
if (!isAvailable()) {
|
|
7632
|
-
console.warn("Embeddings not available; semantic search disabled");
|
|
7633
8472
|
return [];
|
|
7634
8473
|
}
|
|
7635
8474
|
if (query.trim().length === 0) {
|
|
@@ -7638,6 +8477,7 @@ async function searchSemantic(db, query, topK = 8, modelName = DEFAULT_MODEL, ma
|
|
|
7638
8477
|
if (!chunkVectorsTableExists(db)) {
|
|
7639
8478
|
return [];
|
|
7640
8479
|
}
|
|
8480
|
+
ensureEmbeddingProvenance(db, modelName);
|
|
7641
8481
|
const queryEmbeddings = await embedTexts([`${QUERY_INSTRUCTION_PREFIX}${query}`], modelName);
|
|
7642
8482
|
if (queryEmbeddings.length === 0) {
|
|
7643
8483
|
return [];
|
|
@@ -7786,6 +8626,44 @@ function deleteFileEmbeddings(db, filePath) {
|
|
|
7786
8626
|
}
|
|
7787
8627
|
db.prepare(`DELETE FROM chunks WHERE ${pathEqClause("file_path")}`).run(folded);
|
|
7788
8628
|
}
|
|
8629
|
+
function resetAllEmbeddings(db) {
|
|
8630
|
+
const paths = db.prepare("SELECT DISTINCT file_path FROM chunks").pluck().all();
|
|
8631
|
+
const clearEmbedSha = db.prepare(`UPDATE files SET embed_sha = NULL WHERE ${pathEqClause("path")}`);
|
|
8632
|
+
const tx = db.transaction(() => {
|
|
8633
|
+
for (const p of paths) deleteFileEmbeddings(db, p);
|
|
8634
|
+
for (const p of paths) clearEmbedSha.run(foldPath(p));
|
|
8635
|
+
});
|
|
8636
|
+
tx.immediate();
|
|
8637
|
+
return paths.length;
|
|
8638
|
+
}
|
|
8639
|
+
function embeddingProvenance(modelName = DEFAULT_MODEL) {
|
|
8640
|
+
const revision = modelName === DEFAULT_MODEL ? PINNED_MODEL_REVISION.slice(0, 12) : "unpinned";
|
|
8641
|
+
return `${modelName}@${revision}/${backendId()}`;
|
|
8642
|
+
}
|
|
8643
|
+
function backendId() {
|
|
8644
|
+
return `onnxruntime-node@${majorMinor(runtimeVersion())}`;
|
|
8645
|
+
}
|
|
8646
|
+
function majorMinor(version) {
|
|
8647
|
+
const parts = version.split(".");
|
|
8648
|
+
return parts.length >= 2 ? `${parts[0]}.${parts[1]}` : version;
|
|
8649
|
+
}
|
|
8650
|
+
var _provenanceChecked = /* @__PURE__ */ new WeakSet();
|
|
8651
|
+
function ensureEmbeddingProvenance(db, modelName = DEFAULT_MODEL) {
|
|
8652
|
+
if (_provenanceChecked.has(db)) return;
|
|
8653
|
+
_provenanceChecked.add(db);
|
|
8654
|
+
const current = embeddingProvenance(modelName);
|
|
8655
|
+
const stored = db.prepare("SELECT provenance FROM embedding_provenance WHERE id = 1").pluck().get();
|
|
8656
|
+
if (stored === current) return;
|
|
8657
|
+
const cleared = resetAllEmbeddings(db);
|
|
8658
|
+
db.prepare(
|
|
8659
|
+
"INSERT INTO embedding_provenance (id, provenance) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET provenance = excluded.provenance"
|
|
8660
|
+
).run(current);
|
|
8661
|
+
if (cleared > 0) {
|
|
8662
|
+
console.warn(
|
|
8663
|
+
`Embedding stack changed (${stored ?? "unrecorded"} -> ${current}); discarded ${cleared} file${cleared === 1 ? "" : "s"} worth of vectors because old and new ones are not comparable. Run \`token-goat index\` to rebuild them.`
|
|
8664
|
+
);
|
|
8665
|
+
}
|
|
8666
|
+
}
|
|
7789
8667
|
function _extractQueryTokens(query) {
|
|
7790
8668
|
const tokens = /* @__PURE__ */ new Set();
|
|
7791
8669
|
const matches = query.matchAll(_TOKEN_RE);
|
|
@@ -7822,17 +8700,19 @@ function _pathPriorityPenalty(filePath) {
|
|
|
7822
8700
|
}
|
|
7823
8701
|
|
|
7824
8702
|
// src/hooks_read.ts
|
|
7825
|
-
import * as
|
|
7826
|
-
import * as
|
|
8703
|
+
import * as fs23 from "node:fs";
|
|
8704
|
+
import * as path19 from "node:path";
|
|
7827
8705
|
|
|
7828
8706
|
// src/compact.ts
|
|
7829
|
-
import * as
|
|
7830
|
-
import * as
|
|
8707
|
+
import * as fs18 from "node:fs";
|
|
8708
|
+
import * as path14 from "node:path";
|
|
7831
8709
|
|
|
7832
8710
|
// src/overflow_guard.ts
|
|
8711
|
+
function estimateTokensFromLength(length) {
|
|
8712
|
+
return Math.max(1, Math.floor(Math.max(0, length) / 3) + 1);
|
|
8713
|
+
}
|
|
7833
8714
|
function estimateTokens(text) {
|
|
7834
|
-
|
|
7835
|
-
return Math.max(1, Math.floor(stripped.length / 3) + 1);
|
|
8715
|
+
return estimateTokensFromLength(stripAnsiCodes(text).length);
|
|
7836
8716
|
}
|
|
7837
8717
|
function trimToBudget(text, budgetTokens, command) {
|
|
7838
8718
|
const markerMarginTokens = 64;
|
|
@@ -8089,7 +8969,7 @@ function inferSessionGoal(cache, maxTokens = 80) {
|
|
|
8089
8969
|
const dirCounts = new Counter();
|
|
8090
8970
|
for (const fpath of editedPaths) {
|
|
8091
8971
|
try {
|
|
8092
|
-
let parent =
|
|
8972
|
+
let parent = path14.dirname(fpath);
|
|
8093
8973
|
if (parent === ".") {
|
|
8094
8974
|
parent = "root";
|
|
8095
8975
|
} else if (parent.startsWith("./")) {
|
|
@@ -8165,11 +9045,11 @@ function isNoisePath(inputPath) {
|
|
|
8165
9045
|
}
|
|
8166
9046
|
function findLatestSessionId() {
|
|
8167
9047
|
try {
|
|
8168
|
-
const sessionsDir =
|
|
8169
|
-
if (!
|
|
9048
|
+
const sessionsDir = path14.join(tokenGoatHome(), "sessions");
|
|
9049
|
+
if (!fs18.existsSync(sessionsDir)) {
|
|
8170
9050
|
return null;
|
|
8171
9051
|
}
|
|
8172
|
-
const files =
|
|
9052
|
+
const files = fs18.readdirSync(sessionsDir);
|
|
8173
9053
|
const jsonFiles = files.filter((f) => f.endsWith(".json") && !f.includes(AGENT_SALT_MARKER));
|
|
8174
9054
|
if (jsonFiles.length === 0) {
|
|
8175
9055
|
return null;
|
|
@@ -8179,9 +9059,9 @@ function findLatestSessionId() {
|
|
|
8179
9059
|
return null;
|
|
8180
9060
|
}
|
|
8181
9061
|
let latestFile = firstFile;
|
|
8182
|
-
let latestMtime =
|
|
9062
|
+
let latestMtime = fs18.statSync(path14.join(sessionsDir, firstFile)).mtimeMs;
|
|
8183
9063
|
for (const file of jsonFiles) {
|
|
8184
|
-
const mtime =
|
|
9064
|
+
const mtime = fs18.statSync(path14.join(sessionsDir, file)).mtimeMs;
|
|
8185
9065
|
if (mtime > latestMtime) {
|
|
8186
9066
|
latestFile = file;
|
|
8187
9067
|
latestMtime = mtime;
|
|
@@ -8203,37 +9083,37 @@ function eventCount(cache) {
|
|
|
8203
9083
|
function writeSessionManifest(projectHash, sessionId, manifestJson) {
|
|
8204
9084
|
const safeSessionId = sanitizeIdForFilename(sessionId, 64);
|
|
8205
9085
|
if (!safeSessionId) return;
|
|
8206
|
-
const sessionsDir =
|
|
8207
|
-
if (!
|
|
9086
|
+
const sessionsDir = path14.join(dataDir(), "projects", projectHash, "sessions");
|
|
9087
|
+
if (!fs18.existsSync(sessionsDir)) {
|
|
8208
9088
|
ensureDirSync(sessionsDir);
|
|
8209
9089
|
}
|
|
8210
|
-
const dest =
|
|
9090
|
+
const dest = path14.join(sessionsDir, `${safeSessionId}.json`);
|
|
8211
9091
|
atomicWriteText(dest, JSON.stringify(manifestJson));
|
|
8212
9092
|
}
|
|
8213
9093
|
function readAllSessionManifests(projectHash, maxAgeSecs = 3600) {
|
|
8214
|
-
const sessionsDir =
|
|
8215
|
-
if (!
|
|
9094
|
+
const sessionsDir = path14.join(dataDir(), "projects", projectHash, "sessions");
|
|
9095
|
+
if (!fs18.existsSync(sessionsDir)) {
|
|
8216
9096
|
return [];
|
|
8217
9097
|
}
|
|
8218
9098
|
const now = Date.now() / 1e3;
|
|
8219
9099
|
const results = [];
|
|
8220
9100
|
try {
|
|
8221
|
-
const files =
|
|
9101
|
+
const files = fs18.readdirSync(sessionsDir);
|
|
8222
9102
|
for (const file of files) {
|
|
8223
9103
|
if (!file.endsWith(".json")) {
|
|
8224
9104
|
continue;
|
|
8225
9105
|
}
|
|
8226
9106
|
try {
|
|
8227
|
-
const fullPath =
|
|
8228
|
-
const stat2 =
|
|
9107
|
+
const fullPath = path14.join(sessionsDir, file);
|
|
9108
|
+
const stat2 = fs18.statSync(fullPath);
|
|
8229
9109
|
if (now - stat2.mtimeMs / 1e3 > maxAgeSecs) {
|
|
8230
9110
|
try {
|
|
8231
|
-
|
|
9111
|
+
fs18.unlinkSync(fullPath);
|
|
8232
9112
|
} catch {
|
|
8233
9113
|
}
|
|
8234
9114
|
continue;
|
|
8235
9115
|
}
|
|
8236
|
-
const text =
|
|
9116
|
+
const text = fs18.readFileSync(fullPath, "utf8");
|
|
8237
9117
|
const data = JSON.parse(text);
|
|
8238
9118
|
if (typeof data === "object" && data !== null && "files" in data) {
|
|
8239
9119
|
results.push(data);
|
|
@@ -8404,8 +9284,8 @@ function buildManifestWithCount(sessionId, opts) {
|
|
|
8404
9284
|
}
|
|
8405
9285
|
|
|
8406
9286
|
// src/snapshots.ts
|
|
8407
|
-
import * as
|
|
8408
|
-
import * as
|
|
9287
|
+
import * as fs19 from "node:fs";
|
|
9288
|
+
import * as path15 from "node:path";
|
|
8409
9289
|
var MAX_SNAPSHOTS_PER_SESSION = 150;
|
|
8410
9290
|
var MAX_SNAPSHOT_BYTES = 256 * 1024;
|
|
8411
9291
|
var SNAPSHOT_TRUNCATE_BYTES = 50 * 1024;
|
|
@@ -8415,10 +9295,10 @@ var VALID_KINDS = /* @__PURE__ */ new Set([KIND_READ, KIND_PREDICTIVE]);
|
|
|
8415
9295
|
function sessionDir(sessionId) {
|
|
8416
9296
|
if (!sessionId) return null;
|
|
8417
9297
|
const safe = sanitizeIdForFilename(sessionId, 64, "anon");
|
|
8418
|
-
const base =
|
|
8419
|
-
const candidate =
|
|
9298
|
+
const base = path15.join(tokenGoatHome(), "session_snapshots");
|
|
9299
|
+
const candidate = path15.join(base, safe);
|
|
8420
9300
|
try {
|
|
8421
|
-
const rel =
|
|
9301
|
+
const rel = path15.relative(base, candidate);
|
|
8422
9302
|
if (rel.startsWith("..")) return null;
|
|
8423
9303
|
} catch {
|
|
8424
9304
|
return null;
|
|
@@ -8431,7 +9311,7 @@ function pathKey(filePath) {
|
|
|
8431
9311
|
function snapshot_path(sessionId, filePath) {
|
|
8432
9312
|
const d = sessionDir(sessionId);
|
|
8433
9313
|
if (!d) return null;
|
|
8434
|
-
return
|
|
9314
|
+
return path15.join(d, `${pathKey(filePath)}.bin`);
|
|
8435
9315
|
}
|
|
8436
9316
|
function kindSidecarPath(snapshotPath) {
|
|
8437
9317
|
return snapshotPath + ".kind";
|
|
@@ -8439,11 +9319,11 @@ function kindSidecarPath(snapshotPath) {
|
|
|
8439
9319
|
function writeSnapshotKind(sidecarPath, kind) {
|
|
8440
9320
|
try {
|
|
8441
9321
|
const safeKind = VALID_KINDS.has(kind) ? kind : KIND_READ;
|
|
8442
|
-
const dir =
|
|
8443
|
-
if (!
|
|
9322
|
+
const dir = path15.dirname(sidecarPath);
|
|
9323
|
+
if (!fs19.existsSync(dir)) {
|
|
8444
9324
|
ensureDirSync(dir);
|
|
8445
9325
|
}
|
|
8446
|
-
|
|
9326
|
+
fs19.writeFileSync(sidecarPath, safeKind, "utf8");
|
|
8447
9327
|
return true;
|
|
8448
9328
|
} catch {
|
|
8449
9329
|
return false;
|
|
@@ -8452,12 +9332,12 @@ function writeSnapshotKind(sidecarPath, kind) {
|
|
|
8452
9332
|
function evictOldest(d, maxCount) {
|
|
8453
9333
|
try {
|
|
8454
9334
|
const entries = [];
|
|
8455
|
-
const files =
|
|
9335
|
+
const files = fs19.readdirSync(d);
|
|
8456
9336
|
for (const file of files) {
|
|
8457
|
-
const fullPath =
|
|
9337
|
+
const fullPath = path15.join(d, file);
|
|
8458
9338
|
if (!file.endsWith(".bin")) continue;
|
|
8459
9339
|
try {
|
|
8460
|
-
const stat2 =
|
|
9340
|
+
const stat2 = fs19.statSync(fullPath);
|
|
8461
9341
|
entries.push([fullPath, stat2.mtimeMs]);
|
|
8462
9342
|
} catch {
|
|
8463
9343
|
continue;
|
|
@@ -8469,10 +9349,10 @@ function evictOldest(d, maxCount) {
|
|
|
8469
9349
|
const over = entries.length - maxCount;
|
|
8470
9350
|
for (const [p] of entries.slice(0, over)) {
|
|
8471
9351
|
try {
|
|
8472
|
-
|
|
9352
|
+
fs19.unlinkSync(p);
|
|
8473
9353
|
removed++;
|
|
8474
9354
|
try {
|
|
8475
|
-
|
|
9355
|
+
fs19.unlinkSync(kindSidecarPath(p));
|
|
8476
9356
|
} catch {
|
|
8477
9357
|
}
|
|
8478
9358
|
} catch {
|
|
@@ -8501,10 +9381,10 @@ function store(sessionId, filePath, content, opts = {}) {
|
|
|
8501
9381
|
if (!p) return null;
|
|
8502
9382
|
const sha = fingerprintContent(stored);
|
|
8503
9383
|
try {
|
|
8504
|
-
const isNewEntry = !
|
|
9384
|
+
const isNewEntry = !fs19.existsSync(p);
|
|
8505
9385
|
if (!isNewEntry) {
|
|
8506
9386
|
try {
|
|
8507
|
-
const existing =
|
|
9387
|
+
const existing = fs19.readFileSync(p);
|
|
8508
9388
|
if (Buffer.from(existing).equals(stored)) {
|
|
8509
9389
|
return {
|
|
8510
9390
|
path: p,
|
|
@@ -8515,8 +9395,8 @@ function store(sessionId, filePath, content, opts = {}) {
|
|
|
8515
9395
|
} catch {
|
|
8516
9396
|
}
|
|
8517
9397
|
}
|
|
8518
|
-
const dir =
|
|
8519
|
-
if (!
|
|
9398
|
+
const dir = path15.dirname(p);
|
|
9399
|
+
if (!fs19.existsSync(dir)) {
|
|
8520
9400
|
ensureDirSync(dir);
|
|
8521
9401
|
}
|
|
8522
9402
|
if (isNewEntry) {
|
|
@@ -8536,9 +9416,9 @@ function store(sessionId, filePath, content, opts = {}) {
|
|
|
8536
9416
|
}
|
|
8537
9417
|
function load(sessionId, filePath, opts = {}) {
|
|
8538
9418
|
const p = snapshot_path(sessionId, filePath);
|
|
8539
|
-
if (!p || !
|
|
9419
|
+
if (!p || !fs19.existsSync(p)) return null;
|
|
8540
9420
|
try {
|
|
8541
|
-
const stat2 =
|
|
9421
|
+
const stat2 = fs19.statSync(p);
|
|
8542
9422
|
if (stat2.size > MAX_SNAPSHOT_BYTES) {
|
|
8543
9423
|
return null;
|
|
8544
9424
|
}
|
|
@@ -8546,7 +9426,7 @@ function load(sessionId, filePath, opts = {}) {
|
|
|
8546
9426
|
return null;
|
|
8547
9427
|
}
|
|
8548
9428
|
try {
|
|
8549
|
-
const data =
|
|
9429
|
+
const data = fs19.readFileSync(p);
|
|
8550
9430
|
if (opts.expected_sha) {
|
|
8551
9431
|
const actualSha = fingerprintContent(data);
|
|
8552
9432
|
if (actualSha.toLowerCase() !== opts.expected_sha.toLowerCase()) {
|
|
@@ -8559,31 +9439,31 @@ function load(sessionId, filePath, opts = {}) {
|
|
|
8559
9439
|
}
|
|
8560
9440
|
}
|
|
8561
9441
|
function removeEligibleSnapshotFile(fullPath, file, cutoff) {
|
|
8562
|
-
const stat2 =
|
|
9442
|
+
const stat2 = fs19.lstatSync(fullPath);
|
|
8563
9443
|
if ((stat2.mode & 61440) === 40960) return false;
|
|
8564
9444
|
if (cutoff !== void 0 && stat2.mtimeMs >= cutoff) return false;
|
|
8565
|
-
|
|
9445
|
+
fs19.unlinkSync(fullPath);
|
|
8566
9446
|
return file.endsWith(".bin");
|
|
8567
9447
|
}
|
|
8568
9448
|
function cleanup_stale(maxAgeHours = 24) {
|
|
8569
|
-
const base =
|
|
8570
|
-
if (!
|
|
9449
|
+
const base = path15.join(tokenGoatHome(), "session_snapshots");
|
|
9450
|
+
if (!fs19.existsSync(base)) return 0;
|
|
8571
9451
|
const cutoff = Date.now() - maxAgeHours * 3600 * 1e3;
|
|
8572
9452
|
let removed = 0;
|
|
8573
9453
|
try {
|
|
8574
|
-
const sessionDirs =
|
|
9454
|
+
const sessionDirs = fs19.readdirSync(base);
|
|
8575
9455
|
for (const sessionDir2 of sessionDirs) {
|
|
8576
|
-
const sessionPath2 =
|
|
9456
|
+
const sessionPath2 = path15.join(base, sessionDir2);
|
|
8577
9457
|
try {
|
|
8578
|
-
const stat2 =
|
|
9458
|
+
const stat2 = fs19.statSync(sessionPath2);
|
|
8579
9459
|
if (!stat2.isDirectory()) continue;
|
|
8580
9460
|
} catch {
|
|
8581
9461
|
continue;
|
|
8582
9462
|
}
|
|
8583
9463
|
try {
|
|
8584
|
-
const files =
|
|
9464
|
+
const files = fs19.readdirSync(sessionPath2);
|
|
8585
9465
|
for (const file of files) {
|
|
8586
|
-
const fullPath =
|
|
9466
|
+
const fullPath = path15.join(sessionPath2, file);
|
|
8587
9467
|
try {
|
|
8588
9468
|
if (removeEligibleSnapshotFile(fullPath, file, cutoff)) removed++;
|
|
8589
9469
|
} catch {
|
|
@@ -8591,7 +9471,7 @@ function cleanup_stale(maxAgeHours = 24) {
|
|
|
8591
9471
|
}
|
|
8592
9472
|
}
|
|
8593
9473
|
try {
|
|
8594
|
-
|
|
9474
|
+
fs19.rmdirSync(sessionPath2);
|
|
8595
9475
|
} catch {
|
|
8596
9476
|
}
|
|
8597
9477
|
} catch {
|
|
@@ -8627,11 +9507,11 @@ function buildPackageManifestHint(options) {
|
|
|
8627
9507
|
return null;
|
|
8628
9508
|
}
|
|
8629
9509
|
}
|
|
8630
|
-
function _sanitizeHintPath(
|
|
8631
|
-
if (typeof
|
|
9510
|
+
function _sanitizeHintPath(path31) {
|
|
9511
|
+
if (typeof path31 !== "string") {
|
|
8632
9512
|
return "???";
|
|
8633
9513
|
}
|
|
8634
|
-
return
|
|
9514
|
+
return path31.replace(/[\x00]/g, "").slice(0, 200);
|
|
8635
9515
|
}
|
|
8636
9516
|
|
|
8637
9517
|
// src/hints/lang_patterns.ts
|
|
@@ -9259,33 +10139,33 @@ function dispatchFileTypeHandler(filePath, content, contentLengthHint) {
|
|
|
9259
10139
|
}
|
|
9260
10140
|
|
|
9261
10141
|
// src/doc_compact.ts
|
|
9262
|
-
import * as
|
|
9263
|
-
import * as
|
|
10142
|
+
import * as fs20 from "fs";
|
|
10143
|
+
import * as path16 from "path";
|
|
9264
10144
|
var defaultSentencesPerSection = 2;
|
|
9265
10145
|
var headerPrefix = "<!-- token-goat doc-compact source-hash:";
|
|
9266
10146
|
var headerRegex = /^<!-- token-goat doc-compact source-hash:(\S+) source:(.+?) -->\r?$/;
|
|
9267
10147
|
var compactSubdir = "doc_compacts";
|
|
9268
10148
|
function sourceHash(filePath) {
|
|
9269
10149
|
try {
|
|
9270
|
-
return fingerprintContent(
|
|
10150
|
+
return fingerprintContent(fs20.readFileSync(filePath));
|
|
9271
10151
|
} catch {
|
|
9272
10152
|
return "";
|
|
9273
10153
|
}
|
|
9274
10154
|
}
|
|
9275
10155
|
function _compactSlug(absPathStr) {
|
|
9276
10156
|
const h = fingerprintContent(foldPath(absPathStr)).slice(0, 12);
|
|
9277
|
-
const ext =
|
|
9278
|
-
const stem =
|
|
10157
|
+
const ext = path16.extname(absPathStr);
|
|
10158
|
+
const stem = path16.basename(absPathStr, ext);
|
|
9279
10159
|
const safeStem = sanitizeIdForFilename(stem, 32);
|
|
9280
10160
|
return `${h}_${safeStem}`;
|
|
9281
10161
|
}
|
|
9282
10162
|
function compactPathFor(sourcePath) {
|
|
9283
10163
|
const abs = resolveIndexPath(sourcePath);
|
|
9284
|
-
return
|
|
10164
|
+
return path16.join(dataDir(), compactSubdir, `${_compactSlug(abs)}.md`);
|
|
9285
10165
|
}
|
|
9286
10166
|
function readCompactHeader(compactPath) {
|
|
9287
10167
|
try {
|
|
9288
|
-
const text =
|
|
10168
|
+
const text = fs20.readFileSync(compactPath, "utf-8");
|
|
9289
10169
|
const firstLine = text.split("\n")[0] || "";
|
|
9290
10170
|
const m = firstLine.match(headerRegex);
|
|
9291
10171
|
if (!m || !m[1] || !m[2]) return null;
|
|
@@ -9304,8 +10184,8 @@ function isCompactFresh(compactPath, sourcePath) {
|
|
|
9304
10184
|
}
|
|
9305
10185
|
function markCompactStale(compactPath) {
|
|
9306
10186
|
try {
|
|
9307
|
-
if (!
|
|
9308
|
-
const text =
|
|
10187
|
+
if (!fs20.existsSync(compactPath)) return false;
|
|
10188
|
+
const text = fs20.readFileSync(compactPath, "utf-8");
|
|
9309
10189
|
const lines2 = text.split("\n");
|
|
9310
10190
|
if (!lines2[0]) return false;
|
|
9311
10191
|
const m = lines2[0].match(headerRegex);
|
|
@@ -9320,7 +10200,7 @@ function markCompactStale(compactPath) {
|
|
|
9320
10200
|
}
|
|
9321
10201
|
function readCompactBody(compactPath) {
|
|
9322
10202
|
try {
|
|
9323
|
-
const text =
|
|
10203
|
+
const text = fs20.readFileSync(compactPath, "utf-8");
|
|
9324
10204
|
const lines2 = text.split("\n");
|
|
9325
10205
|
if (lines2.length < 2) return null;
|
|
9326
10206
|
const body = lines2.slice(1).join("\n").trimStart();
|
|
@@ -9330,14 +10210,14 @@ function readCompactBody(compactPath) {
|
|
|
9330
10210
|
}
|
|
9331
10211
|
}
|
|
9332
10212
|
function writeCompact(compactPath, sourcePath, compactBody, sourceRel) {
|
|
9333
|
-
const srcPath =
|
|
10213
|
+
const srcPath = path16.resolve(sourcePath);
|
|
9334
10214
|
const sha = sourceHash(srcPath);
|
|
9335
|
-
const displayRel = sourceRel ||
|
|
10215
|
+
const displayRel = sourceRel || path16.basename(srcPath);
|
|
9336
10216
|
const header = `${headerPrefix}${sha} source:${displayRel} -->
|
|
9337
10217
|
`;
|
|
9338
10218
|
const fullText = header + compactBody.trimStart();
|
|
9339
|
-
const dir =
|
|
9340
|
-
if (!
|
|
10219
|
+
const dir = path16.dirname(compactPath);
|
|
10220
|
+
if (!fs20.existsSync(dir)) {
|
|
9341
10221
|
ensureDirSync(dir);
|
|
9342
10222
|
}
|
|
9343
10223
|
atomicWriteText(compactPath, fullText);
|
|
@@ -9461,7 +10341,7 @@ function extractDocCompact(body, heading) {
|
|
|
9461
10341
|
}
|
|
9462
10342
|
function compactDoc(filePath, heading) {
|
|
9463
10343
|
try {
|
|
9464
|
-
const body =
|
|
10344
|
+
const body = fs20.readFileSync(filePath, "utf-8");
|
|
9465
10345
|
const compact = extractDocCompact(body, heading);
|
|
9466
10346
|
return compact || null;
|
|
9467
10347
|
} catch {
|
|
@@ -9471,21 +10351,21 @@ function compactDoc(filePath, heading) {
|
|
|
9471
10351
|
|
|
9472
10352
|
// src/evidence_cache.ts
|
|
9473
10353
|
import crypto2 from "node:crypto";
|
|
9474
|
-
import
|
|
9475
|
-
import
|
|
10354
|
+
import fs21 from "node:fs";
|
|
10355
|
+
import path17 from "node:path";
|
|
9476
10356
|
var MAX_ENTRIES = 500;
|
|
9477
10357
|
var MAX_TEXT_BYTES = 128 * 1024;
|
|
9478
10358
|
var MAX_SEMANTIC_CANDIDATES = 100;
|
|
9479
10359
|
var CACHE_FILE = "workspace-evidence.json";
|
|
9480
10360
|
function cachePath() {
|
|
9481
|
-
return
|
|
10361
|
+
return path17.join(dataDir(), CACHE_FILE);
|
|
9482
10362
|
}
|
|
9483
10363
|
function hash(text) {
|
|
9484
10364
|
return crypto2.createHash("sha256").update(text).digest("hex");
|
|
9485
10365
|
}
|
|
9486
10366
|
function load2() {
|
|
9487
10367
|
try {
|
|
9488
|
-
const parsed = JSON.parse(
|
|
10368
|
+
const parsed = JSON.parse(fs21.readFileSync(cachePath(), "utf8"));
|
|
9489
10369
|
if (!Array.isArray(parsed)) return [];
|
|
9490
10370
|
return parsed.filter(
|
|
9491
10371
|
(entry) => typeof entry === "object" && entry !== null && typeof entry.id === "string" && typeof entry.projectRoot === "string" && typeof entry.source === "string" && (entry.representation === "file" || entry.representation === "tool-output") && typeof entry.contentHash === "string" && typeof entry.text === "string" && typeof entry.createdAt === "number" && (entry.embedding === void 0 || typeof entry.embedding === "string")
|
|
@@ -9497,7 +10377,7 @@ function load2() {
|
|
|
9497
10377
|
function save(entries) {
|
|
9498
10378
|
try {
|
|
9499
10379
|
ensureDirSync(dataDir());
|
|
9500
|
-
|
|
10380
|
+
fs21.writeFileSync(cachePath(), JSON.stringify(entries.slice(0, MAX_ENTRIES)), "utf8");
|
|
9501
10381
|
} catch {
|
|
9502
10382
|
}
|
|
9503
10383
|
}
|
|
@@ -9588,7 +10468,7 @@ function buildDeltaCapsule(projectRoot, limit = 8) {
|
|
|
9588
10468
|
const root = normalizePath(projectRoot);
|
|
9589
10469
|
const changed = load2().filter((entry) => entry.projectRoot === root && entry.representation === "file").filter((entry) => {
|
|
9590
10470
|
try {
|
|
9591
|
-
return hash(
|
|
10471
|
+
return hash(fs21.readFileSync(entry.source, "utf8")) !== entry.contentHash;
|
|
9592
10472
|
} catch {
|
|
9593
10473
|
return true;
|
|
9594
10474
|
}
|
|
@@ -9599,8 +10479,8 @@ ${changed.map((entry) => `- ${entry.source} (use a fresh surgical read)`).join("
|
|
|
9599
10479
|
}
|
|
9600
10480
|
|
|
9601
10481
|
// src/notebook_compact.ts
|
|
9602
|
-
import * as
|
|
9603
|
-
import * as
|
|
10482
|
+
import * as fs22 from "node:fs";
|
|
10483
|
+
import * as path18 from "node:path";
|
|
9604
10484
|
var NB_STRIP_MIN_SAVINGS = 4096;
|
|
9605
10485
|
function stripNotebook(nbDict) {
|
|
9606
10486
|
const cells = [];
|
|
@@ -9620,23 +10500,23 @@ function stripNotebook(nbDict) {
|
|
|
9620
10500
|
var SIDECAR_DEFAULT_MAX_COUNT = 200;
|
|
9621
10501
|
var SIDECAR_DEFAULT_MAX_AGE_MS = 24 * 3600 * 1e3;
|
|
9622
10502
|
function pruneSidecars(cacheRoot, maxCount = SIDECAR_DEFAULT_MAX_COUNT, maxAgeMs = SIDECAR_DEFAULT_MAX_AGE_MS) {
|
|
9623
|
-
const nbStripDir =
|
|
10503
|
+
const nbStripDir = path18.join(cacheRoot, "nb_strip");
|
|
9624
10504
|
let removed = 0;
|
|
9625
10505
|
try {
|
|
9626
|
-
if (!
|
|
10506
|
+
if (!fs22.existsSync(nbStripDir)) return 0;
|
|
9627
10507
|
const cutoff = Date.now() - maxAgeMs;
|
|
9628
10508
|
const kept = [];
|
|
9629
|
-
for (const entry of
|
|
9630
|
-
const dir =
|
|
10509
|
+
for (const entry of fs22.readdirSync(nbStripDir)) {
|
|
10510
|
+
const dir = path18.join(nbStripDir, entry);
|
|
9631
10511
|
let mtime;
|
|
9632
10512
|
try {
|
|
9633
|
-
mtime =
|
|
10513
|
+
mtime = fs22.statSync(dir).mtimeMs;
|
|
9634
10514
|
} catch {
|
|
9635
10515
|
continue;
|
|
9636
10516
|
}
|
|
9637
10517
|
if (mtime < cutoff) {
|
|
9638
10518
|
try {
|
|
9639
|
-
|
|
10519
|
+
fs22.rmSync(dir, { recursive: true, force: true });
|
|
9640
10520
|
removed++;
|
|
9641
10521
|
} catch {
|
|
9642
10522
|
continue;
|
|
@@ -9649,7 +10529,7 @@ function pruneSidecars(cacheRoot, maxCount = SIDECAR_DEFAULT_MAX_COUNT, maxAgeMs
|
|
|
9649
10529
|
kept.sort((a, b) => a[1] - b[1]);
|
|
9650
10530
|
for (const [dir] of kept.slice(0, kept.length - maxCount)) {
|
|
9651
10531
|
try {
|
|
9652
|
-
|
|
10532
|
+
fs22.rmSync(dir, { recursive: true, force: true });
|
|
9653
10533
|
removed++;
|
|
9654
10534
|
} catch {
|
|
9655
10535
|
continue;
|
|
@@ -9663,9 +10543,9 @@ function pruneSidecars(cacheRoot, maxCount = SIDECAR_DEFAULT_MAX_COUNT, maxAgeMs
|
|
|
9663
10543
|
}
|
|
9664
10544
|
function getOrCreateSidecar(rawBytes, cacheRoot, opts = {}) {
|
|
9665
10545
|
const sha = fingerprintContent(rawBytes);
|
|
9666
|
-
const sidecarDir =
|
|
9667
|
-
const sidecarPath =
|
|
9668
|
-
if (
|
|
10546
|
+
const sidecarDir = path18.join(cacheRoot, "nb_strip", sha);
|
|
10547
|
+
const sidecarPath = path18.join(sidecarDir, "stripped.ipynb");
|
|
10548
|
+
if (fs22.existsSync(sidecarPath)) {
|
|
9669
10549
|
return [sidecarPath, false];
|
|
9670
10550
|
}
|
|
9671
10551
|
let nb;
|
|
@@ -9682,7 +10562,7 @@ function getOrCreateSidecar(rawBytes, cacheRoot, opts = {}) {
|
|
|
9682
10562
|
try {
|
|
9683
10563
|
ensureDirSync(sidecarDir);
|
|
9684
10564
|
} catch (err) {
|
|
9685
|
-
if (!
|
|
10565
|
+
if (!fs22.existsSync(sidecarDir)) {
|
|
9686
10566
|
throw err;
|
|
9687
10567
|
}
|
|
9688
10568
|
}
|
|
@@ -9715,8 +10595,8 @@ function isNodeModulesPath(p) {
|
|
|
9715
10595
|
return check.includes("/node_modules/") || check.includes("\\node_modules\\");
|
|
9716
10596
|
}
|
|
9717
10597
|
function relPathWithinRoot(root, target) {
|
|
9718
|
-
const rel =
|
|
9719
|
-
if (rel.startsWith("..") ||
|
|
10598
|
+
const rel = path19.relative(root, target).replace(/\\/g, "/");
|
|
10599
|
+
if (rel.startsWith("..") || path19.isAbsolute(rel)) return null;
|
|
9720
10600
|
return rel;
|
|
9721
10601
|
}
|
|
9722
10602
|
function _isDocFile(filePath) {
|
|
@@ -9747,7 +10627,7 @@ function scanRequestedSlice(absPath, offset, limit) {
|
|
|
9747
10627
|
const windowEnd = offset + limit;
|
|
9748
10628
|
let fd;
|
|
9749
10629
|
try {
|
|
9750
|
-
fd =
|
|
10630
|
+
fd = fs23.openSync(absPath, "r");
|
|
9751
10631
|
} catch {
|
|
9752
10632
|
return null;
|
|
9753
10633
|
}
|
|
@@ -9761,7 +10641,7 @@ function scanRequestedSlice(absPath, offset, limit) {
|
|
|
9761
10641
|
const nearSingleLine = lineNumber < NEAR_SINGLE_LINE_SCAN_THRESHOLD;
|
|
9762
10642
|
return { bytes: sliceBytes, trustworthy: nearSingleLine, nearSingleLine };
|
|
9763
10643
|
}
|
|
9764
|
-
const bytesRead =
|
|
10644
|
+
const bytesRead = fs23.readSync(fd, buf, 0, buf.length, null);
|
|
9765
10645
|
if (bytesRead === 0) {
|
|
9766
10646
|
return {
|
|
9767
10647
|
bytes: sliceBytes,
|
|
@@ -9780,7 +10660,7 @@ function scanRequestedSlice(absPath, offset, limit) {
|
|
|
9780
10660
|
}
|
|
9781
10661
|
} finally {
|
|
9782
10662
|
try {
|
|
9783
|
-
|
|
10663
|
+
fs23.closeSync(fd);
|
|
9784
10664
|
} catch {
|
|
9785
10665
|
}
|
|
9786
10666
|
}
|
|
@@ -9817,7 +10697,7 @@ var BINARY_FILE_TYPE_EXTS = /* @__PURE__ */ new Set(["pdf", "docx", "xlsx", "ppt
|
|
|
9817
10697
|
var TEXT_FILE_TYPE_EXTS = /* @__PURE__ */ new Set(["html", "htm", "xhtml", "txt", "log", "out", "err", "trace", "csv", "tsv", "vtt", "srt"]);
|
|
9818
10698
|
var DISPATCHED_FILE_TYPE_EXTS = /* @__PURE__ */ new Set([...BINARY_FILE_TYPE_EXTS, ...TEXT_FILE_TYPE_EXTS]);
|
|
9819
10699
|
function isDispatchedFileType(basename12) {
|
|
9820
|
-
return DISPATCHED_FILE_TYPE_EXTS.has(
|
|
10700
|
+
return DISPATCHED_FILE_TYPE_EXTS.has(path19.extname(basename12).slice(1).toLowerCase());
|
|
9821
10701
|
}
|
|
9822
10702
|
function surgicalHint(filePath, basename12, lineCount) {
|
|
9823
10703
|
if (lineCount < loadConfig().hints.min_file_lines_for_hint) return "";
|
|
@@ -9880,7 +10760,7 @@ function loadSnapshotDiff(sessionId, normalized, basename12) {
|
|
|
9880
10760
|
try {
|
|
9881
10761
|
const sz = statSize(normalized);
|
|
9882
10762
|
if (sz === null || sz > 256 * 1024) return { kind: "none" };
|
|
9883
|
-
const currentContent =
|
|
10763
|
+
const currentContent = fs23.readFileSync(normalized, "utf8");
|
|
9884
10764
|
const TRUNC_MARKER = "\n<snapshot truncated at ";
|
|
9885
10765
|
const oldRaw = oldSnap.toString("utf8");
|
|
9886
10766
|
const truncIdx = oldRaw.indexOf(TRUNC_MARKER);
|
|
@@ -9966,7 +10846,7 @@ function preReadHandlerInner(event) {
|
|
|
9966
10846
|
try {
|
|
9967
10847
|
const cwd = getCwd(event) ?? process.cwd();
|
|
9968
10848
|
const project = findProject(cwd) ?? makeProjectAt(cwd);
|
|
9969
|
-
const current =
|
|
10849
|
+
const current = fs23.readFileSync(normalized, "utf8");
|
|
9970
10850
|
const evidence = findVerifiedFileEvidence(project.root, normalized, current);
|
|
9971
10851
|
if (evidence !== null) {
|
|
9972
10852
|
recordStat("evidence_cache_hit", 0);
|
|
@@ -9977,7 +10857,7 @@ function preReadHandlerInner(event) {
|
|
|
9977
10857
|
} catch {
|
|
9978
10858
|
}
|
|
9979
10859
|
}
|
|
9980
|
-
const basename12 =
|
|
10860
|
+
const basename12 = path19.basename(normalized);
|
|
9981
10861
|
if (isLockFile(basename12)) {
|
|
9982
10862
|
return denyOutput(
|
|
9983
10863
|
'Lock files are rarely useful to read in full. Use `token-goat section "' + shown + '::<section>"` to extract a specific dependency, or read the relevant manifest instead.'
|
|
@@ -10016,7 +10896,7 @@ function preReadHandlerInner(event) {
|
|
|
10016
10896
|
const skillName = detectSkillFile(normalized);
|
|
10017
10897
|
if (skillName && basename12 === "SKILL.md") {
|
|
10018
10898
|
try {
|
|
10019
|
-
const body =
|
|
10899
|
+
const body = fs23.readFileSync(normalized, "utf-8");
|
|
10020
10900
|
const bodySha = contentHash(body);
|
|
10021
10901
|
const compact = getCompactAnySessionSync(skillName);
|
|
10022
10902
|
const stale = isCompactStale(compact, skillName, bodySha);
|
|
@@ -10047,9 +10927,9 @@ function preReadHandlerInner(event) {
|
|
|
10047
10927
|
const isNotebook = /\.ipynb$/i.test(basename12);
|
|
10048
10928
|
if (event.toolName !== "Grep" && isNotebook) {
|
|
10049
10929
|
try {
|
|
10050
|
-
const rawBytes =
|
|
10930
|
+
const rawBytes = fs23.readFileSync(normalized);
|
|
10051
10931
|
const [sidecarPath] = getOrCreateSidecar(rawBytes, dataDir());
|
|
10052
|
-
const sidecarContent =
|
|
10932
|
+
const sidecarContent = fs23.readFileSync(sidecarPath, "utf-8");
|
|
10053
10933
|
const savedBytes = rawBytes.length - sidecarContent.length;
|
|
10054
10934
|
if (savedBytes >= NB_STRIP_MIN_SAVINGS) {
|
|
10055
10935
|
recordActualRead(event, normalized);
|
|
@@ -10069,7 +10949,7 @@ function preReadHandlerInner(event) {
|
|
|
10069
10949
|
const sz = statSize(normalized);
|
|
10070
10950
|
if (sz !== null && sz >= MARKDOWN_SIZE_THRESHOLD) {
|
|
10071
10951
|
markdownSize = sz;
|
|
10072
|
-
fileContent =
|
|
10952
|
+
fileContent = fs23.readFileSync(normalized, "utf8");
|
|
10073
10953
|
}
|
|
10074
10954
|
} catch {
|
|
10075
10955
|
}
|
|
@@ -10298,7 +11178,7 @@ function preReadHandlerInner(event) {
|
|
|
10298
11178
|
"Note: " + shown + " is large (" + kb + "KB). " + hint + contextPressureAdvisorySuffix()
|
|
10299
11179
|
);
|
|
10300
11180
|
}
|
|
10301
|
-
const fileTypeExt =
|
|
11181
|
+
const fileTypeExt = path19.extname(normalized).slice(1).toLowerCase();
|
|
10302
11182
|
const fileStatSize = size ?? statSize(normalized) ?? 0;
|
|
10303
11183
|
const isKnownFileType = DISPATCHED_FILE_TYPE_EXTS.has(fileTypeExt);
|
|
10304
11184
|
if (event.toolName !== "Grep" && !isImagePath(normalized) && (isKnownFileType || fileStatSize >= FILE_TYPE_THRESHOLDS.generic)) {
|
|
@@ -10307,7 +11187,7 @@ function preReadHandlerInner(event) {
|
|
|
10307
11187
|
let ftContent = "";
|
|
10308
11188
|
if (!BINARY_FILE_TYPE_EXTS.has(fileTypeExt) && fileStatSize <= SLICE_ESTIMATE_SCAN_CAP_BYTES) {
|
|
10309
11189
|
try {
|
|
10310
|
-
ftContent =
|
|
11190
|
+
ftContent = fs23.readFileSync(normalized, "utf8");
|
|
10311
11191
|
} catch {
|
|
10312
11192
|
}
|
|
10313
11193
|
}
|
|
@@ -10337,7 +11217,7 @@ function estimateTruncatedLineCount(normalized) {
|
|
|
10337
11217
|
try {
|
|
10338
11218
|
const sz = statSize(normalized);
|
|
10339
11219
|
if (sz !== null && sz <= SLICE_ESTIMATE_SCAN_CAP_BYTES) {
|
|
10340
|
-
return countTextLines(
|
|
11220
|
+
return countTextLines(fs23.readFileSync(normalized, "utf8"));
|
|
10341
11221
|
}
|
|
10342
11222
|
} catch {
|
|
10343
11223
|
}
|
|
@@ -10360,13 +11240,13 @@ function postReadHandlerInner(event) {
|
|
|
10360
11240
|
if (respText.includes("[Truncated:") || respText.includes("Truncated: PARTIAL view")) {
|
|
10361
11241
|
markFileTruncated(normalized);
|
|
10362
11242
|
}
|
|
10363
|
-
const postBasename =
|
|
11243
|
+
const postBasename = path19.basename(normalized);
|
|
10364
11244
|
const diffSourcesEnabled = loadConfig().hints.serve_diff_on_reread;
|
|
10365
11245
|
if (/\.(md|mdx|markdown|rst|txt)$/i.test(postBasename) || isSessionArtifactFile(normalized) || diffSourcesEnabled && DIFFABLE_SOURCE_RE.test(postBasename)) {
|
|
10366
11246
|
try {
|
|
10367
11247
|
const sz = statSize(normalized);
|
|
10368
11248
|
if (sz !== null && sz <= 256 * 1024) {
|
|
10369
|
-
const content =
|
|
11249
|
+
const content = fs23.readFileSync(normalized);
|
|
10370
11250
|
store(getSessionId(), normalized, content);
|
|
10371
11251
|
}
|
|
10372
11252
|
} catch {
|
|
@@ -10376,7 +11256,7 @@ function postReadHandlerInner(event) {
|
|
|
10376
11256
|
try {
|
|
10377
11257
|
const cwd = getCwd(event) ?? process.cwd();
|
|
10378
11258
|
const project = findProject(cwd) ?? makeProjectAt(cwd);
|
|
10379
|
-
const source = decodeSource(
|
|
11259
|
+
const source = decodeSource(fs23.readFileSync(normalized));
|
|
10380
11260
|
recordEvidence({ projectRoot: project.root, source: normalized, representation: "file", text: source });
|
|
10381
11261
|
} catch {
|
|
10382
11262
|
}
|
|
@@ -10407,7 +11287,7 @@ function postReadHandlerInner(event) {
|
|
|
10407
11287
|
try {
|
|
10408
11288
|
const sz = statSize(normalized);
|
|
10409
11289
|
if (sz !== null && sz <= SLICE_ESTIMATE_SCAN_CAP_BYTES) {
|
|
10410
|
-
const lineCount = countTextLines(
|
|
11290
|
+
const lineCount = countTextLines(fs23.readFileSync(normalized, "utf8"));
|
|
10411
11291
|
const minLines = loadConfig().post_read_code_compress.min_lines;
|
|
10412
11292
|
if (lineCount >= minLines && meetsSavingsFloor(sz)) {
|
|
10413
11293
|
recordStat("session_hint", 0, 0);
|
|
@@ -10427,9 +11307,9 @@ function postReadHandler(event) {
|
|
|
10427
11307
|
registerHook("post_tool_use", postReadHandler, { toolName: "Read" });
|
|
10428
11308
|
|
|
10429
11309
|
// src/cli_context_stats.ts
|
|
10430
|
-
import * as
|
|
11310
|
+
import * as fs25 from "node:fs";
|
|
10431
11311
|
import * as os8 from "node:os";
|
|
10432
|
-
import * as
|
|
11312
|
+
import * as path21 from "node:path";
|
|
10433
11313
|
|
|
10434
11314
|
// src/confirm_apply.ts
|
|
10435
11315
|
import * as readline from "node:readline";
|
|
@@ -10483,8 +11363,8 @@ ${diff}
|
|
|
10483
11363
|
}
|
|
10484
11364
|
|
|
10485
11365
|
// src/memory_prune.ts
|
|
10486
|
-
import * as
|
|
10487
|
-
import * as
|
|
11366
|
+
import * as fs24 from "node:fs";
|
|
11367
|
+
import * as path20 from "node:path";
|
|
10488
11368
|
var ENTRY_RE = /^\s*-\s*\[(?<title>[^\]]+)\]\((?<target>[^)]+?\.md)\)/;
|
|
10489
11369
|
var URL_SCHEME_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
10490
11370
|
function parseIndex(text) {
|
|
@@ -10529,10 +11409,10 @@ function pruneIndex(memoryDir, opts) {
|
|
|
10529
11409
|
changed: false,
|
|
10530
11410
|
tokensSaved: 0
|
|
10531
11411
|
};
|
|
10532
|
-
const memoryMd =
|
|
11412
|
+
const memoryMd = path20.join(memoryDir, "MEMORY.md");
|
|
10533
11413
|
let text;
|
|
10534
11414
|
try {
|
|
10535
|
-
text =
|
|
11415
|
+
text = fs24.readFileSync(memoryMd, "utf-8");
|
|
10536
11416
|
} catch {
|
|
10537
11417
|
return result;
|
|
10538
11418
|
}
|
|
@@ -10543,7 +11423,7 @@ function pruneIndex(memoryDir, opts) {
|
|
|
10543
11423
|
const dups = [];
|
|
10544
11424
|
for (const entry of entries) {
|
|
10545
11425
|
const isUrl = URL_SCHEME_RE.test(entry.target);
|
|
10546
|
-
const targetExists = isUrl ? true :
|
|
11426
|
+
const targetExists = isUrl ? true : path20.isAbsolute(entry.target) ? fs24.existsSync(entry.target) : fs24.existsSync(path20.join(memoryDir, entry.target));
|
|
10547
11427
|
const foldedTarget = foldPath(entry.target);
|
|
10548
11428
|
if (!targetExists) {
|
|
10549
11429
|
dead.push(entry);
|
|
@@ -10597,7 +11477,7 @@ function jaccard(a, b) {
|
|
|
10597
11477
|
function siblingSnippet(filePath) {
|
|
10598
11478
|
let text;
|
|
10599
11479
|
try {
|
|
10600
|
-
text =
|
|
11480
|
+
text = fs24.readFileSync(filePath, { encoding: "utf-8" });
|
|
10601
11481
|
} catch {
|
|
10602
11482
|
return "";
|
|
10603
11483
|
}
|
|
@@ -10687,7 +11567,7 @@ async function tryEmbeddingClusters(siblings, snippets, threshold) {
|
|
|
10687
11567
|
}
|
|
10688
11568
|
async function findContentDuplicates(memoryDir, _opts) {
|
|
10689
11569
|
const threshold = _opts?.threshold ?? 0.92;
|
|
10690
|
-
const siblings =
|
|
11570
|
+
const siblings = fs24.readdirSync(memoryDir).filter((name) => name.toLowerCase().endsWith(".md") && name.toLowerCase() !== "memory.md").map((name) => path20.join(memoryDir, name)).sort();
|
|
10691
11571
|
if (siblings.length < 2) {
|
|
10692
11572
|
return [];
|
|
10693
11573
|
}
|
|
@@ -10711,7 +11591,7 @@ function auditClaudeMd(files) {
|
|
|
10711
11591
|
for (const filePath of files) {
|
|
10712
11592
|
let text;
|
|
10713
11593
|
try {
|
|
10714
|
-
text =
|
|
11594
|
+
text = fs24.readFileSync(filePath, { encoding: "utf-8" });
|
|
10715
11595
|
} catch {
|
|
10716
11596
|
continue;
|
|
10717
11597
|
}
|
|
@@ -10771,7 +11651,7 @@ function auditClaudeMd(files) {
|
|
|
10771
11651
|
const overlaps = [];
|
|
10772
11652
|
for (const [stripped, filesSet] of lineToFiles) {
|
|
10773
11653
|
if (filesSet.has(report.path) && filesSet.size > 1) {
|
|
10774
|
-
const others = Array.from(filesSet).filter((p) => p !== report.path).map((p) =>
|
|
11654
|
+
const others = Array.from(filesSet).filter((p) => p !== report.path).map((p) => path20.basename(p));
|
|
10775
11655
|
if (others.length > 0) {
|
|
10776
11656
|
if (stripped.length > 60) {
|
|
10777
11657
|
overlaps.push(
|
|
@@ -10791,7 +11671,7 @@ function auditClaudeMd(files) {
|
|
|
10791
11671
|
// src/cli_context_stats.ts
|
|
10792
11672
|
function tok(filePath) {
|
|
10793
11673
|
try {
|
|
10794
|
-
const size =
|
|
11674
|
+
const size = fs25.statSync(filePath).size;
|
|
10795
11675
|
return Math.floor(size / 4);
|
|
10796
11676
|
} catch {
|
|
10797
11677
|
return 0;
|
|
@@ -10800,42 +11680,42 @@ function tok(filePath) {
|
|
|
10800
11680
|
function findClaudeMdFiles(projectRoot, homeDir = os8.homedir()) {
|
|
10801
11681
|
const found = [];
|
|
10802
11682
|
const seen = /* @__PURE__ */ new Set();
|
|
10803
|
-
let current =
|
|
11683
|
+
let current = path21.resolve(projectRoot);
|
|
10804
11684
|
while (true) {
|
|
10805
|
-
const candidate =
|
|
10806
|
-
if (!seen.has(candidate) &&
|
|
11685
|
+
const candidate = path21.join(current, "CLAUDE.md");
|
|
11686
|
+
if (!seen.has(candidate) && fs25.existsSync(candidate)) {
|
|
10807
11687
|
found.push(candidate);
|
|
10808
11688
|
seen.add(candidate);
|
|
10809
11689
|
}
|
|
10810
|
-
const parent =
|
|
11690
|
+
const parent = path21.dirname(current);
|
|
10811
11691
|
if (parent === current) break;
|
|
10812
11692
|
current = parent;
|
|
10813
11693
|
}
|
|
10814
|
-
const globalMd =
|
|
10815
|
-
if (!seen.has(globalMd) &&
|
|
11694
|
+
const globalMd = path21.join(homeDir, ".claude", "CLAUDE.md");
|
|
11695
|
+
if (!seen.has(globalMd) && fs25.existsSync(globalMd)) {
|
|
10816
11696
|
found.push(globalMd);
|
|
10817
11697
|
}
|
|
10818
11698
|
return found;
|
|
10819
11699
|
}
|
|
10820
11700
|
function findMemoryMd(projectRoot, homeDir = os8.homedir(), alternateRoots = []) {
|
|
10821
11701
|
try {
|
|
10822
|
-
const projectsDir =
|
|
10823
|
-
if (!
|
|
10824
|
-
const rootStr =
|
|
11702
|
+
const projectsDir = path21.join(homeDir, ".claude", "projects");
|
|
11703
|
+
if (!fs25.existsSync(projectsDir)) return null;
|
|
11704
|
+
const rootStr = path21.resolve(projectRoot);
|
|
10825
11705
|
const candidateRoots = [rootStr];
|
|
10826
11706
|
try {
|
|
10827
|
-
const realRoot =
|
|
11707
|
+
const realRoot = fs25.realpathSync.native(rootStr);
|
|
10828
11708
|
if (realRoot !== rootStr) candidateRoots.push(realRoot);
|
|
10829
11709
|
} catch {
|
|
10830
11710
|
}
|
|
10831
11711
|
for (const alternate of alternateRoots) {
|
|
10832
|
-
const resolved =
|
|
11712
|
+
const resolved = path21.resolve(alternate);
|
|
10833
11713
|
if (!candidateRoots.includes(resolved)) candidateRoots.push(resolved);
|
|
10834
11714
|
}
|
|
10835
11715
|
for (const root of candidateRoots) {
|
|
10836
11716
|
const expectedSlug = root.replace(/[^A-Za-z0-9]/g, "-");
|
|
10837
|
-
const candidate =
|
|
10838
|
-
if (
|
|
11717
|
+
const candidate = path21.join(projectsDir, expectedSlug, "memory", "MEMORY.md");
|
|
11718
|
+
if (fs25.existsSync(candidate)) return candidate;
|
|
10839
11719
|
}
|
|
10840
11720
|
return null;
|
|
10841
11721
|
} catch {
|
|
@@ -10849,10 +11729,10 @@ function buildStats(projectRoot, homeDir = os8.homedir(), alternateRoots = []) {
|
|
|
10849
11729
|
for (const p of claudeMds) {
|
|
10850
11730
|
const t = tok(p);
|
|
10851
11731
|
claudeMdTotal += t;
|
|
10852
|
-
const parentDir =
|
|
11732
|
+
const parentDir = path21.basename(path21.dirname(p));
|
|
10853
11733
|
const label = parentDir === ".claude" ? "~/.claude/CLAUDE.md" : (() => {
|
|
10854
11734
|
try {
|
|
10855
|
-
return
|
|
11735
|
+
return path21.relative(projectRoot, p);
|
|
10856
11736
|
} catch {
|
|
10857
11737
|
return p;
|
|
10858
11738
|
}
|
|
@@ -10916,11 +11796,11 @@ async function runContextStats(opts = {}) {
|
|
|
10916
11796
|
process.stdout.write("[--fix] No MEMORY.md found; nothing to prune.\n");
|
|
10917
11797
|
} else {
|
|
10918
11798
|
const memPath = result.memory_md_path;
|
|
10919
|
-
const pruneResult = pruneIndex(
|
|
11799
|
+
const pruneResult = pruneIndex(path21.dirname(memPath), { dryRun: true });
|
|
10920
11800
|
if (!pruneResult.changed || pruneResult.after === void 0) {
|
|
10921
11801
|
process.stdout.write("[--fix] MEMORY.md already clean; nothing to prune.\n");
|
|
10922
11802
|
} else {
|
|
10923
|
-
const before =
|
|
11803
|
+
const before = fs25.readFileSync(memPath, "utf-8");
|
|
10924
11804
|
const applyResult = await confirmAndApply(
|
|
10925
11805
|
[{ path: memPath, before, after: pruneResult.after, label: "MEMORY.md" }],
|
|
10926
11806
|
opts.yes === true ? { yes: true } : {}
|
|
@@ -10947,8 +11827,8 @@ async function runContextStats(opts = {}) {
|
|
|
10947
11827
|
}
|
|
10948
11828
|
|
|
10949
11829
|
// src/baseline.ts
|
|
10950
|
-
import * as
|
|
10951
|
-
import * as
|
|
11830
|
+
import * as fs26 from "node:fs";
|
|
11831
|
+
import * as path22 from "node:path";
|
|
10952
11832
|
var SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
10953
11833
|
"node_modules",
|
|
10954
11834
|
".git",
|
|
@@ -10986,12 +11866,12 @@ function walkProject(rootDir, opts = {}) {
|
|
|
10986
11866
|
if (dir === void 0) break;
|
|
10987
11867
|
let entries;
|
|
10988
11868
|
try {
|
|
10989
|
-
entries =
|
|
11869
|
+
entries = fs26.readdirSync(dir, { withFileTypes: true });
|
|
10990
11870
|
} catch {
|
|
10991
11871
|
continue;
|
|
10992
11872
|
}
|
|
10993
11873
|
for (const entry of entries) {
|
|
10994
|
-
const full =
|
|
11874
|
+
const full = path22.join(dir, entry.name);
|
|
10995
11875
|
if (entry.isDirectory()) {
|
|
10996
11876
|
if (SKIP_DIRS.has(entry.name) || extraSkipDirs.includes(entry.name)) continue;
|
|
10997
11877
|
if (entry.name.startsWith(".") && entry.name !== ".") {
|
|
@@ -11066,7 +11946,7 @@ function fetchTopSymbols(limit, dbPath, rootDir) {
|
|
|
11066
11946
|
}
|
|
11067
11947
|
}
|
|
11068
11948
|
function buildProjectMap(rootDir = process.cwd(), opts = {}) {
|
|
11069
|
-
const root =
|
|
11949
|
+
const root = path22.resolve(rootDir);
|
|
11070
11950
|
const config = loadConfig();
|
|
11071
11951
|
const { files, languages } = walkProject(root, { excludeTests: config.repomap.exclude_tests });
|
|
11072
11952
|
const compact = opts.compact === true || files.length > config.repomap.compact_file_threshold;
|
|
@@ -11075,12 +11955,12 @@ function buildProjectMap(rootDir = process.cwd(), opts = {}) {
|
|
|
11075
11955
|
const recentFiles = files.map((f) => {
|
|
11076
11956
|
let mtime;
|
|
11077
11957
|
try {
|
|
11078
|
-
mtime =
|
|
11958
|
+
mtime = fs26.statSync(f).mtimeMs;
|
|
11079
11959
|
} catch {
|
|
11080
11960
|
mtime = 0;
|
|
11081
11961
|
}
|
|
11082
11962
|
return { f, mtime };
|
|
11083
|
-
}).sort((a, b) => b.mtime - a.mtime).slice(0, compact ? 5 : 15).map((x) =>
|
|
11963
|
+
}).sort((a, b) => b.mtime - a.mtime).slice(0, compact ? 5 : 15).map((x) => path22.relative(root, x.f));
|
|
11084
11964
|
return {
|
|
11085
11965
|
rootDir: root,
|
|
11086
11966
|
fileCount: files.length,
|
|
@@ -11092,7 +11972,7 @@ function buildProjectMap(rootDir = process.cwd(), opts = {}) {
|
|
|
11092
11972
|
}
|
|
11093
11973
|
function formatProjectMap(map, compact = false) {
|
|
11094
11974
|
const lines2 = [];
|
|
11095
|
-
const rel =
|
|
11975
|
+
const rel = path22.basename(map.rootDir);
|
|
11096
11976
|
lines2.push(`# Project map: ${rel}`);
|
|
11097
11977
|
lines2.push(`Files: ${map.fileCount}`);
|
|
11098
11978
|
const langPairs = Object.entries(map.languages).sort((a, b) => b[1] - a[1]);
|
|
@@ -11120,13 +12000,13 @@ function formatProjectMap(map, compact = false) {
|
|
|
11120
12000
|
}
|
|
11121
12001
|
function mapLookupBytesSaved(map, emittedText) {
|
|
11122
12002
|
const referencedFiles = /* @__PURE__ */ new Set([
|
|
11123
|
-
...map.recentFiles.map((f) => normalizePath(
|
|
12003
|
+
...map.recentFiles.map((f) => normalizePath(path22.resolve(map.rootDir, f))),
|
|
11124
12004
|
...map.topSymbols.map((s) => normalizePath(s.filePath))
|
|
11125
12005
|
]);
|
|
11126
12006
|
let fullSourceBytes = 0;
|
|
11127
12007
|
for (const fp of referencedFiles) {
|
|
11128
12008
|
try {
|
|
11129
|
-
fullSourceBytes +=
|
|
12009
|
+
fullSourceBytes += fs26.statSync(fp).size;
|
|
11130
12010
|
} catch {
|
|
11131
12011
|
}
|
|
11132
12012
|
}
|
|
@@ -11140,14 +12020,14 @@ function findMemSuggestionCandidates(projectRoot) {
|
|
|
11140
12020
|
const claudeMdFiles = findClaudeMdFiles(projectRoot);
|
|
11141
12021
|
const candidateFiles = new Set(claudeMdFiles);
|
|
11142
12022
|
for (const claudeMd of claudeMdFiles) {
|
|
11143
|
-
const agentsMd =
|
|
11144
|
-
if (
|
|
12023
|
+
const agentsMd = path22.join(path22.dirname(claudeMd), "AGENTS.md");
|
|
12024
|
+
if (fs26.existsSync(agentsMd)) candidateFiles.add(agentsMd);
|
|
11145
12025
|
}
|
|
11146
12026
|
const suggestions = [];
|
|
11147
12027
|
for (const filePath of candidateFiles) {
|
|
11148
12028
|
let text;
|
|
11149
12029
|
try {
|
|
11150
|
-
text =
|
|
12030
|
+
text = fs26.readFileSync(filePath, { encoding: "utf-8" });
|
|
11151
12031
|
} catch {
|
|
11152
12032
|
continue;
|
|
11153
12033
|
}
|
|
@@ -11176,7 +12056,7 @@ function formatMemSuggestions(projectRoot) {
|
|
|
11176
12056
|
if (suggestions.length === 0) return "";
|
|
11177
12057
|
const lines2 = ["", "## mem suggestions"];
|
|
11178
12058
|
for (const s of suggestions) {
|
|
11179
|
-
const basename12 =
|
|
12059
|
+
const basename12 = path22.basename(s.path);
|
|
11180
12060
|
lines2.push(
|
|
11181
12061
|
"Consider: mem import --from-md " + s.path + " # migrates " + s.count + " preference-shaped lines from " + basename12 + " as pending facts for review"
|
|
11182
12062
|
);
|
|
@@ -11316,9 +12196,9 @@ function getFileEntry(filePath, dbPath = globalDbPath()) {
|
|
|
11316
12196
|
embedSha: row.embed_sha ?? ""
|
|
11317
12197
|
};
|
|
11318
12198
|
}
|
|
11319
|
-
function sanitizeFtsQuery(query,
|
|
12199
|
+
function sanitizeFtsQuery(query, join23 = "AND") {
|
|
11320
12200
|
const terms = query.split(/\s+/).filter((t) => t.length > 0).map((t) => '"' + t.replace(/"/g, '""') + '"');
|
|
11321
|
-
return terms.join(
|
|
12201
|
+
return terms.join(join23 === "OR" ? " OR " : " ");
|
|
11322
12202
|
}
|
|
11323
12203
|
function runFtsQuery(db, match, limit, scope, rootDir) {
|
|
11324
12204
|
const sql = `SELECT s.file_path, s.name, s.kind, s.line_start, s.line_end, s.body, s.docstring, s.parent FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid WHERE symbols_fts MATCH ?${scope !== void 0 ? ` AND ${scope.clause}` : ""} ORDER BY bm25(symbols_fts) LIMIT ?`;
|
|
@@ -11347,9 +12227,9 @@ function searchSymbolsFts(query, limit = 50, dbPath = globalDbPath(), rootDir) {
|
|
|
11347
12227
|
}
|
|
11348
12228
|
|
|
11349
12229
|
// src/parser.ts
|
|
11350
|
-
import * as
|
|
12230
|
+
import * as fs27 from "node:fs";
|
|
11351
12231
|
import { createRequire as createRequire4 } from "node:module";
|
|
11352
|
-
import * as
|
|
12232
|
+
import * as path27 from "node:path";
|
|
11353
12233
|
|
|
11354
12234
|
// src/languages/csharp.ts
|
|
11355
12235
|
var USING_RE = /^(?:global\s+)?using\s+(?:static\s+)?([A-Za-z_][A-Za-z0-9_.]*)\s*(?:=\s*([A-Za-z_][A-Za-z0-9_.<>,\s]*))?\s*;/;
|
|
@@ -11782,7 +12662,7 @@ function extractHtml(content, filePath) {
|
|
|
11782
12662
|
}
|
|
11783
12663
|
|
|
11784
12664
|
// src/languages/liquid.ts
|
|
11785
|
-
import * as
|
|
12665
|
+
import * as path23 from "node:path";
|
|
11786
12666
|
var INCLUDE_RE = /{%-?\s*include\s+(['"])((?:(?!\1)[\s\S])+?)\1/gi;
|
|
11787
12667
|
var SECTION_RE = /{%-?\s*section\s+(['"])((?:(?!\1)[\s\S])+?)\1/gi;
|
|
11788
12668
|
var RENDER_RE = /{%-?\s*render\s+(['"])((?:(?!\1)[\s\S])+?)\1/gi;
|
|
@@ -11825,7 +12705,7 @@ function extractLiquid(content, filePath, relPath) {
|
|
|
11825
12705
|
const resolvedRel = relPath ?? filePath;
|
|
11826
12706
|
const relPosix = resolvedRel.replace(/\\/g, "/");
|
|
11827
12707
|
if (relPosix.startsWith("sections/") || relPosix.includes("/sections/")) {
|
|
11828
|
-
const stem =
|
|
12708
|
+
const stem = path23.basename(resolvedRel, path23.extname(resolvedRel));
|
|
11829
12709
|
symbols.push({ filePath, name: stem, kind: "liquid_section_file", lineStart: 1, lineEnd: 1, body: "", docstring: "", parent: "" });
|
|
11830
12710
|
}
|
|
11831
12711
|
const totalLines = countContentLines(content);
|
|
@@ -12862,7 +13742,7 @@ function bracedBodyEndLine(content, lineIndex, parenIndex, totalLines, fallback)
|
|
|
12862
13742
|
j++;
|
|
12863
13743
|
}
|
|
12864
13744
|
if (content[j] !== "{") return fallback;
|
|
12865
|
-
return findMatchingBraceEndLine(content, j, totalLines, lineIndex, "#");
|
|
13745
|
+
return findMatchingBraceEndLine(content, j, totalLines, lineIndex, "#", { backtickQuote: true });
|
|
12866
13746
|
}
|
|
12867
13747
|
function callEndLine(content, lineIndex, parenIndex, fallback) {
|
|
12868
13748
|
const close = matchingParenIndex(content, parenIndex);
|
|
@@ -14128,7 +15008,7 @@ function extractApex(content, filePath) {
|
|
|
14128
15008
|
}
|
|
14129
15009
|
|
|
14130
15010
|
// src/languages/salesforce_metadata.ts
|
|
14131
|
-
import * as
|
|
15011
|
+
import * as path24 from "node:path";
|
|
14132
15012
|
var MAX_SYMBOLS10 = 1e3;
|
|
14133
15013
|
var MAX_REFS = 1e3;
|
|
14134
15014
|
var FLOW_TAG_KIND = {
|
|
@@ -14186,7 +15066,7 @@ function normalizedPath(filePath) {
|
|
|
14186
15066
|
return filePath.replace(/\\/g, "/");
|
|
14187
15067
|
}
|
|
14188
15068
|
function basenameWithout(filePath, suffix) {
|
|
14189
|
-
const base =
|
|
15069
|
+
const base = path24.basename(filePath);
|
|
14190
15070
|
return base.toLowerCase().endsWith(suffix.toLowerCase()) ? base.slice(0, base.length - suffix.length) : base;
|
|
14191
15071
|
}
|
|
14192
15072
|
function objectNameFromPath(filePath) {
|
|
@@ -14225,12 +15105,12 @@ function snakeCase(value) {
|
|
|
14225
15105
|
return value.replace(/([a-z0-9])([A-Z])/g, "$1_$2").replace(/([A-Z]+)([A-Z][a-z])/g, "$1_$2").replace(/[^A-Za-z0-9]+/g, "_").replace(/^_+|_+$/g, "").toLowerCase();
|
|
14226
15106
|
}
|
|
14227
15107
|
function companionName(filePath) {
|
|
14228
|
-
const base =
|
|
15108
|
+
const base = path24.basename(filePath);
|
|
14229
15109
|
const match = /^(.+)\.(?:cls|trigger|page|component|cmp|app|evt|intf|design|auradoc|tokens|js)-meta\.xml$/i.exec(base);
|
|
14230
15110
|
return match?.[1] === void 0 ? null : `${match[1]}.metadata`;
|
|
14231
15111
|
}
|
|
14232
15112
|
function metadataArtifactName(filePath) {
|
|
14233
|
-
const base =
|
|
15113
|
+
const base = path24.basename(filePath);
|
|
14234
15114
|
const match = /^(.+)\.[^.]+-meta\.xml$/i.exec(base);
|
|
14235
15115
|
return match?.[1] ?? basenameWithout(filePath, "-meta.xml");
|
|
14236
15116
|
}
|
|
@@ -14321,7 +15201,7 @@ function extractSalesforceMetadata(rawContent, filePath) {
|
|
|
14321
15201
|
const seen = /* @__PURE__ */ new Set();
|
|
14322
15202
|
const refs = [];
|
|
14323
15203
|
const seenRefs = /* @__PURE__ */ new Set();
|
|
14324
|
-
const base =
|
|
15204
|
+
const base = path24.basename(filePath).toLowerCase();
|
|
14325
15205
|
const whole = wholeFileSpan(content);
|
|
14326
15206
|
const root = rootElement(content);
|
|
14327
15207
|
if (root === null) return { symbols, refs };
|
|
@@ -14465,14 +15345,14 @@ function extractSalesforceMetadata(rawContent, filePath) {
|
|
|
14465
15345
|
}
|
|
14466
15346
|
|
|
14467
15347
|
// src/languages/salesforce_frontend.ts
|
|
14468
|
-
import * as
|
|
15348
|
+
import * as path25 from "node:path";
|
|
14469
15349
|
function lines(content) {
|
|
14470
15350
|
return content.split("\n");
|
|
14471
15351
|
}
|
|
14472
15352
|
function bundleName(filePath) {
|
|
14473
15353
|
const normalized = filePath.replaceAll("\\", "/");
|
|
14474
|
-
const parent =
|
|
14475
|
-
const base =
|
|
15354
|
+
const parent = path25.posix.basename(path25.posix.dirname(normalized));
|
|
15355
|
+
const base = path25.posix.basename(normalized).replace(/\.[^.]+$/, "");
|
|
14476
15356
|
return parent === "lwc" || parent === "aura" ? base : parent;
|
|
14477
15357
|
}
|
|
14478
15358
|
function lwcTagAlias(name) {
|
|
@@ -14576,7 +15456,7 @@ var MARKUP_KIND = {
|
|
|
14576
15456
|
};
|
|
14577
15457
|
function markupArtifactName(filePath, extension) {
|
|
14578
15458
|
if (extension === ".page" || extension === ".component" || extension === ".email") {
|
|
14579
|
-
return
|
|
15459
|
+
return path25.posix.basename(filePath.replaceAll("\\", "/")).replace(new RegExp(`${extension.replace(".", "\\.")}$`, "i"), "");
|
|
14580
15460
|
}
|
|
14581
15461
|
return bundleName(filePath);
|
|
14582
15462
|
}
|
|
@@ -14596,7 +15476,7 @@ function attributeRefs(refs, content, filePath, attribute, split = false) {
|
|
|
14596
15476
|
}
|
|
14597
15477
|
function extractSalesforceMarkup(content, filePath) {
|
|
14598
15478
|
const normalized = filePath.replaceAll("\\", "/");
|
|
14599
|
-
const extension =
|
|
15479
|
+
const extension = path25.posix.extname(normalized).toLowerCase();
|
|
14600
15480
|
const kind = MARKUP_KIND[extension] ?? "salesforce_markup";
|
|
14601
15481
|
const symbols = [
|
|
14602
15482
|
symbol(filePath, markupArtifactName(normalized, extension), kind, 1, countContentLines(content))
|
|
@@ -14628,7 +15508,7 @@ function extractSalesforceMarkup(content, filePath) {
|
|
|
14628
15508
|
}
|
|
14629
15509
|
|
|
14630
15510
|
// src/languages/sfc_idx.ts
|
|
14631
|
-
import * as
|
|
15511
|
+
import * as path26 from "node:path";
|
|
14632
15512
|
var MAX_SYMBOLS11 = 500;
|
|
14633
15513
|
function dedupe2(values, key) {
|
|
14634
15514
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -14647,7 +15527,7 @@ function finalize(symbols, refs) {
|
|
|
14647
15527
|
}
|
|
14648
15528
|
function componentName(filePath) {
|
|
14649
15529
|
const normalized = filePath.replaceAll("\\", "/");
|
|
14650
|
-
return
|
|
15530
|
+
return path26.posix.basename(normalized).replace(/\.[^.]+$/, "");
|
|
14651
15531
|
}
|
|
14652
15532
|
function matchLine2(content, offset) {
|
|
14653
15533
|
return content.slice(0, offset).split("\n").length;
|
|
@@ -14747,21 +15627,22 @@ function maskSpans(content, spans) {
|
|
|
14747
15627
|
}
|
|
14748
15628
|
return chars.join("");
|
|
14749
15629
|
}
|
|
14750
|
-
function
|
|
14751
|
-
|
|
15630
|
+
function componentSymbols(filePath, name, kind, totalLines) {
|
|
15631
|
+
if (totalLines < 1) return [];
|
|
15632
|
+
return [{ filePath, name, kind, lineStart: 1, lineEnd: totalLines, body: "", docstring: "", parent: "" }];
|
|
14752
15633
|
}
|
|
14753
15634
|
function extractVue(content, filePath) {
|
|
14754
15635
|
const totalLines = countContentLines(content);
|
|
14755
15636
|
const lineIndex = buildLineIndex(content);
|
|
14756
15637
|
const name = componentName(filePath);
|
|
14757
|
-
const symbols =
|
|
15638
|
+
const symbols = componentSymbols(filePath, name, "vue_component", totalLines);
|
|
14758
15639
|
const refs = [];
|
|
14759
15640
|
for (const block of extractTagBlocks(content, lineIndex, "script")) {
|
|
14760
|
-
symbols
|
|
15641
|
+
pushAll(symbols, extractTopLevelDeclarations(block.content, filePath, block.contentStartLine));
|
|
14761
15642
|
}
|
|
14762
15643
|
for (const block of extractTagBlocks(content, lineIndex, "template")) {
|
|
14763
15644
|
const markup = stripXmlComments(block.content);
|
|
14764
|
-
refs
|
|
15645
|
+
pushAll(refs, extractComponentRefs(markup, filePath, block.contentStartLine, true));
|
|
14765
15646
|
}
|
|
14766
15647
|
return finalize(symbols, refs);
|
|
14767
15648
|
}
|
|
@@ -14769,11 +15650,11 @@ function extractSvelte(content, filePath) {
|
|
|
14769
15650
|
const totalLines = countContentLines(content);
|
|
14770
15651
|
const lineIndex = buildLineIndex(content);
|
|
14771
15652
|
const name = componentName(filePath);
|
|
14772
|
-
const symbols =
|
|
15653
|
+
const symbols = componentSymbols(filePath, name, "svelte_component", totalLines);
|
|
14773
15654
|
const refs = [];
|
|
14774
15655
|
const scriptBlocks = extractTagBlocks(content, lineIndex, "script");
|
|
14775
15656
|
for (const block of scriptBlocks) {
|
|
14776
|
-
symbols
|
|
15657
|
+
pushAll(symbols, extractTopLevelDeclarations(block.content, filePath, block.contentStartLine));
|
|
14777
15658
|
}
|
|
14778
15659
|
const styleBlocks = extractTagBlocks(content, lineIndex, "style");
|
|
14779
15660
|
const spans = [...scriptBlocks, ...styleBlocks].map((b) => [
|
|
@@ -14781,7 +15662,7 @@ function extractSvelte(content, filePath) {
|
|
|
14781
15662
|
b.matchEnd
|
|
14782
15663
|
]);
|
|
14783
15664
|
const markup = stripXmlComments(maskSpans(content, spans));
|
|
14784
|
-
refs
|
|
15665
|
+
pushAll(refs, extractComponentRefs(markup, filePath, 1, true));
|
|
14785
15666
|
return finalize(symbols, refs);
|
|
14786
15667
|
}
|
|
14787
15668
|
function detectAstroFrontmatter(content) {
|
|
@@ -14802,7 +15683,7 @@ function extractAstro(content, filePath) {
|
|
|
14802
15683
|
const totalLines = countContentLines(content);
|
|
14803
15684
|
const lineIndex = buildLineIndex(content);
|
|
14804
15685
|
const name = componentName(filePath);
|
|
14805
|
-
const symbols =
|
|
15686
|
+
const symbols = componentSymbols(filePath, name, "astro_component", totalLines);
|
|
14806
15687
|
const refs = [];
|
|
14807
15688
|
const lines2 = content.split("\n");
|
|
14808
15689
|
const fm = detectAstroFrontmatter(content);
|
|
@@ -14810,7 +15691,7 @@ function extractAstro(content, filePath) {
|
|
|
14810
15691
|
if (fm) {
|
|
14811
15692
|
const frontmatterContent = lines2.slice(fm.openLine + 1, fm.closeLine).join("\n");
|
|
14812
15693
|
const contentStartLine = fm.openLine + 2;
|
|
14813
|
-
symbols
|
|
15694
|
+
pushAll(symbols, extractTopLevelDeclarations(frontmatterContent, filePath, contentStartLine));
|
|
14814
15695
|
const fenceStartOffset = lineIndex[fm.openLine] ?? 0;
|
|
14815
15696
|
const fenceEndOffset = lineIndex[fm.closeLine + 1] ?? content.length;
|
|
14816
15697
|
spans.push([fenceStartOffset, fenceEndOffset]);
|
|
@@ -14818,7 +15699,7 @@ function extractAstro(content, filePath) {
|
|
|
14818
15699
|
const styleBlocks = extractTagBlocks(content, lineIndex, "style");
|
|
14819
15700
|
for (const block of styleBlocks) spans.push([block.matchStart, block.matchEnd]);
|
|
14820
15701
|
const markup = stripXmlComments(maskSpans(content, spans));
|
|
14821
|
-
refs
|
|
15702
|
+
pushAll(refs, extractComponentRefs(markup, filePath, 1, false));
|
|
14822
15703
|
return finalize(symbols, refs);
|
|
14823
15704
|
}
|
|
14824
15705
|
|
|
@@ -14958,8 +15839,8 @@ function countNewlines(s) {
|
|
|
14958
15839
|
return n;
|
|
14959
15840
|
}
|
|
14960
15841
|
function loadGrammar(lang, filePath, content) {
|
|
14961
|
-
const useTsx = lang === "typescript" && filePath !== void 0 &&
|
|
14962
|
-
const useCppHeader = lang === "c" && filePath !== void 0 &&
|
|
15842
|
+
const useTsx = lang === "typescript" && filePath !== void 0 && path27.extname(filePath).toLowerCase() === ".tsx";
|
|
15843
|
+
const useCppHeader = lang === "c" && filePath !== void 0 && path27.extname(filePath).toLowerCase() === ".h" && content !== void 0 && CPP_HEADER_SNIFF_RE.test(content);
|
|
14963
15844
|
const cacheKey = useTsx ? "typescript:tsx" : useCppHeader ? "c:cpp-header" : lang;
|
|
14964
15845
|
const cached = _grammarCache.get(cacheKey);
|
|
14965
15846
|
if (cached !== void 0) return cached;
|
|
@@ -15107,6 +15988,9 @@ function makeSymbol(filePath, name, kind, node, lines2, style) {
|
|
|
15107
15988
|
parent: ""
|
|
15108
15989
|
};
|
|
15109
15990
|
}
|
|
15991
|
+
function fanOutElidesBodies(nameCount, declarationChars) {
|
|
15992
|
+
return nameCount > 1 && nameCount * declarationChars > MAX_SYMBOL_BODY_CHARS;
|
|
15993
|
+
}
|
|
15110
15994
|
function collectPatternBindings(node) {
|
|
15111
15995
|
const names = [];
|
|
15112
15996
|
const walk = (n) => {
|
|
@@ -15114,6 +15998,16 @@ function collectPatternBindings(node) {
|
|
|
15114
15998
|
if (n.text !== "") names.push(n.text);
|
|
15115
15999
|
return;
|
|
15116
16000
|
}
|
|
16001
|
+
if (n.type === "assignment_pattern" || n.type === "object_assignment_pattern") {
|
|
16002
|
+
const left = n.childForFieldName("left");
|
|
16003
|
+
if (left !== null) walk(left);
|
|
16004
|
+
return;
|
|
16005
|
+
}
|
|
16006
|
+
if (n.type === "pair_pattern") {
|
|
16007
|
+
const value = n.childForFieldName("value");
|
|
16008
|
+
if (value !== null) walk(value);
|
|
16009
|
+
return;
|
|
16010
|
+
}
|
|
15117
16011
|
for (const child of n.namedChildren) walk(child);
|
|
15118
16012
|
};
|
|
15119
16013
|
walk(node);
|
|
@@ -15175,7 +16069,7 @@ function extractTsJsSymbols(root, filePath, lines2) {
|
|
|
15175
16069
|
out.push(makeSymbol(filePath, name.text, isFn ? "function" : "variable", child, lines2, "c"));
|
|
15176
16070
|
} else {
|
|
15177
16071
|
const bindings = collectPatternBindings(name);
|
|
15178
|
-
const elideBodies = bindings.length
|
|
16072
|
+
const elideBodies = fanOutElidesBodies(bindings.length, child.text.length);
|
|
15179
16073
|
for (const bound of bindings) {
|
|
15180
16074
|
const sym = makeSymbol(filePath, bound, "variable", child, lines2, "c");
|
|
15181
16075
|
out.push(elideBodies ? { ...sym, body: "" } : sym);
|
|
@@ -15293,14 +16187,26 @@ var GO_LOCAL_KINDS = /* @__PURE__ */ new Set([
|
|
|
15293
16187
|
// the interface type declaring them does not.
|
|
15294
16188
|
"method_elem"
|
|
15295
16189
|
]);
|
|
16190
|
+
var GO_MULTI_NAME_SPECS = /* @__PURE__ */ new Set(["var_spec", "const_spec"]);
|
|
15296
16191
|
function extractGoSymbols(root, filePath, lines2) {
|
|
15297
16192
|
const out = [];
|
|
15298
16193
|
const visit = (node, insideFunction) => {
|
|
15299
16194
|
const kind = GO_KIND_BY_TYPE.get(node.type);
|
|
15300
16195
|
if (kind !== void 0 && !(insideFunction && GO_LOCAL_KINDS.has(node.type))) {
|
|
15301
|
-
|
|
15302
|
-
|
|
15303
|
-
|
|
16196
|
+
if (GO_MULTI_NAME_SPECS.has(node.type)) {
|
|
16197
|
+
const declared = node.namedChildren.filter(
|
|
16198
|
+
(c) => c.type === "identifier" && c.text !== "" && c.text !== "_"
|
|
16199
|
+
);
|
|
16200
|
+
const elideBodies = fanOutElidesBodies(declared.length, node.text.length);
|
|
16201
|
+
for (const child of declared) {
|
|
16202
|
+
const sym = makeSymbol(filePath, child.text, kind, node, lines2, "c");
|
|
16203
|
+
out.push(elideBodies ? { ...sym, body: "" } : sym);
|
|
16204
|
+
}
|
|
16205
|
+
} else {
|
|
16206
|
+
const name = nodeName(node);
|
|
16207
|
+
if (name !== null && name !== "") {
|
|
16208
|
+
out.push(makeSymbol(filePath, name, kind, node, lines2, "c"));
|
|
16209
|
+
}
|
|
15304
16210
|
}
|
|
15305
16211
|
}
|
|
15306
16212
|
const childInside = insideFunction || GO_FN_SCOPE_TYPES.has(node.type);
|
|
@@ -16626,9 +17532,9 @@ function isUnderSkipDir(filePath, skipDirs) {
|
|
|
16626
17532
|
}
|
|
16627
17533
|
function isParseSkipEligible(filePath, cfg) {
|
|
16628
17534
|
if (isUnderSkipDir(filePath, cfg.skip_dirs)) return true;
|
|
16629
|
-
if (cfg.skip_files.includes(
|
|
17535
|
+
if (cfg.skip_files.includes(path27.basename(filePath))) return true;
|
|
16630
17536
|
try {
|
|
16631
|
-
const stat2 =
|
|
17537
|
+
const stat2 = fs27.statSync(filePath);
|
|
16632
17538
|
if (stat2.size > cfg.large_file_skip_kb * 1024) return true;
|
|
16633
17539
|
} catch {
|
|
16634
17540
|
}
|
|
@@ -16668,9 +17574,10 @@ function writeParseResult(filePath, content, result, dbPath) {
|
|
|
16668
17574
|
insRef.run(r.filePath, r.name, r.line, r.col, r.context);
|
|
16669
17575
|
}
|
|
16670
17576
|
});
|
|
16671
|
-
writeAll();
|
|
17577
|
+
writeAll.immediate();
|
|
16672
17578
|
}
|
|
16673
|
-
function indexFileSync(
|
|
17579
|
+
function indexFileSync(rawPath, dbPath = globalDbPath(), preReadBytes) {
|
|
17580
|
+
const filePath = canonicalizeIndexPath(rawPath);
|
|
16674
17581
|
const ixCfg = loadConfig().indexing;
|
|
16675
17582
|
if (ixCfg !== void 0 && isParseSkipEligible(filePath, ixCfg)) {
|
|
16676
17583
|
const db = getDb(dbPath);
|
|
@@ -16684,7 +17591,7 @@ function indexFileSync(filePath, dbPath = globalDbPath(), preReadBytes) {
|
|
|
16684
17591
|
raw = preReadBytes;
|
|
16685
17592
|
} else {
|
|
16686
17593
|
try {
|
|
16687
|
-
raw =
|
|
17594
|
+
raw = fs27.readFileSync(filePath);
|
|
16688
17595
|
} catch (err) {
|
|
16689
17596
|
if (err.code === "ENOENT") return;
|
|
16690
17597
|
throw err;
|
|
@@ -16715,14 +17622,32 @@ var UNAVAILABLE_EMBED_SHA_PREFIX = "unavailable:";
|
|
|
16715
17622
|
function unavailableEmbedSha(sha) {
|
|
16716
17623
|
return UNAVAILABLE_EMBED_SHA_PREFIX + sha;
|
|
16717
17624
|
}
|
|
17625
|
+
function canonicalizeIndexPath(absPath) {
|
|
17626
|
+
if (!isCaseInsensitiveFs()) return absPath;
|
|
17627
|
+
let real;
|
|
17628
|
+
try {
|
|
17629
|
+
real = fs27.realpathSync.native(absPath);
|
|
17630
|
+
} catch {
|
|
17631
|
+
return absPath;
|
|
17632
|
+
}
|
|
17633
|
+
const cut = Math.max(absPath.lastIndexOf("/"), absPath.lastIndexOf("\\"));
|
|
17634
|
+
const base = absPath.slice(cut + 1);
|
|
17635
|
+
const realNorm = normalizePath(real);
|
|
17636
|
+
const realBase = path27.basename(realNorm);
|
|
17637
|
+
if (base === realBase) return absPath;
|
|
17638
|
+
if (foldPath(base) !== foldPath(realBase)) return absPath;
|
|
17639
|
+
const callerDir = cut < 0 ? normalizePath(path27.resolve(".")) : normalizePath(absPath.slice(0, cut));
|
|
17640
|
+
if (foldPath(callerDir) !== foldPath(path27.dirname(realNorm))) return absPath;
|
|
17641
|
+
return absPath.slice(0, cut + 1) + realBase;
|
|
17642
|
+
}
|
|
16718
17643
|
function indexedPathSpellingIsStale(storedPath, absPath) {
|
|
16719
17644
|
if (!isCaseInsensitiveFs()) return false;
|
|
16720
17645
|
const stored = normalizePath(storedPath);
|
|
16721
|
-
const candidate = normalizePath(
|
|
17646
|
+
const candidate = normalizePath(path27.resolve(absPath));
|
|
16722
17647
|
if (foldPath(stored) !== foldPath(candidate)) return false;
|
|
16723
17648
|
let real;
|
|
16724
17649
|
try {
|
|
16725
|
-
real = normalizePath(
|
|
17650
|
+
real = normalizePath(fs27.realpathSync.native(absPath));
|
|
16726
17651
|
} catch {
|
|
16727
17652
|
return false;
|
|
16728
17653
|
}
|
|
@@ -16751,7 +17676,8 @@ function isEmbedFresh(storedEmbedSha, sha, embeddingsEnabled, depsAvailable) {
|
|
|
16751
17676
|
if (!depsAvailable && storedEmbedSha === unavailableEmbedSha(sha)) return true;
|
|
16752
17677
|
return false;
|
|
16753
17678
|
}
|
|
16754
|
-
async function indexFileEmbeddings(
|
|
17679
|
+
async function indexFileEmbeddings(rawPath, dbPath = globalDbPath(), sha, onError) {
|
|
17680
|
+
const filePath = canonicalizeIndexPath(rawPath);
|
|
16755
17681
|
const ixCfg = loadConfig().indexing;
|
|
16756
17682
|
if (!ixCfg.embeddings_enabled) {
|
|
16757
17683
|
stampEmbedSha(getDb(dbPath), filePath, sha, disabledEmbedSha);
|
|
@@ -16788,7 +17714,7 @@ async function indexFileEmbeddings(filePath, dbPath = globalDbPath(), sha, onErr
|
|
|
16788
17714
|
}
|
|
16789
17715
|
let content;
|
|
16790
17716
|
try {
|
|
16791
|
-
content = decodeSource(await
|
|
17717
|
+
content = decodeSource(await fs27.promises.readFile(filePath));
|
|
16792
17718
|
} catch {
|
|
16793
17719
|
return;
|
|
16794
17720
|
}
|
|
@@ -16834,25 +17760,27 @@ function stampEmbedSha(db, filePath, sha, makeValue) {
|
|
|
16834
17760
|
}
|
|
16835
17761
|
function safeMtime(filePath) {
|
|
16836
17762
|
try {
|
|
16837
|
-
return
|
|
17763
|
+
return fs27.statSync(filePath).mtimeMs / 1e3;
|
|
16838
17764
|
} catch {
|
|
16839
17765
|
return 0;
|
|
16840
17766
|
}
|
|
16841
17767
|
}
|
|
16842
17768
|
|
|
16843
17769
|
// src/index_prune.ts
|
|
16844
|
-
import * as
|
|
16845
|
-
import * as
|
|
17770
|
+
import * as fs28 from "node:fs";
|
|
17771
|
+
import * as path28 from "node:path";
|
|
16846
17772
|
function removeFileFromIndex(db, filePath) {
|
|
16847
17773
|
const tx = db.transaction(() => {
|
|
16848
17774
|
deleteFileRows(db, filePath);
|
|
16849
17775
|
deleteFileEmbeddings(db, filePath);
|
|
16850
17776
|
});
|
|
16851
|
-
tx();
|
|
17777
|
+
tx.immediate();
|
|
16852
17778
|
}
|
|
16853
17779
|
function isTooShallowToPrune(rootPrefix) {
|
|
16854
|
-
const
|
|
17780
|
+
const normalized = normalizePath(rootPrefix);
|
|
17781
|
+
const segments = normalized.split("/").filter((s) => s.length > 0 && !/^[a-z]:$/i.test(s));
|
|
16855
17782
|
if (segments.length === 2 && segments[0]?.toLowerCase() === "mnt" && /^[a-z]$/i.test(segments[1] ?? "")) return true;
|
|
17783
|
+
if (normalized.startsWith("//") && segments.length <= 2) return true;
|
|
16856
17784
|
return segments.length === 0;
|
|
16857
17785
|
}
|
|
16858
17786
|
function foldedBounds(rootPrefix) {
|
|
@@ -16877,7 +17805,8 @@ function findDeletablePaths(rootPrefix, dbPath) {
|
|
|
16877
17805
|
for (const p of foldedPathsUnderRoot(rootPrefix, dbPath)) {
|
|
16878
17806
|
let stillExists;
|
|
16879
17807
|
try {
|
|
16880
|
-
|
|
17808
|
+
const st = fs28.statSync(p, { throwIfNoEntry: false });
|
|
17809
|
+
stillExists = st !== void 0 && st.isFile();
|
|
16881
17810
|
} catch {
|
|
16882
17811
|
continue;
|
|
16883
17812
|
}
|
|
@@ -16900,14 +17829,23 @@ function removeFilesBestEffort(db, paths) {
|
|
|
16900
17829
|
return removed;
|
|
16901
17830
|
}
|
|
16902
17831
|
function removeDeletedFilesBestEffort(db, paths) {
|
|
16903
|
-
const
|
|
17832
|
+
const removed = [];
|
|
17833
|
+
for (const p of paths) {
|
|
17834
|
+
let gone;
|
|
16904
17835
|
try {
|
|
16905
|
-
|
|
17836
|
+
const st = fs28.statSync(p, { throwIfNoEntry: false });
|
|
17837
|
+
gone = st === void 0 || !st.isFile();
|
|
16906
17838
|
} catch {
|
|
16907
|
-
|
|
17839
|
+
continue;
|
|
16908
17840
|
}
|
|
16909
|
-
|
|
16910
|
-
|
|
17841
|
+
if (!gone) continue;
|
|
17842
|
+
try {
|
|
17843
|
+
removeFileFromIndex(db, p);
|
|
17844
|
+
removed.push(p);
|
|
17845
|
+
} catch {
|
|
17846
|
+
}
|
|
17847
|
+
}
|
|
17848
|
+
return removed;
|
|
16911
17849
|
}
|
|
16912
17850
|
function pruneDeletedFiles(rootPrefix, dbPath = globalDbPath()) {
|
|
16913
17851
|
if (isTooShallowToPrune(rootPrefix)) return 0;
|
|
@@ -16962,7 +17900,7 @@ function pruneOrphanedChunks(dbPath = globalDbPath()) {
|
|
|
16962
17900
|
return removed;
|
|
16963
17901
|
}
|
|
16964
17902
|
function recordKnownRoot(filePath, dbPath = globalDbPath()) {
|
|
16965
|
-
const project = findProject(
|
|
17903
|
+
const project = findProject(path28.dirname(filePath));
|
|
16966
17904
|
if (project === null || isTooShallowToPrune(project.root)) return;
|
|
16967
17905
|
const db = getDb(dbPath);
|
|
16968
17906
|
db.prepare(
|
|
@@ -16985,7 +17923,7 @@ function sweepKnownRoots(dbPath = globalDbPath(), opts) {
|
|
|
16985
17923
|
if (isTooShallowToPrune(root)) continue;
|
|
16986
17924
|
let reachable;
|
|
16987
17925
|
try {
|
|
16988
|
-
reachable =
|
|
17926
|
+
reachable = fs28.statSync(root, { throwIfNoEntry: false })?.isDirectory() === true;
|
|
16989
17927
|
} catch {
|
|
16990
17928
|
reachable = false;
|
|
16991
17929
|
}
|
|
@@ -17020,19 +17958,19 @@ function sweepKnownRoots(dbPath = globalDbPath(), opts) {
|
|
|
17020
17958
|
}
|
|
17021
17959
|
var KNOWN_ROOT_RECORD_MIN_INTERVAL_MS = 60 * 60 * 1e3;
|
|
17022
17960
|
function knownRootRecordMarkerPath(dir, filePath) {
|
|
17023
|
-
return
|
|
17961
|
+
return path28.join(dir, `known-root-record-${shortFingerprint(path28.dirname(filePath))}.marker`);
|
|
17024
17962
|
}
|
|
17025
17963
|
var KNOWN_ROOT_MARKER_PREFIX = "known-root-record-";
|
|
17026
17964
|
function sweepExpiredKnownRootMarkers(dir = dataDir()) {
|
|
17027
17965
|
let removed = 0;
|
|
17028
17966
|
try {
|
|
17029
17967
|
const cutoff = Date.now() - KNOWN_ROOT_RECORD_MIN_INTERVAL_MS;
|
|
17030
|
-
for (const file of
|
|
17968
|
+
for (const file of fs28.readdirSync(dir)) {
|
|
17031
17969
|
if (!file.startsWith(KNOWN_ROOT_MARKER_PREFIX) || !file.endsWith(".marker")) continue;
|
|
17032
|
-
const full =
|
|
17970
|
+
const full = path28.join(dir, file);
|
|
17033
17971
|
try {
|
|
17034
|
-
if (
|
|
17035
|
-
|
|
17972
|
+
if (fs28.statSync(full).mtimeMs < cutoff) {
|
|
17973
|
+
fs28.unlinkSync(full);
|
|
17036
17974
|
removed += 1;
|
|
17037
17975
|
}
|
|
17038
17976
|
} catch {
|
|
@@ -17045,13 +17983,13 @@ function sweepExpiredKnownRootMarkers(dir = dataDir()) {
|
|
|
17045
17983
|
function recordKnownRootThrottled(filePath, dir = dataDir(), dbPath = globalDbPath()) {
|
|
17046
17984
|
const markerPath = knownRootRecordMarkerPath(dir, filePath);
|
|
17047
17985
|
try {
|
|
17048
|
-
const stat2 =
|
|
17986
|
+
const stat2 = fs28.statSync(markerPath);
|
|
17049
17987
|
if (Date.now() - stat2.mtimeMs < KNOWN_ROOT_RECORD_MIN_INTERVAL_MS) return;
|
|
17050
17988
|
} catch {
|
|
17051
17989
|
}
|
|
17052
17990
|
try {
|
|
17053
17991
|
ensureDirSync(dir);
|
|
17054
|
-
|
|
17992
|
+
fs28.writeFileSync(markerPath, "");
|
|
17055
17993
|
} catch {
|
|
17056
17994
|
}
|
|
17057
17995
|
recordKnownRoot(filePath, dbPath);
|
|
@@ -17059,8 +17997,8 @@ function recordKnownRootThrottled(filePath, dir = dataDir(), dbPath = globalDbPa
|
|
|
17059
17997
|
|
|
17060
17998
|
// src/worker.ts
|
|
17061
17999
|
import { spawn as spawn2 } from "node:child_process";
|
|
17062
|
-
import * as
|
|
17063
|
-
import * as
|
|
18000
|
+
import * as fs29 from "node:fs";
|
|
18001
|
+
import * as path29 from "node:path";
|
|
17064
18002
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
17065
18003
|
var DEFAULT_POLL_INTERVAL_MS = 2e3;
|
|
17066
18004
|
function resolvePollIntervalMs(explicit) {
|
|
@@ -17080,7 +18018,7 @@ var unclearedDrainingSnapshots = /* @__PURE__ */ new Map();
|
|
|
17080
18018
|
function drainingSnapshotStamp(file, content) {
|
|
17081
18019
|
let identity = "unknown";
|
|
17082
18020
|
try {
|
|
17083
|
-
const stat2 =
|
|
18021
|
+
const stat2 = fs29.statSync(file);
|
|
17084
18022
|
identity = `${stat2.mtimeMs}:${stat2.size}:${stat2.ino}`;
|
|
17085
18023
|
} catch {
|
|
17086
18024
|
}
|
|
@@ -17089,15 +18027,15 @@ function drainingSnapshotStamp(file, content) {
|
|
|
17089
18027
|
var DRAINING_READ_ATTEMPTS = 5;
|
|
17090
18028
|
var DRAINING_READ_RETRY_DELAY_MS = 50;
|
|
17091
18029
|
function listDrainingFiles(queuePath) {
|
|
17092
|
-
const dir =
|
|
17093
|
-
const base = `${
|
|
18030
|
+
const dir = path29.dirname(queuePath);
|
|
18031
|
+
const base = `${path29.basename(queuePath)}.draining`;
|
|
17094
18032
|
let entries;
|
|
17095
18033
|
try {
|
|
17096
|
-
entries =
|
|
18034
|
+
entries = fs29.readdirSync(dir);
|
|
17097
18035
|
} catch {
|
|
17098
18036
|
return [];
|
|
17099
18037
|
}
|
|
17100
|
-
return entries.filter((name) => name === base || name.startsWith(`${base}.alt-`) && !name.includes(".corrupt-")).sort().map((name) =>
|
|
18038
|
+
return entries.filter((name) => name === base || name.startsWith(`${base}.alt-`) && !name.includes(".corrupt-")).sort().map((name) => path29.join(dir, name));
|
|
17101
18039
|
}
|
|
17102
18040
|
var MAX_TRANSIENT_RETRIES = 5;
|
|
17103
18041
|
function bumpRetryCount(dbPath, absPath) {
|
|
@@ -17114,7 +18052,7 @@ function bumpRetryCount(dbPath, absPath) {
|
|
|
17114
18052
|
db.prepare("INSERT INTO files (path, retry_count) VALUES (?, 1)").run(normalized);
|
|
17115
18053
|
return 1;
|
|
17116
18054
|
});
|
|
17117
|
-
return tx();
|
|
18055
|
+
return tx.immediate();
|
|
17118
18056
|
}
|
|
17119
18057
|
function clearRetryCount(dbPath, absPath) {
|
|
17120
18058
|
try {
|
|
@@ -17125,17 +18063,17 @@ function clearRetryCount(dbPath, absPath) {
|
|
|
17125
18063
|
}
|
|
17126
18064
|
}
|
|
17127
18065
|
function dirtyQueuePathFor(dir) {
|
|
17128
|
-
return
|
|
18066
|
+
return path29.join(dir, "queue", "dirty.txt");
|
|
17129
18067
|
}
|
|
17130
18068
|
function drainHeartbeatPathFor(dir) {
|
|
17131
|
-
return
|
|
18069
|
+
return path29.join(dir, "queue", "drain-heartbeat");
|
|
17132
18070
|
}
|
|
17133
18071
|
function writeDrainHeartbeat(dir, force = false) {
|
|
17134
18072
|
const now = Date.now();
|
|
17135
18073
|
if (!force && now - (heartbeatWriteTimes.get(dir) ?? 0) < WORKER_HEARTBEAT_REFRESH_MS) return;
|
|
17136
18074
|
try {
|
|
17137
|
-
ensureDirSync(
|
|
17138
|
-
|
|
18075
|
+
ensureDirSync(path29.dirname(drainHeartbeatPathFor(dir)));
|
|
18076
|
+
fs29.writeFileSync(drainHeartbeatPathFor(dir), `${process.pid}
|
|
17139
18077
|
`);
|
|
17140
18078
|
heartbeatWriteTimes.set(dir, now);
|
|
17141
18079
|
} catch {
|
|
@@ -17144,15 +18082,15 @@ function writeDrainHeartbeat(dir, force = false) {
|
|
|
17144
18082
|
function hasFreshWorkerHeartbeat(dir, pid) {
|
|
17145
18083
|
try {
|
|
17146
18084
|
const heartbeatPath = drainHeartbeatPathFor(dir);
|
|
17147
|
-
if (Date.now() -
|
|
17148
|
-
return
|
|
18085
|
+
if (Date.now() - fs29.statSync(heartbeatPath).mtimeMs > WORKER_HEARTBEAT_STALE_MS) return false;
|
|
18086
|
+
return fs29.readFileSync(heartbeatPath, "utf8").trim() === String(pid);
|
|
17149
18087
|
} catch {
|
|
17150
18088
|
return false;
|
|
17151
18089
|
}
|
|
17152
18090
|
}
|
|
17153
18091
|
function pidFileIsWithinStartupGrace(dir) {
|
|
17154
18092
|
try {
|
|
17155
|
-
return Date.now() -
|
|
18093
|
+
return Date.now() - fs29.statSync(workerPidPath(dir)).mtimeMs < WORKER_STARTUP_GRACE_MS;
|
|
17156
18094
|
} catch {
|
|
17157
18095
|
return false;
|
|
17158
18096
|
}
|
|
@@ -17186,28 +18124,28 @@ function parseDirtyQueueLines(raw) {
|
|
|
17186
18124
|
return out;
|
|
17187
18125
|
}
|
|
17188
18126
|
function workerPidPath(dir = dataDir()) {
|
|
17189
|
-
return
|
|
18127
|
+
return path29.join(dir, "worker.pid");
|
|
17190
18128
|
}
|
|
17191
18129
|
function getDirtyPathsFor(dir) {
|
|
17192
18130
|
let raw;
|
|
17193
18131
|
try {
|
|
17194
|
-
raw =
|
|
18132
|
+
raw = fs29.readFileSync(dirtyQueuePathFor(dir), "utf8");
|
|
17195
18133
|
} catch {
|
|
17196
18134
|
return [];
|
|
17197
18135
|
}
|
|
17198
18136
|
return parseDirtyQueueLines(raw);
|
|
17199
18137
|
}
|
|
17200
18138
|
function workerErrorLogPath(dir) {
|
|
17201
|
-
return
|
|
18139
|
+
return path29.join(dir, "worker-errors.log");
|
|
17202
18140
|
}
|
|
17203
18141
|
var WORKER_ERROR_LOG_MAX_BYTES = 5 * 1024 * 1024;
|
|
17204
18142
|
var CORRUPT_QUARANTINE_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1e3;
|
|
17205
18143
|
function cleanupWorkerStateFiles(dir) {
|
|
17206
18144
|
try {
|
|
17207
18145
|
const logPath = workerErrorLogPath(dir);
|
|
17208
|
-
const stat2 =
|
|
18146
|
+
const stat2 = fs29.statSync(logPath);
|
|
17209
18147
|
if (stat2.size > WORKER_ERROR_LOG_MAX_BYTES) {
|
|
17210
|
-
|
|
18148
|
+
fs29.writeFileSync(
|
|
17211
18149
|
logPath,
|
|
17212
18150
|
`${(/* @__PURE__ */ new Date()).toISOString()} worker-errors.log rotated (exceeded ${WORKER_ERROR_LOG_MAX_BYTES} bytes)
|
|
17213
18151
|
`
|
|
@@ -17216,13 +18154,13 @@ function cleanupWorkerStateFiles(dir) {
|
|
|
17216
18154
|
} catch {
|
|
17217
18155
|
}
|
|
17218
18156
|
try {
|
|
17219
|
-
const queueDir =
|
|
18157
|
+
const queueDir = path29.dirname(dirtyQueuePathFor(dir));
|
|
17220
18158
|
const cutoff = Date.now() - CORRUPT_QUARANTINE_MAX_AGE_MS;
|
|
17221
|
-
for (const file of
|
|
18159
|
+
for (const file of fs29.readdirSync(queueDir)) {
|
|
17222
18160
|
if (!file.includes(".corrupt-")) continue;
|
|
17223
|
-
const full =
|
|
18161
|
+
const full = path29.join(queueDir, file);
|
|
17224
18162
|
try {
|
|
17225
|
-
if (
|
|
18163
|
+
if (fs29.statSync(full).mtimeMs < cutoff) fs29.unlinkSync(full);
|
|
17226
18164
|
} catch {
|
|
17227
18165
|
}
|
|
17228
18166
|
}
|
|
@@ -17253,7 +18191,7 @@ registerReset(() => {
|
|
|
17253
18191
|
function embedFileSerialized(absPath, dbPath, sha) {
|
|
17254
18192
|
const key = foldPath(absPath);
|
|
17255
18193
|
const prior = inFlightEmbeddings.get(key);
|
|
17256
|
-
const dir =
|
|
18194
|
+
const dir = path29.dirname(dbPath);
|
|
17257
18195
|
const onEmbedError = (err) => {
|
|
17258
18196
|
const message = extractErrorMessage(err);
|
|
17259
18197
|
appendWorkerErrorLog(dir, `${(/* @__PURE__ */ new Date()).toISOString()} indexFileEmbeddings failed for ${absPath}: ${message}
|
|
@@ -17287,7 +18225,7 @@ function embedFileSerialized(absPath, dbPath, sha) {
|
|
|
17287
18225
|
}
|
|
17288
18226
|
function appendWorkerErrorLog(dir, line) {
|
|
17289
18227
|
try {
|
|
17290
|
-
|
|
18228
|
+
fs29.appendFileSync(workerErrorLogPath(dir), line);
|
|
17291
18229
|
} catch {
|
|
17292
18230
|
}
|
|
17293
18231
|
}
|
|
@@ -17304,7 +18242,7 @@ function logTransientReadFailure(dir, absPath) {
|
|
|
17304
18242
|
);
|
|
17305
18243
|
}
|
|
17306
18244
|
function bumpAndCheckRetry(dir, absPath) {
|
|
17307
|
-
const dbPath =
|
|
18245
|
+
const dbPath = path29.join(dir, "global.db");
|
|
17308
18246
|
const attempts = bumpRetryCount(dbPath, absPath);
|
|
17309
18247
|
if (attempts > MAX_TRANSIENT_RETRIES) {
|
|
17310
18248
|
if (attempts === MAX_TRANSIENT_RETRIES + 1) {
|
|
@@ -17321,14 +18259,14 @@ function bumpAndCheckRetry(dir, absPath) {
|
|
|
17321
18259
|
function appendToDirtyQueue(dir, absPath) {
|
|
17322
18260
|
const queuePath = dirtyQueuePathFor(dir);
|
|
17323
18261
|
try {
|
|
17324
|
-
ensureDirSync(
|
|
18262
|
+
ensureDirSync(path29.dirname(queuePath));
|
|
17325
18263
|
let leadingNewline = "";
|
|
17326
18264
|
try {
|
|
17327
|
-
const existing =
|
|
18265
|
+
const existing = fs29.readFileSync(queuePath, "utf8");
|
|
17328
18266
|
if (existing.length > 0 && !existing.endsWith("\n")) leadingNewline = "\n";
|
|
17329
18267
|
} catch {
|
|
17330
18268
|
}
|
|
17331
|
-
|
|
18269
|
+
fs29.appendFileSync(queuePath, `${leadingNewline}${encodeDirtyQueueLine(absPath)}
|
|
17332
18270
|
`);
|
|
17333
18271
|
} catch {
|
|
17334
18272
|
}
|
|
@@ -17337,7 +18275,7 @@ function requeueDirtyPath(dir, absPath) {
|
|
|
17337
18275
|
if (bumpAndCheckRetry(dir, absPath)) appendToDirtyQueue(dir, absPath);
|
|
17338
18276
|
}
|
|
17339
18277
|
function makeIndexer(dbPath) {
|
|
17340
|
-
const dir =
|
|
18278
|
+
const dir = path29.dirname(dbPath);
|
|
17341
18279
|
return (absPath, sha) => {
|
|
17342
18280
|
try {
|
|
17343
18281
|
const ixCfgForSkip = loadConfig().indexing;
|
|
@@ -17389,16 +18327,16 @@ function processDirtyBatch(paths, index = makeIndexer(globalDbPath()), remove =
|
|
|
17389
18327
|
requeue(dir, p);
|
|
17390
18328
|
continue;
|
|
17391
18329
|
}
|
|
17392
|
-
clearRetryCount(
|
|
18330
|
+
clearRetryCount(path29.join(dir, "global.db"), p);
|
|
17393
18331
|
try {
|
|
17394
|
-
const
|
|
18332
|
+
const dirname19 = path29.dirname(p);
|
|
17395
18333
|
let root;
|
|
17396
|
-
if (projectRootCache.has(
|
|
17397
|
-
root = projectRootCache.get(
|
|
18334
|
+
if (projectRootCache.has(dirname19)) {
|
|
18335
|
+
root = projectRootCache.get(dirname19) ?? null;
|
|
17398
18336
|
} else {
|
|
17399
|
-
const project = findProject(
|
|
18337
|
+
const project = findProject(dirname19);
|
|
17400
18338
|
root = project?.root ?? null;
|
|
17401
|
-
projectRootCache.set(
|
|
18339
|
+
projectRootCache.set(dirname19, root);
|
|
17402
18340
|
}
|
|
17403
18341
|
if (root) lastKnownProjectRoots.set(dir, root);
|
|
17404
18342
|
} catch {
|
|
@@ -17417,7 +18355,7 @@ function sleepSyncMs(ms) {
|
|
|
17417
18355
|
function drainOnce(dir, index, remove) {
|
|
17418
18356
|
const queuePath = dirtyQueuePathFor(dir);
|
|
17419
18357
|
const draining = `${queuePath}.draining`;
|
|
17420
|
-
const dbPath =
|
|
18358
|
+
const dbPath = path29.join(dir, "global.db");
|
|
17421
18359
|
const indexFn = index ?? makeIndexer(dbPath);
|
|
17422
18360
|
const removeFn = remove ?? makeRemover(dbPath);
|
|
17423
18361
|
let processed = 0;
|
|
@@ -17434,7 +18372,7 @@ function drainOnce(dir, index, remove) {
|
|
|
17434
18372
|
let drainingContent = null;
|
|
17435
18373
|
for (let attempt = 0; attempt < DRAINING_READ_ATTEMPTS; attempt++) {
|
|
17436
18374
|
try {
|
|
17437
|
-
drainingContent =
|
|
18375
|
+
drainingContent = fs29.readFileSync(drainingFile, "utf8");
|
|
17438
18376
|
break;
|
|
17439
18377
|
} catch {
|
|
17440
18378
|
if (attempt < DRAINING_READ_ATTEMPTS - 1) sleepSyncMs(DRAINING_READ_RETRY_DELAY_MS);
|
|
@@ -17442,7 +18380,7 @@ function drainOnce(dir, index, remove) {
|
|
|
17442
18380
|
}
|
|
17443
18381
|
if (drainingContent === null) {
|
|
17444
18382
|
try {
|
|
17445
|
-
|
|
18383
|
+
fs29.renameSync(drainingFile, `${drainingFile}.corrupt-${Date.now()}`);
|
|
17446
18384
|
unclearedDrainingSnapshots.delete(drainingFile);
|
|
17447
18385
|
} catch {
|
|
17448
18386
|
}
|
|
@@ -17452,23 +18390,23 @@ function drainOnce(dir, index, remove) {
|
|
|
17452
18390
|
processed += processDirtyBatch(parseDirtyQueueLines(drainingContent), indexFn, removeFn, dir, requeueFn);
|
|
17453
18391
|
}
|
|
17454
18392
|
try {
|
|
17455
|
-
|
|
18393
|
+
fs29.rmSync(drainingFile, { force: true });
|
|
17456
18394
|
unclearedDrainingSnapshots.delete(drainingFile);
|
|
17457
18395
|
} catch {
|
|
17458
18396
|
try {
|
|
17459
|
-
|
|
18397
|
+
fs29.renameSync(drainingFile, `${drainingFile}.corrupt-${Date.now()}`);
|
|
17460
18398
|
unclearedDrainingSnapshots.delete(drainingFile);
|
|
17461
18399
|
} catch {
|
|
17462
18400
|
unclearedDrainingSnapshots.set(drainingFile, drainingSnapshotStamp(drainingFile, drainingContent));
|
|
17463
18401
|
}
|
|
17464
18402
|
}
|
|
17465
18403
|
}
|
|
17466
|
-
if (
|
|
17467
|
-
const claimTarget =
|
|
18404
|
+
if (fs29.existsSync(queuePath)) {
|
|
18405
|
+
const claimTarget = fs29.existsSync(draining) ? `${draining}.alt-${Date.now()}` : draining;
|
|
17468
18406
|
let claimed = false;
|
|
17469
18407
|
for (let attempt = 0; attempt < 5; attempt++) {
|
|
17470
18408
|
try {
|
|
17471
|
-
|
|
18409
|
+
fs29.renameSync(queuePath, claimTarget);
|
|
17472
18410
|
claimed = true;
|
|
17473
18411
|
break;
|
|
17474
18412
|
} catch {
|
|
@@ -17479,14 +18417,14 @@ function drainOnce(dir, index, remove) {
|
|
|
17479
18417
|
let claimedContent = "";
|
|
17480
18418
|
let readOk = false;
|
|
17481
18419
|
try {
|
|
17482
|
-
claimedContent =
|
|
18420
|
+
claimedContent = fs29.readFileSync(claimTarget, "utf8");
|
|
17483
18421
|
readOk = true;
|
|
17484
18422
|
} catch {
|
|
17485
18423
|
}
|
|
17486
18424
|
if (readOk) {
|
|
17487
18425
|
processed += processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn);
|
|
17488
18426
|
try {
|
|
17489
|
-
const recheck =
|
|
18427
|
+
const recheck = fs29.readFileSync(claimTarget, "utf8");
|
|
17490
18428
|
if (recheck !== claimedContent) {
|
|
17491
18429
|
const extra = recheck.startsWith(claimedContent) ? recheck.slice(claimedContent.length) : recheck;
|
|
17492
18430
|
for (const p of parseDirtyQueueLines(extra)) appendToDirtyQueue(dir, p);
|
|
@@ -17494,10 +18432,10 @@ function drainOnce(dir, index, remove) {
|
|
|
17494
18432
|
} catch {
|
|
17495
18433
|
}
|
|
17496
18434
|
try {
|
|
17497
|
-
|
|
18435
|
+
fs29.rmSync(claimTarget, { force: true });
|
|
17498
18436
|
} catch {
|
|
17499
18437
|
try {
|
|
17500
|
-
|
|
18438
|
+
fs29.renameSync(claimTarget, `${claimTarget}.corrupt-${Date.now()}`);
|
|
17501
18439
|
} catch {
|
|
17502
18440
|
unclearedDrainingSnapshots.set(claimTarget, drainingSnapshotStamp(claimTarget, claimedContent));
|
|
17503
18441
|
}
|
|
@@ -17530,7 +18468,7 @@ function pidAlive(pid) {
|
|
|
17530
18468
|
}
|
|
17531
18469
|
function readPidFile(dir) {
|
|
17532
18470
|
try {
|
|
17533
|
-
const raw =
|
|
18471
|
+
const raw = fs29.readFileSync(workerPidPath(dir), "utf8").trim();
|
|
17534
18472
|
if (!/^\d+$/.test(raw)) return null;
|
|
17535
18473
|
return parseInt(raw, 10);
|
|
17536
18474
|
} catch {
|
|
@@ -17543,20 +18481,20 @@ function isWorkerRunning(dir = dataDir()) {
|
|
|
17543
18481
|
return pidAlive(pid) && hasFreshWorkerHeartbeat(dir, pid);
|
|
17544
18482
|
}
|
|
17545
18483
|
function workerHealthCheckMarkerPath(dir) {
|
|
17546
|
-
return
|
|
18484
|
+
return path29.join(dir, "worker-healthcheck.marker");
|
|
17547
18485
|
}
|
|
17548
18486
|
var WORKER_HEALTHCHECK_MIN_INTERVAL_MS = 5 * 60 * 1e3;
|
|
17549
18487
|
function ensureWorkerAlive(dir = dataDir()) {
|
|
17550
18488
|
if (process.env["TOKEN_GOAT_NO_WORKER_SPAWN"] === "1") return;
|
|
17551
18489
|
const markerPath = workerHealthCheckMarkerPath(dir);
|
|
17552
18490
|
try {
|
|
17553
|
-
const stat2 =
|
|
18491
|
+
const stat2 = fs29.statSync(markerPath);
|
|
17554
18492
|
if (Date.now() - stat2.mtimeMs < WORKER_HEALTHCHECK_MIN_INTERVAL_MS) return;
|
|
17555
18493
|
} catch {
|
|
17556
18494
|
}
|
|
17557
18495
|
try {
|
|
17558
18496
|
ensureDirSync(dir);
|
|
17559
|
-
|
|
18497
|
+
fs29.writeFileSync(markerPath, "");
|
|
17560
18498
|
} catch {
|
|
17561
18499
|
}
|
|
17562
18500
|
if (isWorkerRunning(dir)) return;
|
|
@@ -17586,7 +18524,7 @@ function stopWorker(dir = dataDir()) {
|
|
|
17586
18524
|
}
|
|
17587
18525
|
if (readPidFile(dir) === pid) {
|
|
17588
18526
|
try {
|
|
17589
|
-
|
|
18527
|
+
fs29.rmSync(workerPidPath(dir), { force: true });
|
|
17590
18528
|
} catch {
|
|
17591
18529
|
}
|
|
17592
18530
|
}
|
|
@@ -17601,7 +18539,7 @@ var WorkerAlreadyRunningError = class extends Error {
|
|
|
17601
18539
|
function claimWorkerPidFile(dir, pid) {
|
|
17602
18540
|
const pidPath = workerPidPath(dir);
|
|
17603
18541
|
try {
|
|
17604
|
-
|
|
18542
|
+
fs29.writeFileSync(pidPath, `${pid}
|
|
17605
18543
|
`, { flag: "wx" });
|
|
17606
18544
|
return true;
|
|
17607
18545
|
} catch (e) {
|
|
@@ -17618,11 +18556,11 @@ function claimWorkerPidFile(dir, pid) {
|
|
|
17618
18556
|
}
|
|
17619
18557
|
}
|
|
17620
18558
|
try {
|
|
17621
|
-
|
|
18559
|
+
fs29.rmSync(pidPath, { force: true });
|
|
17622
18560
|
} catch {
|
|
17623
18561
|
}
|
|
17624
18562
|
try {
|
|
17625
|
-
|
|
18563
|
+
fs29.writeFileSync(pidPath, `${pid}
|
|
17626
18564
|
`, { flag: "wx" });
|
|
17627
18565
|
return true;
|
|
17628
18566
|
} catch (e2) {
|
|
@@ -17632,9 +18570,9 @@ function claimWorkerPidFile(dir, pid) {
|
|
|
17632
18570
|
}
|
|
17633
18571
|
function daemonEntryScript() {
|
|
17634
18572
|
const self = fileURLToPath2(import.meta.url);
|
|
17635
|
-
const launcher =
|
|
18573
|
+
const launcher = path29.join(path29.dirname(self), "token-goat.mjs");
|
|
17636
18574
|
try {
|
|
17637
|
-
if (
|
|
18575
|
+
if (fs29.existsSync(launcher)) return launcher;
|
|
17638
18576
|
} catch {
|
|
17639
18577
|
}
|
|
17640
18578
|
return self;
|
|
@@ -17645,7 +18583,7 @@ function startDetachedWorker(opts) {
|
|
|
17645
18583
|
try {
|
|
17646
18584
|
ensureDirSync(dir);
|
|
17647
18585
|
} catch (e) {
|
|
17648
|
-
if (e.code !== "EEXIST" || !
|
|
18586
|
+
if (e.code !== "EEXIST" || !fs29.existsSync(dir)) throw e;
|
|
17649
18587
|
}
|
|
17650
18588
|
const child = spawn2(
|
|
17651
18589
|
process.execPath,
|
|
@@ -17682,7 +18620,7 @@ async function runWorkerLoop(dir, pollIntervalMs, shouldStop = () => false) {
|
|
|
17682
18620
|
let lastKnownRootsSweepMs = 0;
|
|
17683
18621
|
let ownedPidFile = false;
|
|
17684
18622
|
while (!shouldStop()) {
|
|
17685
|
-
if (!
|
|
18623
|
+
if (!fs29.existsSync(dir)) break;
|
|
17686
18624
|
const pidOwner = readPidFile(dir);
|
|
17687
18625
|
if (pidOwner === process.pid) ownedPidFile = true;
|
|
17688
18626
|
else if (ownedPidFile && pidOwner !== null) break;
|
|
@@ -17707,7 +18645,7 @@ async function runWorkerLoop(dir, pollIntervalMs, shouldStop = () => false) {
|
|
|
17707
18645
|
}
|
|
17708
18646
|
if (Date.now() - lastKnownRootsSweepMs >= KNOWN_ROOTS_SWEEP_INTERVAL_MS) {
|
|
17709
18647
|
try {
|
|
17710
|
-
const result = sweepKnownRoots(
|
|
18648
|
+
const result = sweepKnownRoots(path29.join(dir, "global.db"));
|
|
17711
18649
|
if (result.flaggedRoots.length > 0) {
|
|
17712
18650
|
appendWorkerErrorLog(
|
|
17713
18651
|
dir,
|
|
@@ -17730,7 +18668,7 @@ function runDetachedWorkerDaemon() {
|
|
|
17730
18668
|
process.on("exit", () => {
|
|
17731
18669
|
if (readPidFile(dir) === process.pid) {
|
|
17732
18670
|
try {
|
|
17733
|
-
|
|
18671
|
+
fs29.rmSync(workerPidPath(dir), { force: true });
|
|
17734
18672
|
} catch {
|
|
17735
18673
|
}
|
|
17736
18674
|
}
|
|
@@ -17740,26 +18678,26 @@ function runDetachedWorkerDaemon() {
|
|
|
17740
18678
|
}
|
|
17741
18679
|
|
|
17742
18680
|
// src/hooks_index.ts
|
|
17743
|
-
import * as
|
|
17744
|
-
import * as
|
|
18681
|
+
import * as fs30 from "node:fs";
|
|
18682
|
+
import * as path30 from "node:path";
|
|
17745
18683
|
function dirtyQueuePath() {
|
|
17746
|
-
return
|
|
18684
|
+
return path30.join(dataDir(), "queue", "dirty.txt");
|
|
17747
18685
|
}
|
|
17748
18686
|
function appendDirtyPath(normalizedPath2) {
|
|
17749
18687
|
const queuePath = dirtyQueuePath();
|
|
17750
|
-
const dir =
|
|
18688
|
+
const dir = path30.dirname(queuePath);
|
|
17751
18689
|
try {
|
|
17752
18690
|
ensureDirSync(dir);
|
|
17753
18691
|
} catch (e) {
|
|
17754
|
-
if (e.code !== "EEXIST" || !
|
|
18692
|
+
if (e.code !== "EEXIST" || !fs30.existsSync(dir)) throw e;
|
|
17755
18693
|
}
|
|
17756
18694
|
let leadingNewline = "";
|
|
17757
18695
|
try {
|
|
17758
|
-
const existing =
|
|
18696
|
+
const existing = fs30.readFileSync(queuePath, "utf8");
|
|
17759
18697
|
if (existing.length > 0 && !existing.endsWith("\n")) leadingNewline = "\n";
|
|
17760
18698
|
} catch {
|
|
17761
18699
|
}
|
|
17762
|
-
|
|
18700
|
+
fs30.appendFileSync(queuePath, `${leadingNewline}${encodeDirtyQueueLine(normalizedPath2)}
|
|
17763
18701
|
`);
|
|
17764
18702
|
}
|
|
17765
18703
|
function enqueueDirtyPathSafe(filePath, opts) {
|
|
@@ -17772,7 +18710,7 @@ function getDirtyPaths() {
|
|
|
17772
18710
|
const queuePath = dirtyQueuePath();
|
|
17773
18711
|
let raw;
|
|
17774
18712
|
try {
|
|
17775
|
-
raw =
|
|
18713
|
+
raw = fs30.readFileSync(queuePath, "utf8");
|
|
17776
18714
|
} catch {
|
|
17777
18715
|
return [];
|
|
17778
18716
|
}
|
|
@@ -17781,9 +18719,9 @@ function getDirtyPaths() {
|
|
|
17781
18719
|
function preCompactIndexHandler(_event) {
|
|
17782
18720
|
const paths = getDirtyPaths();
|
|
17783
18721
|
if (paths.length > 0) {
|
|
17784
|
-
const sidecar =
|
|
18722
|
+
const sidecar = path30.join(dataDir(), "queue", "pending.txt");
|
|
17785
18723
|
try {
|
|
17786
|
-
ensureDirSync(
|
|
18724
|
+
ensureDirSync(path30.dirname(sidecar));
|
|
17787
18725
|
atomicWriteBytes(sidecar, Buffer.from(`${paths.join("\n")}
|
|
17788
18726
|
`, "utf8"));
|
|
17789
18727
|
} catch {
|
|
@@ -17977,6 +18915,8 @@ export {
|
|
|
17977
18915
|
getGlobMatchCount,
|
|
17978
18916
|
setLastTabContext,
|
|
17979
18917
|
getLastTabContext,
|
|
18918
|
+
hasSeenImage,
|
|
18919
|
+
recordSeenImage,
|
|
17980
18920
|
recordOutstandingAgentSpawn,
|
|
17981
18921
|
getOutstandingAgentSpawns,
|
|
17982
18922
|
removeOutstandingAgentSpawn,
|
|
@@ -18000,6 +18940,8 @@ export {
|
|
|
18000
18940
|
installCodex,
|
|
18001
18941
|
uninstallCodex,
|
|
18002
18942
|
isCodexInstalled,
|
|
18943
|
+
copilotCliUserRoot,
|
|
18944
|
+
copilotCliMcpToolsDir,
|
|
18003
18945
|
copilotCliConfigPath,
|
|
18004
18946
|
copilotCliScriptPath,
|
|
18005
18947
|
installCopilotCli,
|
|
@@ -18023,11 +18965,13 @@ export {
|
|
|
18023
18965
|
loadBlob,
|
|
18024
18966
|
listBlobs,
|
|
18025
18967
|
pruneBlobs,
|
|
18968
|
+
estimateTokensFromLength,
|
|
18026
18969
|
estimateTokens,
|
|
18027
18970
|
trimToBudget,
|
|
18028
18971
|
capJsonRows,
|
|
18029
18972
|
SESSIONS_SUBDIR,
|
|
18030
18973
|
AGENT_SALT_MARKER,
|
|
18974
|
+
sessionSidecarPath,
|
|
18031
18975
|
listSiblingSessionStates,
|
|
18032
18976
|
loadSessionState,
|
|
18033
18977
|
saveSessionState,
|
|
@@ -18066,6 +19010,7 @@ export {
|
|
|
18066
19010
|
formatShrinkSummary,
|
|
18067
19011
|
isImagePath,
|
|
18068
19012
|
probeImageMeta,
|
|
19013
|
+
imageQualifiesForShrink,
|
|
18069
19014
|
shrinkImage,
|
|
18070
19015
|
compactPathFor,
|
|
18071
19016
|
isCompactFresh,
|
|
@@ -18076,6 +19021,8 @@ export {
|
|
|
18076
19021
|
compactDoc,
|
|
18077
19022
|
OVER_FETCH_FACTOR,
|
|
18078
19023
|
MAX_OVER_FETCH,
|
|
19024
|
+
isAvailable,
|
|
19025
|
+
embeddingBackendLoadError,
|
|
18079
19026
|
searchSemantic,
|
|
18080
19027
|
mergeNearbyHits,
|
|
18081
19028
|
embeddingsDepsAvailable,
|