token-goat 2.9.18 → 2.9.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/{token-goat-chunk-V3ZWBEHM.mjs → token-goat-chunk-2GTB5RBZ.mjs} +365 -26
- package/dist/{token-goat-chunk-ARNRFR6M.mjs → token-goat-chunk-3W5P673V.mjs} +1 -1
- package/dist/{token-goat-chunk-YQZE5NS6.mjs → token-goat-chunk-62VJ47WN.mjs} +6 -6
- package/dist/{token-goat-chunk-CTOYJ232.mjs → token-goat-chunk-7DGTVBWP.mjs} +45 -8
- package/dist/{token-goat-chunk-5MMJAJF3.mjs → token-goat-chunk-AVRD7SBO.mjs} +60 -8
- package/dist/{token-goat-chunk-7VLWNQDQ.mjs → token-goat-chunk-BTKDZW7F.mjs} +1 -1
- package/dist/{token-goat-chunk-LMFO66YD.mjs → token-goat-chunk-DA43OE7L.mjs} +1 -1
- package/dist/{token-goat-chunk-6LGVCCMY.mjs → token-goat-chunk-DA6TIDLI.mjs} +21 -16
- package/dist/{token-goat-chunk-UENEL6KX.mjs → token-goat-chunk-DHIXE6GV.mjs} +2 -2
- package/dist/{token-goat-chunk-W3E65QOW.mjs → token-goat-chunk-EP7TCWHM.mjs} +304 -46
- package/dist/{token-goat-chunk-HIZXQVNU.mjs → token-goat-chunk-EXLLLIC3.mjs} +4 -4
- package/dist/{token-goat-chunk-H3FONFBO.mjs → token-goat-chunk-F24NE3FO.mjs} +3 -3
- package/dist/{token-goat-chunk-LA3O4JGD.mjs → token-goat-chunk-FJZ5MZK7.mjs} +3 -3
- package/dist/{token-goat-chunk-BFEWOMA3.mjs → token-goat-chunk-FQYMZASI.mjs} +632 -352
- package/dist/{token-goat-chunk-OWSJJY5R.mjs → token-goat-chunk-FX5DHKGF.mjs} +5 -5
- package/dist/{token-goat-chunk-K4DG6EEF.mjs → token-goat-chunk-GYC34EGE.mjs} +7 -8
- package/dist/{token-goat-chunk-WFX2V3OF.mjs → token-goat-chunk-HPMMLV37.mjs} +358 -116
- package/dist/{token-goat-chunk-I67CPTDD.mjs → token-goat-chunk-IBTFGZBX.mjs} +19 -14
- package/dist/{token-goat-chunk-L7VCX33I.mjs → token-goat-chunk-JNIFSVWM.mjs} +36 -10
- package/dist/{token-goat-chunk-EPXNOKIV.mjs → token-goat-chunk-KWM6KBPA.mjs} +2 -2
- package/dist/{token-goat-chunk-BWCRC23L.mjs → token-goat-chunk-L2UOJBV4.mjs} +21 -20
- package/dist/{token-goat-chunk-52IMXJ2L.mjs → token-goat-chunk-LACH3WKS.mjs} +4 -4
- package/dist/token-goat-chunk-MU6I4IDI.mjs +337 -0
- package/dist/{token-goat-chunk-GRI6HWX3.mjs → token-goat-chunk-NJKNAC37.mjs} +1 -1
- package/dist/{token-goat-chunk-ERTXEKB6.mjs → token-goat-chunk-NMTKNYGF.mjs} +2 -0
- package/dist/{token-goat-chunk-34OUJ2IE.mjs → token-goat-chunk-NYFYIRG3.mjs} +12 -12
- package/dist/{token-goat-chunk-4EXFN2AW.mjs → token-goat-chunk-OZRSREKR.mjs} +2 -2
- package/dist/token-goat-chunk-P46RTOAQ.mjs +27 -0
- package/dist/{token-goat-chunk-QML7ITHJ.mjs → token-goat-chunk-PIAESKVU.mjs} +139 -23
- package/dist/{token-goat-chunk-HPOFR6QX.mjs → token-goat-chunk-S7RFIX4Y.mjs} +180 -50
- package/dist/{token-goat-chunk-CA6V36KW.mjs → token-goat-chunk-SOKAPOOC.mjs} +23 -4
- package/dist/token-goat-chunk-VJQBYSXG.mjs +20 -0
- package/dist/{token-goat-chunk-LTYWLKAG.mjs → token-goat-chunk-WKO5TVNA.mjs} +922 -611
- package/dist/{token-goat-chunk-QIDLJGXZ.mjs → token-goat-chunk-ZGQXRYKE.mjs} +29 -5
- package/dist/token-goat-hook.mjs +12 -12
- package/dist/token-goat.core.mjs +21 -20
- package/docs/cli.md +2 -1
- package/docs/security.md +1 -1
- package/package.json +1 -1
- package/dist/token-goat-chunk-2KTZ6J7T.mjs +0 -173
- package/dist/token-goat-chunk-RHHFEXKF.mjs +0 -93
package/README.md
CHANGED
|
@@ -255,6 +255,8 @@ $ token-goat read "src/auth.py::login"
|
|
|
255
255
|
out: 38 lines # ~300 tokens (97% smaller)
|
|
256
256
|
```
|
|
257
257
|
|
|
258
|
+
A line number works in place of a name, which is what you are holding when a grep hit, a stack frame, or a diff hunk points at code: `token-goat read "src/auth.py:142"` returns the symbol enclosing line 142, and `read "src/auth.py:142-190"` returns every symbol the range touches. A Windows drive letter stays part of the path, so `C:/src/auth.py:142` reads line 142 of that file, and a bare `C:142` stays a drive-relative path rather than a line number.
|
|
259
|
+
|
|
258
260
|
Same applies to `token-goat section "README.md::Install"` — one heading instead of the whole document. Anchor IDs and setext headings resolve too, so `section "doc.md::Quick-start"` works when the file uses `Quick start` as an `<h2>` with an explicit `{#quick-start}` anchor.
|
|
259
261
|
|
|
260
262
|
### 3. Compact manifest — preserve what mattered
|
|
@@ -1,34 +1,47 @@
|
|
|
1
1
|
import { createRequire as __cjsRequire } from 'node:module';
|
|
2
2
|
const require = __cjsRequire(import.meta.url);
|
|
3
3
|
import {
|
|
4
|
-
|
|
4
|
+
accumulateResidentLine,
|
|
5
|
+
claudeConfigDir,
|
|
6
|
+
createResidentContextStats,
|
|
7
|
+
enqueueDirtyPathsSafe,
|
|
8
|
+
estimateTokens,
|
|
9
|
+
fenceUntrusted,
|
|
5
10
|
getToolName,
|
|
6
|
-
getTrackedFiles
|
|
7
|
-
|
|
11
|
+
getTrackedFiles,
|
|
12
|
+
scanAndRecord,
|
|
13
|
+
summarizeResidentContext
|
|
14
|
+
} from "./token-goat-chunk-FQYMZASI.mjs";
|
|
8
15
|
import {
|
|
9
|
-
|
|
10
|
-
|
|
16
|
+
RECALL_DEFAULT_LIMIT,
|
|
17
|
+
commandHash,
|
|
18
|
+
getBashOutput,
|
|
19
|
+
indexRecallEntry,
|
|
20
|
+
listRecentRecall,
|
|
21
|
+
normalizeCommandForCacheKey,
|
|
22
|
+
searchRecall
|
|
23
|
+
} from "./token-goat-chunk-7DGTVBWP.mjs";
|
|
11
24
|
import {
|
|
12
25
|
isBlobStale,
|
|
13
26
|
loadBlob,
|
|
14
27
|
redactSecrets,
|
|
15
28
|
storeBlob
|
|
16
|
-
} from "./token-goat-chunk-
|
|
29
|
+
} from "./token-goat-chunk-S7RFIX4Y.mjs";
|
|
17
30
|
import {
|
|
18
|
-
|
|
19
|
-
} from "./token-goat-chunk-
|
|
31
|
+
parserFingerprintForLanguage
|
|
32
|
+
} from "./token-goat-chunk-EP7TCWHM.mjs";
|
|
20
33
|
import {
|
|
21
34
|
getProjectFileEntries,
|
|
22
35
|
projectScopeClause
|
|
23
|
-
} from "./token-goat-chunk-
|
|
36
|
+
} from "./token-goat-chunk-IBTFGZBX.mjs";
|
|
24
37
|
import {
|
|
25
38
|
getDb
|
|
26
|
-
} from "./token-goat-chunk-
|
|
39
|
+
} from "./token-goat-chunk-JNIFSVWM.mjs";
|
|
27
40
|
import {
|
|
28
41
|
fingerprintFile,
|
|
29
42
|
getDisplayRoot,
|
|
30
43
|
shortFingerprint
|
|
31
|
-
} from "./token-goat-chunk-
|
|
44
|
+
} from "./token-goat-chunk-SOKAPOOC.mjs";
|
|
32
45
|
import {
|
|
33
46
|
registerReset
|
|
34
47
|
} from "./token-goat-chunk-EEIDFMEM.mjs";
|
|
@@ -37,14 +50,18 @@ import {
|
|
|
37
50
|
countNoun,
|
|
38
51
|
extractErrorMessage,
|
|
39
52
|
foldPath,
|
|
40
|
-
globalDbPath
|
|
41
|
-
|
|
53
|
+
globalDbPath,
|
|
54
|
+
pad
|
|
55
|
+
} from "./token-goat-chunk-ZGQXRYKE.mjs";
|
|
42
56
|
import {
|
|
57
|
+
UNTRUSTED_TOOL_TAG,
|
|
58
|
+
UNTRUSTED_WEB_TAG,
|
|
43
59
|
displaySafeJson,
|
|
60
|
+
fenceUntrustedContent,
|
|
44
61
|
normalizePath,
|
|
45
62
|
resolveIndexPath,
|
|
46
63
|
toDisplayPath
|
|
47
|
-
} from "./token-goat-chunk-
|
|
64
|
+
} from "./token-goat-chunk-NMTKNYGF.mjs";
|
|
48
65
|
import {
|
|
49
66
|
compileGuardedRegex
|
|
50
67
|
} from "./token-goat-chunk-GMOUBOX4.mjs";
|
|
@@ -115,7 +132,7 @@ var DEFAULT_STDIN_TIMEOUT_MS = 5e3;
|
|
|
115
132
|
var MAX_STDIN_WALL_MS = 6e4;
|
|
116
133
|
var MAX_STDIN_BYTES = 64 * 1024 * 1024;
|
|
117
134
|
function readStdinJson(timeoutMs = DEFAULT_STDIN_TIMEOUT_MS, maxBytes = MAX_STDIN_BYTES) {
|
|
118
|
-
return new Promise((
|
|
135
|
+
return new Promise((resolve3, reject) => {
|
|
119
136
|
const chunks = [];
|
|
120
137
|
let totalBytes = 0;
|
|
121
138
|
let settled = false;
|
|
@@ -161,7 +178,7 @@ function readStdinJson(timeoutMs = DEFAULT_STDIN_TIMEOUT_MS, maxBytes = MAX_STDI
|
|
|
161
178
|
return;
|
|
162
179
|
}
|
|
163
180
|
try {
|
|
164
|
-
|
|
181
|
+
resolve3(JSON.parse(text));
|
|
165
182
|
} catch (err) {
|
|
166
183
|
reject(err instanceof Error ? err : new Error(String(err)));
|
|
167
184
|
}
|
|
@@ -525,7 +542,7 @@ async function describeTarget(target, table, opts) {
|
|
|
525
542
|
const absPath = resolve(process.cwd(), target);
|
|
526
543
|
if (existsSync(absPath)) {
|
|
527
544
|
try {
|
|
528
|
-
const { isSqliteFile, getSqliteSchema, formatSqliteSchema } = await import("./token-goat-chunk-
|
|
545
|
+
const { isSqliteFile, getSqliteSchema, formatSqliteSchema } = await import("./token-goat-chunk-OZRSREKR.mjs");
|
|
529
546
|
if (isSqliteFile(absPath)) {
|
|
530
547
|
const schema = getSqliteSchema(absPath);
|
|
531
548
|
if (table) {
|
|
@@ -752,6 +769,10 @@ function checkSymbolBodySize(dbPath) {
|
|
|
752
769
|
}
|
|
753
770
|
}
|
|
754
771
|
|
|
772
|
+
// src/mcp_tool_pattern.ts
|
|
773
|
+
init_define_import_meta_env();
|
|
774
|
+
var MCP_TOOL_PATTERN = "^mcp__";
|
|
775
|
+
|
|
755
776
|
// src/reconcile.ts
|
|
756
777
|
init_define_import_meta_env();
|
|
757
778
|
import * as fs2 from "node:fs";
|
|
@@ -821,7 +842,7 @@ function runReconcile(opts = {}) {
|
|
|
821
842
|
lines.push(`${countNoun(result.embedStale, "file")} of the above ${result.embedStale === 1 ? "was" : "were"} unchanged on disk but held out-of-date embeddings (${reembedVerb}).`);
|
|
822
843
|
}
|
|
823
844
|
if (result.trackedUnavailable) {
|
|
824
|
-
lines.push("This project has an index but git listed no files in it, so nothing
|
|
845
|
+
lines.push("This project has an index but git listed no files in it, so nothing on disk was compared against it and changed or new files could not be found. Files gone from disk were still detected. Run token-goat inside the repository, or reindex with --walk if this directory is deliberately not under git.");
|
|
825
846
|
}
|
|
826
847
|
if (result.budgetExhausted) {
|
|
827
848
|
lines.push(`Stopped at the ${result.elapsedMs}ms budget with ${countNoun(result.unscanned, "file")} unchecked, so there may be more drift; deletions were not computed at all, because an unchecked file is indistinguishable from a deleted one. Raise --budget-ms for a complete sweep.`);
|
|
@@ -877,7 +898,7 @@ function reconcileProject(opts = {}) {
|
|
|
877
898
|
added.push(file);
|
|
878
899
|
continue;
|
|
879
900
|
}
|
|
880
|
-
if (entry.parserSha !==
|
|
901
|
+
if (entry.parserSha !== parserFingerprintForLanguage(entry.language)) {
|
|
881
902
|
changed.push(file);
|
|
882
903
|
parserStale++;
|
|
883
904
|
continue;
|
|
@@ -908,7 +929,7 @@ function reconcileProject(opts = {}) {
|
|
|
908
929
|
}
|
|
909
930
|
const trackedUnavailable = tracked.length === 0 && indexed.size > 0;
|
|
910
931
|
const removed = [];
|
|
911
|
-
if (!budgetExhausted
|
|
932
|
+
if (!budgetExhausted) {
|
|
912
933
|
for (const [folded, entry] of indexed) {
|
|
913
934
|
if (seenOnDisk.has(folded)) continue;
|
|
914
935
|
if (Date.now() - startedAt > budgetMs) {
|
|
@@ -918,17 +939,17 @@ function reconcileProject(opts = {}) {
|
|
|
918
939
|
}
|
|
919
940
|
try {
|
|
920
941
|
fs2.statSync(entry.filePath);
|
|
921
|
-
} catch {
|
|
922
|
-
|
|
942
|
+
} catch (err) {
|
|
943
|
+
const code = err?.code;
|
|
944
|
+
if (code === "ENOENT" || code === "ENOTDIR") removed.push(entry.filePath);
|
|
923
945
|
}
|
|
924
946
|
}
|
|
925
947
|
}
|
|
926
948
|
let enqueued = 0;
|
|
927
949
|
if (opts.dryRun !== true) {
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
}
|
|
950
|
+
const toEnqueue = [...changed, ...added, ...removed].map((p) => resolveIndexPath(p, cwd));
|
|
951
|
+
enqueueDirtyPathsSafe(toEnqueue, { alreadyResolved: true });
|
|
952
|
+
enqueued = toEnqueue.length;
|
|
932
953
|
if (budgetExhausted) {
|
|
933
954
|
if (lastScanned !== null) writeReconcileCursor(dbPath, projectRoot, lastScanned);
|
|
934
955
|
} else if (tracked.length > 0) {
|
|
@@ -951,8 +972,319 @@ function reconcileProject(opts = {}) {
|
|
|
951
972
|
};
|
|
952
973
|
}
|
|
953
974
|
|
|
975
|
+
// src/waste.ts
|
|
976
|
+
init_define_import_meta_env();
|
|
977
|
+
import * as fs3 from "node:fs";
|
|
978
|
+
import * as path from "node:path";
|
|
979
|
+
function projectTranscriptsDir(projectRoot) {
|
|
980
|
+
const rootStr = path.resolve(projectRoot);
|
|
981
|
+
const slug = rootStr.replace(/[^A-Za-z0-9]/g, "-");
|
|
982
|
+
return path.join(claudeConfigDir(), "projects", slug);
|
|
983
|
+
}
|
|
984
|
+
function findLatestTranscript(projectRoot) {
|
|
985
|
+
const dir = projectTranscriptsDir(projectRoot);
|
|
986
|
+
let entries;
|
|
987
|
+
try {
|
|
988
|
+
entries = fs3.readdirSync(dir).filter((f) => f.endsWith(".jsonl"));
|
|
989
|
+
} catch {
|
|
990
|
+
return null;
|
|
991
|
+
}
|
|
992
|
+
let best = null;
|
|
993
|
+
for (const name of entries) {
|
|
994
|
+
const full = path.join(dir, name);
|
|
995
|
+
let stat;
|
|
996
|
+
try {
|
|
997
|
+
stat = fs3.statSync(full);
|
|
998
|
+
} catch {
|
|
999
|
+
continue;
|
|
1000
|
+
}
|
|
1001
|
+
if (best === null || stat.mtimeMs > best.mtimeMs) best = { file: full, mtimeMs: stat.mtimeMs };
|
|
1002
|
+
}
|
|
1003
|
+
return best?.file ?? null;
|
|
1004
|
+
}
|
|
1005
|
+
function safeStringify(value) {
|
|
1006
|
+
try {
|
|
1007
|
+
return JSON.stringify(value) ?? "";
|
|
1008
|
+
} catch {
|
|
1009
|
+
return "";
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
var FILE_PATH_TOOLS = /* @__PURE__ */ new Set(["Read", "Edit", "Write", "NotebookEdit", "MultiEdit"]);
|
|
1013
|
+
function extractFilePath(name, input) {
|
|
1014
|
+
if (!FILE_PATH_TOOLS.has(name)) return null;
|
|
1015
|
+
if (input === null || typeof input !== "object") return null;
|
|
1016
|
+
const o = input;
|
|
1017
|
+
const fp = o["file_path"];
|
|
1018
|
+
if (typeof fp === "string") return fp;
|
|
1019
|
+
const notebookPath = o["notebook_path"];
|
|
1020
|
+
return typeof notebookPath === "string" ? notebookPath : null;
|
|
1021
|
+
}
|
|
1022
|
+
function extractCommand(name, input) {
|
|
1023
|
+
if (name !== "Bash") return null;
|
|
1024
|
+
if (input === null || typeof input !== "object") return null;
|
|
1025
|
+
const cmd = input["command"];
|
|
1026
|
+
return typeof cmd === "string" ? cmd : null;
|
|
1027
|
+
}
|
|
1028
|
+
function summarizeInput(name, input, filePath, command) {
|
|
1029
|
+
if (filePath !== null) return filePath;
|
|
1030
|
+
if (command !== null) return command.length > 100 ? `${command.slice(0, 100)}\u2026` : command;
|
|
1031
|
+
if (input === null || typeof input !== "object") return "";
|
|
1032
|
+
const o = input;
|
|
1033
|
+
for (const key of ["pattern", "query", "path", "description", "prompt"]) {
|
|
1034
|
+
const v = o[key];
|
|
1035
|
+
if (typeof v === "string") return v.length > 100 ? `${v.slice(0, 100)}\u2026` : v;
|
|
1036
|
+
}
|
|
1037
|
+
const s = safeStringify(input);
|
|
1038
|
+
return s.length > 100 ? `${s.slice(0, 100)}\u2026` : s;
|
|
1039
|
+
}
|
|
1040
|
+
function extractResultText(content) {
|
|
1041
|
+
if (typeof content === "string") return content;
|
|
1042
|
+
if (Array.isArray(content)) {
|
|
1043
|
+
return content.map((b) => {
|
|
1044
|
+
if (b !== null && typeof b === "object" && b["type"] === "text") {
|
|
1045
|
+
const t = b["text"];
|
|
1046
|
+
return typeof t === "string" ? t : "";
|
|
1047
|
+
}
|
|
1048
|
+
return "";
|
|
1049
|
+
}).join("\n");
|
|
1050
|
+
}
|
|
1051
|
+
return "";
|
|
1052
|
+
}
|
|
1053
|
+
function parseTranscript(transcriptPath) {
|
|
1054
|
+
const raw = fs3.readFileSync(transcriptPath, "utf-8");
|
|
1055
|
+
const calls = [];
|
|
1056
|
+
const resultTextById = /* @__PURE__ */ new Map();
|
|
1057
|
+
const assistantTurns = [];
|
|
1058
|
+
const resident = createResidentContextStats();
|
|
1059
|
+
let seq = 0;
|
|
1060
|
+
for (const line of raw.split("\n")) {
|
|
1061
|
+
const trimmed = line.trim();
|
|
1062
|
+
if (trimmed === "") continue;
|
|
1063
|
+
let obj;
|
|
1064
|
+
try {
|
|
1065
|
+
obj = JSON.parse(trimmed);
|
|
1066
|
+
} catch {
|
|
1067
|
+
continue;
|
|
1068
|
+
}
|
|
1069
|
+
if (obj === null || typeof obj !== "object") continue;
|
|
1070
|
+
const o = obj;
|
|
1071
|
+
accumulateResidentLine(resident, o, trimmed.length);
|
|
1072
|
+
const message = o["message"];
|
|
1073
|
+
if (message === null || typeof message !== "object") continue;
|
|
1074
|
+
const messageObj = message;
|
|
1075
|
+
const content = messageObj["content"];
|
|
1076
|
+
if (!Array.isArray(content)) continue;
|
|
1077
|
+
const role = typeof messageObj["role"] === "string" ? messageObj["role"] : null;
|
|
1078
|
+
const cwd = typeof o["cwd"] === "string" ? o["cwd"] : null;
|
|
1079
|
+
let turnTextTokens = 0;
|
|
1080
|
+
let sawTextBlock = false;
|
|
1081
|
+
for (const block of content) {
|
|
1082
|
+
if (block === null || typeof block !== "object") continue;
|
|
1083
|
+
const b = block;
|
|
1084
|
+
if (b["type"] === "tool_use" && typeof b["id"] === "string" && typeof b["name"] === "string") {
|
|
1085
|
+
const name = b["name"];
|
|
1086
|
+
const input = b["input"];
|
|
1087
|
+
const filePath = extractFilePath(name, input);
|
|
1088
|
+
const command = extractCommand(name, input);
|
|
1089
|
+
calls.push({
|
|
1090
|
+
seq: seq++,
|
|
1091
|
+
id: b["id"],
|
|
1092
|
+
name,
|
|
1093
|
+
input,
|
|
1094
|
+
filePath,
|
|
1095
|
+
command,
|
|
1096
|
+
summary: summarizeInput(name, input, filePath, command),
|
|
1097
|
+
cwd
|
|
1098
|
+
});
|
|
1099
|
+
} else if (b["type"] === "tool_result" && typeof b["tool_use_id"] === "string") {
|
|
1100
|
+
const id = b["tool_use_id"];
|
|
1101
|
+
if (!resultTextById.has(id)) {
|
|
1102
|
+
resultTextById.set(id, extractResultText(b["content"]));
|
|
1103
|
+
}
|
|
1104
|
+
} else if (role === "assistant" && b["type"] === "text" && typeof b["text"] === "string") {
|
|
1105
|
+
sawTextBlock = true;
|
|
1106
|
+
turnTextTokens += estimateTokens(b["text"]);
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
if (role === "assistant" && sawTextBlock) assistantTurns.push(turnTextTokens);
|
|
1110
|
+
}
|
|
1111
|
+
return { calls, resultTextById, assistantTurns, resident };
|
|
1112
|
+
}
|
|
1113
|
+
function costPerCall(parsed) {
|
|
1114
|
+
return parsed.calls.map((c) => ({
|
|
1115
|
+
...c,
|
|
1116
|
+
tokens: estimateTokens(parsed.resultTextById.get(c.id) ?? "")
|
|
1117
|
+
}));
|
|
1118
|
+
}
|
|
1119
|
+
function aggregateSortedDesc(pairs) {
|
|
1120
|
+
return [...pairs].map(([key, tokens]) => ({ key, tokens })).sort((a, b) => b.tokens - a.tokens);
|
|
1121
|
+
}
|
|
1122
|
+
function tokensByTool(costs) {
|
|
1123
|
+
const m = /* @__PURE__ */ new Map();
|
|
1124
|
+
for (const c of costs) m.set(c.name, (m.get(c.name) ?? 0) + c.tokens);
|
|
1125
|
+
return aggregateSortedDesc(m);
|
|
1126
|
+
}
|
|
1127
|
+
function tokensByFile(costs) {
|
|
1128
|
+
const m = /* @__PURE__ */ new Map();
|
|
1129
|
+
for (const c of costs) {
|
|
1130
|
+
if (c.filePath === null) continue;
|
|
1131
|
+
m.set(c.filePath, (m.get(c.filePath) ?? 0) + c.tokens);
|
|
1132
|
+
}
|
|
1133
|
+
return aggregateSortedDesc(m);
|
|
1134
|
+
}
|
|
1135
|
+
function topExpensiveCalls(costs, n) {
|
|
1136
|
+
return [...costs].sort((a, b) => b.tokens - a.tokens).slice(0, n);
|
|
1137
|
+
}
|
|
1138
|
+
function neverTouchedAgain(costs) {
|
|
1139
|
+
const readsByFile = /* @__PURE__ */ new Map();
|
|
1140
|
+
for (const c of costs) {
|
|
1141
|
+
if (c.name !== "Read" || c.filePath === null) continue;
|
|
1142
|
+
const arr = readsByFile.get(c.filePath) ?? [];
|
|
1143
|
+
arr.push(c);
|
|
1144
|
+
readsByFile.set(c.filePath, arr);
|
|
1145
|
+
}
|
|
1146
|
+
const out = [];
|
|
1147
|
+
for (const [filePath, reads] of readsByFile) {
|
|
1148
|
+
const firstSeq = Math.min(...reads.map((r) => r.seq));
|
|
1149
|
+
const touchedLater = costs.some((c) => {
|
|
1150
|
+
if (c.seq <= firstSeq) return false;
|
|
1151
|
+
if (c.filePath === filePath) return true;
|
|
1152
|
+
return safeStringify(c.input).includes(filePath);
|
|
1153
|
+
});
|
|
1154
|
+
if (!touchedLater) {
|
|
1155
|
+
out.push({ filePath, tokens: reads.reduce((n, r) => n + r.tokens, 0) });
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
return out.sort((a, b) => b.tokens - a.tokens);
|
|
1159
|
+
}
|
|
1160
|
+
async function repeatedUncompressedBashCommands(costs) {
|
|
1161
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1162
|
+
for (const c of costs) {
|
|
1163
|
+
if (c.name !== "Bash" || c.command === null) continue;
|
|
1164
|
+
const norm = normalizeCommandForCacheKey(c.command);
|
|
1165
|
+
const arr = groups.get(norm) ?? [];
|
|
1166
|
+
arr.push(c);
|
|
1167
|
+
groups.set(norm, arr);
|
|
1168
|
+
}
|
|
1169
|
+
const out = [];
|
|
1170
|
+
for (const [norm, calls] of groups) {
|
|
1171
|
+
if (calls.length < 2) continue;
|
|
1172
|
+
let cached = false;
|
|
1173
|
+
for (const c of calls) {
|
|
1174
|
+
try {
|
|
1175
|
+
const hash = await commandHash(c.command, c.cwd);
|
|
1176
|
+
if (getBashOutput(hash) !== null) {
|
|
1177
|
+
cached = true;
|
|
1178
|
+
break;
|
|
1179
|
+
}
|
|
1180
|
+
} catch {
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
if (cached) continue;
|
|
1184
|
+
const totalTokens = calls.reduce((n, c) => n + c.tokens, 0);
|
|
1185
|
+
out.push({
|
|
1186
|
+
normalized: norm,
|
|
1187
|
+
count: calls.length,
|
|
1188
|
+
totalTokens,
|
|
1189
|
+
avgTokens: Math.round(totalTokens / calls.length)
|
|
1190
|
+
});
|
|
1191
|
+
}
|
|
1192
|
+
return out.sort((a, b) => b.totalTokens - a.totalTokens);
|
|
1193
|
+
}
|
|
1194
|
+
function assistantOutputCost(assistantTurns) {
|
|
1195
|
+
const turnCount = assistantTurns.length;
|
|
1196
|
+
let generatedTokens = 0;
|
|
1197
|
+
let resendCeilingTokens = 0;
|
|
1198
|
+
for (let i = 0; i < turnCount; i++) {
|
|
1199
|
+
const tokens = assistantTurns[i] ?? 0;
|
|
1200
|
+
generatedTokens += tokens;
|
|
1201
|
+
resendCeilingTokens += tokens * (turnCount - 1 - i);
|
|
1202
|
+
}
|
|
1203
|
+
return { turnCount, generatedTokens, resendCeilingTokens };
|
|
1204
|
+
}
|
|
1205
|
+
async function buildWasteReport(transcriptPath, opts = {}) {
|
|
1206
|
+
const parsed = parseTranscript(transcriptPath);
|
|
1207
|
+
const costs = costPerCall(parsed);
|
|
1208
|
+
const topN = opts.topN ?? 10;
|
|
1209
|
+
return {
|
|
1210
|
+
transcriptPath,
|
|
1211
|
+
totalTokens: costs.reduce((n, c) => n + c.tokens, 0),
|
|
1212
|
+
tokensByTool: tokensByTool(costs),
|
|
1213
|
+
tokensByFile: tokensByFile(costs),
|
|
1214
|
+
topCalls: topExpensiveCalls(costs, topN).map((c) => ({ seq: c.seq, name: c.name, summary: c.summary, tokens: c.tokens })),
|
|
1215
|
+
neverTouchedAgain: neverTouchedAgain(costs),
|
|
1216
|
+
repeatedUncompressedBash: await repeatedUncompressedBashCommands(costs),
|
|
1217
|
+
assistantOutput: assistantOutputCost(parsed.assistantTurns),
|
|
1218
|
+
residentContext: summarizeResidentContext(parsed.resident)
|
|
1219
|
+
};
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
// src/cli_recall.ts
|
|
1223
|
+
init_define_import_meta_env();
|
|
1224
|
+
var RECALL_COMMAND = {
|
|
1225
|
+
bash: "bash-output",
|
|
1226
|
+
web: "web-output",
|
|
1227
|
+
mcp: "mcp-output"
|
|
1228
|
+
};
|
|
1229
|
+
function fenceTagForCacheType(cacheType) {
|
|
1230
|
+
return cacheType === "web" ? UNTRUSTED_WEB_TAG : UNTRUSTED_TOOL_TAG;
|
|
1231
|
+
}
|
|
1232
|
+
function fenceRecallListing(text, hits) {
|
|
1233
|
+
const tags = new Set(hits.map((hit) => fenceTagForCacheType(hit.cacheType)));
|
|
1234
|
+
const tag = tags.size === 1 ? [...tags][0] ?? UNTRUSTED_TOOL_TAG : UNTRUSTED_TOOL_TAG;
|
|
1235
|
+
return fenceUntrusted(text, tag);
|
|
1236
|
+
}
|
|
1237
|
+
function fenceSnippetIfMatched(hit) {
|
|
1238
|
+
const matches = scanAndRecord(hit.snippet);
|
|
1239
|
+
if (matches.length === 0) return hit.snippet;
|
|
1240
|
+
return fenceUntrustedContent(hit.snippet, matches, fenceTagForCacheType(hit.cacheType));
|
|
1241
|
+
}
|
|
1242
|
+
function printHits(query, hits) {
|
|
1243
|
+
const w = (text) => {
|
|
1244
|
+
process.stdout.write(text);
|
|
1245
|
+
};
|
|
1246
|
+
if (hits.length === 0) {
|
|
1247
|
+
w(query === void 0 ? "No cache entries yet.\n" : `No cache entries match: ${query}
|
|
1248
|
+
`);
|
|
1249
|
+
return;
|
|
1250
|
+
}
|
|
1251
|
+
const body = hits.map((hit) => {
|
|
1252
|
+
const label = hit.label.length > 80 ? hit.label.slice(0, 77) + "..." : hit.label;
|
|
1253
|
+
const pointer = hit.blobPresent ? ` (token-goat ${RECALL_COMMAND[hit.cacheType]} ${hit.id})` : " (blob expired -- content below is all that remains)";
|
|
1254
|
+
return `[${pad(hit.cacheType, 4)}] ${hit.id}
|
|
1255
|
+
${pointer}
|
|
1256
|
+
${label}
|
|
1257
|
+
${hit.snippet}
|
|
1258
|
+
`;
|
|
1259
|
+
}).join("\n");
|
|
1260
|
+
w(`${fenceRecallListing(body, hits)}
|
|
1261
|
+
`);
|
|
1262
|
+
}
|
|
1263
|
+
function runRecallCommand(query, opts = {}) {
|
|
1264
|
+
const browse = query === void 0 || query.trim() === "";
|
|
1265
|
+
const scope = {
|
|
1266
|
+
...opts.type !== void 0 ? { type: opts.type } : {},
|
|
1267
|
+
...opts.limit !== void 0 ? { limit: opts.limit } : {}
|
|
1268
|
+
};
|
|
1269
|
+
const effectiveLimit = opts.limit ?? RECALL_DEFAULT_LIMIT;
|
|
1270
|
+
const fetched = browse ? listRecentRecall({ ...scope, limit: effectiveLimit + 1 }) : searchRecall(query, { ...scope, limit: effectiveLimit + 1 });
|
|
1271
|
+
const hits = fetched.slice(0, effectiveLimit);
|
|
1272
|
+
if (fetched.length > hits.length) {
|
|
1273
|
+
process.stderr.write(`Showing ${hits.length} entries; more are available (raise --limit to see them).
|
|
1274
|
+
`);
|
|
1275
|
+
}
|
|
1276
|
+
if (opts.json === true) {
|
|
1277
|
+
const fenced = hits.map((hit) => ({ ...hit, snippet: fenceSnippetIfMatched(hit) }));
|
|
1278
|
+
process.stdout.write(`${displaySafeJson(fenced, 0)}
|
|
1279
|
+
`);
|
|
1280
|
+
return;
|
|
1281
|
+
}
|
|
1282
|
+
printHits(browse ? void 0 : query, hits);
|
|
1283
|
+
}
|
|
1284
|
+
|
|
954
1285
|
export {
|
|
955
1286
|
checkSymbolBodySize,
|
|
1287
|
+
MCP_TOOL_PATTERN,
|
|
956
1288
|
WEB_OUTPUT_SUBDIR,
|
|
957
1289
|
storeWebOutput,
|
|
958
1290
|
getWebOutput,
|
|
@@ -962,6 +1294,13 @@ export {
|
|
|
962
1294
|
runReconcile,
|
|
963
1295
|
isReconcileClean,
|
|
964
1296
|
reconcileProject,
|
|
1297
|
+
projectTranscriptsDir,
|
|
1298
|
+
findLatestTranscript,
|
|
1299
|
+
safeStringify,
|
|
1300
|
+
extractResultText,
|
|
1301
|
+
buildWasteReport,
|
|
1302
|
+
RECALL_COMMAND,
|
|
1303
|
+
runRecallCommand,
|
|
965
1304
|
MAX_STDIN_BYTES,
|
|
966
1305
|
readStdinJson,
|
|
967
1306
|
diagnoseSqlFailure,
|
|
@@ -20,18 +20,18 @@ import {
|
|
|
20
20
|
setOcrTimeoutForTesting,
|
|
21
21
|
setTesseractEntryForTesting,
|
|
22
22
|
verifyOcrLangCache
|
|
23
|
-
} from "./token-goat-chunk-
|
|
24
|
-
import "./token-goat-chunk-
|
|
25
|
-
import "./token-goat-chunk-
|
|
23
|
+
} from "./token-goat-chunk-F24NE3FO.mjs";
|
|
24
|
+
import "./token-goat-chunk-S7RFIX4Y.mjs";
|
|
25
|
+
import "./token-goat-chunk-JNIFSVWM.mjs";
|
|
26
26
|
import {
|
|
27
27
|
DEFAULT_OCR_LANG,
|
|
28
28
|
isSupportedOcrLang,
|
|
29
29
|
resolveOcrLangs
|
|
30
|
-
} from "./token-goat-chunk-
|
|
30
|
+
} from "./token-goat-chunk-SOKAPOOC.mjs";
|
|
31
31
|
import "./token-goat-chunk-EEIDFMEM.mjs";
|
|
32
32
|
import "./token-goat-chunk-OSUFN2FV.mjs";
|
|
33
|
-
import "./token-goat-chunk-
|
|
34
|
-
import "./token-goat-chunk-
|
|
33
|
+
import "./token-goat-chunk-ZGQXRYKE.mjs";
|
|
34
|
+
import "./token-goat-chunk-NMTKNYGF.mjs";
|
|
35
35
|
import "./token-goat-chunk-GMOUBOX4.mjs";
|
|
36
36
|
import "./token-goat-chunk-A37V4PBF.mjs";
|
|
37
37
|
export {
|
|
@@ -9,28 +9,28 @@ import {
|
|
|
9
9
|
savedTokensFromBytes,
|
|
10
10
|
storeBlob,
|
|
11
11
|
stripAnsiEscapes
|
|
12
|
-
} from "./token-goat-chunk-
|
|
12
|
+
} from "./token-goat-chunk-S7RFIX4Y.mjs";
|
|
13
13
|
import {
|
|
14
14
|
sanitizeFtsQuery
|
|
15
|
-
} from "./token-goat-chunk-
|
|
15
|
+
} from "./token-goat-chunk-IBTFGZBX.mjs";
|
|
16
16
|
import {
|
|
17
17
|
getDb
|
|
18
|
-
} from "./token-goat-chunk-
|
|
18
|
+
} from "./token-goat-chunk-JNIFSVWM.mjs";
|
|
19
19
|
import {
|
|
20
20
|
loadConfig,
|
|
21
21
|
shortFingerprint
|
|
22
|
-
} from "./token-goat-chunk-
|
|
22
|
+
} from "./token-goat-chunk-SOKAPOOC.mjs";
|
|
23
23
|
import {
|
|
24
24
|
registerReset
|
|
25
25
|
} from "./token-goat-chunk-EEIDFMEM.mjs";
|
|
26
26
|
import {
|
|
27
27
|
globalDbPath,
|
|
28
28
|
runGit
|
|
29
|
-
} from "./token-goat-chunk-
|
|
29
|
+
} from "./token-goat-chunk-ZGQXRYKE.mjs";
|
|
30
30
|
import {
|
|
31
31
|
normalizePath,
|
|
32
32
|
resolveIndexPath
|
|
33
|
-
} from "./token-goat-chunk-
|
|
33
|
+
} from "./token-goat-chunk-NMTKNYGF.mjs";
|
|
34
34
|
import {
|
|
35
35
|
init_define_import_meta_env
|
|
36
36
|
} from "./token-goat-chunk-A37V4PBF.mjs";
|
|
@@ -1520,6 +1520,9 @@ function commandHashSync(command, cwd = null) {
|
|
|
1520
1520
|
function commandHash(command, cwd = null) {
|
|
1521
1521
|
return Promise.resolve(commandHashSync(command, cwd));
|
|
1522
1522
|
}
|
|
1523
|
+
function bashOutputIdSync(command, output, cwd = null) {
|
|
1524
|
+
return shortFingerprint(`${commandHashSync(command, cwd)}\0${output}`);
|
|
1525
|
+
}
|
|
1523
1526
|
function computeBashFingerprints(command, cwd) {
|
|
1524
1527
|
const fingerprints = {};
|
|
1525
1528
|
if (cwd && (isGitMutableCommand(command) || isGitPushCommand(command) || isTestRunnerCommand2(command) || isLintCommand(command) || isBuildCommand(command) || isNpmRunScriptCommand(command))) {
|
|
@@ -1630,7 +1633,8 @@ function summarizeOutputDelta(oldOutput, newOutput) {
|
|
|
1630
1633
|
return `[token-goat: delta] output changed: ${oldLines.length} -> ${newLines.length} lines`;
|
|
1631
1634
|
}
|
|
1632
1635
|
function storeBashOutputSync(command, output, exitCode, cwd = null) {
|
|
1633
|
-
const id =
|
|
1636
|
+
const id = bashOutputIdSync(command, output, cwd);
|
|
1637
|
+
const commandKey = commandHashSync(command, cwd);
|
|
1634
1638
|
const fingerprints = computeBashFingerprints(command, cwd);
|
|
1635
1639
|
const redactedOutput = redactSecrets(output).text;
|
|
1636
1640
|
const redactedCommand = redactSecrets(command).text;
|
|
@@ -1644,11 +1648,18 @@ function storeBashOutputSync(command, output, exitCode, cwd = null) {
|
|
|
1644
1648
|
...fingerprints ? { fingerprints } : {}
|
|
1645
1649
|
};
|
|
1646
1650
|
_byId.set(id, entry);
|
|
1651
|
+
_byId.set(commandKey, entry);
|
|
1647
1652
|
storeBlob(BASH_OUTPUT_SUBDIR, id, entry);
|
|
1653
|
+
if (commandKey !== id) storeBlob(BASH_OUTPUT_SUBDIR, commandKey, { latestId: id });
|
|
1648
1654
|
indexRecallEntry("bash", id, redactedCommand, `${redactedCommand}
|
|
1649
1655
|
${redactedOutput}`, entry.storedAt);
|
|
1650
1656
|
return id;
|
|
1651
1657
|
}
|
|
1658
|
+
function coerceBashRedirect(raw) {
|
|
1659
|
+
if (raw === null || typeof raw !== "object") return null;
|
|
1660
|
+
const latest = raw["latestId"];
|
|
1661
|
+
return typeof latest === "string" && latest.length > 0 ? latest : null;
|
|
1662
|
+
}
|
|
1652
1663
|
function storeBashOutput(command, output, exitCode, cwd = null) {
|
|
1653
1664
|
return Promise.resolve(storeBashOutputSync(command, output, exitCode, cwd));
|
|
1654
1665
|
}
|
|
@@ -1682,7 +1693,17 @@ function getBashOutput(id) {
|
|
|
1682
1693
|
const hit = _byId.get(id);
|
|
1683
1694
|
if (hit !== void 0) return hit;
|
|
1684
1695
|
if (isBlobStale(BASH_OUTPUT_SUBDIR, id)) return null;
|
|
1685
|
-
const
|
|
1696
|
+
const raw = loadBlob(BASH_OUTPUT_SUBDIR, id);
|
|
1697
|
+
const target = coerceBashRedirect(raw);
|
|
1698
|
+
if (target !== null) {
|
|
1699
|
+
if (isBlobStale(BASH_OUTPUT_SUBDIR, target)) return null;
|
|
1700
|
+
const latest = coerceBashEntry(loadBlob(BASH_OUTPUT_SUBDIR, target));
|
|
1701
|
+
if (latest === null) return null;
|
|
1702
|
+
_byId.set(target, latest);
|
|
1703
|
+
_byId.set(id, latest);
|
|
1704
|
+
return latest;
|
|
1705
|
+
}
|
|
1706
|
+
const entry = coerceBashEntry(raw);
|
|
1686
1707
|
if (entry === null) return null;
|
|
1687
1708
|
_byId.set(id, entry);
|
|
1688
1709
|
return entry;
|
|
@@ -6651,6 +6672,8 @@ var GrepFilter = class extends ToolFilter {
|
|
|
6651
6672
|
if (nonEmpty.length <= _GREP_COMPRESS_THRESHOLD) return clipGrepLines(text, argv);
|
|
6652
6673
|
const fileCounts = /* @__PURE__ */ new Map();
|
|
6653
6674
|
let unattributed = 0;
|
|
6675
|
+
const searchFiles = positionalArgs(argv.slice(1)).slice(1);
|
|
6676
|
+
const singleFile = searchFiles.length === 1 ? searchFiles[0] : void 0;
|
|
6654
6677
|
for (const line of nonEmpty) {
|
|
6655
6678
|
if (line.startsWith("Binary file ") && line.includes(" matches")) {
|
|
6656
6679
|
const fname = line.slice("Binary file ".length).replace(/ matches$/, "");
|
|
@@ -6667,6 +6690,10 @@ var GrepFilter = class extends ToolFilter {
|
|
|
6667
6690
|
continue;
|
|
6668
6691
|
}
|
|
6669
6692
|
}
|
|
6693
|
+
if (singleFile !== void 0 && (/^\d+[:-]/.test(line) || colonIdx <= 0)) {
|
|
6694
|
+
fileCounts.set(singleFile, (fileCounts.get(singleFile) ?? 0) + 1);
|
|
6695
|
+
continue;
|
|
6696
|
+
}
|
|
6670
6697
|
unattributed++;
|
|
6671
6698
|
}
|
|
6672
6699
|
const totalMatches = [...fileCounts.values()].reduce((a, b) => a + b, 0) + unattributed;
|
|
@@ -14913,6 +14940,12 @@ var PytestFilter = class extends ToolFilter {
|
|
|
14913
14940
|
}
|
|
14914
14941
|
};
|
|
14915
14942
|
var pytestFilter = new PytestFilter();
|
|
14943
|
+
function looksLikePytestOutput(text) {
|
|
14944
|
+
const lines = text.split("\n").filter((line) => line.trim() !== "");
|
|
14945
|
+
if (lines.length === 0) return false;
|
|
14946
|
+
const progressLines = lines.filter((line) => DOTS_RE.test(line) || FILE_DOTS_RE.test(line) || HEADER_RE.test(line) || XDIST_PREFIX_RE.test(line));
|
|
14947
|
+
return progressLines.length / lines.length >= 0.5;
|
|
14948
|
+
}
|
|
14916
14949
|
|
|
14917
14950
|
// src/tool_filters/test_runners.ts
|
|
14918
14951
|
init_define_import_meta_env();
|
|
@@ -15094,6 +15127,7 @@ export {
|
|
|
15094
15127
|
splitOwnTrailingNotices,
|
|
15095
15128
|
dedupeConsecutive,
|
|
15096
15129
|
capTokens,
|
|
15130
|
+
pathStem,
|
|
15097
15131
|
shlexSplit,
|
|
15098
15132
|
clipLongMatchLine,
|
|
15099
15133
|
isLockFile,
|
|
@@ -15112,6 +15146,7 @@ export {
|
|
|
15112
15146
|
isScopedGitStatusOrDiffStatCommand,
|
|
15113
15147
|
normalizeCommandForCacheKey,
|
|
15114
15148
|
commandHash,
|
|
15149
|
+
bashOutputIdSync,
|
|
15115
15150
|
isBashEntryStale,
|
|
15116
15151
|
summarizeOutputDelta,
|
|
15117
15152
|
storeBashOutputSync,
|
|
@@ -15122,6 +15157,8 @@ export {
|
|
|
15122
15157
|
compressedTokensSaved,
|
|
15123
15158
|
CompressedOutput,
|
|
15124
15159
|
ToolFilter,
|
|
15160
|
+
pytestFilter,
|
|
15161
|
+
looksLikePytestOutput,
|
|
15125
15162
|
TOOL_FILTERS,
|
|
15126
15163
|
dispatchArgv,
|
|
15127
15164
|
selectFilter,
|