token-goat 2.9.17 → 2.9.19

package/README.md

@@ -255,6 +255,8 @@ $ token-goat read "src/auth.py::login"
 out: 38 lines              # ~300 tokens   (97% smaller)
 ```
 
+A line number works in place of a name, which is what you are holding when a grep hit, a stack frame, or a diff hunk points at code: `token-goat read "src/auth.py:142"` returns the symbol enclosing line 142, and `read "src/auth.py:142-190"` returns every symbol the range touches. A Windows drive letter stays part of the path, so `C:/src/auth.py:142` reads line 142 of that file, and a bare `C:142` stays a drive-relative path rather than a line number.
+
 Same applies to `token-goat section "README.md::Install"` — one heading instead of the whole document. Anchor IDs and setext headings resolve too, so `section "doc.md::Quick-start"` works when the file uses `Quick start` as an `<h2>` with an explicit `{#quick-start}` anchor.
 
 ### 3. Compact manifest — preserve what mattered

package/dist/{token-goat-chunk-HRUGS6CO.mjs → token-goat-chunk-2GTB5RBZ.mjs}

@@ -1,34 +1,47 @@
 import { createRequire as __cjsRequire } from 'node:module';
 const require = __cjsRequire(import.meta.url);
 import {
-  enqueueDirtyPathSafe,
+  accumulateResidentLine,
+  claudeConfigDir,
+  createResidentContextStats,
+  enqueueDirtyPathsSafe,
+  estimateTokens,
+  fenceUntrusted,
   getToolName,
-  getTrackedFiles
-} from "./token-goat-chunk-PKNOHVWP.mjs";
+  getTrackedFiles,
+  scanAndRecord,
+  summarizeResidentContext
+} from "./token-goat-chunk-FQYMZASI.mjs";
 import {
-  indexRecallEntry
-} from "./token-goat-chunk-C26ZBGBC.mjs";
+  RECALL_DEFAULT_LIMIT,
+  commandHash,
+  getBashOutput,
+  indexRecallEntry,
+  listRecentRecall,
+  normalizeCommandForCacheKey,
+  searchRecall
+} from "./token-goat-chunk-7DGTVBWP.mjs";
 import {
   isBlobStale,
   loadBlob,
   redactSecrets,
   storeBlob
-} from "./token-goat-chunk-26YUO7MP.mjs";
+} from "./token-goat-chunk-S7RFIX4Y.mjs";
 import {
-  PARSER_FINGERPRINT
-} from "./token-goat-chunk-73GXYYZR.mjs";
+  parserFingerprintForLanguage
+} from "./token-goat-chunk-EP7TCWHM.mjs";
 import {
   getProjectFileEntries,
   projectScopeClause
-} from "./token-goat-chunk-BK3RPPTK.mjs";
+} from "./token-goat-chunk-IBTFGZBX.mjs";
 import {
   getDb
-} from "./token-goat-chunk-MWSVUKVV.mjs";
+} from "./token-goat-chunk-JNIFSVWM.mjs";
 import {
   fingerprintFile,
   getDisplayRoot,
   shortFingerprint
-} from "./token-goat-chunk-2VASS5VR.mjs";
+} from "./token-goat-chunk-SOKAPOOC.mjs";
 import {
   registerReset
 } from "./token-goat-chunk-EEIDFMEM.mjs";
@@ -37,14 +50,18 @@ import {
   countNoun,
   extractErrorMessage,
   foldPath,
-  globalDbPath
-} from "./token-goat-chunk-UBZCON6N.mjs";
+  globalDbPath,
+  pad
+} from "./token-goat-chunk-ZGQXRYKE.mjs";
 import {
+  UNTRUSTED_TOOL_TAG,
+  UNTRUSTED_WEB_TAG,
   displaySafeJson,
+  fenceUntrustedContent,
   normalizePath,
   resolveIndexPath,
   toDisplayPath
-} from "./token-goat-chunk-ERTXEKB6.mjs";
+} from "./token-goat-chunk-NMTKNYGF.mjs";
 import {
   compileGuardedRegex
 } from "./token-goat-chunk-GMOUBOX4.mjs";
@@ -115,7 +132,7 @@ var DEFAULT_STDIN_TIMEOUT_MS = 5e3;
 var MAX_STDIN_WALL_MS = 6e4;
 var MAX_STDIN_BYTES = 64 * 1024 * 1024;
 function readStdinJson(timeoutMs = DEFAULT_STDIN_TIMEOUT_MS, maxBytes = MAX_STDIN_BYTES) {
-  return new Promise((resolve2, reject) => {
+  return new Promise((resolve3, reject) => {
     const chunks = [];
     let totalBytes = 0;
     let settled = false;
@@ -161,7 +178,7 @@ function readStdinJson(timeoutMs = DEFAULT_STDIN_TIMEOUT_MS, maxBytes = MAX_STDI
           return;
         }
         try {
-          resolve2(JSON.parse(text));
+          resolve3(JSON.parse(text));
         } catch (err) {
           reject(err instanceof Error ? err : new Error(String(err)));
         }
@@ -525,7 +542,7 @@ async function describeTarget(target, table, opts) {
   const absPath = resolve(process.cwd(), target);
   if (existsSync(absPath)) {
     try {
-      const { isSqliteFile, getSqliteSchema, formatSqliteSchema } = await import("./token-goat-chunk-4EXFN2AW.mjs");
+      const { isSqliteFile, getSqliteSchema, formatSqliteSchema } = await import("./token-goat-chunk-OZRSREKR.mjs");
       if (isSqliteFile(absPath)) {
         const schema = getSqliteSchema(absPath);
         if (table) {
@@ -752,6 +769,10 @@ function checkSymbolBodySize(dbPath) {
   }
 }
 
+// src/mcp_tool_pattern.ts
+init_define_import_meta_env();
+var MCP_TOOL_PATTERN = "^mcp__";
+
 // src/reconcile.ts
 init_define_import_meta_env();
 import * as fs2 from "node:fs";
@@ -821,7 +842,7 @@ function runReconcile(opts = {}) {
     lines.push(`${countNoun(result.embedStale, "file")} of the above ${result.embedStale === 1 ? "was" : "were"} unchanged on disk but held out-of-date embeddings (${reembedVerb}).`);
   }
   if (result.trackedUnavailable) {
-    lines.push("This project has an index but git listed no files in it, so nothing could be compared and no deletions were computed. Run token-goat inside the repository, or reindex with --walk if this directory is deliberately not under git.");
+    lines.push("This project has an index but git listed no files in it, so nothing on disk was compared against it and changed or new files could not be found. Files gone from disk were still detected. Run token-goat inside the repository, or reindex with --walk if this directory is deliberately not under git.");
   }
   if (result.budgetExhausted) {
     lines.push(`Stopped at the ${result.elapsedMs}ms budget with ${countNoun(result.unscanned, "file")} unchecked, so there may be more drift; deletions were not computed at all, because an unchecked file is indistinguishable from a deleted one. Raise --budget-ms for a complete sweep.`);
@@ -877,7 +898,7 @@ function reconcileProject(opts = {}) {
       added.push(file);
       continue;
     }
-    if (entry.parserSha !== PARSER_FINGERPRINT) {
+    if (entry.parserSha !== parserFingerprintForLanguage(entry.language)) {
       changed.push(file);
       parserStale++;
       continue;
@@ -908,7 +929,7 @@ function reconcileProject(opts = {}) {
   }
   const trackedUnavailable = tracked.length === 0 && indexed.size > 0;
   const removed = [];
-  if (!budgetExhausted && !trackedUnavailable) {
+  if (!budgetExhausted) {
     for (const [folded, entry] of indexed) {
       if (seenOnDisk.has(folded)) continue;
       if (Date.now() - startedAt > budgetMs) {
@@ -918,17 +939,17 @@ function reconcileProject(opts = {}) {
       }
       try {
         fs2.statSync(entry.filePath);
-      } catch {
-        removed.push(entry.filePath);
+      } catch (err) {
+        const code = err?.code;
+        if (code === "ENOENT" || code === "ENOTDIR") removed.push(entry.filePath);
       }
     }
   }
   let enqueued = 0;
   if (opts.dryRun !== true) {
-    for (const p of [...changed, ...added, ...removed]) {
-      enqueueDirtyPathSafe(resolveIndexPath(p, cwd), { alreadyResolved: true });
-      enqueued++;
-    }
+    const toEnqueue = [...changed, ...added, ...removed].map((p) => resolveIndexPath(p, cwd));
+    enqueueDirtyPathsSafe(toEnqueue, { alreadyResolved: true });
+    enqueued = toEnqueue.length;
     if (budgetExhausted) {
       if (lastScanned !== null) writeReconcileCursor(dbPath, projectRoot, lastScanned);
     } else if (tracked.length > 0) {
@@ -951,8 +972,319 @@ function reconcileProject(opts = {}) {
   };
 }
 
+// src/waste.ts
+init_define_import_meta_env();
+import * as fs3 from "node:fs";
+import * as path from "node:path";
+function projectTranscriptsDir(projectRoot) {
+  const rootStr = path.resolve(projectRoot);
+  const slug = rootStr.replace(/[^A-Za-z0-9]/g, "-");
+  return path.join(claudeConfigDir(), "projects", slug);
+}
+function findLatestTranscript(projectRoot) {
+  const dir = projectTranscriptsDir(projectRoot);
+  let entries;
+  try {
+    entries = fs3.readdirSync(dir).filter((f) => f.endsWith(".jsonl"));
+  } catch {
+    return null;
+  }
+  let best = null;
+  for (const name of entries) {
+    const full = path.join(dir, name);
+    let stat;
+    try {
+      stat = fs3.statSync(full);
+    } catch {
+      continue;
+    }
+    if (best === null || stat.mtimeMs > best.mtimeMs) best = { file: full, mtimeMs: stat.mtimeMs };
+  }
+  return best?.file ?? null;
+}
+function safeStringify(value) {
+  try {
+    return JSON.stringify(value) ?? "";
+  } catch {
+    return "";
+  }
+}
+var FILE_PATH_TOOLS = /* @__PURE__ */ new Set(["Read", "Edit", "Write", "NotebookEdit", "MultiEdit"]);
+function extractFilePath(name, input) {
+  if (!FILE_PATH_TOOLS.has(name)) return null;
+  if (input === null || typeof input !== "object") return null;
+  const o = input;
+  const fp = o["file_path"];
+  if (typeof fp === "string") return fp;
+  const notebookPath = o["notebook_path"];
+  return typeof notebookPath === "string" ? notebookPath : null;
+}
+function extractCommand(name, input) {
+  if (name !== "Bash") return null;
+  if (input === null || typeof input !== "object") return null;
+  const cmd = input["command"];
+  return typeof cmd === "string" ? cmd : null;
+}
+function summarizeInput(name, input, filePath, command) {
+  if (filePath !== null) return filePath;
+  if (command !== null) return command.length > 100 ? `${command.slice(0, 100)}\u2026` : command;
+  if (input === null || typeof input !== "object") return "";
+  const o = input;
+  for (const key of ["pattern", "query", "path", "description", "prompt"]) {
+    const v = o[key];
+    if (typeof v === "string") return v.length > 100 ? `${v.slice(0, 100)}\u2026` : v;
+  }
+  const s = safeStringify(input);
+  return s.length > 100 ? `${s.slice(0, 100)}\u2026` : s;
+}
+function extractResultText(content) {
+  if (typeof content === "string") return content;
+  if (Array.isArray(content)) {
+    return content.map((b) => {
+      if (b !== null && typeof b === "object" && b["type"] === "text") {
+        const t = b["text"];
+        return typeof t === "string" ? t : "";
+      }
+      return "";
+    }).join("\n");
+  }
+  return "";
+}
+function parseTranscript(transcriptPath) {
+  const raw = fs3.readFileSync(transcriptPath, "utf-8");
+  const calls = [];
+  const resultTextById = /* @__PURE__ */ new Map();
+  const assistantTurns = [];
+  const resident = createResidentContextStats();
+  let seq = 0;
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (trimmed === "") continue;
+    let obj;
+    try {
+      obj = JSON.parse(trimmed);
+    } catch {
+      continue;
+    }
+    if (obj === null || typeof obj !== "object") continue;
+    const o = obj;
+    accumulateResidentLine(resident, o, trimmed.length);
+    const message = o["message"];
+    if (message === null || typeof message !== "object") continue;
+    const messageObj = message;
+    const content = messageObj["content"];
+    if (!Array.isArray(content)) continue;
+    const role = typeof messageObj["role"] === "string" ? messageObj["role"] : null;
+    const cwd = typeof o["cwd"] === "string" ? o["cwd"] : null;
+    let turnTextTokens = 0;
+    let sawTextBlock = false;
+    for (const block of content) {
+      if (block === null || typeof block !== "object") continue;
+      const b = block;
+      if (b["type"] === "tool_use" && typeof b["id"] === "string" && typeof b["name"] === "string") {
+        const name = b["name"];
+        const input = b["input"];
+        const filePath = extractFilePath(name, input);
+        const command = extractCommand(name, input);
+        calls.push({
+          seq: seq++,
+          id: b["id"],
+          name,
+          input,
+          filePath,
+          command,
+          summary: summarizeInput(name, input, filePath, command),
+          cwd
+        });
+      } else if (b["type"] === "tool_result" && typeof b["tool_use_id"] === "string") {
+        const id = b["tool_use_id"];
+        if (!resultTextById.has(id)) {
+          resultTextById.set(id, extractResultText(b["content"]));
+        }
+      } else if (role === "assistant" && b["type"] === "text" && typeof b["text"] === "string") {
+        sawTextBlock = true;
+        turnTextTokens += estimateTokens(b["text"]);
+      }
+    }
+    if (role === "assistant" && sawTextBlock) assistantTurns.push(turnTextTokens);
+  }
+  return { calls, resultTextById, assistantTurns, resident };
+}
+function costPerCall(parsed) {
+  return parsed.calls.map((c) => ({
+    ...c,
+    tokens: estimateTokens(parsed.resultTextById.get(c.id) ?? "")
+  }));
+}
+function aggregateSortedDesc(pairs) {
+  return [...pairs].map(([key, tokens]) => ({ key, tokens })).sort((a, b) => b.tokens - a.tokens);
+}
+function tokensByTool(costs) {
+  const m = /* @__PURE__ */ new Map();
+  for (const c of costs) m.set(c.name, (m.get(c.name) ?? 0) + c.tokens);
+  return aggregateSortedDesc(m);
+}
+function tokensByFile(costs) {
+  const m = /* @__PURE__ */ new Map();
+  for (const c of costs) {
+    if (c.filePath === null) continue;
+    m.set(c.filePath, (m.get(c.filePath) ?? 0) + c.tokens);
+  }
+  return aggregateSortedDesc(m);
+}
+function topExpensiveCalls(costs, n) {
+  return [...costs].sort((a, b) => b.tokens - a.tokens).slice(0, n);
+}
+function neverTouchedAgain(costs) {
+  const readsByFile = /* @__PURE__ */ new Map();
+  for (const c of costs) {
+    if (c.name !== "Read" || c.filePath === null) continue;
+    const arr = readsByFile.get(c.filePath) ?? [];
+    arr.push(c);
+    readsByFile.set(c.filePath, arr);
+  }
+  const out = [];
+  for (const [filePath, reads] of readsByFile) {
+    const firstSeq = Math.min(...reads.map((r) => r.seq));
+    const touchedLater = costs.some((c) => {
+      if (c.seq <= firstSeq) return false;
+      if (c.filePath === filePath) return true;
+      return safeStringify(c.input).includes(filePath);
+    });
+    if (!touchedLater) {
+      out.push({ filePath, tokens: reads.reduce((n, r) => n + r.tokens, 0) });
+    }
+  }
+  return out.sort((a, b) => b.tokens - a.tokens);
+}
+async function repeatedUncompressedBashCommands(costs) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const c of costs) {
+    if (c.name !== "Bash" || c.command === null) continue;
+    const norm = normalizeCommandForCacheKey(c.command);
+    const arr = groups.get(norm) ?? [];
+    arr.push(c);
+    groups.set(norm, arr);
+  }
+  const out = [];
+  for (const [norm, calls] of groups) {
+    if (calls.length < 2) continue;
+    let cached = false;
+    for (const c of calls) {
+      try {
+        const hash = await commandHash(c.command, c.cwd);
+        if (getBashOutput(hash) !== null) {
+          cached = true;
+          break;
+        }
+      } catch {
+      }
+    }
+    if (cached) continue;
+    const totalTokens = calls.reduce((n, c) => n + c.tokens, 0);
+    out.push({
+      normalized: norm,
+      count: calls.length,
+      totalTokens,
+      avgTokens: Math.round(totalTokens / calls.length)
+    });
+  }
+  return out.sort((a, b) => b.totalTokens - a.totalTokens);
+}
+function assistantOutputCost(assistantTurns) {
+  const turnCount = assistantTurns.length;
+  let generatedTokens = 0;
+  let resendCeilingTokens = 0;
+  for (let i = 0; i < turnCount; i++) {
+    const tokens = assistantTurns[i] ?? 0;
+    generatedTokens += tokens;
+    resendCeilingTokens += tokens * (turnCount - 1 - i);
+  }
+  return { turnCount, generatedTokens, resendCeilingTokens };
+}
+async function buildWasteReport(transcriptPath, opts = {}) {
+  const parsed = parseTranscript(transcriptPath);
+  const costs = costPerCall(parsed);
+  const topN = opts.topN ?? 10;
+  return {
+    transcriptPath,
+    totalTokens: costs.reduce((n, c) => n + c.tokens, 0),
+    tokensByTool: tokensByTool(costs),
+    tokensByFile: tokensByFile(costs),
+    topCalls: topExpensiveCalls(costs, topN).map((c) => ({ seq: c.seq, name: c.name, summary: c.summary, tokens: c.tokens })),
+    neverTouchedAgain: neverTouchedAgain(costs),
+    repeatedUncompressedBash: await repeatedUncompressedBashCommands(costs),
+    assistantOutput: assistantOutputCost(parsed.assistantTurns),
+    residentContext: summarizeResidentContext(parsed.resident)
+  };
+}
+
+// src/cli_recall.ts
+init_define_import_meta_env();
+var RECALL_COMMAND = {
+  bash: "bash-output",
+  web: "web-output",
+  mcp: "mcp-output"
+};
+function fenceTagForCacheType(cacheType) {
+  return cacheType === "web" ? UNTRUSTED_WEB_TAG : UNTRUSTED_TOOL_TAG;
+}
+function fenceRecallListing(text, hits) {
+  const tags = new Set(hits.map((hit) => fenceTagForCacheType(hit.cacheType)));
+  const tag = tags.size === 1 ? [...tags][0] ?? UNTRUSTED_TOOL_TAG : UNTRUSTED_TOOL_TAG;
+  return fenceUntrusted(text, tag);
+}
+function fenceSnippetIfMatched(hit) {
+  const matches = scanAndRecord(hit.snippet);
+  if (matches.length === 0) return hit.snippet;
+  return fenceUntrustedContent(hit.snippet, matches, fenceTagForCacheType(hit.cacheType));
+}
+function printHits(query, hits) {
+  const w = (text) => {
+    process.stdout.write(text);
+  };
+  if (hits.length === 0) {
+    w(query === void 0 ? "No cache entries yet.\n" : `No cache entries match: ${query}
+`);
+    return;
+  }
+  const body = hits.map((hit) => {
+    const label = hit.label.length > 80 ? hit.label.slice(0, 77) + "..." : hit.label;
+    const pointer = hit.blobPresent ? `  (token-goat ${RECALL_COMMAND[hit.cacheType]} ${hit.id})` : "  (blob expired -- content below is all that remains)";
+    return `[${pad(hit.cacheType, 4)}] ${hit.id}
+${pointer}
+  ${label}
+  ${hit.snippet}
+`;
+  }).join("\n");
+  w(`${fenceRecallListing(body, hits)}
+`);
+}
+function runRecallCommand(query, opts = {}) {
+  const browse = query === void 0 || query.trim() === "";
+  const scope = {
+    ...opts.type !== void 0 ? { type: opts.type } : {},
+    ...opts.limit !== void 0 ? { limit: opts.limit } : {}
+  };
+  const effectiveLimit = opts.limit ?? RECALL_DEFAULT_LIMIT;
+  const fetched = browse ? listRecentRecall({ ...scope, limit: effectiveLimit + 1 }) : searchRecall(query, { ...scope, limit: effectiveLimit + 1 });
+  const hits = fetched.slice(0, effectiveLimit);
+  if (fetched.length > hits.length) {
+    process.stderr.write(`Showing ${hits.length} entries; more are available (raise --limit to see them).
+`);
+  }
+  if (opts.json === true) {
+    const fenced = hits.map((hit) => ({ ...hit, snippet: fenceSnippetIfMatched(hit) }));
+    process.stdout.write(`${displaySafeJson(fenced, 0)}
+`);
+    return;
+  }
+  printHits(browse ? void 0 : query, hits);
+}
+
 export {
   checkSymbolBodySize,
+  MCP_TOOL_PATTERN,
   WEB_OUTPUT_SUBDIR,
   storeWebOutput,
   getWebOutput,
@@ -962,6 +1294,13 @@ export {
   runReconcile,
   isReconcileClean,
   reconcileProject,
+  projectTranscriptsDir,
+  findLatestTranscript,
+  safeStringify,
+  extractResultText,
+  buildWasteReport,
+  RECALL_COMMAND,
+  runRecallCommand,
   MAX_STDIN_BYTES,
   readStdinJson,
   diagnoseSqlFailure,

package/dist/{token-goat-chunk-W2GUUQAQ.mjs → token-goat-chunk-3W5P673V.mjs}

@@ -12,7 +12,7 @@ import {
   parseOoxmlPart,
   readOoxmlZip,
   sortNumberedParts
-} from "./token-goat-chunk-66224KCZ.mjs";
+} from "./token-goat-chunk-BTKDZW7F.mjs";
 import {
   compileGuardedRegex
 } from "./token-goat-chunk-GMOUBOX4.mjs";

package/dist/{token-goat-chunk-5WB7GDH4.mjs → token-goat-chunk-62VJ47WN.mjs}

@@ -20,18 +20,18 @@ import {
   setOcrTimeoutForTesting,
   setTesseractEntryForTesting,
   verifyOcrLangCache
-} from "./token-goat-chunk-CICJFVIH.mjs";
-import "./token-goat-chunk-26YUO7MP.mjs";
-import "./token-goat-chunk-MWSVUKVV.mjs";
+} from "./token-goat-chunk-F24NE3FO.mjs";
+import "./token-goat-chunk-S7RFIX4Y.mjs";
+import "./token-goat-chunk-JNIFSVWM.mjs";
 import {
   DEFAULT_OCR_LANG,
   isSupportedOcrLang,
   resolveOcrLangs
-} from "./token-goat-chunk-2VASS5VR.mjs";
+} from "./token-goat-chunk-SOKAPOOC.mjs";
 import "./token-goat-chunk-EEIDFMEM.mjs";
 import "./token-goat-chunk-OSUFN2FV.mjs";
-import "./token-goat-chunk-UBZCON6N.mjs";
-import "./token-goat-chunk-ERTXEKB6.mjs";
+import "./token-goat-chunk-ZGQXRYKE.mjs";
+import "./token-goat-chunk-NMTKNYGF.mjs";
 import "./token-goat-chunk-GMOUBOX4.mjs";
 import "./token-goat-chunk-A37V4PBF.mjs";
 export {