token-goat 2.6.33 → 2.6.35

package/dist/{token-goat-hook-chunk-WBBHV6TZ.mjs → token-goat-hook-chunk-3AAH72SO.mjs}

@@ -76,13 +76,15 @@ import {
   runSymbol,
   runTestFor,
   runTypes,
+  runXmlOutline,
+  runXmlQuery,
   runYamlOutline,
   runYamlQuery,
   runZipList,
   runZipRead,
   symbolNamesInFile,
   upsertNote
-} from "./token-goat-hook-chunk-VDXQXYJK.mjs";
+} from "./token-goat-hook-chunk-NXVT4F3S.mjs";
 import {
   BASH_OUTPUT_SUBDIR,
   GEMINI_TOOL_NAME_MAP,
@@ -103,7 +105,7 @@ import {
   readStdinJson,
   searchRecall,
   storeWebOutput
-} from "./token-goat-hook-chunk-ZT47M3IA.mjs";
+} from "./token-goat-hook-chunk-CRBMPT74.mjs";
 import {
   AGENT_SALT_MARKER,
   CONTEXT_AUTOCOMPACT_TOKENS,
@@ -149,6 +151,7 @@ import {
   findLatestSessionId,
   findMemSuggestionCandidates,
   findMemoryMd,
+  findOrphanedChunkPaths,
   findStrayClaudeMdBlocks,
   findSystemTempFiles,
   formatAge,
@@ -170,6 +173,7 @@ import {
   incrementSkillHit,
   indexFileEmbeddings,
   indexFileSync,
+  indexedPathSpellingIsStale,
   installClaudeMd,
   installCodex,
   installCopilotCli,
@@ -207,6 +211,7 @@ import {
   pruneBlobs,
   pruneBlockedRoot,
   pruneDeletedFiles,
+  pruneOrphanedChunks,
   pruneSkillOutputs,
   pruneSystemTempFiles,
   querySheet,
@@ -238,7 +243,7 @@ import {
   vscodeDecoderConfigured,
   walkProject,
   writeCompact
-} from "./token-goat-hook-chunk-UWV4MTYI.mjs";
+} from "./token-goat-hook-chunk-XM4W3WCO.mjs";
 import {
   C,
   CONFIG_KEY_ENV_OVERRIDES,
@@ -255,13 +260,17 @@ import {
   configPath,
   countNoun,
   dataDir,
+  decodeSource,
   defaultConfig,
   detectLanguage,
+  detectSourceEncoding,
+  encodeSource,
   ensureDirSync,
   ensureNewline,
   escapeRegExp,
   extractErrorMessage,
   fg,
+  fileIsAbsent,
   findProject,
   fingerprintContent,
   fingerprintFile,
@@ -313,7 +322,7 @@ import {
   withFileLock,
   withRetryOnLock,
   writeJsonSettings
-} from "./token-goat-hook-chunk-E6AIPKNF.mjs";
+} from "./token-goat-hook-chunk-2O7P4Z6Q.mjs";
 import "./token-goat-hook-chunk-BUOCULAM.mjs";
 import {
   __export
@@ -5460,7 +5469,7 @@ function purgeRoots() {
 function purgeDataDirectories() {
   const result = { removed: [], absent: [], failed: [] };
   for (const root of purgeRoots()) {
-    if (!fs10.existsSync(root)) {
+    if (fileIsAbsent(root)) {
       result.absent.push(root);
       continue;
     }
@@ -6140,7 +6149,7 @@ async function runDoctorAndExit(opts) {
 }
 
 // src/webfetch.ts
-import { existsSync as existsSync8, readdirSync as readdirSync5, statSync as statSync4, unlinkSync as unlinkSync2 } from "fs";
+import { existsSync as existsSync7, readdirSync as readdirSync5, statSync as statSync4, unlinkSync as unlinkSync2 } from "fs";
 import * as http from "http";
 import * as https from "https";
 import { isIPv4, isIPv6 } from "net";
@@ -6327,7 +6336,7 @@ function isPrivateIPv6(ip) {
 var STALE_DOWNLOAD_AGE_MS = 10 * 60 * 1e3;
 function cleanupStaleDownloads() {
   const cacheDir = webCacheDir();
-  if (!existsSync8(cacheDir)) return 0;
+  if (!existsSync7(cacheDir)) return 0;
   let removed = 0;
   try {
     const files = readdirSync5(cacheDir);
@@ -8719,7 +8728,7 @@ function collectFiles(projectRoot, patterns, opts = {}) {
       }
       let content;
       try {
-        content = redactIfDotenv(p, fs13.readFileSync(fd, "utf8"));
+        content = redactIfDotenv(p, decodeSource(fs13.readFileSync(fd)));
       } catch {
         result.skipped.push(`${rel} (unreadable)`);
         continue;
@@ -9526,7 +9535,7 @@ function isMarkerOccurrence(matched, hasColon) {
 function scanFileForTodos(filePath, kindSet) {
   let text;
   try {
-    text = fs16.readFileSync(filePath, "utf8");
+    text = decodeSource(fs16.readFileSync(filePath));
   } catch {
     return [];
   }
@@ -11834,9 +11843,10 @@ function cmdProject(opts) {
     const removed = before.length - after.length;
     const stale = before.filter((r) => !after.includes(r));
     const staleTempFiles = findSystemTempFiles();
+    const orphanChunkPaths = findOrphanedChunkPaths();
     if (opts.dryRun === true) {
       if (opts.json === true) {
-        emit(JSON.stringify({ dryRun: true, wouldPrune: removed, stale, wouldPruneTempFiles: staleTempFiles.length, staleTempFiles, blocked_roots: before }, null, 2));
+        emit(JSON.stringify({ dryRun: true, wouldPrune: removed, stale, wouldPruneTempFiles: staleTempFiles.length, staleTempFiles, wouldPruneOrphanChunkFiles: orphanChunkPaths.length, orphanChunkPaths, blocked_roots: before }, null, 2));
         return;
       }
       if (removed === 0) {
@@ -11851,18 +11861,26 @@ function cmdProject(opts) {
         emit(`Would prune ${staleTempFiles.length} stale indexed temp-dir file(s):`);
         for (const p of staleTempFiles) emit(`  ${p}`);
       }
+      if (orphanChunkPaths.length === 0) {
+        emit("Would prune 0 orphaned embedding chunk file(s). Nothing to do.");
+      } else {
+        emit(`Would prune orphaned embedding chunks for ${orphanChunkPaths.length} file(s):`);
+        for (const p of orphanChunkPaths) emit(`  ${p}`);
+      }
       return;
     }
     cfg.worker.blocked_roots = after;
     saveConfigSafe(cfg);
     invalidateConfigCache();
     const prunedTempFiles = pruneSystemTempFiles();
+    const prunedOrphanChunks = pruneOrphanedChunks();
     if (opts.json === true) {
-      emit(JSON.stringify({ pruned: removed, blocked_roots: after, prunedTempFiles: prunedTempFiles.length }, null, 2));
+      emit(JSON.stringify({ pruned: removed, blocked_roots: after, prunedTempFiles: prunedTempFiles.length, prunedOrphanChunkFiles: prunedOrphanChunks.length }, null, 2));
       return;
     }
     emit(`Pruned ${removed} stale root(s). Remaining: ${after.length}`);
     emit(`Pruned ${prunedTempFiles.length} stale indexed temp-dir file(s).`);
+    emit(`Pruned orphaned embedding chunks for ${prunedOrphanChunks.length} file(s).`);
     return;
   }
   emitErr2(`project: unknown action '${action}'. Use list, exclude, or prune.`);
@@ -13047,15 +13065,19 @@ function formatSpentCell(row) {
   if (row.bytesEmitted === null) return "n/a (legacy)";
   return row.legacyEmissions > 0 ? `${row.bytesEmitted} (${row.legacyEmissions} legacy)` : String(row.bytesEmitted);
 }
+function suppressedCell(row) {
+  if (!row.suppressed) return "no";
+  return row.suppressionPermanent ? "yes (permanent)" : "yes";
+}
 function printSummary(rows) {
   const w = (text) => {
     process.stdout.write(text);
   };
-  w(pad("category", 22) + pad("emitted", 9) + pad("acted-on", 10) + pad("efficacy", 10) + pad("suppressed", 12) + pad("manual+", 9) + pad("manual-", 9) + "spent\n");
+  w(pad("category", 22) + pad("emitted", 9) + pad("acted-on", 10) + pad("efficacy", 10) + pad("suppressed", 17) + pad("manual+", 9) + pad("manual-", 9) + "spent\n");
   for (const row of rows) {
     const pct = row.efficacyPct === null ? "n/a" : `${row.efficacyPct}%`;
     w(
-      pad(row.category, 22) + pad(String(row.emitted), 9) + pad(String(row.actedOn), 10) + pad(pct, 10) + pad(row.suppressed ? "yes" : "no", 12) + pad(String(row.manualEffective), 9) + pad(String(row.manualIneffective), 9) + formatSpentCell(row) + "\n"
+      pad(row.category, 22) + pad(String(row.emitted), 9) + pad(String(row.actedOn), 10) + pad(pct, 10) + pad(suppressedCell(row), 17) + pad(String(row.manualEffective), 9) + pad(String(row.manualIneffective), 9) + formatSpentCell(row) + "\n"
     );
   }
 }
@@ -13095,6 +13117,13 @@ function runHintStatsCommand(opts = {}) {
     process.stdout.write("No hint emissions recorded yet \u2014 the zeros below are absence of data, not measured ineffectiveness.\n");
   }
   printSummary(rows);
+  const permanent = rows.filter((r) => r.suppressionPermanent);
+  if (permanent.length > 0) {
+    const names = permanent.map((r) => r.category).join(", ");
+    process.stdout.write(`
+hints.backoff_thresholds is empty, so suppression here is permanent rather than a self-healing throttle: ${names} will not emit again on any occasion. Set backoff_thresholds to re-enable probe emissions, or run \`token-goat hint-stats --reset\` to clear the ledger.
+`);
+  }
   printTotals(getHintStatsTotals());
 }
 
@@ -13334,7 +13363,8 @@ async function cmdIndex(pathArg, opts = {}) {
     const sha = fingerprintFile(key);
     if (sha === null && !fs26.existsSync(key)) continue;
     const entry = sha !== null ? getFileEntry(key, dbPath) : null;
-    const parseUnchanged = !force && sha !== null && entry?.sha === sha;
+    const spellingStale = entry !== null && indexedPathSpellingIsStale(entry.filePath, key);
+    const parseUnchanged = !force && !spellingStale && sha !== null && entry?.sha === sha;
     const embeddingsEnabled = loadConfig().indexing?.embeddings_enabled ?? true;
     const depsAvailable = embeddingsEnabled && embeddingsDepsAvailable(getDb(dbPath));
     const embedUnchanged = !force && parseUnchanged && sha !== null && isEmbedFresh(entry?.embedSha, sha, embeddingsEnabled, depsAvailable);
@@ -13417,7 +13447,7 @@ async function cmdMcpServe() {
   let StdioServerTransport;
   try {
     ;
-    ({ createMcpServer } = await import("./token-goat-hook-chunk-EKGDLI4D.mjs"));
+    ({ createMcpServer } = await import("./token-goat-hook-chunk-VU6VGZBM.mjs"));
     ({ StdioServerTransport } = await import("@modelcontextprotocol/sdk/server/stdio.js"));
   } catch (err2) {
     process.stderr.write(
@@ -13438,7 +13468,7 @@ async function cmdHook(event, opts) {
   if (typeof opts.harness === "string" && opts.harness.length > 0) {
     process.env[ENV_KEYS.HARNESS_OVERRIDE] = opts.harness;
   }
-  const { relay } = await import("./token-goat-hook-chunk-XBOWDNJB.mjs");
+  const { relay } = await import("./token-goat-hook-chunk-J4HKWUWZ.mjs");
   await relay(event);
 }
 async function cmdInstall(opts) {
@@ -13902,7 +13932,7 @@ function cmdBashOutput(id, opts) {
       if (st.isFIFO() || st.isSocket()) {
         throw new CliError(`--file '${opts.file}' is a special file (FIFO or socket) \u2014 only regular files are supported`);
       }
-      content = redactIfDotenv(opts.file, fs26.readFileSync(opts.file, "utf-8"));
+      content = redactIfDotenv(opts.file, decodeSource(fs26.readFileSync(opts.file)));
     } catch (e) {
       if (e instanceof CliError) throw e;
       throw new CliError(`cannot read file: ${opts.file}`);
@@ -14217,6 +14247,16 @@ function cmdYamlOutline(file, opts) {
 function cmdYamlQuery(file, yamlPath, opts) {
   process.exitCode = runYamlQuery({ file, path: yamlPath, ...opts });
 }
+function cmdXmlOutline(file, opts) {
+  process.exitCode = runXmlOutline({
+    file,
+    ...opts.json === true ? { json: true } : {},
+    ...opts.maxDepth !== void 0 ? { maxDepth: requireNonNegativeInt("--max-depth", opts.maxDepth) } : {}
+  });
+}
+function cmdXmlQuery(file, xmlPath, opts) {
+  process.exitCode = runXmlQuery({ file, path: xmlPath, ...opts });
+}
 function cmdOpenApiOutline(file, opts) {
   process.exitCode = runOpenApiOutline({ file, ...opts });
 }
@@ -14285,7 +14325,7 @@ function emitExtraFileArgsNote(command, first, extras, opts = {}) {
 }
 async function cmdCompress(opts) {
   try {
-    const bashRunner = await import("./token-goat-hook-chunk-26DLRODR.mjs");
+    const bashRunner = await import("./token-goat-hook-chunk-D6UJEFHX.mjs");
     if (opts.compress === false) {
       process.exitCode = bashRunner.runRaw(opts.cmd, parseTimeout(opts.timeout, bashRunner.DEFAULT_TIMEOUT_SECONDS));
       return;
@@ -14311,7 +14351,7 @@ async function cmdSkillBody(name, opts) {
   if (filePath === null) {
     throw new CliError(`skill '${name}' not found`);
   }
-  const body = fs26.readFileSync(filePath, "utf-8");
+  const body = decodeSource(fs26.readFileSync(filePath));
   if (opts.compact === true) {
     out(extractCompactFromMarker(body) ?? body);
   } else {
@@ -14328,7 +14368,7 @@ async function cmdSkillCompact(name, opts) {
     for (const skill of skills) {
       const filePath = await getSkillFilePath(skill.name);
       if (!filePath) continue;
-      const body2 = fs26.readFileSync(filePath, "utf-8");
+      const body2 = decodeSource(fs26.readFileSync(filePath));
       const compact2 = extractCompactFromMarker(body2);
       if (compact2 === null) continue;
       const sourceSha2 = contentHash(body2);
@@ -14532,7 +14572,7 @@ async function cmdSkillSection(nameHeading, headingArg) {
   if (!filePath) {
     throw new CliError(`skill '${skillName}' not found`);
   }
-  const body = fs26.readFileSync(filePath, "utf-8");
+  const body = decodeSource(fs26.readFileSync(filePath));
   const extracted = extractNamedSection(body, heading);
   if (!extracted) {
     process.exitCode = 1;
@@ -14963,7 +15003,13 @@ function writeReplacedBuffer(file, replacedBuf, preWriteStat) {
     const testDelayMs = Number(process.env["TOKEN_GOAT_TEST_REPLACE_DELAY_MS"] ?? "");
     if (Number.isFinite(testDelayMs) && testDelayMs > 0) {
       process.stderr.write("TOKEN_GOAT_TEST_REPLACE_DELAY_READY\n");
-      sleepSync(testDelayMs);
+      const until = process.env["TOKEN_GOAT_TEST_REPLACE_DELAY_UNTIL"];
+      const deadline = Date.now() + testDelayMs;
+      if (until !== void 0 && until !== "") {
+        while (Date.now() < deadline && !fs26.existsSync(until)) sleepSync(5);
+      } else {
+        sleepSync(testDelayMs);
+      }
     }
     let preRenameStat;
     try {
@@ -15114,13 +15160,14 @@ function cmdInsertSection(file, opts) {
     else if (allHeadings.length > 0) messages.push(`Try: token-goat outline ${file}`);
     throw new CliError(messages.join("\n"));
   }
-  let rawText;
+  let rawBytes;
   try {
-    rawText = fs26.readFileSync(file, "utf-8");
+    rawBytes = fs26.readFileSync(file);
   } catch (e) {
     mapFsError(e, void 0, file);
   }
-  if (rawText.charCodeAt(0) === 65279) rawText = rawText.slice(1);
+  const sourceEncoding = detectSourceEncoding(rawBytes);
+  const rawText = decodeSource(rawBytes);
   const eol = detectDominantEol(Buffer.from(rawText, "utf8"));
   const lfLines = rawText.replace(/\r\n/g, "\n").split("\n");
   const insertAt = result.lineEnd;
@@ -15139,7 +15186,7 @@ function cmdInsertSection(file, opts) {
     }
   }
   try {
-    atomicWriteBuffer(file, Buffer.from(mergedText, "utf8"));
+    atomicWriteBuffer(file, encodeSource(mergedText, sourceEncoding));
   } catch (e) {
     mapFsError(e, void 0, file);
   }
@@ -15810,6 +15857,10 @@ function buildProgram() {
   program2.command("yaml-query <file> <path>").description(
     "extract one value or a projected/filtered subset from a YAML document by dot-path instead of a raw Read (same grammar as json-query)\n\npath grammar: dot-separated keys with optional bracket segments -- [n] index, [*] wildcard (projects every element/value), [field=value] filter (keeps array elements whose field stringifies to value). Examples: spec.containers[0].image, items[*].name, items[kind=Service]"
   ).option("--head <n>", "limit a projected/filtered result to the first N items").option("--json", "emit the result as JSON instead of text").action(guard(cmdYamlQuery));
+  program2.command("xml-outline <file>").description("structural summary of an XML document (element hierarchy / attribute names / child counts) instead of a raw Read").option("--json", "emit the outline as JSON instead of text").option("--max-depth <n>", "max depth of element hierarchy to show").action(guard(cmdXmlOutline));
+  program2.command("xml-query <file> <path>").description(
+    "extract elements or attributes from an XML document by tag path instead of a raw Read\n\npath grammar: slash- or dot-separated tag names with optional bracket segments and attribute selectors -- [n] index, [*] wildcard, [@attr] or [@attr=value] filter, and trailing @attr to extract attribute value. Examples: root.child, catalog/book[@id=101]/title, /feed/entry[*]/@href"
+  ).option("--head <n>", "limit a matching result list to the first N items").option("--json", "emit the result as JSON instead of text").action(guard(cmdXmlQuery));
   program2.command("openapi-outline <file>").description("per-operation listing (method, path, operationId, summary, tags) of an OpenAPI 3.x / Swagger 2.0 spec (JSON or YAML) instead of a raw Read").option("--json", "emit the operation list as JSON instead of text").action(guard(cmdOpenApiOutline));
   program2.command("openapi-op <file> <operation>").description(
     'full detail (parameters, request body schema, response schemas, description) for exactly one OpenAPI operation instead of a raw Read\n\noperation may be an operationId (exact match) or a "METHOD path" spec, e.g. "GET /users/{id}"'