token-goat 2.9.19 → 2.9.21

package/dist/{token-goat-chunk-3W5P673V.mjs → token-goat-chunk-MZQORUX5.mjs}

@@ -12,7 +12,7 @@ import {
   parseOoxmlPart,
   readOoxmlZip,
   sortNumberedParts
-} from "./token-goat-chunk-BTKDZW7F.mjs";
+} from "./token-goat-chunk-QTIRVK3M.mjs";
 import {
   compileGuardedRegex
 } from "./token-goat-chunk-GMOUBOX4.mjs";

package/dist/{token-goat-chunk-DHIXE6GV.mjs → token-goat-chunk-ORRDWKLA.mjs}

@@ -11,7 +11,7 @@ import {
   ooxmlPartBudget,
   parseOoxmlPart,
   readOoxmlZip
-} from "./token-goat-chunk-BTKDZW7F.mjs";
+} from "./token-goat-chunk-QTIRVK3M.mjs";
 import {
   displaySafeText
 } from "./token-goat-chunk-NMTKNYGF.mjs";

package/dist/{token-goat-chunk-F24NE3FO.mjs → token-goat-chunk-PCR43MCM.mjs}

@@ -5,11 +5,11 @@ import {
   isSupportedOcrLang,
   loadConfig,
   resolveOcrLangs
-} from "./token-goat-chunk-SOKAPOOC.mjs";
+} from "./token-goat-chunk-M6CDVASA.mjs";
 import {
   ensureDirSync,
   tokenGoatHome
-} from "./token-goat-chunk-ZGQXRYKE.mjs";
+} from "./token-goat-chunk-TGBSMHPM.mjs";
 import {
   fenceUntrustedOcrText
 } from "./token-goat-chunk-NMTKNYGF.mjs";

package/dist/token-goat-chunk-PDYQA3AP.mjs

@@ -0,0 +1,248 @@
+import { createRequire as __cjsRequire } from 'node:module';
+const require = __cjsRequire(import.meta.url);
+import {
+  WEB_FETCH_KEY_SEP,
+  computeAdaptiveBudget,
+  getContextPressure,
+  getSessionBashOutputs,
+  getSessionBashReruns,
+  getSessionFiles,
+  getSessionWebFetches,
+  isNoisePath,
+  listSiblingSessionStates,
+  loadSessionCache
+} from "./token-goat-chunk-MECDAPXB.mjs";
+import {
+  getBashOutput
+} from "./token-goat-chunk-M43UJVYQ.mjs";
+import {
+  loadConfig
+} from "./token-goat-chunk-M6CDVASA.mjs";
+import {
+  foldPath,
+  runGit,
+  toKB
+} from "./token-goat-chunk-TGBSMHPM.mjs";
+import {
+  displaySafePath,
+  displaySafeText,
+  neutralizeSpokenMarkers
+} from "./token-goat-chunk-NMTKNYGF.mjs";
+import {
+  init_define_import_meta_env
+} from "./token-goat-chunk-A37V4PBF.mjs";
+
+// src/manifest.ts
+init_define_import_meta_env();
+import { spawnSync } from "node:child_process";
+var MEM_EPOCH_TIMEOUT_MS = 800;
+var MAX_ROWS = 40;
+function appendCappedSection(lines, header, rows, cap) {
+  if (rows.length === 0) return;
+  lines.push("");
+  lines.push(header);
+  for (const row of rows.slice(0, cap)) lines.push(row);
+  if (rows.length > cap) lines.push(`- ...and ${rows.length - cap} more`);
+}
+function renderReadRow(entry) {
+  const kb = Math.max(1, toKB(entry.sizeBytes));
+  const plural = entry.readCount === 1 ? "read" : "reads";
+  const edited = entry.wasEdited ? ", edited" : "";
+  return `- ${displaySafePath(entry.path)} (${kb}kb, ${entry.readCount} ${plural}${edited})`;
+}
+function renderSymbolReadRow(entry) {
+  return `- ${displaySafePath(entry.path)} (symbols: ${(entry.symbols_read ?? []).map(displaySafeText).join(", ")})`;
+}
+function mergeManifestFiles(parent, siblingFiles) {
+  const byPath = /* @__PURE__ */ new Map();
+  for (const f of parent) byPath.set(foldPath(f.path), f);
+  for (const f of siblingFiles) {
+    const key = foldPath(f.path);
+    const prev = byPath.get(key);
+    if (prev === void 0) {
+      byPath.set(key, f);
+      continue;
+    }
+    const mergedSymbols = [.../* @__PURE__ */ new Set([...prev.symbols_read ?? [], ...f.symbols_read ?? []])];
+    byPath.set(key, {
+      path: prev.path,
+      readCount: Math.max(prev.readCount, f.readCount),
+      lastReadAt: Math.max(prev.lastReadAt, f.lastReadAt),
+      wasEdited: prev.wasEdited || f.wasEdited,
+      sizeBytes: f.lastReadAt >= prev.lastReadAt ? f.sizeBytes : prev.sizeBytes,
+      ...prev.wasTruncated || f.wasTruncated ? { wasTruncated: true } : {},
+      ...mergedSymbols.length > 0 ? { symbols_read: mergedSymbols } : {}
+    });
+  }
+  return Array.from(byPath.values());
+}
+function renderWebFetchRow(key, cacheId) {
+  const [url = key, prompt = ""] = key.split(WEB_FETCH_KEY_SEP);
+  const promptSuffix = prompt ? `, prompt: ${neutralizeSpokenMarkers(JSON.stringify(prompt))}` : "";
+  return `- ${displaySafeText(url)} (cacheId: ${cacheId}${promptSuffix})`;
+}
+function selectManifestFiles(siblings) {
+  const ownFiles = [...getSessionFiles().values()];
+  const siblingFiles = siblings.flatMap((s) => s.files);
+  const files = siblingFiles.length > 0 ? mergeManifestFiles(ownFiles, siblingFiles) : ownFiles;
+  const editedFiles = files.filter((f) => f.wasEdited);
+  const readFiles = files.filter((f) => f.readCount > 0 && !f.wasEdited && !isNoisePath(f.path));
+  const symbolOnlyFiles = files.filter((f) => f.readCount === 0 && !f.wasEdited && (f.symbols_read?.length ?? 0) > 0);
+  return { files, readFiles, editedFiles, symbolOnlyFiles };
+}
+function selectManifestWebFetches(siblings) {
+  const merged = /* @__PURE__ */ new Map();
+  for (const sibling of siblings) for (const [key, cacheId] of sibling.webFetches) merged.set(key, cacheId);
+  for (const [key, cacheId] of getSessionWebFetches()) merged.set(key, cacheId);
+  return [...merged.entries()];
+}
+function manifestPrintedPaths(sessionId, limit, cwd, transcriptPath) {
+  const { printed } = buildManifestParts(sessionId, cwd, transcriptPath);
+  return printed.slice(0, limit);
+}
+function buildManifestParts(sessionId, cwd, transcriptPath) {
+  const siblings = sessionId !== void 0 ? listSiblingSessionStates(sessionId) : [];
+  const { files, readFiles, editedFiles, symbolOnlyFiles } = selectManifestFiles(siblings);
+  const webFetches = selectManifestWebFetches(siblings);
+  const lines = [];
+  lines.push("## Session context");
+  lines.push(`Files read: ${readFiles.length}`);
+  lines.push(`Files edited: ${editedFiles.length}`);
+  const fileRows = [];
+  const appendFileSection = (header, entries, render) => {
+    const rows = entries.map(render);
+    appendCappedSection(lines, header, rows, MAX_ROWS);
+    for (let i = 0; i < Math.min(rows.length, MAX_ROWS); i++) fileRows.push({ path: displaySafePath(entries[i].path), row: rows[i] });
+  };
+  appendFileSection("### Edited files", editedFiles, (entry) => `- ${displaySafePath(entry.path)}`);
+  appendFileSection("### Surgically read files (symbol/section reads, never read whole)", symbolOnlyFiles, renderSymbolReadRow);
+  appendFileSection("### Read files", readFiles, renderReadRow);
+  appendCappedSection(lines, "### Web URLs fetched", webFetches.map(([key, cacheId]) => renderWebFetchRow(key, cacheId)), MAX_ROWS);
+  lines.push(...buildSafeToDiscardSection(files));
+  lines.push(...buildMemEpochSection());
+  const text = capManifestChars(lines.join("\n"), sessionId, cwd, transcriptPath);
+  const seen = /* @__PURE__ */ new Set();
+  const printed = [];
+  for (const { path, row } of fileRows) {
+    if (seen.has(path) || !(text.includes(row + "\n") || text.endsWith(row))) continue;
+    seen.add(path);
+    printed.push(path);
+  }
+  return { text, printed };
+}
+function buildManifest(sessionId, cwd, transcriptPath) {
+  return buildManifestParts(sessionId, cwd, transcriptPath).text;
+}
+function gitDirtySignals(cwd) {
+  const timeoutMs = loadConfig().hints.git_hint_max_ms;
+  let hasPendingDiff = false;
+  try {
+    const diffResult = runGit(["diff", "--no-color", "--stat", "HEAD"], { cwd, timeoutMs });
+    hasPendingDiff = diffResult.exitCode === 0 && diffResult.stdout.trim() !== "";
+  } catch {
+  }
+  let hasUncommittedChanges = false;
+  try {
+    const statusResult = runGit(["status", "--porcelain"], { cwd, timeoutMs });
+    hasUncommittedChanges = statusResult.exitCode === 0 && statusResult.stdout.trim() !== "";
+  } catch {
+  }
+  return { hasPendingDiff, hasUncommittedChanges };
+}
+function adaptiveCharBonus(sessionId, cwd, transcriptPath) {
+  if (!cwd) return 0;
+  const cache = loadSessionCache(sessionId ?? "") ?? {};
+  const ageSecs = cache.created_ts !== void 0 ? Math.max(0, Date.now() / 1e3 - cache.created_ts) : 0;
+  const contextPressure = getContextPressure(cache, transcriptPath);
+  const { hasPendingDiff, hasUncommittedChanges } = gitDirtySignals(cwd);
+  if (!hasPendingDiff && !hasUncommittedChanges) return 0;
+  const baseline = computeAdaptiveBudget(cache, ageSecs, { contextPressure });
+  const withGitSignal = computeAdaptiveBudget(cache, ageSecs, { hasPendingDiff, hasUncommittedChanges, contextPressure });
+  const deltaTokens = Math.max(0, withGitSignal - baseline);
+  return deltaTokens * 3;
+}
+function capManifestChars(manifest, sessionId, cwd, transcriptPath) {
+  const cap = loadConfig().compact_assist.max_manifest_chars;
+  if (cap <= 0) return manifest;
+  if (manifest.length <= cap) return manifest;
+  const effectiveCap = cap + adaptiveCharBonus(sessionId, cwd, transcriptPath);
+  if (manifest.length <= effectiveCap) return manifest;
+  const omitted = manifest.length - effectiveCap;
+  return manifest.slice(0, effectiveCap) + `
+...(manifest truncated at ${effectiveCap} chars; ${omitted} chars omitted)`;
+}
+function buildSafeToDiscardSection(files) {
+  const rerunHashes = getSessionBashReruns();
+  const bashOutputs = getSessionBashOutputs();
+  const rerunHashSet = new Set(rerunHashes);
+  const rerunRows = [];
+  for (const hash of rerunHashes) {
+    const id = bashOutputs.find(([h]) => h === hash)?.[1];
+    if (id === void 0) continue;
+    const entry = getBashOutput(id);
+    if (entry === null) continue;
+    const flatCommand = neutralizeSpokenMarkers(entry.command.replace(/[\t\r\n]+/g, " "));
+    rerunRows.push("- `" + flatCommand + "` \u2014 an older run of this exact command was superseded; recall the surviving copy with `bash-output " + id + " --full`");
+  }
+  const supersededReadRows = [];
+  for (const f of files) {
+    if (f.readCount > 1 || f.wasEdited) {
+      const reason = f.wasEdited ? "edited after being read" : "re-read " + f.readCount + "x";
+      supersededReadRows.push("- " + displaySafePath(f.path) + " (" + reason + " \u2014 only the latest content already in context is current)");
+    }
+  }
+  const cachedOutputRows = [];
+  for (const [hash, id] of bashOutputs) {
+    if (rerunHashSet.has(hash)) continue;
+    const entry = getBashOutput(id);
+    if (entry === null) continue;
+    const flatCommand = neutralizeSpokenMarkers(entry.command.replace(/[\t\r\n]+/g, " "));
+    cachedOutputRows.push("- `" + flatCommand + "` \u2014 recallable via `bash-output " + id + " --full`");
+  }
+  const total = rerunRows.length + supersededReadRows.length + cachedOutputRows.length;
+  if (total === 0) return [];
+  const lines = [];
+  lines.push("");
+  lines.push("### SAFE_TO_DISCARD (" + total + " items \u2014 provably inert; each is recallable, not gone)");
+  appendCappedSection(lines, "Superseded reruns (" + rerunRows.length + "):", rerunRows, MAX_ROWS);
+  appendCappedSection(lines, "Superseded file reads (" + supersededReadRows.length + "):", supersededReadRows, MAX_ROWS);
+  appendCappedSection(lines, "Other cached bash outputs (" + cachedOutputRows.length + "):", cachedOutputRows, MAX_ROWS);
+  return lines;
+}
+function buildMemEpochSection() {
+  let result;
+  try {
+    result = spawnSync("mem", ["epoch"], {
+      encoding: "utf-8",
+      timeout: MEM_EPOCH_TIMEOUT_MS,
+      windowsHide: true
+    });
+  } catch {
+    return [];
+  }
+  if (result.error !== void 0 || result.status !== 0) return [];
+  const stdout = typeof result.stdout === "string" ? result.stdout.trim() : "";
+  if (!/^\d+$/.test(stdout)) return [];
+  return [
+    "",
+    "### mem epoch",
+    `mem epoch: ${stdout} (no live TGMEM block is tracked in this session -- only the epoch counter is folded in; see buildMemEpochSection's doc comment)`
+  ];
+}
+var MANIFEST_PREAMBLE = "When summarizing this session, keep the file paths and symbol names below exactly as written -- they are the handles the next turn needs to resume work. Do not paraphrase them into prose.";
+var MANIFEST_RECOVERY_PREAMBLE = "The conversation was just compacted. Below is what this session touched, recovered from token-goat's own ledger rather than from the summary -- use it to re-establish paths and symbol names the summary may have dropped.";
+var BUDGET_ESCALATION_MARKER = "TG-BUDGET-ESCALATION:";
+function summaryBudgetDirective(budgetChars) {
+  if (budgetChars <= 0) return "";
+  return ` Aim for at most ${budgetChars} characters. Prefer a shorter summary that keeps every path, symbol, command and decision over a longer one that reproduces tool output verbatim: cite the recall id (\`token-goat bash-output <id> --full\`) rather than quoting the output again. If this session genuinely cannot be summarized within that target without losing state the next turn needs, exceed it and open the summary with a single line reading \`${BUDGET_ESCALATION_MARKER} <one-sentence reason>\`.`;
+}
+
+export {
+  renderWebFetchRow,
+  manifestPrintedPaths,
+  buildManifest,
+  MANIFEST_PREAMBLE,
+  MANIFEST_RECOVERY_PREAMBLE,
+  BUDGET_ESCALATION_MARKER,
+  summaryBudgetDirective
+};

package/dist/{token-goat-chunk-S7RFIX4Y.mjs → token-goat-chunk-PHDNWISV.mjs}

@@ -2,11 +2,11 @@ import { createRequire as __cjsRequire } from 'node:module';
 const require = __cjsRequire(import.meta.url);
 import {
   getDb
-} from "./token-goat-chunk-JNIFSVWM.mjs";
+} from "./token-goat-chunk-2ZHD3XCD.mjs";
 import {
   getHarnessName,
   loadConfig
-} from "./token-goat-chunk-SOKAPOOC.mjs";
+} from "./token-goat-chunk-M6CDVASA.mjs";
 import {
   registerReset
 } from "./token-goat-chunk-EEIDFMEM.mjs";
@@ -23,7 +23,7 @@ import {
   extractErrorMessage,
   sanitizeIdForFilename,
   tokenGoatHome
-} from "./token-goat-chunk-ZGQXRYKE.mjs";
+} from "./token-goat-chunk-TGBSMHPM.mjs";
 import {
   displaySafeText
 } from "./token-goat-chunk-NMTKNYGF.mjs";
@@ -54,12 +54,6 @@ function stripAnsiEscapes(s) {
   }
   return s.replace(_ANSI_ESCAPE_RE, "");
 }
-function stripAnsi(s) {
-  if (!s.includes("\x1B")) {
-    return s;
-  }
-  return stripAnsiEscapes(s).replace(_PUA_RE, "");
-}
 function fmtBytes(n) {
   let value = n;
   const units = ["B", "KB", "MB", "GB", "TB"];
@@ -75,7 +69,7 @@ function fg(r, g, b) {
   return `${_E}[38;2;${r};${g};${b}m`;
 }
 function vlen(s) {
-  return stripAnsi(s).length;
+  return stripAnsiEscapes(s).replace(_PUA_RE, "").length;
 }
 function padR(s, w) {
   return s + " ".repeat(Math.max(0, w - vlen(s)));
@@ -115,6 +109,41 @@ init_define_import_meta_env();
 import * as fs from "node:fs";
 import * as path from "node:path";
 
+// src/token_estimate.ts
+init_define_import_meta_env();
+var BYTES_PER_TOKEN = {
+  text: 4,
+  dense: 1.45
+};
+var GUARD_MARGIN = 0.75;
+function creditDivisor(cls = "text") {
+  return BYTES_PER_TOKEN[cls];
+}
+function guardDivisor(cls = "text") {
+  return BYTES_PER_TOKEN[cls] * GUARD_MARGIN;
+}
+var DENSE_ALPHABET = /^[A-Za-z0-9+/=_-]$/;
+var MIN_DENSE_SAMPLE = 200;
+var MIN_DENSE_DISTINCT = 16;
+var CLASSIFY_SAMPLE_CHARS = 4096;
+function classifyContent(text) {
+  const sample = text.slice(0, CLASSIFY_SAMPLE_CHARS);
+  if (sample.length < MIN_DENSE_SAMPLE) return "text";
+  let whitespace = 0;
+  let dense = 0;
+  const distinct = /* @__PURE__ */ new Set();
+  for (const ch of sample) {
+    if (ch === " " || ch === "\n" || ch === "	" || ch === "\r") whitespace++;
+    else if (DENSE_ALPHABET.test(ch)) {
+      dense++;
+      distinct.add(ch);
+    }
+  }
+  if (whitespace / sample.length >= 0.05) return "text";
+  if (distinct.size < MIN_DENSE_DISTINCT) return "text";
+  return dense / sample.length >= 0.95 ? "dense" : "text";
+}
+
 // src/render/stats_renderer.ts
 init_define_import_meta_env();
 var _STATS_MESSAGES_FALLBACK = {
@@ -777,7 +806,7 @@ function _renderByProjectSection(stats) {
         nameColor: C.TEXT_PRIMARY
       })
     );
-    lines.push(`${_M}  ${fg(...C.TEXT_DIM)}\u2514\u2500 ${p.hash}  ${displaySafeText(stripAnsi(p.path))}${RESET}`);
+    lines.push(`${_M}  ${fg(...C.TEXT_DIM)}\u2514\u2500 ${p.hash}  ${displaySafeText(stripAnsiEscapes(p.path))}${RESET}`);
   }
   return lines;
 }
@@ -871,8 +900,8 @@ var SOURCE_CONTENT = "content";
 var SOURCE_OTHER = "other";
 var _BYTES_MODE_ONLY_KINDS = /* @__PURE__ */ new Set(["webfetch_image", "gdrive_image"]);
 var COUNT_ONLY_KINDS = /* @__PURE__ */ new Set(["secret_redacted"]);
-function savedTokensFromBytes(bytes) {
-  return Math.round(Math.max(0, bytes) / 4);
+function savedTokensFromBytes(bytes, cls = "text") {
+  return Math.round(Math.max(0, bytes) / creditDivisor(cls));
 }
 var KIND_TO_SOURCE = {
   image_shrink: SOURCE_IMAGE,
@@ -2141,10 +2170,12 @@ function sweepCacheRoots(extraRoots = []) {
 }
 
 export {
+  BYTES_PER_TOKEN,
+  guardDivisor,
+  classifyContent,
   colorStdout,
   RESET,
   stripAnsiEscapes,
-  stripAnsi,
   fg,
   C,
   SOURCE_HINT,