token-goat 2.9.18 → 2.9.19

package/dist/{token-goat-chunk-HPOFR6QX.mjs → token-goat-chunk-S7RFIX4Y.mjs}

@@ -2,14 +2,17 @@ import { createRequire as __cjsRequire } from 'node:module';
 const require = __cjsRequire(import.meta.url);
 import {
   getDb
-} from "./token-goat-chunk-L7VCX33I.mjs";
+} from "./token-goat-chunk-JNIFSVWM.mjs";
 import {
   getHarnessName,
   loadConfig
-} from "./token-goat-chunk-CA6V36KW.mjs";
+} from "./token-goat-chunk-SOKAPOOC.mjs";
 import {
   registerReset
 } from "./token-goat-chunk-EEIDFMEM.mjs";
+import {
+  Database
+} from "./token-goat-chunk-OSUFN2FV.mjs";
 import {
   VERSION,
   atomicWriteText,
@@ -17,12 +20,13 @@ import {
   dataDir,
   dataDirForHome,
   ensureDirSync,
+  extractErrorMessage,
   sanitizeIdForFilename,
   tokenGoatHome
-} from "./token-goat-chunk-QIDLJGXZ.mjs";
+} from "./token-goat-chunk-ZGQXRYKE.mjs";
 import {
   displaySafeText
-} from "./token-goat-chunk-ERTXEKB6.mjs";
+} from "./token-goat-chunk-NMTKNYGF.mjs";
 import {
   growsExponentially,
   hasNestedQuantifier
@@ -108,6 +112,7 @@ var C = {
 
 // src/stats.ts
 init_define_import_meta_env();
+import * as fs from "node:fs";
 import * as path from "node:path";
 
 // src/render/stats_renderer.ts
@@ -954,11 +959,7 @@ var KIND_TO_SOURCE = {
   compact_doc: SOURCE_READ,
   note_read: SOURCE_READ,
   note_list: SOURCE_READ,
-  // note-add is a write (like insert-section/replace, which record no stat at all -- neither
-  // has a "full source it replaces" savings concept). It still gets an event-only entry here
-  // (no bytesSaved/tokensSaved argument, same as skill_load) purely so `token-goat note-add`
-  // usage is visible in `token-goat stats --full` at all -- SOURCE_OTHER, not SOURCE_READ,
-  // since it is not a token-savings substitute for a read.
+  // note-add is a write (like insert-section/replace, which record no stat at all -- neither has a "full source it replaces" savings concept). It still gets an event-only entry here (no bytesSaved/tokensSaved argument, same as skill_load) purely so `token-goat note-add` usage is visible in `token-goat stats --full` at all -- SOURCE_OTHER, not SOURCE_READ, since it is not a token-savings substitute for a read.
   note_write: SOURCE_OTHER,
   web_fetch: SOURCE_WEB,
   injection_detected: SOURCE_WEB,
@@ -973,6 +974,8 @@ var KIND_TO_SOURCE = {
   dirty_queue_append_failed: SOURCE_OTHER,
   worker_healthcheck_failed: SOURCE_OTHER,
   known_root_record_failed: SOURCE_OTHER,
+  // Same fail-soft shape, from hooks_session_start.ts's reconcileNote: a thrown reconcile sweep must not block session start, but it also must not vanish silently, so the catch that swallows it records why instead of returning null with nothing recorded.
+  reconcile_note_failed: SOURCE_OTHER,
   // Measurement of what a compaction produced (hooks_compact.ts postCompactHandler): summary size and how many manifest paths survived into it. SOURCE_OTHER and always recorded at (0, 0) -- the summary was written whether or not token-goat was watching, so there is no counterfactual in which those bytes were saved. Filing it anywhere with a savings total would credit token-goat for the whole summary, which is the accounting mistake this registry exists to prevent.
   compact_summary: SOURCE_OTHER,
   // Envelope compaction of an oversized subagent report (hooks_agent_spawn.ts). SOURCE_CONTENT, not SOURCE_HINT: the handler's sibling session_hint entry is advisory (it only appends a recall pointer and genuinely saves nothing), whereas this kind records a real rewrite with real bytes removed, so filing it under the advisory bucket would understate the compaction and repeat the zero-savings desync this registry keeps getting bitten by.
@@ -986,12 +989,7 @@ var KIND_TO_SOURCE = {
   plan_echo_collapse: SOURCE_CONTENT,
   // Lossless re-layout of Grep content-mode output (hooks_grep.ts foldGrepContentHandler). SOURCE_CONTENT, not SOURCE_HINT, for the same reason as agent_report_compact above: its sibling grep_dedup_hint is advisory and saves nothing directly, whereas this is a real rewrite with real bytes removed. Filing it under the advisory bucket would silently add non-hint savings to hint_stats.ts's savedBytes (which reads by_source[SOURCE_HINT] wholesale) and overstate the hint ledger's net benefit.
   "grep:fold": SOURCE_CONTENT,
-  // Withholding of already-served stretches from a completed Read (hooks_read.ts
-  // elideAlreadyServedLines). SOURCE_CONTENT, not SOURCE_HINT, for the same reason as
-  // grep:fold above: its siblings read_count_deny and read_served_deny are decisions about
-  // whether a read happens at all, whereas this is a rewrite of a result that did happen,
-  // with real bytes removed from it. Filing it under the advisory bucket would add non-hint
-  // savings to hint_stats.ts's savedBytes, which reads by_source[SOURCE_HINT] wholesale.
+  // Withholding of already-served stretches from a completed Read (hooks_read.ts elideAlreadyServedLines). SOURCE_CONTENT, not SOURCE_HINT, for the same reason as grep:fold above: its siblings read_count_deny and read_served_deny are decisions about whether a read happens at all, whereas this is a rewrite of a result that did happen, with real bytes removed from it. Filing it under the advisory bucket would add non-hint savings to hint_stats.ts's savedBytes, which reads by_source[SOURCE_HINT] wholesale.
   "read:served_elide": SOURCE_CONTENT,
   // Same bucket and same reasoning as read:served_elide directly above: a rewrite of a Read that did happen, with real bytes removed, not an advisory about whether to read at all.
   "read:body_fold": SOURCE_CONTENT,
@@ -1011,7 +1009,9 @@ var KIND_PREFIX_TO_SOURCE = [
   ["skill_body:", SOURCE_SKILL],
   ["skill_compact:", SOURCE_SKILL],
   ["bashoutput:", SOURCE_BASH],
-  ["taskoutput:", SOURCE_CONTENT]
+  ["taskoutput:", SOURCE_CONTENT],
+  // Hook wall-clock timing (relay.ts's relayInProcess), one row per invocation, always 0 bytes/0 tokens: it measures token-goat's own overhead, not a saving. See hook_latency.ts's hookLatencyBreakdown() for the dedicated read path and pruneHookStats() below for its own (shorter) retention.
+  ["hook:", SOURCE_OTHER]
 ];
 var COMMAND_KINDS = {
   symbol: /* @__PURE__ */ new Set(["symbol_lookup"]),
@@ -1132,7 +1132,8 @@ CREATE TABLE IF NOT EXISTS stats (
   detail TEXT,
   harness TEXT,
   traceparent TEXT,
-  tg_version TEXT
+  tg_version TEXT,
+  duration_ms INTEGER
 );
 CREATE INDEX IF NOT EXISTS idx_stats_ts ON stats(ts);
 CREATE INDEX IF NOT EXISTS idx_stats_kind ON stats(kind);
@@ -1190,6 +1191,29 @@ function migrateGlobalSchema(db) {
   } catch (err) {
     if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
   }
+  try {
+    db.exec("ALTER TABLE stats ADD COLUMN duration_ms INTEGER");
+  } catch (err) {
+    if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
+  }
+}
+var TEST_ISOLATION_LEAK_WINDOW_START_TS = 1780406895;
+var TEST_ISOLATION_LEAK_WINDOW_END_TS = 1782119331;
+function pruneTestIsolationLeakRows(db) {
+  try {
+    db.prepare(
+      `DELETE FROM stats WHERE tg_version IS NULL AND ts BETWEEN ? AND ? AND (detail LIKE '%pytest-of-%' OR detail LIKE '%/fake/%')`
+    ).run(TEST_ISOLATION_LEAK_WINDOW_START_TS, TEST_ISOLATION_LEAK_WINDOW_END_TS);
+  } catch {
+  }
+}
+function dropRetiredPythonTables(db) {
+  try {
+    db.prepare("DROP TABLE IF EXISTS grep_patterns").run();
+    db.prepare("DROP TABLE IF EXISTS miss_patterns").run();
+    db.prepare("DROP TABLE IF EXISTS wal_bloat").run();
+  } catch {
+  }
 }
 var _harnessColumnByDb = /* @__PURE__ */ new WeakMap();
 function statsHasHarnessColumn(db) {
@@ -1236,6 +1260,21 @@ function statsHasVersionColumn(db) {
   _versionColumnByDb.set(db, present);
   return present;
 }
+var _durationColumnByDb = /* @__PURE__ */ new WeakMap();
+function statsHasDurationColumn(db) {
+  const cached = _durationColumnByDb.get(db);
+  if (cached !== void 0) return cached;
+  let present;
+  try {
+    present = db.prepare("PRAGMA table_info(stats)").all().some(
+      (c) => c.name === "duration_ms"
+    );
+  } catch {
+    present = false;
+  }
+  _durationColumnByDb.set(db, present);
+  return present;
+}
 function getGlobalDb(homeDir) {
   const basePath = homeDir ? dataDirForHome(homeDir) : dataDir();
   const dbPath = path.join(basePath, "global.db");
@@ -1248,6 +1287,7 @@ function getGlobalDb(homeDir) {
   return db;
 }
 var STATS_RETENTION_DAYS = 180;
+var UNMAPPED_TOOL_RETENTION_DAYS = 30;
 var STATS_ROLLUP_INTERVAL_MS = 6 * 60 * 60 * 1e3;
 function rollupAndPruneStats(db, retentionDays = STATS_RETENTION_DAYS) {
   try {
@@ -1269,11 +1309,31 @@ function rollupAndPruneStats(db, retentionDays = STATS_RETENTION_DAYS) {
            tokens_saved = tokens_saved + excluded.tokens_saved`
       ).run(cutoff);
       db.prepare(`DELETE FROM stats WHERE ts < ?`).run(cutoff);
+      try {
+        const unmappedCutoff = Math.floor(Date.now() / 1e3) - UNMAPPED_TOOL_RETENTION_DAYS * 86400;
+        db.prepare(`DELETE FROM unmapped_tools WHERE last_seen < ?`).run(unmappedCutoff);
+      } catch {
+      }
     });
     run(cutoffTs);
   } catch {
   }
 }
+function pruneHintEmissions(db, retentionDays = STATS_RETENTION_DAYS) {
+  try {
+    const cutoffMs = Date.now() - retentionDays * 86400 * 1e3;
+    db.prepare(`DELETE FROM hint_emissions WHERE emitted_at < ?`).run(cutoffMs);
+  } catch {
+  }
+}
+var HOOK_STATS_RETENTION_DAYS = 7;
+function pruneHookStats(db, retentionDays = HOOK_STATS_RETENTION_DAYS) {
+  try {
+    const cutoffTs = Math.floor(Date.now() / 1e3) - retentionDays * 86400;
+    db.prepare(`DELETE FROM stats WHERE kind LIKE 'hook:%' AND ts < ?`).run(cutoffTs);
+  } catch {
+  }
+}
 function maybeRunStatsMaintenance(db) {
   try {
     const now = Date.now();
@@ -1284,7 +1344,11 @@ function maybeRunStatsMaintenance(db) {
       `INSERT INTO stats_maintenance (id, last_rollup_ts) VALUES (1, ?)
        ON CONFLICT(id) DO UPDATE SET last_rollup_ts = excluded.last_rollup_ts`
     ).run(now);
+    pruneHookStats(db);
     rollupAndPruneStats(db);
+    pruneHintEmissions(db);
+    pruneTestIsolationLeakRows(db);
+    dropRetiredPythonTables(db);
   } catch {
   }
 }
@@ -1303,9 +1367,33 @@ function noStatsMessage(windowDays, homeDir) {
   if (total === 0) return "No stats recorded yet.";
   return `No stats in the last ${countNoun(windowDays, "day")} (${total} recorded outside this window; use --window-days 0 for all time).`;
 }
-function recordStat(kind, bytesSaved = 0, tokensSaved = 0, _testDb, detail, traceparent) {
+var STATS_WRITE_FAILURE_LOG_MIN_INTERVAL_MS = 60 * 1e3;
+function statsWriteFailureMarkerPath(dir) {
+  return path.join(dir, "stats-write-failed.marker");
+}
+function statsWriteFailureLogPath(dir) {
+  return path.join(dir, "stats-write-failed.log");
+}
+function recordStatWriteFailure(kind, err, dir = dataDir()) {
   try {
-    const db = _testDb ?? getGlobalDb();
+    const markerPath = statsWriteFailureMarkerPath(dir);
+    try {
+      if (Date.now() - fs.statSync(markerPath).mtimeMs < STATS_WRITE_FAILURE_LOG_MIN_INTERVAL_MS) return;
+    } catch {
+    }
+    ensureDirSync(dir);
+    fs.writeFileSync(markerPath, "");
+    const line = `${(/* @__PURE__ */ new Date()).toISOString()} recordStat write failed for kind=${kind}: ${extractErrorMessage(err)}`;
+    fs.appendFileSync(statsWriteFailureLogPath(dir), displaySafeText(line.replace(/[\n\r]+$/, "")) + "\n");
+  } catch {
+  }
+}
+var STATS_WRITE_BUSY_TIMEOUT_MS = 200;
+function recordStat(kind, bytesSaved = 0, tokensSaved = 0, _testDb, detail, traceparent, durationMs) {
+  let db;
+  try {
+    if (!_testDb) getGlobalDb();
+    db = _testDb ?? new Database(path.join(dataDir(), "global.db"), { timeout: STATS_WRITE_BUSY_TIMEOUT_MS });
     const ts = Math.floor(Date.now() / 1e3);
     const tp = traceparent ?? process.env["TRACEPARENT"] ?? process.env["traceparent"] ?? null;
     const cols = ["ts", "kind", "bytes_saved", "tokens_saved", "detail"];
@@ -1322,11 +1410,23 @@ function recordStat(kind, bytesSaved = 0, tokensSaved = 0, _testDb, detail, trac
       cols.push("tg_version");
       vals.push(VERSION);
     }
+    if (durationMs !== void 0 && statsHasDurationColumn(db)) {
+      cols.push("duration_ms");
+      vals.push(Math.round(durationMs));
+    }
     db.prepare(
       `INSERT INTO stats (${cols.join(", ")}) VALUES (${cols.map(() => "?").join(", ")})`
     ).run(...vals);
     maybeRunStatsMaintenance(db);
-  } catch {
+  } catch (e) {
+    recordStatWriteFailure(kind, e);
+  } finally {
+    if (db && !_testDb) {
+      try {
+        db.close();
+      } catch {
+      }
+    }
   }
 }
 var MAX_TOOL_NAME_CHARS = 200;
@@ -1356,6 +1456,18 @@ function readUnmappedTools(dbPath, homeDir) {
     return [];
   }
 }
+function pruneStalePatternCoveredUnmappedTools(db, patterns) {
+  if (patterns.length === 0) return;
+  const regexes = patterns.map((p) => new RegExp(p));
+  try {
+    const rows = db.prepare("SELECT DISTINCT tool_name FROM unmapped_tools").all();
+    const stale = rows.filter((r) => regexes.some((re) => re.test(r.tool_name)));
+    if (stale.length === 0) return;
+    const del = db.prepare("DELETE FROM unmapped_tools WHERE tool_name = ?");
+    for (const r of stale) del.run(r.tool_name);
+  } catch {
+  }
+}
 function summarize(windowDays = 30, testDb, homeDir) {
   const t0 = Date.now();
   const sinceTs = windowDays > 0 ? Math.floor((Date.now() - windowDays * 24 * 60 * 60 * 1e3) / 1e3) : null;
@@ -1496,18 +1608,11 @@ function _totalsLines(summary) {
     `Total events:   ${summary.total_events}`,
     `Bytes saved:    ${fmtBytes(summary.total_bytes_saved)}`,
     `Tokens saved:   ${summary.total_tokens_saved}`,
-    // Disclosure, not a correction: `total_tokens_saved` above sums rows written under whichever
-    // pricing formula was live when each was recorded, and `tg_version` cannot be read back into
-    // "which formula" for a row from before this disclosure existed (PRICING_VERSION_UNRECORDED is
-    // the overwhelming majority of all-time rows). Excluding those rows from the headline would
-    // discard nearly the whole figure rather than fix it, so the honest move is to keep the sum and
-    // say plainly that it spans more than one era, not to quietly present a mixed total as single-formula.
+    // Disclosure, not a correction: `total_tokens_saved` above sums rows written under whichever pricing formula was live when each was recorded, and `tg_version` cannot be read back into "which formula" for a row from before this disclosure existed (PRICING_VERSION_UNRECORDED is the overwhelming majority of all-time rows). Excluding those rows from the headline would discard nearly the whole figure rather than fix it, so the honest move is to keep the sum and say plainly that it spans more than one era, not to quietly present a mixed total as single-formula.
     ...hasMixedPricingEras(summary) ? [
       `Pricing note:   totals mix ${countNoun(Object.keys(summary.by_pricing_version).length, "tg_version era")} (${countNoun(summary.by_pricing_version[PRICING_VERSION_UNRECORDED]?.events ?? 0, "row")} unrecorded); see 'token-goat stats --json' -> by_pricing_version for the breakdown`
     ] : [],
-    // Printed on its own line, below the token total and never inside it, because it counts
-    // placeholders rather than tokens. Omitted entirely when nothing was redacted, so the line is
-    // information rather than a permanent zero. See COUNT_ONLY_KINDS.
+    // Printed on its own line, below the token total and never inside it, because it counts placeholders rather than tokens. Omitted entirely when nothing was redacted, so the line is information rather than a permanent zero. See COUNT_ONLY_KINDS.
     ...summary.counts["secret_redacted"] ? [`Secrets hidden: ${summary.counts["secret_redacted"]} (a count, not tokens)`] : [],
     `Window:         ${summary.window_days} days`
   ];
@@ -1515,7 +1620,8 @@ function _totalsLines(summary) {
 function _useRichStats() {
   if (process.env["NO_COLOR"]) return false;
   if (process.stdout.isTTY === true) return true;
-  return process.stdout.isTTY === void 0 && !process.env["CI"];
+  const forceColor = process.env["FORCE_COLOR"];
+  return forceColor !== void 0 && forceColor !== "0";
 }
 function _renderShortTotals(summary) {
   const lines = [
@@ -1678,9 +1784,7 @@ var SECRET_PATTERNS = [
   // Redacts only the token itself, not the "Authorization: Bearer " prefix -- the lookbehind anchors on the header name and scheme so the surrounding request-log line stays readable, matching how AWS_ACCESS_KEY_ID=... above keeps its own prefix intact. The optional quotes on either side of the colon are what let this see a header carried in JSON rather than in raw wire format. Without them the lookbehind demanded the colon sit directly against the header name and the scheme directly against the space, so a body like {"Authorization": "Bearer <token>"} -- the shape any logged fetch or MCP result arrives in -- matched nothing and the token was cached verbatim. `token` is the second scheme spelling in wide use -- it is what curl and gh examples pass for GitHub and many other APIs -- and an opaque value behind it carries exactly the same authority as one behind `Bearer`. The trailing gap is `{1,8}` rather than a single space for the same reason every other gap in this lookbehind already is: a hand-aligned or reformatted header ("Authorization: Bearer <token>") is ordinary, and demanding exactly one space there made this the one position in the pattern that a second space defeated.
   ["auth_bearer_token", /(?<=Authorization["']?[ \t]{0,8}:[ \t]{0,8}["']?[ \t]{0,8}(?:Bearer|token)[ \t]{1,8})[A-Za-z0-9\-._~+/]{10,}=*/gi],
   ["auth_basic_token", /(?<=Authorization["']?[ \t]{0,8}:[ \t]{0,8}["']?[ \t]{0,8}Basic[ \t])[A-Za-z0-9+/]{6,}=*/gi],
-  // JWTs have no distinctive prefix of their own, but the base64url encoding of the smallest realistic header ('{"alg":' or similar) always starts with "eyJ", so that's the practical anchor here -- each of the three dot-separated segments requires a minimum length to avoid matching a short, coincidentally dotted token.
-  //
-  // The trailing `={0,2}` on each segment is what makes a padded token match. Base64url as the JWT spec defines it drops the `=` padding, but producers that reach for a plain base64 encoder emit it anyway, and a `=` in the header or payload segment used to defeat the match outright: not a partial redaction, but none at all, so the entire token was printed. `=` cannot appear anywhere except the end of a segment, since it is not one of the characters the segment body allows, so accepting it here cannot widen the match onto anything else.
+  // JWTs have no distinctive prefix of their own, but the base64url encoding of the smallest realistic header ('{"alg":' or similar) always starts with "eyJ", so that's the practical anchor here -- each of the three dot-separated segments requires a minimum length to avoid matching a short, coincidentally dotted token. The trailing `={0,2}` on each segment is what makes a padded token match. Base64url as the JWT spec defines it drops the `=` padding, but producers that reach for a plain base64 encoder emit it anyway, and a `=` in the header or payload segment used to defeat the match outright: not a partial redaction, but none at all, so the entire token was printed. `=` cannot appear anywhere except the end of a segment, since it is not one of the characters the segment body allows, so accepting it here cannot widen the match onto anything else.
   ["jwt", /eyJ[A-Za-z0-9_-]{10,}={0,2}\.[A-Za-z0-9_-]{10,}={0,2}\.[A-Za-z0-9_-]{10,}={0,2}/g],
   ["npm_token", /npm_[A-Za-z0-9]{36}/g],
   // rk_live_ (restricted keys) share the sk_live_/sk_test_ secret-key shape and risk level, so one pattern covers all three rather than adding a near-duplicate entry.
@@ -1692,9 +1796,7 @@ var SECRET_PATTERNS = [
   ["url_credentials", /(?<=:\/\/[^\s:@/]{1,64}:)[^\s:@/]{1,256}(?=@)/g],
   // Azure storage account connection strings (`DefaultEndpointsProtocol=https;AccountName=...; AccountKey=<base64>==;EndpointSuffix=core.windows.net`) carry a full read/write key to the account in the AccountKey field, and none of the generic patterns above catch it: generic_secret_assignment's keyword list (password|passwd|secret|api[_-]?key| access[_-]?token|refresh[_-]?token|id[_-]?token) has nothing that matches "AccountKey", and an unanchored base64-shape pattern was deliberately rejected -- this module's header already warns that a bare high-entropy-blob heuristic false-fires on ordinary code, JSON and log output, and an 88-char base64 value is exactly the shape a hash, a compiled asset digest or a generated id can also take. Anchoring on the literal `AccountKey=` field name instead keeps the match specific to this one connection-string field. The value class is base64 proper (letters, digits, `+`, `/`, trailing `=` padding) and none of those characters include `;`, so the match terminates on its own at the `;` that starts the next `Name=` field -- unlike generic_secret_assignment's separator characters (`& ; # , :`), which double as ordinary credential characters and need a lookahead to tell the two roles apart, `;` is never valid base64 and needs no such lookahead here. The lookbehind keeps `AccountKey=` itself in the output, matching auth_bearer_token and presigned_signature above, so `;EndpointSuffix=core.windows.net` after it stays fully readable too. `SharedAccessKey` is the same credential one Azure service over: Service Bus, Event Hubs and Relay spell it that way (`Endpoint=sb://ns.servicebus.windows.net/;SharedAccessKeyName=Root; SharedAccessKey=<base64>`) and it carries the same authority over that namespace that AccountKey does over a storage account. `SharedAccessKeyName` is a plain identifier rather than a secret, and never matches: the separator in the lookbehind sits directly against the key name, so the `Name` in between stops it dead. The separator is spelled the way auth_bearer_token above spells its own, for the same reasons that comment records having learned the hard way. Optional quotes on either side of it, so the JSON and YAML forms a logged MCP result or api response actually arrives in are matched rather than stopped dead at the opening quote. A bounded gap rather than exactly one space, so a hand-aligned or reformatted `AccountKey = ...` in an appsettings file is not the single variant that defeats the whole pattern. Case-insensitive for the same reason presigned_signature is. The leading word boundary is what keeps the widened name from reaching into the middle of a longer identifier such as `myaccountkey=`.
   ["azure_storage_key", /(?<=\b(?:AccountKey|SharedAccessKey)["']?[ \t]{0,8}[:=][ \t]{0,8}["']?)[A-Za-z0-9+/]{40,}=*/gi],
-  // Generic key=value assignments in .env-file and connection-string/query-string shape. The lookbehind again redacts only the value, and the value's character class deliberately excludes whitespace, '&', ';', '#', quote characters, and '[' ']' ':' -- that exclusion is what stops this from swallowing the rest of the line (a trailing comment or the next key=value pair) or the remainder of a query string past the matched parameter, which is exactly the kind of over-eager match this module's own design note above warns broad heuristics produce. The '[' ']' ':' exclusion also matters because this pattern runs last: an earlier pattern's own "OPENAI_API_KEY=[REDACTED:openai_project_key]" replacement text contains "API_KEY=" too, and without excluding those characters this pattern would re-match and double-redact its own placeholder. The length has a lower bound only (no upper bound): capping it at 64 used to leave the tail of any longer secret unredacted in plain text, which is worse than no redaction because it looks handled. A single negated-class quantifier like this cannot backtrack catastrophically -- there is no nested or overlapping quantifier for the engine to explore multiple ways of matching, so removing the upper bound does not introduce a ReDoS risk. Quotes are permitted around the separator, but never inside the value class. A quoted value is the ordinary way secrets are written -- .env files, JSON, YAML, TOML all quote by default -- and the lookbehind used to stop dead at the opening quote, so `API_KEY="..."` passed through in full while the bare `API_KEY=...` was caught. The closing quote of the key name blocked it from the other side too, which is what kept every JSON body unredacted. Keeping quotes out of the value class is still what stops the match running past the closing quote. The keyword may be a prefix of a longer key name rather than the whole of it, so the trailing identifier class below is load-bearing: without it the lookbehind required the keyword to sit immediately before the separator, and AWS_SECRET_ACCESS_KEY=, SECRET_KEY=, and DB_PASSWORD_HASH= all passed through in full. That class matches identifier characters only, so prose that merely mentions a keyword still never reaches a separator and stays unredacted. api[_-]?key covers the apikey and api-key spellings too. `& ; # , :` play two incompatible roles. They separate one field from the next (a query string, a cookie header, an inline env list), and they are also perfectly ordinary credential characters. Rejecting them outright got the first role right and the second badly wrong: the match stopped at the first one and left everything after it in plain text, so `password=corr&horse&battery` redacted four characters and printed the rest, and `DB_PASSWORD=Aa1:xyz` matched nothing at all because the run before the `:` was under the four-character floor. A tail left sitting in the open is the outcome this module's header calls worse than no redaction, because it reads as handled.
-  //
-  // So the separator role is decided by what follows rather than assumed: one of these characters ends the value only when the next thing along is another `name=` / `name:` pair, which is what an actual field separator is always followed by. `,OTHER=public` and `; other=1` still end it; the `&` in the middle of a passphrase does not. Whitespace, quotes and brackets are unchanged -- they end a value unconditionally, which is also what keeps this pattern from re-matching the `[REDACTED:...]` placeholder it just wrote.
+  // Generic key=value assignments in .env-file and connection-string/query-string shape. The lookbehind again redacts only the value, and the value's character class deliberately excludes whitespace, '&', ';', '#', quote characters, and '[' ']' ':' -- that exclusion is what stops this from swallowing the rest of the line (a trailing comment or the next key=value pair) or the remainder of a query string past the matched parameter, which is exactly the kind of over-eager match this module's own design note above warns broad heuristics produce. The '[' ']' ':' exclusion also matters because this pattern runs last: an earlier pattern's own "OPENAI_API_KEY=[REDACTED:openai_project_key]" replacement text contains "API_KEY=" too, and without excluding those characters this pattern would re-match and double-redact its own placeholder. The length has a lower bound only (no upper bound): capping it at 64 used to leave the tail of any longer secret unredacted in plain text, which is worse than no redaction because it looks handled. A single negated-class quantifier like this cannot backtrack catastrophically -- there is no nested or overlapping quantifier for the engine to explore multiple ways of matching, so removing the upper bound does not introduce a ReDoS risk. Quotes are permitted around the separator, but never inside the value class. A quoted value is the ordinary way secrets are written -- .env files, JSON, YAML, TOML all quote by default -- and the lookbehind used to stop dead at the opening quote, so `API_KEY="..."` passed through in full while the bare `API_KEY=...` was caught. The closing quote of the key name blocked it from the other side too, which is what kept every JSON body unredacted. Keeping quotes out of the value class is still what stops the match running past the closing quote. The keyword may be a prefix of a longer key name rather than the whole of it, so the trailing identifier class below is load-bearing: without it the lookbehind required the keyword to sit immediately before the separator, and AWS_SECRET_ACCESS_KEY=, SECRET_KEY=, and DB_PASSWORD_HASH= all passed through in full. That class matches identifier characters only, so prose that merely mentions a keyword still never reaches a separator and stays unredacted. api[_-]?key covers the apikey and api-key spellings too. `& ; # , :` play two incompatible roles. They separate one field from the next (a query string, a cookie header, an inline env list), and they are also perfectly ordinary credential characters. Rejecting them outright got the first role right and the second badly wrong: the match stopped at the first one and left everything after it in plain text, so `password=corr&horse&battery` redacted four characters and printed the rest, and `DB_PASSWORD=Aa1:xyz` matched nothing at all because the run before the `:` was under the four-character floor. A tail left sitting in the open is the outcome this module's header calls worse than no redaction, because it reads as handled. So the separator role is decided by what follows rather than assumed: one of these characters ends the value only when the next thing along is another `name=` / `name:` pair, which is what an actual field separator is always followed by. `,OTHER=public` and `; other=1` still end it; the `&` in the middle of a passphrase does not. Whitespace, quotes and brackets are unchanged -- they end a value unconditionally, which is also what keeps this pattern from re-matching the `[REDACTED:...]` placeholder it just wrote.
   ["generic_secret_assignment", /(?<=(?:password|passwd|secret|api[_-]?key|access[_-]?token|refresh[_-]?token|id[_-]?token)[a-z0-9_-]{0,64}["']?[ \t]{0,8}[:=][ \t]{0,8}["']?)(?:\\[^\n]|[^\s\\&;#,:'"[\]{}]|[&;#,:](?![ \t]*[A-Za-z_][A-Za-z0-9_.-]*[ \t]*[:=])){4,}/gi]
 ];
 function countRedactionPlaceholders(text) {
@@ -1777,6 +1879,28 @@ function characterClasses(s) {
 function looksLikeCredential(s) {
   return characterClasses(s) >= 3 && entropyBitsPerChar(s) >= 3.5;
 }
+var RECALL_TUNED_KIND = "generic_secret_assignment";
+function matchesAnywhere(pattern, text) {
+  pattern.lastIndex = 0;
+  const hit = pattern.test(text);
+  pattern.lastIndex = 0;
+  return hit;
+}
+function hasPreciseSecret(text, config = loadConfig()) {
+  for (const [kind, pattern] of SECRET_PATTERNS) {
+    if (kind === RECALL_TUNED_KIND) continue;
+    if (matchesAnywhere(pattern, text)) return true;
+  }
+  for (const custom of compileCustomPatterns(config.redaction.custom_patterns).patterns) {
+    if (matchesAnywhere(custom, text)) return true;
+  }
+  if (config.redaction.strict) {
+    for (const candidate of text.match(STRICT_CANDIDATE) ?? []) {
+      if (looksLikeCredential(candidate)) return true;
+    }
+  }
+  return false;
+}
 function redactSecrets(text, config = loadConfig()) {
   let count = 0;
   let out = text;
@@ -1804,7 +1928,7 @@ function redactSecrets(text, config = loadConfig()) {
 
 // src/disk_cache.ts
 init_define_import_meta_env();
-import * as fs from "node:fs";
+import * as fs2 from "node:fs";
 import * as path2 from "node:path";
 var DEFAULT_MAX_COUNT = 200;
 var DEFAULT_MAX_AGE_MS = 24 * 3600 * 1e3;
@@ -1831,7 +1955,7 @@ function isBlobStale(subdir, id) {
   const p = blobPath(subdir, id);
   if (p === null) return false;
   try {
-    const stat = fs.statSync(p);
+    const stat = fs2.statSync(p);
     return Date.now() - stat.mtimeMs > DEFAULT_MAX_AGE_MS;
   } catch {
     return false;
@@ -1873,7 +1997,7 @@ function storeBlob(subdir, id, value, opts = {}) {
   if (Number.isFinite(maxBytesPerItem) && Buffer.byteLength(json, "utf-8") > maxBytesPerItem) return false;
   try {
     const dir = path2.dirname(p);
-    if (!fs.existsSync(dir)) ensureDirSync(dir);
+    if (!fs2.existsSync(dir)) ensureDirSync(dir);
     atomicWriteText(p, json);
   } catch {
     return false;
@@ -1891,8 +2015,8 @@ function loadBlob(subdir, id) {
   const p = blobPath(subdir, id);
   if (!p) return null;
   try {
-    if (!fs.existsSync(p)) return null;
-    return JSON.parse(fs.readFileSync(p, "utf8"));
+    if (!fs2.existsSync(p)) return null;
+    return JSON.parse(fs2.readFileSync(p, "utf8"));
   } catch {
     return null;
   }
@@ -1901,13 +2025,13 @@ function listBlobs(subdir) {
   const dir = blobDir(subdir);
   const out = [];
   try {
-    if (!fs.existsSync(dir)) return out;
-    for (const file of fs.readdirSync(dir)) {
+    if (!fs2.existsSync(dir)) return out;
+    for (const file of fs2.readdirSync(dir)) {
       if (!file.endsWith(".json")) continue;
       const id = file.slice(0, -5);
       let mtime = 0;
       try {
-        mtime = fs.statSync(path2.join(dir, file)).mtimeMs;
+        mtime = fs2.statSync(path2.join(dir, file)).mtimeMs;
       } catch {
       }
       const value = loadBlob(subdir, id);
@@ -1924,15 +2048,15 @@ function pruneBlobs(subdir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX
 function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_AGE_MS, maxBytes = Number.POSITIVE_INFINITY, protectedPath) {
   let removed = 0;
   try {
-    if (!fs.existsSync(dir)) return 0;
+    if (!fs2.existsSync(dir)) return 0;
     const cutoff = Date.now() - maxAgeMs;
     let kept = [];
     let protectedEntry;
-    for (const file of fs.readdirSync(dir)) {
+    for (const file of fs2.readdirSync(dir)) {
       const full = path2.join(dir, file);
       let stat;
       try {
-        stat = fs.statSync(full);
+        stat = fs2.statSync(full);
       } catch {
         continue;
       }
@@ -1943,7 +2067,7 @@ function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
       }
       if (stat.mtimeMs < cutoff) {
         try {
-          fs.unlinkSync(full);
+          fs2.unlinkSync(full);
           removed++;
         } catch {
           continue;
@@ -1959,7 +2083,7 @@ function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
       kept = kept.slice(kept.length - countBudget);
       for (const [full] of excess) {
         try {
-          fs.unlinkSync(full);
+          fs2.unlinkSync(full);
           removed++;
         } catch {
           continue;
@@ -1974,7 +2098,7 @@ function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
         if (!oldest) break;
         const [full, , size] = oldest;
         try {
-          fs.unlinkSync(full);
+          fs2.unlinkSync(full);
           removed++;
           total -= size;
         } catch {
@@ -2026,15 +2150,21 @@ export {
   SOURCE_HINT,
   savedTokensFromBytes,
   formatLocalTimestamp,
+  statsHasHarnessColumn,
+  statsHasDurationColumn,
+  getGlobalDb,
+  HOOK_STATS_RETENTION_DAYS,
   recordStat,
   recordUnmappedTool,
   readUnmappedTools,
+  pruneStalePatternCoveredUnmappedTools,
   summarize,
   _useRichStats,
   renderShortStats,
   renderStats2 as renderStats,
   countRedactionPlaceholders,
   compileCustomPatterns,
+  hasPreciseSecret,
   redactSecrets,
   DEFAULT_MAX_COUNT,
   DEFAULT_MAX_AGE_MS,

package/dist/{token-goat-chunk-CA6V36KW.mjs → token-goat-chunk-SOKAPOOC.mjs}

@@ -13,10 +13,10 @@ import {
   foldPath,
   projectConfigPath,
   runGit
-} from "./token-goat-chunk-QIDLJGXZ.mjs";
+} from "./token-goat-chunk-ZGQXRYKE.mjs";
 import {
   normalizeDarwinSystemAlias
-} from "./token-goat-chunk-ERTXEKB6.mjs";
+} from "./token-goat-chunk-NMTKNYGF.mjs";
 import {
   init_define_import_meta_env
 } from "./token-goat-chunk-A37V4PBF.mjs";
@@ -1202,6 +1202,8 @@ var PROJECT_LOCKED_KEYS = [
   "indexing.skip_files",
   "indexing.large_file_skip_kb",
   "indexing.large_file_symbol_only_kb",
+  // Same reasoning as the two thresholds above, on the chunk axis: a checked-in `.token-goat.toml` dropping this to 1 would take every file in the repository out of `semantic` for a reviewing agent, and `semantic` would answer "no matches" in the words it uses for a genuine absence.
+  "indexing.max_chunks_per_file",
   "indexing.cross_project_symbols",
   "worker.blocked_roots"
 ];
@@ -1514,7 +1516,8 @@ var CONFIG_DEFAULTS = {
     session_start_reminder: true
   },
   hooks: {
-    watchdog_ms: 700
+    watchdog_ms: 700,
+    latency_budget_ms: 1500
   },
   webfetch: {
     allow: [],
@@ -1542,6 +1545,8 @@ var CONFIG_DEFAULTS = {
   indexing: {
     large_file_symbol_only_kb: 500,
     large_file_skip_kb: 2048,
+    // 600 comes from a census of one real machine-wide index: 243,603 chunks over 62 projects and roughly 4,400 files. The largest hand-written source file anywhere in it produced 387 chunks (a 4,000-line test file); the next tier up starts at 513 and is entirely generated data snapshots, minified vendor assets and one HTML draft. 600 sits in that gap with about 1.5x headroom over the largest real file, and above it 32 files held 54,674 chunks -- 22% of the whole index from 0.7% of its files.
+    max_chunks_per_file: 600,
     skip_dirs: [],
     skip_files: ["coverage.json", "coverage-final.json"],
     embeddings_enabled: true,
@@ -1572,6 +1577,7 @@ var CONFIG_DEFAULTS = {
   },
   hint_stats: {
     suppress_threshold_pct: 15,
+    defiance_threshold_pct: 85,
     min_sample_size: 5
   },
   semantic: {
@@ -1710,6 +1716,7 @@ var NUMERIC_FIELD_BOUNDS = {
   "hints.cross_session_read_dedup_ttl_secs": { min: 1, max: 86400 },
   "hints.mcp_dedup_ttl_secs": { min: 1, max: 3600 },
   "hooks.watchdog_ms": { min: 100, max: 3e4 },
+  "hooks.latency_budget_ms": { min: 1, max: 6e5 },
   "webfetch.max_file_count": { min: 0, max: 1e7 },
   "webfetch.max_bytes": { min: 0, max: 100 * 1024 * 1024 * 1024 },
   "webfetch.compress_min_bytes": { min: 1024, max: 10 * 1024 * 1024 },
@@ -1717,8 +1724,10 @@ var NUMERIC_FIELD_BOUNDS = {
   "worker.embed_threads": { min: 1, max: 16 },
   "indexing.large_file_symbol_only_kb": { min: 1, max: 1048576, clampTo: "indexing.large_file_skip_kb" },
   "indexing.large_file_skip_kb": { min: 1, max: 1048576 },
+  "indexing.max_chunks_per_file": { min: 1, max: 1e6 },
   "context.model_window_tokens": { min: 1e4, max: 1e7 },
   "hint_stats.suppress_threshold_pct": { min: 0, max: 100 },
+  "hint_stats.defiance_threshold_pct": { min: 0, max: 100 },
   "hint_stats.min_sample_size": { min: 1, max: 1e4 },
   "semantic.archive_weight": { min: 0.05, max: 1 },
   "semantic.docs_weight": { min: 0.05, max: 1 }
@@ -2182,6 +2191,8 @@ function _buildConfig(raw, projectRaw = {}) {
   const hk = getDefaultConfig("hooks");
   hk.watchdog_ms = validatedInt(hk_raw["watchdog_ms"], hk.watchdog_ms, ...boundsOf("hooks.watchdog_ms"));
   hk.watchdog_ms = envInt("TOKEN_GOAT_HOOK_WATCHDOG_MS", hk.watchdog_ms, ...boundsOf("hooks.watchdog_ms"));
+  hk.latency_budget_ms = validatedInt(hk_raw["latency_budget_ms"], hk.latency_budget_ms, ...boundsOf("hooks.latency_budget_ms"));
+  hk.latency_budget_ms = envInt("TOKEN_GOAT_HOOK_LATENCY_BUDGET_MS", hk.latency_budget_ms, ...boundsOf("hooks.latency_budget_ms"));
   const wf_raw = section(raw, "webfetch");
   const wf = getDefaultConfig("webfetch");
   wf.allow = validatedStrList(wf_raw["allow"], wf.allow);
@@ -2209,6 +2220,7 @@ function _buildConfig(raw, projectRaw = {}) {
   ix.large_file_symbol_only_kb = validatedInt(ix_raw["large_file_symbol_only_kb"], ix.large_file_symbol_only_kb, ...boundsOf("indexing.large_file_symbol_only_kb"));
   ix.large_file_skip_kb = validatedInt(ix_raw["large_file_skip_kb"], ix.large_file_skip_kb, ...boundsOf("indexing.large_file_skip_kb"));
   ix.large_file_symbol_only_kb = Math.min(ix.large_file_symbol_only_kb, ix.large_file_skip_kb);
+  ix.max_chunks_per_file = validatedInt(ix_raw["max_chunks_per_file"], ix.max_chunks_per_file, ...boundsOf("indexing.max_chunks_per_file"));
   ix.skip_dirs = validatedStrList(ix_raw["skip_dirs"], ix.skip_dirs);
   ix.skip_files = validatedStrList(ix_raw["skip_files"], ix.skip_files);
   ix.embeddings_enabled = validatedBool(ix_raw["embeddings_enabled"], ix.embeddings_enabled);
@@ -2250,6 +2262,8 @@ function _buildConfig(raw, projectRaw = {}) {
   const hs_raw = section(raw, "hint_stats");
   const hs = getDefaultConfig("hint_stats");
   hs.suppress_threshold_pct = validatedInt(hs_raw["suppress_threshold_pct"], hs.suppress_threshold_pct, ...boundsOf("hint_stats.suppress_threshold_pct"));
+  hs.defiance_threshold_pct = validatedInt(hs_raw["defiance_threshold_pct"], 100 - hs.suppress_threshold_pct, ...boundsOf("hint_stats.defiance_threshold_pct"));
+  hs.defiance_threshold_pct = envInt("TOKEN_GOAT_HINT_DEFIANCE_THRESHOLD_PCT", hs.defiance_threshold_pct, ...boundsOf("hint_stats.defiance_threshold_pct"));
   hs.min_sample_size = validatedInt(hs_raw["min_sample_size"], hs.min_sample_size, ...boundsOf("hint_stats.min_sample_size"));
   const sem_raw = section(raw, "semantic");
   const sem = getDefaultConfig("semantic");
@@ -2345,6 +2359,7 @@ var CONFIG_KEY_ENV_OVERRIDES = {
   "hints.pre_skill_advisory": ["TOKEN_GOAT_PRE_SKILL_ADVISORY"],
   "hints.quiet_hours": ["TOKEN_GOAT_QUIET_HOURS"],
   "hooks.watchdog_ms": ["TOKEN_GOAT_HOOK_WATCHDOG_MS"],
+  "hooks.latency_budget_ms": ["TOKEN_GOAT_HOOK_LATENCY_BUDGET_MS"],
   "webfetch.max_file_count": ["TOKEN_GOAT_WEB_CACHE_MAX_FILES"],
   "webfetch.max_bytes": ["TOKEN_GOAT_WEB_CACHE_MAX_BYTES"],
   "webfetch.compress_bodies": ["TOKEN_GOAT_WEB_COMPRESS"],
@@ -2359,6 +2374,7 @@ var CONFIG_KEY_ENV_OVERRIDES = {
   "redaction.custom_patterns": ["TOKEN_GOAT_REDACTION_CUSTOM_PATTERNS"],
   "redaction.strict": ["TOKEN_GOAT_REDACTION_STRICT"],
   "network.offline": ["TOKEN_GOAT_OFFLINE"],
+  "hint_stats.defiance_threshold_pct": ["TOKEN_GOAT_HINT_DEFIANCE_THRESHOLD_PCT"],
   "mcp.confine_reads_to_project_root": ["TOKEN_GOAT_MCP_CONFINE_READS"],
   "mcp.allowed_roots": ["TOKEN_GOAT_MCP_ALLOWED_ROOTS"],
   "webfetch.allow": ["TOKEN_GOAT_WEBFETCH_ALLOW"],
@@ -2498,7 +2514,8 @@ function saveConfig(config) {
       session_start_reminder: config.hints.session_start_reminder
     },
     hooks: {
-      watchdog_ms: config.hooks.watchdog_ms
+      watchdog_ms: config.hooks.watchdog_ms,
+      latency_budget_ms: config.hooks.latency_budget_ms
     },
     webfetch: {
       allow: config.webfetch.allow,
@@ -2517,6 +2534,7 @@ function saveConfig(config) {
     indexing: {
       large_file_symbol_only_kb: config.indexing.large_file_symbol_only_kb,
       large_file_skip_kb: config.indexing.large_file_skip_kb,
+      max_chunks_per_file: config.indexing.max_chunks_per_file,
       skip_dirs: config.indexing.skip_dirs,
       skip_files: config.indexing.skip_files,
       embeddings_enabled: config.indexing.embeddings_enabled,
@@ -2546,6 +2564,7 @@ function saveConfig(config) {
     },
     hint_stats: {
       suppress_threshold_pct: config.hint_stats.suppress_threshold_pct,
+      defiance_threshold_pct: config.hint_stats.defiance_threshold_pct,
       min_sample_size: config.hint_stats.min_sample_size
     },
     semantic: {

package/dist/token-goat-chunk-VJQBYSXG.mjs

@@ -0,0 +1,20 @@
+import { createRequire as __cjsRequire } from 'node:module';
+const require = __cjsRequire(import.meta.url);
+import {
+  checkUpdateStatus,
+  cmdUpgrade,
+  compareSemver,
+  fetchLatestVersion
+} from "./token-goat-chunk-GYC34EGE.mjs";
+import "./token-goat-chunk-SOKAPOOC.mjs";
+import "./token-goat-chunk-EEIDFMEM.mjs";
+import "./token-goat-chunk-ZGQXRYKE.mjs";
+import "./token-goat-chunk-NMTKNYGF.mjs";
+import "./token-goat-chunk-GMOUBOX4.mjs";
+import "./token-goat-chunk-A37V4PBF.mjs";
+export {
+  checkUpdateStatus,
+  cmdUpgrade,
+  compareSemver,
+  fetchLatestVersion
+};