token-goat 2.9.19 → 2.9.21

package/dist/{token-goat-chunk-SOKAPOOC.mjs → token-goat-chunk-M6CDVASA.mjs}

@@ -13,7 +13,7 @@ import {
   foldPath,
   projectConfigPath,
   runGit
-} from "./token-goat-chunk-ZGQXRYKE.mjs";
+} from "./token-goat-chunk-TGBSMHPM.mjs";
 import {
   normalizeDarwinSystemAlias
 } from "./token-goat-chunk-NMTKNYGF.mjs";
@@ -1205,7 +1205,14 @@ var PROJECT_LOCKED_KEYS = [
   // Same reasoning as the two thresholds above, on the chunk axis: a checked-in `.token-goat.toml` dropping this to 1 would take every file in the repository out of `semantic` for a reviewing agent, and `semantic` would answer "no matches" in the words it uses for a genuine absence.
   "indexing.max_chunks_per_file",
   "indexing.cross_project_symbols",
-  "worker.blocked_roots"
+  // The same blast radius as the indexing keys above, applied to retrieval rather than to what gets indexed: `semantic` drops a dense hit whose distance exceeds this, so a checked-in `.token-goat.toml` setting it near the floor removes this repository's own code from the vector half of every search a reviewing agent runs, and the command then answers "no matches" in the words it uses for a name that genuinely is not there. The keyword pass would still answer, which makes it worse rather than better -- the result looks like a working search. The user's own global config still sets it freely; only the project-supplied layer is refused.
+  "semantic.max_distance",
+  "worker.blocked_roots",
+  // global.db is a machine-wide shared database across every project on the host. A checked-in
+  // per-project .token-goat.toml must not be able to lower the size warning threshold or trigger
+  // auto-reclaim/purging of embeddings across other repositories sharing the same database.
+  "indexing.max_db_size_mb",
+  "indexing.auto_reclaim_embeddings"
 ];
 var _lastProjectConfigLockedKeys = [];
 function lastProjectConfigLockedKeys() {
@@ -1344,6 +1351,13 @@ function isAutoTriggerMultiplierExplicit() {
 init_define_import_meta_env();
 import * as path2 from "node:path";
 
+// src/own_lookup.ts
+init_define_import_meta_env();
+function ownGet(map, key) {
+  if (!map || !Object.hasOwn(map, key)) return void 0;
+  return map[key];
+}
+
 // src/config_types.ts
 init_define_import_meta_env();
 
@@ -1352,16 +1366,7 @@ init_define_import_meta_env();
 var CONFIG_DEFAULTS = {
   compact_assist: {
     enabled: true,
-    triggers: ["manual", "auto"],
-    min_events: 3,
-    max_manifest_tokens: 400,
     auto_trigger_multiplier: 2,
-    compact_skip_ttl_secs: 300,
-    noise_floor_tokens: 0,
-    edited_dir_group_threshold: 3,
-    max_section_lines: 0,
-    wide_session_threshold: 15,
-    orchestrator_commit_threshold: 5,
     max_manifest_chars: 1600,
     summary_budget_chars: 24e3,
     harness: "auto"
@@ -1550,7 +1555,9 @@ var CONFIG_DEFAULTS = {
     skip_dirs: [],
     skip_files: ["coverage.json", "coverage-final.json"],
     embeddings_enabled: true,
-    cross_project_symbols: true
+    cross_project_symbols: true,
+    max_db_size_mb: 1500,
+    auto_reclaim_embeddings: false
   },
   compression: {
     profile: "auto"
@@ -1582,7 +1589,9 @@ var CONFIG_DEFAULTS = {
   },
   semantic: {
     archive_weight: 0.7,
-    docs_weight: 0.92
+    docs_weight: 0.92,
+    // Deliberately loose enough to admit everything this metric produces, which is a measured conclusion rather than a placeholder. Against a large index, 12 genuine questions about code that is present had best hits from 0.635 to 0.813 and 11 off-corpus ones (nonsense strings, digits, a cake recipe) from 0.820 to 1.017, which looks separable at about 0.9. It is not: on a small corpus the nearest genuine match is much further away, and a two-file project in this repo's own suite matches its target at 0.934 -- above that cut and inside the noise band. So the two populations overlap across corpus sizes and no fixed distance tells them apart, while getting it wrong removes real answers and says nothing. Left at the retrieval bound's value so the default filters nothing; it exists to be lowered per-project by someone who has measured their own corpus, and runSemantic says which distance it refused when a lowered value empties the vector half.
+    max_distance: 1.2
   }
 };
 function getDefaultConfig(section2) {
@@ -1663,15 +1672,7 @@ function section(raw, key) {
   return val !== null && typeof val === "object" && !Array.isArray(val) ? val : {};
 }
 var NUMERIC_FIELD_BOUNDS = {
-  "compact_assist.min_events": { min: 0, max: 1e3 },
-  "compact_assist.max_manifest_tokens": { min: 50, max: 1e4 },
   "compact_assist.auto_trigger_multiplier": { min: 1, max: 10 },
-  "compact_assist.compact_skip_ttl_secs": { min: 1, max: 3600 },
-  "compact_assist.noise_floor_tokens": { min: 0, max: 1e4 },
-  "compact_assist.edited_dir_group_threshold": { min: 0, max: 100 },
-  "compact_assist.max_section_lines": { min: 0, max: 1e4 },
-  "compact_assist.wide_session_threshold": { min: 1, max: 1e4 },
-  "compact_assist.orchestrator_commit_threshold": { min: 1, max: 1e4 },
   "compact_assist.max_manifest_chars": { min: 0, max: 16e3 },
   "compact_assist.summary_budget_chars": { min: 0, max: 2e5 },
   "bash_compress.max_lines": { min: 50, max: 1e5 },
@@ -1725,15 +1726,18 @@ var NUMERIC_FIELD_BOUNDS = {
   "indexing.large_file_symbol_only_kb": { min: 1, max: 1048576, clampTo: "indexing.large_file_skip_kb" },
   "indexing.large_file_skip_kb": { min: 1, max: 1048576 },
   "indexing.max_chunks_per_file": { min: 1, max: 1e6 },
+  "indexing.max_db_size_mb": { min: 1, max: 1e5 },
   "context.model_window_tokens": { min: 1e4, max: 1e7 },
   "hint_stats.suppress_threshold_pct": { min: 0, max: 100 },
   "hint_stats.defiance_threshold_pct": { min: 0, max: 100 },
   "hint_stats.min_sample_size": { min: 1, max: 1e4 },
   "semantic.archive_weight": { min: 0.05, max: 1 },
-  "semantic.docs_weight": { min: 0.05, max: 1 }
+  "semantic.docs_weight": { min: 0.05, max: 1 },
+  // Upper bound is DEFAULT_DISTANCE_THRESHOLD, not the metric's own ceiling of 2. An L2 distance between unit vectors cannot exceed 2, so 2 reads like 'admit everything' -- but this floor only ever narrows what searchSemantic already returned, and runSemantic lets that scan keep its own 1.2 bound (see the max_distance comment on SemanticConfig for why the scan is the wrong place for a relevance decision). Anything between 1.2 and 2 was therefore discarded inside the scan before the floor could see it: a user raising this to widen recall got the setting accepted, validated and persisted, and identical results, with no diagnostic able to say why. validatedFloat clamps rather than rejects, so an existing value above the new bound becomes 1.2 and behaves exactly as it already did. The lower bound is not 0: a floor of 0 admits only an exact vector match, which would disable the vector half entirely while `semantic` went on answering in the words it uses for a genuine absence.
+  "semantic.max_distance": { min: 0.05, max: 1.2 }
 };
 function boundsOf(key) {
-  const b = NUMERIC_FIELD_BOUNDS[key];
+  const b = ownGet(NUMERIC_FIELD_BOUNDS, key);
   if (!b) throw new Error(`token-goat: no NUMERIC_FIELD_BOUNDS entry for '${key}'`);
   return [b.min, b.max];
 }
@@ -1747,7 +1751,7 @@ function walkGetNumeric(obj, parts) {
   return typeof cur === "number" ? cur : void 0;
 }
 function validateNumericField(fieldKey, value, cfg) {
-  const bounds = NUMERIC_FIELD_BOUNDS[fieldKey];
+  const bounds = ownGet(NUMERIC_FIELD_BOUNDS, fieldKey);
   if (!bounds) return void 0;
   let clamped = Math.max(bounds.min, Math.min(bounds.max, value));
   if (bounds.clampTo) {
@@ -1773,7 +1777,7 @@ function validateEnumField(fieldKey, value) {
     const allowed2 = [...SUPPORTED_OCR_LANG_CODES];
     return tokens.length > 0 && tokens.every((t) => isSupportedOcrLang(t)) ? void 0 : allowed2;
   }
-  const allowed = ENUM_FIELD_VALUES[fieldKey];
+  const allowed = ownGet(ENUM_FIELD_VALUES, fieldKey);
   if (!allowed) return void 0;
   return allowed.includes(value) ? void 0 : allowed;
 }
@@ -1822,7 +1826,9 @@ var ENV_KEYS2 = [
   "TOKEN_GOAT_WEB_CACHE_MAX_FILES",
   "TOKEN_GOAT_WEB_CACHE_MAX_BYTES",
   "TOKEN_GOAT_WRITE_REWRITE_MIN_LINES",
-  "TOKEN_GOAT_WRITE_REWRITE_UNCHANGED_PCT"
+  "TOKEN_GOAT_WRITE_REWRITE_UNCHANGED_PCT",
+  "TOKEN_GOAT_INDEXING_MAX_DB_SIZE_MB",
+  "TOKEN_GOAT_INDEXING_AUTO_RECLAIM_EMBEDDINGS"
 ];
 function allEnvKeys() {
   return [.../* @__PURE__ */ new Set([...ENV_KEYS2, ...Object.values(CONFIG_KEY_ENV_OVERRIDES).flat()])];
@@ -1868,7 +1874,7 @@ function rawValueEquals(a, b) {
 }
 function rejectionReason(key, rawValue, effectiveValue, cfg) {
   if (typeof rawValue === "number") {
-    const bounds = NUMERIC_FIELD_BOUNDS[key];
+    const bounds = ownGet(NUMERIC_FIELD_BOUNDS, key);
     if (bounds !== void 0) {
       if (rawValue < bounds.min || rawValue > bounds.max) return `outside the allowed range ${bounds.min}-${bounds.max}`;
       if (!Number.isInteger(rawValue) && Number.isInteger(effectiveValue)) return "not a whole number";
@@ -2001,16 +2007,7 @@ function _buildConfig(raw, projectRaw = {}) {
   const ca_raw = section(raw, "compact_assist");
   const ca = getDefaultConfig("compact_assist");
   ca.enabled = validatedBool(ca_raw["enabled"], ca.enabled);
-  ca.triggers = validatedStrList(ca_raw["triggers"], ca.triggers);
-  ca.min_events = validatedInt(ca_raw["min_events"], ca.min_events, ...boundsOf("compact_assist.min_events"));
-  ca.max_manifest_tokens = validatedInt(ca_raw["max_manifest_tokens"], ca.max_manifest_tokens, ...boundsOf("compact_assist.max_manifest_tokens"));
   ca.auto_trigger_multiplier = validatedFloat(ca_raw["auto_trigger_multiplier"], ca.auto_trigger_multiplier, ...boundsOf("compact_assist.auto_trigger_multiplier"));
-  ca.compact_skip_ttl_secs = validatedFloat(ca_raw["compact_skip_ttl_secs"], ca.compact_skip_ttl_secs, ...boundsOf("compact_assist.compact_skip_ttl_secs"));
-  ca.noise_floor_tokens = validatedInt(ca_raw["noise_floor_tokens"], ca.noise_floor_tokens, ...boundsOf("compact_assist.noise_floor_tokens"));
-  ca.edited_dir_group_threshold = validatedInt(ca_raw["edited_dir_group_threshold"], ca.edited_dir_group_threshold, ...boundsOf("compact_assist.edited_dir_group_threshold"));
-  ca.max_section_lines = validatedInt(ca_raw["max_section_lines"], ca.max_section_lines, ...boundsOf("compact_assist.max_section_lines"));
-  ca.wide_session_threshold = validatedInt(ca_raw["wide_session_threshold"], ca.wide_session_threshold, ...boundsOf("compact_assist.wide_session_threshold"));
-  ca.orchestrator_commit_threshold = validatedInt(ca_raw["orchestrator_commit_threshold"], ca.orchestrator_commit_threshold, ...boundsOf("compact_assist.orchestrator_commit_threshold"));
   ca.max_manifest_chars = validatedInt(ca_raw["max_manifest_chars"], ca.max_manifest_chars, ...boundsOf("compact_assist.max_manifest_chars"));
   ca.summary_budget_chars = validatedInt(ca_raw["summary_budget_chars"], ca.summary_budget_chars, ...boundsOf("compact_assist.summary_budget_chars"));
   ca.harness = validatedStr(ca_raw["harness"], ca.harness);
@@ -2227,6 +2224,10 @@ function _buildConfig(raw, projectRaw = {}) {
   ix.embeddings_enabled = envBool("TOKEN_GOAT_EMBEDDINGS_ENABLED", ix.embeddings_enabled);
   ix.cross_project_symbols = validatedBool(ix_raw["cross_project_symbols"], ix.cross_project_symbols);
   ix.cross_project_symbols = envBool("TOKEN_GOAT_CROSS_PROJECT_SYMBOLS", ix.cross_project_symbols);
+  ix.max_db_size_mb = validatedInt(ix_raw["max_db_size_mb"], ix.max_db_size_mb, ...boundsOf("indexing.max_db_size_mb"));
+  ix.max_db_size_mb = envInt("TOKEN_GOAT_INDEXING_MAX_DB_SIZE_MB", ix.max_db_size_mb, ...boundsOf("indexing.max_db_size_mb"));
+  ix.auto_reclaim_embeddings = validatedBool(ix_raw["auto_reclaim_embeddings"], ix.auto_reclaim_embeddings);
+  ix.auto_reclaim_embeddings = envBool("TOKEN_GOAT_INDEXING_AUTO_RECLAIM_EMBEDDINGS", ix.auto_reclaim_embeddings);
   const cpr_raw = section(raw, "compression");
   const cpr = getDefaultConfig("compression");
   cpr.profile = validatedStr(cpr_raw["profile"], cpr.profile);
@@ -2269,6 +2270,7 @@ function _buildConfig(raw, projectRaw = {}) {
   const sem = getDefaultConfig("semantic");
   sem.archive_weight = validatedFloat(sem_raw["archive_weight"], sem.archive_weight, ...boundsOf("semantic.archive_weight"));
   sem.docs_weight = validatedFloat(sem_raw["docs_weight"], sem.docs_weight, ...boundsOf("semantic.docs_weight"));
+  sem.max_distance = validatedFloat(sem_raw["max_distance"], sem.max_distance, ...boundsOf("semantic.max_distance"));
   return {
     compact_assist: ca,
     bash_compress: bc,
@@ -2379,7 +2381,9 @@ var CONFIG_KEY_ENV_OVERRIDES = {
   "mcp.allowed_roots": ["TOKEN_GOAT_MCP_ALLOWED_ROOTS"],
   "webfetch.allow": ["TOKEN_GOAT_WEBFETCH_ALLOW"],
   "webfetch.deny": ["TOKEN_GOAT_WEBFETCH_DENY"],
-  "indexing.embeddings_enabled": ["TOKEN_GOAT_EMBEDDINGS_ENABLED"]
+  "indexing.embeddings_enabled": ["TOKEN_GOAT_EMBEDDINGS_ENABLED"],
+  "indexing.max_db_size_mb": ["TOKEN_GOAT_INDEXING_MAX_DB_SIZE_MB"],
+  "indexing.auto_reclaim_embeddings": ["TOKEN_GOAT_INDEXING_AUTO_RECLAIM_EMBEDDINGS"]
 };
 function saveConfig(config) {
   const ca = config.compact_assist;
@@ -2392,16 +2396,7 @@ function saveConfig(config) {
   const data = {
     compact_assist: {
       enabled: ca.enabled,
-      triggers: ca.triggers,
-      min_events: ca.min_events,
-      max_manifest_tokens: ca.max_manifest_tokens,
       ...keepsMultiplierDefault ? {} : { auto_trigger_multiplier: ca.auto_trigger_multiplier },
-      compact_skip_ttl_secs: ca.compact_skip_ttl_secs,
-      noise_floor_tokens: ca.noise_floor_tokens,
-      edited_dir_group_threshold: ca.edited_dir_group_threshold,
-      max_section_lines: ca.max_section_lines,
-      wide_session_threshold: ca.wide_session_threshold,
-      orchestrator_commit_threshold: ca.orchestrator_commit_threshold,
       max_manifest_chars: ca.max_manifest_chars,
       summary_budget_chars: ca.summary_budget_chars,
       harness: ca.harness
@@ -2538,7 +2533,9 @@ function saveConfig(config) {
       skip_dirs: config.indexing.skip_dirs,
       skip_files: config.indexing.skip_files,
       embeddings_enabled: config.indexing.embeddings_enabled,
-      cross_project_symbols: config.indexing.cross_project_symbols
+      cross_project_symbols: config.indexing.cross_project_symbols,
+      max_db_size_mb: config.indexing.max_db_size_mb,
+      auto_reclaim_embeddings: config.indexing.auto_reclaim_embeddings
     },
     compression: {
       profile: config.compression.profile
@@ -2569,6 +2566,7 @@ function saveConfig(config) {
     },
     semantic: {
       archive_weight: config.semantic.archive_weight,
+      max_distance: config.semantic.max_distance,
       docs_weight: config.semantic.docs_weight
     }
   };
@@ -2593,6 +2591,7 @@ export {
   SUPPORTED_OCR_LANG_CODES,
   isSupportedOcrLang,
   resolveOcrLangs,
+  ownGet,
   defaultConfig,
   makeProjectAt,
   findProject,