token-goat 2.9.15 → 2.9.16

package/dist/{token-goat-chunk-2X2EBBC6.mjs → token-goat-chunk-63QWJ6UM.mjs}

@@ -2,12 +2,12 @@ import { createRequire as __cjsRequire } from 'node:module';
 const require = __cjsRequire(import.meta.url);
 import {
   getDb
-} from "./token-goat-chunk-K7F2BFIK.mjs";
+} from "./token-goat-chunk-R3X5TCPB.mjs";
 import {
   foldPath,
   globalDbPath,
   isCaseInsensitiveFs
-} from "./token-goat-chunk-PM76YS22.mjs";
+} from "./token-goat-chunk-3H6FQT62.mjs";
 import {
   normalizePath
 } from "./token-goat-chunk-ERTXEKB6.mjs";

package/dist/{token-goat-chunk-NDRP4KJQ.mjs → token-goat-chunk-BFNTLGVA.mjs}

@@ -4,11 +4,7 @@ import {
   pathEqClause,
   projectScopeClause,
   querySymbols
-} from "./token-goat-chunk-2X2EBBC6.mjs";
-import {
-  DocumentRefusedError,
-  MAX_DOCUMENT_WORK_MILLIS
-} from "./token-goat-chunk-Y4AFKTHK.mjs";
+} from "./token-goat-chunk-63QWJ6UM.mjs";
 import {
   TREE_SITTER_LANGUAGES,
   _detectOpenQuote,
@@ -25,11 +21,11 @@ import {
   stripJsComments,
   stripStringLiterals,
   stripXmlComments
-} from "./token-goat-chunk-K7F2BFIK.mjs";
+} from "./token-goat-chunk-R3X5TCPB.mjs";
 import {
   fingerprintContent,
   loadConfig
-} from "./token-goat-chunk-S4XRY446.mjs";
+} from "./token-goat-chunk-H3DXI6VD.mjs";
 import {
   registerReset
 } from "./token-goat-chunk-EEIDFMEM.mjs";
@@ -43,11 +39,15 @@ import {
   foldPath,
   globalDbPath,
   isCaseInsensitiveFs
-} from "./token-goat-chunk-PM76YS22.mjs";
+} from "./token-goat-chunk-3H6FQT62.mjs";
 import {
   displaySafeText,
   normalizePath
 } from "./token-goat-chunk-ERTXEKB6.mjs";
+import {
+  DocumentRefusedError,
+  MAX_DOCUMENT_WORK_MILLIS
+} from "./token-goat-chunk-Y4AFKTHK.mjs";
 import {
   init_define_import_meta_env
 } from "./token-goat-chunk-A37V4PBF.mjs";
@@ -69,16 +69,16 @@ function isTransientDocumentRefusal(err) {
 async function extractEmbeddableDocumentText(filePath) {
   switch (path.extname(filePath).toLowerCase()) {
     case ".pdf": {
-      const { extractPdfText, readPdfFileWithinBounds } = await import("./token-goat-chunk-Z6UXPYJA.mjs");
+      const { extractPdfText, readPdfFileWithinBounds } = await import("./token-goat-chunk-F2ARFHLJ.mjs");
       const { text } = await extractPdfText(await readPdfFileWithinBounds(filePath));
       return text;
     }
     case ".docx":
-      return await (await import("./token-goat-chunk-LT7JRU6K.mjs")).docxText(filePath);
+      return await (await import("./token-goat-chunk-VVIYEFBI.mjs")).docxText(filePath);
     case ".pptx":
-      return await (await import("./token-goat-chunk-7ZYK25AO.mjs")).pptxAllSlidesText(filePath, true);
+      return await (await import("./token-goat-chunk-OCI25AXY.mjs")).pptxAllSlidesText(filePath, true);
     case ".xlsx":
-      return await (await import("./token-goat-chunk-3NSDDTGL.mjs")).allSheetsHeadText(filePath, XLSX_SHEET_ROW_CAP);
+      return await (await import("./token-goat-chunk-ZEHFYJ7Z.mjs")).allSheetsHeadText(filePath, XLSX_SHEET_ROW_CAP);
     default:
       return null;
   }
@@ -297,6 +297,12 @@ var BertWordPiece = class _BertWordPiece {
     ids.push(this.sepId);
     return ids.length > maxLength ? ids.slice(0, maxLength) : ids;
   }
+  /** How many wordpieces `text` becomes, markers excluded and nothing cut: what {@link encode} would read before its limit, for a caller sizing text to fit under it. Words never span whitespace, so the count of a newline-joined text is the sum of its lines' counts. */
+  countTokens(text) {
+    const ids = [];
+    for (const word of preTokenize(normalize(text))) this.wordToIds(word, ids);
+    return ids.length;
+  }
 };
 
 // src/embed_model.ts
@@ -558,6 +564,10 @@ var EmbeddingModel = class _EmbeddingModel {
     });
     return new _EmbeddingModel(tokenizer, session, ort.Tensor);
   }
+  /** Wordpieces `text` becomes under this model's own tokenizer, markers excluded. */
+  countTokens(text) {
+    return this.tokenizer.countTokens(text);
+  }
   /** Embed one text. Sequences are run singly, so there is no padding and no mask to get wrong. */
   async embed(text) {
     const ids = this.tokenizer.encode(text);
@@ -687,14 +697,14 @@ init_define_import_meta_env();
 
 // src/embed_fingerprint.ts
 init_define_import_meta_env();
-var EMBED_FINGERPRINT = "b1fd4a80b27f2ca1";
+var EMBED_FINGERPRINT = "b7b2ff71de288d13";
 
 // src/embeddings.ts
 var QUERY_INSTRUCTION_PREFIX = "Represent this sentence for searching relevant passages: ";
 var _extractorCache = /* @__PURE__ */ new Map();
 var inHousePipelineFn = async (_task, modelName) => {
   const model = await EmbeddingModel.load(modelName);
-  return async (text) => ({ data: await model.embed(text) });
+  return Object.assign(async (text) => ({ data: await model.embed(text) }), { countTokens: (text) => model.countTokens(text) });
 };
 var _pipelineFnOverride = null;
 registerReset(() => {
@@ -724,6 +734,7 @@ async function buildExtractorWithRetry(pipelineFn, modelName) {
 }
 var MIN_CHUNK_CHARS = 50;
 var MAX_CHUNK_CHARS = 8e3;
+var MAX_CHUNK_TOKENS = MAX_SEQUENCE_TOKENS - 2;
 var DEFAULT_DISTANCE_THRESHOLD = 1.2;
 var _GENERATED_PATH_SEGMENTS = /* @__PURE__ */ new Set([
   "node_modules",
@@ -772,15 +783,7 @@ function isAvailable() {
 function embeddingBackendLoadError() {
   return runtimeLoadError();
 }
-async function embedTexts(texts, modelName = DEFAULT_MODEL) {
-  if (!isAvailable()) {
-    throw new Error(
-      `Embedding backend not available: ${runtimeLoadError()?.message ?? "unknown error"}`
-    );
-  }
-  if (texts.length === 0) {
-    return [];
-  }
+function getExtractor(modelName) {
   let extractorPromise = _extractorCache.get(modelName);
   if (!extractorPromise) {
     const pipelineFn = _pipelineFnOverride ?? inHousePipelineFn;
@@ -790,7 +793,18 @@ async function embedTexts(texts, modelName = DEFAULT_MODEL) {
       if (_extractorCache.get(modelName) === extractorPromise) _extractorCache.delete(modelName);
     });
   }
-  const extractor = await extractorPromise;
+  return extractorPromise;
+}
+async function embedTexts(texts, modelName = DEFAULT_MODEL) {
+  if (!isAvailable()) {
+    throw new Error(
+      `Embedding backend not available: ${runtimeLoadError()?.message ?? "unknown error"}`
+    );
+  }
+  if (texts.length === 0) {
+    return [];
+  }
+  const extractor = await getExtractor(modelName);
   const vecs = [];
   const expectedDim = DEFAULT_DIM;
   try {
@@ -840,17 +854,29 @@ function packVec(vec) {
   }
   return Buffer.from(view.buffer);
 }
-function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize, overlap, kind) {
+function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize, overlap, kind, countTokens) {
   const chunks = [];
+  const lineTokenCache = /* @__PURE__ */ new Map();
+  const tokensOf = (lineNo) => {
+    if (countTokens === void 0) return 0;
+    let n = lineTokenCache.get(lineNo);
+    if (n === void 0) {
+      n = countTokens(lines2[lineNo - 1] ?? "");
+      lineTokenCache.set(lineNo, n);
+    }
+    return n;
+  };
   let currentChunk = "";
+  let currentTokens = 0;
   let startLine = rangeStart;
   let currentLine = rangeStart;
   for (let lineNo = rangeStart; lineNo <= rangeEnd; lineNo++) {
     const line = lines2[lineNo - 1] ?? "";
     const lineWithNewline = line + "\n";
-    if (currentChunk.length + lineWithNewline.length > chunkSize && currentChunk.length > 0) {
+    const lineTokens = tokensOf(lineNo);
+    if ((currentChunk.length + lineWithNewline.length > chunkSize || currentTokens + lineTokens > MAX_CHUNK_TOKENS) && currentChunk.length > 0) {
       const trimmedLength = currentChunk.trim().length;
-      const currentChunkTooSmall = trimmedLength < MIN_CHUNK_CHARS;
+      const currentChunkTooSmall = trimmedLength === 0 || trimmedLength < MIN_CHUNK_CHARS && countTokens === void 0;
       if (!currentChunkTooSmall) {
         chunks.push({
           filePath,
@@ -861,11 +887,15 @@ function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize,
         });
       }
       let overlapChars = 0;
+      let overlapTokens = 0;
       let computedOverlapStart = currentLine;
       while (computedOverlapStart > rangeStart) {
         const candidateChars = (lines2[computedOverlapStart - 2] ?? "").length + 1;
         if (overlapChars + candidateChars > overlap) break;
+        const candidateTokens = tokensOf(computedOverlapStart - 1);
+        if (overlapTokens + candidateTokens + lineTokens > MAX_CHUNK_TOKENS) break;
         overlapChars += candidateChars;
+        overlapTokens += candidateTokens;
         computedOverlapStart--;
       }
       const droppedChunkHadContent = currentChunkTooSmall && trimmedLength > 0;
@@ -873,10 +903,24 @@ function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize,
       if (overlapStart !== startLine) {
         const overlapText = lines2.slice(overlapStart - 1, currentLine - 1).join("\n");
         currentChunk = overlapText + "\n";
+        currentTokens = overlapTokens;
         startLine = overlapStart;
       }
     }
+    if (countTokens !== void 0 && (lineTokens > MAX_CHUNK_TOKENS || lineWithNewline.length > chunkSize)) {
+      const pieces = splitToFit(line, (piece) => piece.length + 1 <= chunkSize && countTokens(piece) <= MAX_CHUNK_TOKENS);
+      const last = pieces.pop() ?? "";
+      for (const piece of pieces) {
+        if (piece.trim().length > 0) chunks.push({ filePath, startLine: lineNo, endLine: lineNo, text: piece.trim(), kind });
+      }
+      currentChunk = last + "\n";
+      currentTokens = countTokens(last);
+      startLine = lineNo;
+      currentLine++;
+      continue;
+    }
     currentChunk += lineWithNewline;
+    currentTokens += lineTokens;
     currentLine++;
   }
   if (currentChunk.trim().length >= MIN_CHUNK_CHARS) {
@@ -889,17 +933,42 @@ function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize,
     });
   } else if (currentChunk.length > 0 && chunks.length > 0) {
     const last = chunks[chunks.length - 1];
-    last.endLine = rangeEnd;
-    last.text = lines2.slice(last.startLine - 1, rangeEnd).join("\n").trim();
+    let mergedTokens = 0;
+    for (let lineNo = last.startLine; lineNo <= rangeEnd; lineNo++) mergedTokens += tokensOf(lineNo);
+    if (mergedTokens <= MAX_CHUNK_TOKENS) {
+      last.endLine = rangeEnd;
+      last.text = lines2.slice(last.startLine - 1, rangeEnd).join("\n").trim();
+    } else if (currentChunk.trim().length > 0) {
+      chunks.push({ filePath, startLine, endLine: rangeEnd, text: currentChunk.trim(), kind });
+    }
   }
   return chunks;
 }
-function chunkFile(filePath, content, chunkSize = MAX_CHUNK_CHARS, overlap = 200, boundaries = []) {
+function splitToFit(text, fits) {
+  if (text.length < 2 || fits(text)) return [text];
+  const mid = Math.floor(text.length / 2);
+  let cut = mid;
+  for (let d = 0; d <= Math.floor(text.length / 4); d++) {
+    if (/\s/.test(text[mid - d] ?? "")) {
+      cut = mid - d;
+      break;
+    }
+    if (/\s/.test(text[mid + d] ?? "")) {
+      cut = mid + d;
+      break;
+    }
+  }
+  const before = text.charCodeAt(cut - 1);
+  if (before >= 55296 && before <= 56319) cut++;
+  if (cut <= 0 || cut >= text.length) return [text];
+  return [...splitToFit(text.slice(0, cut), fits), ...splitToFit(text.slice(cut), fits)];
+}
+function chunkFile(filePath, content, chunkSize = MAX_CHUNK_CHARS, overlap = 200, boundaries = [], countTokens) {
   const lines2 = content.split(/\r?\n/);
   if (lines2.length > 1 && lines2[lines2.length - 1] === "") lines2.pop();
   const totalLines = lines2.length;
   if (boundaries.length === 0) {
-    return splitRangeIntoChunks(filePath, lines2, 1, totalLines, chunkSize, overlap, "window");
+    return splitRangeIntoChunks(filePath, lines2, 1, totalLines, chunkSize, overlap, "window", countTokens);
   }
   const clipped = boundaries.map((b) => ({
     start: Math.max(1, Math.min(b.start, totalLines)),
@@ -907,7 +976,7 @@ function chunkFile(filePath, content, chunkSize = MAX_CHUNK_CHARS, overlap = 200
     kind: b.kind
   })).filter((b) => b.end >= b.start).sort((a, b) => a.start - b.start || b.end - a.end);
   if (clipped.length === 0) {
-    return splitRangeIntoChunks(filePath, lines2, 1, totalLines, chunkSize, overlap, "window");
+    return splitRangeIntoChunks(filePath, lines2, 1, totalLines, chunkSize, overlap, "window", countTokens);
   }
   const flattened = [];
   let openEnd = 0;
@@ -974,7 +1043,7 @@ function chunkFile(filePath, content, chunkSize = MAX_CHUNK_CHARS, overlap = 200
   }
   const chunks = [];
   for (const r of ranges) {
-    chunks.push(...splitRangeIntoChunks(filePath, lines2, r.start, r.end, chunkSize, overlap, r.kind));
+    chunks.push(...splitRangeIntoChunks(filePath, lines2, r.start, r.end, chunkSize, overlap, r.kind, countTokens));
   }
   return chunks;
 }
@@ -1225,8 +1294,12 @@ function mergeNearbyHits(hits, proximity = 20) {
   return merged;
 }
 async function indexFile(db, filePath, content, boundaries = []) {
-  const chunks = chunkFile(filePath, content, void 0, void 0, boundaries);
+  let chunks = chunkFile(filePath, content, void 0, void 0, boundaries);
   if (chunks.length > 0) {
+    if (embeddingsDepsAvailable(db)) {
+      const { countTokens } = await getExtractor(DEFAULT_MODEL);
+      if (countTokens !== void 0) chunks = chunkFile(filePath, content, void 0, void 0, boundaries, countTokens);
+    }
     return upsertChunks(db, chunks);
   }
   deleteFileEmbeddings(db, filePath);
@@ -1260,11 +1333,11 @@ function deleteFileEmbeddings(db, filePath) {
   }
   db.prepare(`DELETE FROM chunks WHERE ${pathEqClause("file_path")}`).run(folded);
 }
-function resetAllEmbeddings(db) {
+function resetAllEmbeddings(db, keepVectors = false) {
   const paths = db.prepare("SELECT DISTINCT file_path FROM chunks").pluck().all();
   const clearEmbedSha = db.prepare(`UPDATE files SET embed_sha = NULL WHERE ${pathEqClause("path")}`);
   const tx = db.transaction(() => {
-    for (const p of paths) deleteFileEmbeddings(db, p);
+    if (!keepVectors) for (const p of paths) deleteFileEmbeddings(db, p);
     for (const p of paths) clearEmbedSha.run(foldPath(p));
   });
   tx.immediate();
@@ -1282,17 +1355,22 @@ function majorMinor(version) {
   return parts.length >= 2 ? `${parts[0]}.${parts[1]}` : version;
 }
 var _provenanceChecked = /* @__PURE__ */ new WeakSet();
+function vectorSpaceOf(provenance) {
+  const at = provenance.lastIndexOf("/embed-");
+  return at === -1 ? provenance : provenance.slice(0, at);
+}
 function ensureEmbeddingProvenance(db, modelName = DEFAULT_MODEL) {
   if (_provenanceChecked.has(db)) return;
   _provenanceChecked.add(db);
   const current = embeddingProvenance(modelName);
   const stored = db.prepare("SELECT provenance FROM embedding_provenance WHERE id = 1").pluck().get();
   if (stored === current) return;
-  const cleared = resetAllEmbeddings(db);
+  const keepVectors = stored !== void 0 && vectorSpaceOf(stored) === vectorSpaceOf(current);
+  const cleared = resetAllEmbeddings(db, keepVectors);
   db.prepare(
     "INSERT INTO embedding_provenance (id, provenance) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET provenance = excluded.provenance"
   ).run(current);
-  if (cleared > 0) {
+  if (cleared > 0 && !keepVectors) {
     console.warn(
       `Embedding stack changed (${stored ?? "unrecorded"} -> ${current}); discarded ${cleared} file${cleared === 1 ? "" : "s"} worth of vectors because they no longer reliably describe this file's current chunks. Run \`token-goat index\` to rebuild them.`
     );
@@ -1335,7 +1413,7 @@ function _pathPriorityPenalty(filePath) {
 
 // src/parser_fingerprint.ts
 init_define_import_meta_env();
-var PARSER_FINGERPRINT = "ef41adb465538db6";
+var PARSER_FINGERPRINT = "e70e91822f151d92";
 
 // src/parser.ts
 init_define_import_meta_env();
@@ -1374,7 +1452,7 @@ function* eachUnfencedLine(lines2) {
 }
 
 // src/hints/markdown_hints.ts
-var MARKDOWN_SIZE_THRESHOLD = 8e3;
+var MARKDOWN_SIZE_THRESHOLD = 4e3;
 var MAX_HEADINGS = 40;
 var MAX_OUTPUT_LINES = 60;
 function extractMarkdownHeadings(content, limit = MAX_HEADINGS) {
@@ -3594,7 +3672,7 @@ function parseSourceSymbolsTreeSitterOnly(content, filePath, language) {
 var REGEX_ADAPTERS_SLOT = /* @__PURE__ */ Symbol.for("token-goat.regex-adapters");
 var adapterSlot = globalThis;
 async function loadRegexExtractors() {
-  adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-GIIHUSZX.mjs");
+  adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-MIL62GMN.mjs");
 }
 function regexAdapters() {
   const loaded = adapterSlot[REGEX_ADAPTERS_SLOT];
@@ -4166,7 +4244,7 @@ function tableSectionEndIndex(headers, headerPos, totalLines) {
   }
   return totalLines;
 }
-function resolveHeaderPos(headers, base, ordinal) {
+function resolveHeaderPos(headers, base, ordinal, allowFuzzy = true) {
   const target = base.toLowerCase();
   const normalizedTarget = normalizeHeading(base).toLowerCase();
   const strippedTarget = normalizeHeadingStrip(base).toLowerCase();
@@ -4227,10 +4305,106 @@ function resolveHeaderPos(headers, base, ordinal) {
     const allWordsMatch = queryWords.every((qw) => headingWords.some((hw) => hw.includes(qw)));
     if (allWordsMatch) widenedMatches.push(i);
   }
-  if (widenedMatches.length !== 1) return null;
-  const widenedPos = widenedMatches[0];
-  if (widenedPos === void 0) return null;
-  return { headerPos: widenedPos, redirectedFrom: base, occurrences: null };
+  if (widenedMatches.length === 1) {
+    const widenedPos = widenedMatches[0];
+    if (widenedPos !== void 0) {
+      return { headerPos: widenedPos, redirectedFrom: base, occurrences: null };
+    }
+  }
+  function sectionLevenshtein(a, b) {
+    if (a === b) return 0;
+    const la = a.length;
+    const lb = b.length;
+    if (la === 0) return lb;
+    if (lb === 0) return la;
+    const v0 = new Array(lb + 1);
+    const v1 = new Array(lb + 1);
+    for (let i = 0; i <= lb; i++) v0[i] = i;
+    for (let i = 0; i < la; i++) {
+      v1[0] = i + 1;
+      for (let j = 0; j < lb; j++) {
+        const cost = a.charCodeAt(i) === b.charCodeAt(j) ? 0 : 1;
+        v1[j + 1] = Math.min(v1[j] + 1, v0[j + 1] + 1, v0[j] + cost);
+      }
+      for (let j = 0; j <= lb; j++) v0[j] = v1[j];
+    }
+    return v0[lb];
+  }
+  const STOP_WORDS = /* @__PURE__ */ new Set(["a", "an", "the", "and", "or", "in", "on", "at", "to", "for", "of", "with", "by", "is", "&"]);
+  const cleanTokens = (str) => str.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 0);
+  const rawQTokens = cleanTokens(base);
+  const qTokens = rawQTokens.filter((t) => !STOP_WORDS.has(t) || rawQTokens.length <= 2);
+  if (allowFuzzy && qTokens.length > 0) {
+    const candidates = [];
+    for (let i = 0; i < headers.length; i++) {
+      const h = headers[i];
+      if (h === void 0) continue;
+      const rawHTokens = cleanTokens(h.heading);
+      const hTokens = rawHTokens.filter((t) => !STOP_WORDS.has(t) || rawHTokens.length <= 2);
+      if (hTokens.length === 0) continue;
+      const normH = normalizeHeading(h.heading).toLowerCase();
+      const maxLen = Math.max(normalizedTarget.length, normH.length);
+      const editDist = sectionLevenshtein(normalizedTarget, normH);
+      const strSim = maxLen > 0 ? 1 - editDist / maxLen : 0;
+      let matchedQ = 0;
+      for (const qw of qTokens) {
+        let best = 0;
+        for (const hw of hTokens) {
+          if (qw === hw) {
+            best = 1;
+            break;
+          }
+          if (qw.length >= 3 && hw.length >= 3) {
+            if (hw.startsWith(qw) || qw.startsWith(hw)) {
+              best = Math.max(best, 0.9);
+            } else if (sectionLevenshtein(qw, hw) <= 1) {
+              best = Math.max(best, 0.85);
+            }
+          }
+        }
+        matchedQ += best;
+      }
+      let matchedH = 0;
+      for (const hw of hTokens) {
+        let best = 0;
+        for (const qw of qTokens) {
+          if (qw === hw) {
+            best = 1;
+            break;
+          }
+          if (qw.length >= 3 && hw.length >= 3) {
+            if (hw.startsWith(qw) || qw.startsWith(hw)) {
+              best = Math.max(best, 0.9);
+            } else if (sectionLevenshtein(qw, hw) <= 1) {
+              best = Math.max(best, 0.85);
+            }
+          }
+        }
+        matchedH += best;
+      }
+      const wordRecall = matchedQ / qTokens.length;
+      const wordPrecision = matchedH / hTokens.length;
+      const tokenDice = 2 * matchedQ / (qTokens.length + hTokens.length);
+      let score = 0;
+      if (strSim >= 0.8) {
+        score = strSim;
+      } else if (tokenDice >= 0.75 && wordRecall >= 0.7 && wordPrecision >= 0.7) {
+        score = (tokenDice + wordRecall + wordPrecision) / 3;
+      }
+      if (score >= 0.75) {
+        candidates.push({ index: i, score });
+      }
+    }
+    if (candidates.length > 0) {
+      candidates.sort((a, b) => b.score - a.score);
+      const best = candidates[0];
+      const runnerUp = candidates[1];
+      if (candidates.length === 1 || runnerUp !== void 0 && best.score - runnerUp.score >= 0.08) {
+        return { headerPos: best.index, redirectedFrom: base, occurrences: null };
+      }
+    }
+  }
+  return null;
 }
 function buildSectionResult(headers, kind, lines2, headerPos, redirectedFrom) {
   const header = headers[headerPos];
@@ -4249,12 +4423,12 @@ function buildSectionResult(headers, kind, lines2, headerPos, redirectedFrom) {
   };
   return redirectedFrom === null ? result : { ...result, redirectedFrom };
 }
-function resolveSectionFromText(text, headingSpec, language) {
+function resolveSectionFromText(text, headingSpec, language, allowFuzzy = true) {
   const { headers, kind } = findHeaders(text, language);
   const { base, ordinal } = parseHeadingSpec(headingSpec, headers);
   if (base.length === 0) return null;
   const lines2 = text.split("\n");
-  const resolved = resolveHeaderPos(headers, base, ordinal);
+  const resolved = resolveHeaderPos(headers, base, ordinal, allowFuzzy);
   if (resolved === null) return null;
   const built = buildSectionResult(headers, kind, lines2, resolved.headerPos, resolved.redirectedFrom);
   if (built === null || resolved.occurrences === null) return built;
@@ -4281,10 +4455,10 @@ function readTextForSections(filePath, readFn) {
   }
   return redactIfDotenv(filePath, text);
 }
-function readSection(filePath, headingSpec, readFn) {
+function readSection(filePath, headingSpec, readFn, allowFuzzy = true) {
   const text = readTextForSections(filePath, readFn);
   if (text === null) return null;
-  return resolveSectionFromText(text, headingSpec, refineLanguageByContent(filePath, detectLanguage(filePath), text));
+  return resolveSectionFromText(text, headingSpec, refineLanguageByContent(filePath, detectLanguage(filePath), text), allowFuzzy);
 }
 function findContainingSection(filePath, lineStart, lineEnd, readFn) {
   const text = readTextForSections(filePath, readFn);
@@ -4334,6 +4508,8 @@ export {
   isEmbeddableDocument,
   ipynbToVirtualSource,
   eachUnfencedLine,
+  modelFilesPresent,
+  ensureModelFiles,
   checkEmbeddingPreflight,
   OVER_FETCH_FACTOR,
   MAX_OVER_FETCH,

package/dist/token-goat-chunk-BGHSSJUA.mjs

@@ -0,0 +1,102 @@
+import { createRequire as __cjsRequire } from 'node:module';
+const require = __cjsRequire(import.meta.url);
+import {
+  GIF_WEB_PALETTE,
+  assertDecodableSize,
+  quantizeRgbaToIndexed,
+  require_omggif,
+  resizeRgba
+} from "./token-goat-chunk-RRNZMM3A.mjs";
+import {
+  __toESM,
+  init_define_import_meta_env
+} from "./token-goat-chunk-A37V4PBF.mjs";
+
+// src/image_gif_encode.ts
+init_define_import_meta_env();
+var import_omggif = __toESM(require_omggif(), 1);
+function unionRects(a, b) {
+  const x = Math.min(a.x, b.x);
+  const y = Math.min(a.y, b.y);
+  return { x, y, width: Math.max(a.x + a.width, b.x + b.width) - x, height: Math.max(a.y + a.height, b.y + b.height) - y };
+}
+function gifFrameDelta(prev, prevTransparentIndex, cur, curTransparentIndex, width, height) {
+  let minX = width;
+  let minY = height;
+  let maxX = -1;
+  let maxY = -1;
+  let cleared = false;
+  for (let y = 0; y < height; y++) {
+    const row = y * width;
+    for (let x = 0; x < width; x++) {
+      const i = row + x;
+      const curIdx = cur[i] ?? 0;
+      const prevIdx = prev[i] ?? 0;
+      const curClear = curTransparentIndex !== null && curIdx === curTransparentIndex;
+      if (curClear === (prevTransparentIndex !== null && prevIdx === prevTransparentIndex) && (curClear || curIdx === prevIdx)) continue;
+      if (curClear) cleared = true;
+      if (x < minX) minX = x;
+      if (x > maxX) maxX = x;
+      if (y < minY) minY = y;
+      if (y > maxY) maxY = y;
+    }
+  }
+  if (maxX < 0) return { changed: { x: 0, y: 0, width: 1, height: 1 }, cleared: false };
+  return { changed: { x: minX, y: minY, width: maxX - minX + 1, height: maxY - minY + 1 }, cleared };
+}
+function sliceIndexedRect(indexed, canvasWidth, rect) {
+  if (rect.x === 0 && rect.y === 0 && rect.width === canvasWidth && rect.height * canvasWidth === indexed.length) return indexed;
+  const out = new Array(rect.width * rect.height);
+  for (let row = 0; row < rect.height; row++) {
+    const src = (rect.y + row) * canvasWidth + rect.x;
+    const dst = row * rect.width;
+    for (let col = 0; col < rect.width; col++) out[dst + col] = indexed[src + col] ?? 0;
+  }
+  return out;
+}
+function encodeAnimatedGifDelta(decoded, targetW, targetH) {
+  assertDecodableSize("GIF output", targetW, targetH, 5, decoded.frames.length);
+  const outBuf = Buffer.alloc(targetW * targetH * 5 * decoded.frames.length + 4096);
+  const gifWriter = new import_omggif.default.GifWriter(outBuf, targetW, targetH, { loop: 0, palette: GIF_WEB_PALETTE });
+  let pending = null;
+  const writePending = (disposal) => {
+    if (pending === null) return;
+    gifWriter.addFrame(pending.rect.x, pending.rect.y, pending.rect.width, pending.rect.height, pending.pixels, {
+      delay: pending.delay,
+      disposal,
+      ...pending.transparent === null ? {} : { transparent: pending.transparent }
+    });
+    pending = null;
+  };
+  for (const frame of decoded.frames) {
+    const resizedFrameRgba = resizeRgba(frame.data, frame.width, frame.height, targetW, targetH);
+    const { indexedPixels, transparentIndex } = quantizeRgbaToIndexed(resizedFrameRgba, targetW, targetH);
+    let rect = { x: 0, y: 0, width: targetW, height: targetH };
+    if (pending !== null) {
+      const delta = gifFrameDelta(pending.canvas, pending.transparent, indexedPixels, transparentIndex, targetW, targetH);
+      rect = delta.changed;
+      if (delta.cleared) {
+        rect = unionRects(pending.rect, delta.changed);
+        pending = { ...pending, rect, pixels: sliceIndexedRect(pending.canvas, targetW, rect) };
+        writePending(2);
+      } else {
+        writePending(1);
+      }
+    }
+    pending = {
+      rect,
+      pixels: sliceIndexedRect(indexedPixels, targetW, rect),
+      canvas: indexedPixels,
+      transparent: transparentIndex,
+      delay: frame.delay
+    };
+  }
+  writePending(1);
+  return outBuf.subarray(0, gifWriter.end());
+}
+export {
+  encodeAnimatedGifDelta,
+  gifFrameDelta,
+  sliceIndexedRect,
+  unionRects
+};

package/dist/{token-goat-chunk-IVUQLQWN.mjs → token-goat-chunk-DMQ32ZSR.mjs}

@@ -2,11 +2,11 @@ import { createRequire as __cjsRequire } from 'node:module';
 const require = __cjsRequire(import.meta.url);
 import {
   getDb
-} from "./token-goat-chunk-K7F2BFIK.mjs";
+} from "./token-goat-chunk-R3X5TCPB.mjs";
 import {
   getHarnessName,
   loadConfig
-} from "./token-goat-chunk-S4XRY446.mjs";
+} from "./token-goat-chunk-H3DXI6VD.mjs";
 import {
   registerReset
 } from "./token-goat-chunk-EEIDFMEM.mjs";
@@ -19,7 +19,7 @@ import {
   ensureDirSync,
   sanitizeIdForFilename,
   tokenGoatHome
-} from "./token-goat-chunk-PM76YS22.mjs";
+} from "./token-goat-chunk-3H6FQT62.mjs";
 import {
   displaySafeText
 } from "./token-goat-chunk-ERTXEKB6.mjs";
@@ -473,6 +473,7 @@ var _KIND_GROUPS = [
       "gdrive_image",
       "webfetch_image",
       "image_shrink_skipped",
+      "image_shrink_over_pixel_limit",
       "image_shrink_cache_hit",
       "image_ocr"
     ])
@@ -872,6 +873,7 @@ var KIND_TO_SOURCE = {
   image_shrink: SOURCE_IMAGE,
   image_shrink_cache_hit: SOURCE_IMAGE,
   image_shrink_skipped: SOURCE_IMAGE,
+  image_shrink_over_pixel_limit: SOURCE_IMAGE,
   image_ocr: SOURCE_IMAGE,
   webfetch_image: SOURCE_IMAGE,
   gdrive_image: SOURCE_IMAGE,