token-goat 2.9.15 → 2.9.17

package/dist/{token-goat-chunk-NDRP4KJQ.mjs → token-goat-chunk-73GXYYZR.mjs}

@@ -4,11 +4,7 @@ import {
   pathEqClause,
   projectScopeClause,
   querySymbols
-} from "./token-goat-chunk-2X2EBBC6.mjs";
-import {
-  DocumentRefusedError,
-  MAX_DOCUMENT_WORK_MILLIS
-} from "./token-goat-chunk-Y4AFKTHK.mjs";
+} from "./token-goat-chunk-BK3RPPTK.mjs";
 import {
   TREE_SITTER_LANGUAGES,
   _detectOpenQuote,
@@ -25,11 +21,11 @@ import {
   stripJsComments,
   stripStringLiterals,
   stripXmlComments
-} from "./token-goat-chunk-K7F2BFIK.mjs";
+} from "./token-goat-chunk-MWSVUKVV.mjs";
 import {
   fingerprintContent,
   loadConfig
-} from "./token-goat-chunk-S4XRY446.mjs";
+} from "./token-goat-chunk-2VASS5VR.mjs";
 import {
   registerReset
 } from "./token-goat-chunk-EEIDFMEM.mjs";
@@ -43,11 +39,15 @@ import {
   foldPath,
   globalDbPath,
   isCaseInsensitiveFs
-} from "./token-goat-chunk-PM76YS22.mjs";
+} from "./token-goat-chunk-UBZCON6N.mjs";
 import {
   displaySafeText,
   normalizePath
 } from "./token-goat-chunk-ERTXEKB6.mjs";
+import {
+  DocumentRefusedError,
+  MAX_DOCUMENT_WORK_MILLIS
+} from "./token-goat-chunk-Y4AFKTHK.mjs";
 import {
   init_define_import_meta_env
 } from "./token-goat-chunk-A37V4PBF.mjs";
@@ -69,16 +69,16 @@ function isTransientDocumentRefusal(err) {
 async function extractEmbeddableDocumentText(filePath) {
   switch (path.extname(filePath).toLowerCase()) {
     case ".pdf": {
-      const { extractPdfText, readPdfFileWithinBounds } = await import("./token-goat-chunk-Z6UXPYJA.mjs");
+      const { extractPdfText, readPdfFileWithinBounds } = await import("./token-goat-chunk-F2ARFHLJ.mjs");
       const { text } = await extractPdfText(await readPdfFileWithinBounds(filePath));
       return text;
     }
     case ".docx":
-      return await (await import("./token-goat-chunk-LT7JRU6K.mjs")).docxText(filePath);
+      return await (await import("./token-goat-chunk-EKGVCAES.mjs")).docxText(filePath);
     case ".pptx":
-      return await (await import("./token-goat-chunk-7ZYK25AO.mjs")).pptxAllSlidesText(filePath, true);
+      return await (await import("./token-goat-chunk-2XA5ZW4T.mjs")).pptxAllSlidesText(filePath, true);
     case ".xlsx":
-      return await (await import("./token-goat-chunk-3NSDDTGL.mjs")).allSheetsHeadText(filePath, XLSX_SHEET_ROW_CAP);
+      return await (await import("./token-goat-chunk-RFFUCQK7.mjs")).allSheetsHeadText(filePath, XLSX_SHEET_ROW_CAP);
     default:
       return null;
   }
@@ -297,6 +297,12 @@ var BertWordPiece = class _BertWordPiece {
     ids.push(this.sepId);
     return ids.length > maxLength ? ids.slice(0, maxLength) : ids;
   }
+  /** How many wordpieces `text` becomes, markers excluded and nothing cut: what {@link encode} would read before its limit, for a caller sizing text to fit under it. Words never span whitespace, so the count of a newline-joined text is the sum of its lines' counts. */
+  countTokens(text) {
+    const ids = [];
+    for (const word of preTokenize(normalize(text))) this.wordToIds(word, ids);
+    return ids.length;
+  }
 };
 
 // src/embed_model.ts
@@ -558,6 +564,10 @@ var EmbeddingModel = class _EmbeddingModel {
     });
     return new _EmbeddingModel(tokenizer, session, ort.Tensor);
   }
+  /** Wordpieces `text` becomes under this model's own tokenizer, markers excluded. */
+  countTokens(text) {
+    return this.tokenizer.countTokens(text);
+  }
   /** Embed one text. Sequences are run singly, so there is no padding and no mask to get wrong. */
   async embed(text) {
     const ids = this.tokenizer.encode(text);
@@ -687,14 +697,14 @@ init_define_import_meta_env();
 
 // src/embed_fingerprint.ts
 init_define_import_meta_env();
-var EMBED_FINGERPRINT = "b1fd4a80b27f2ca1";
+var EMBED_FINGERPRINT = "b7b2ff71de288d13";
 
 // src/embeddings.ts
 var QUERY_INSTRUCTION_PREFIX = "Represent this sentence for searching relevant passages: ";
 var _extractorCache = /* @__PURE__ */ new Map();
 var inHousePipelineFn = async (_task, modelName) => {
   const model = await EmbeddingModel.load(modelName);
-  return async (text) => ({ data: await model.embed(text) });
+  return Object.assign(async (text) => ({ data: await model.embed(text) }), { countTokens: (text) => model.countTokens(text) });
 };
 var _pipelineFnOverride = null;
 registerReset(() => {
@@ -724,6 +734,7 @@ async function buildExtractorWithRetry(pipelineFn, modelName) {
 }
 var MIN_CHUNK_CHARS = 50;
 var MAX_CHUNK_CHARS = 8e3;
+var MAX_CHUNK_TOKENS = MAX_SEQUENCE_TOKENS - 2;
 var DEFAULT_DISTANCE_THRESHOLD = 1.2;
 var _GENERATED_PATH_SEGMENTS = /* @__PURE__ */ new Set([
   "node_modules",
@@ -772,15 +783,7 @@ function isAvailable() {
 function embeddingBackendLoadError() {
   return runtimeLoadError();
 }
-async function embedTexts(texts, modelName = DEFAULT_MODEL) {
-  if (!isAvailable()) {
-    throw new Error(
-      `Embedding backend not available: ${runtimeLoadError()?.message ?? "unknown error"}`
-    );
-  }
-  if (texts.length === 0) {
-    return [];
-  }
+function getExtractor(modelName) {
   let extractorPromise = _extractorCache.get(modelName);
   if (!extractorPromise) {
     const pipelineFn = _pipelineFnOverride ?? inHousePipelineFn;
@@ -790,7 +793,18 @@ async function embedTexts(texts, modelName = DEFAULT_MODEL) {
       if (_extractorCache.get(modelName) === extractorPromise) _extractorCache.delete(modelName);
     });
   }
-  const extractor = await extractorPromise;
+  return extractorPromise;
+}
+async function embedTexts(texts, modelName = DEFAULT_MODEL) {
+  if (!isAvailable()) {
+    throw new Error(
+      `Embedding backend not available: ${runtimeLoadError()?.message ?? "unknown error"}`
+    );
+  }
+  if (texts.length === 0) {
+    return [];
+  }
+  const extractor = await getExtractor(modelName);
   const vecs = [];
   const expectedDim = DEFAULT_DIM;
   try {
@@ -840,17 +854,29 @@ function packVec(vec) {
   }
   return Buffer.from(view.buffer);
 }
-function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize, overlap, kind) {
+function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize, overlap, kind, countTokens) {
   const chunks = [];
+  const lineTokenCache = /* @__PURE__ */ new Map();
+  const tokensOf = (lineNo) => {
+    if (countTokens === void 0) return 0;
+    let n = lineTokenCache.get(lineNo);
+    if (n === void 0) {
+      n = countTokens(lines2[lineNo - 1] ?? "");
+      lineTokenCache.set(lineNo, n);
+    }
+    return n;
+  };
   let currentChunk = "";
+  let currentTokens = 0;
   let startLine = rangeStart;
   let currentLine = rangeStart;
   for (let lineNo = rangeStart; lineNo <= rangeEnd; lineNo++) {
     const line = lines2[lineNo - 1] ?? "";
     const lineWithNewline = line + "\n";
-    if (currentChunk.length + lineWithNewline.length > chunkSize && currentChunk.length > 0) {
+    const lineTokens = tokensOf(lineNo);
+    if ((currentChunk.length + lineWithNewline.length > chunkSize || currentTokens + lineTokens > MAX_CHUNK_TOKENS) && currentChunk.length > 0) {
       const trimmedLength = currentChunk.trim().length;
-      const currentChunkTooSmall = trimmedLength < MIN_CHUNK_CHARS;
+      const currentChunkTooSmall = trimmedLength === 0 || trimmedLength < MIN_CHUNK_CHARS && countTokens === void 0;
       if (!currentChunkTooSmall) {
         chunks.push({
           filePath,
@@ -861,11 +887,15 @@ function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize,
         });
       }
       let overlapChars = 0;
+      let overlapTokens = 0;
       let computedOverlapStart = currentLine;
       while (computedOverlapStart > rangeStart) {
         const candidateChars = (lines2[computedOverlapStart - 2] ?? "").length + 1;
         if (overlapChars + candidateChars > overlap) break;
+        const candidateTokens = tokensOf(computedOverlapStart - 1);
+        if (overlapTokens + candidateTokens + lineTokens > MAX_CHUNK_TOKENS) break;
         overlapChars += candidateChars;
+        overlapTokens += candidateTokens;
         computedOverlapStart--;
       }
       const droppedChunkHadContent = currentChunkTooSmall && trimmedLength > 0;
@@ -873,10 +903,24 @@ function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize,
       if (overlapStart !== startLine) {
         const overlapText = lines2.slice(overlapStart - 1, currentLine - 1).join("\n");
         currentChunk = overlapText + "\n";
+        currentTokens = overlapTokens;
         startLine = overlapStart;
       }
     }
+    if (countTokens !== void 0 && (lineTokens > MAX_CHUNK_TOKENS || lineWithNewline.length > chunkSize)) {
+      const pieces = splitToFit(line, (piece) => piece.length + 1 <= chunkSize && countTokens(piece) <= MAX_CHUNK_TOKENS);
+      const last = pieces.pop() ?? "";
+      for (const piece of pieces) {
+        if (piece.trim().length > 0) chunks.push({ filePath, startLine: lineNo, endLine: lineNo, text: piece.trim(), kind });
+      }
+      currentChunk = last + "\n";
+      currentTokens = countTokens(last);
+      startLine = lineNo;
+      currentLine++;
+      continue;
+    }
     currentChunk += lineWithNewline;
+    currentTokens += lineTokens;
     currentLine++;
   }
   if (currentChunk.trim().length >= MIN_CHUNK_CHARS) {
@@ -889,17 +933,42 @@ function splitRangeIntoChunks(filePath, lines2, rangeStart, rangeEnd, chunkSize,
     });
   } else if (currentChunk.length > 0 && chunks.length > 0) {
     const last = chunks[chunks.length - 1];
-    last.endLine = rangeEnd;
-    last.text = lines2.slice(last.startLine - 1, rangeEnd).join("\n").trim();
+    let mergedTokens = 0;
+    for (let lineNo = last.startLine; lineNo <= rangeEnd; lineNo++) mergedTokens += tokensOf(lineNo);
+    if (mergedTokens <= MAX_CHUNK_TOKENS) {
+      last.endLine = rangeEnd;
+      last.text = lines2.slice(last.startLine - 1, rangeEnd).join("\n").trim();
+    } else if (currentChunk.trim().length > 0) {
+      chunks.push({ filePath, startLine, endLine: rangeEnd, text: currentChunk.trim(), kind });
+    }
   }
   return chunks;
 }
-function chunkFile(filePath, content, chunkSize = MAX_CHUNK_CHARS, overlap = 200, boundaries = []) {
+function splitToFit(text, fits) {
+  if (text.length < 2 || fits(text)) return [text];
+  const mid = Math.floor(text.length / 2);
+  let cut = mid;
+  for (let d = 0; d <= Math.floor(text.length / 4); d++) {
+    if (/\s/.test(text[mid - d] ?? "")) {
+      cut = mid - d;
+      break;
+    }
+    if (/\s/.test(text[mid + d] ?? "")) {
+      cut = mid + d;
+      break;
+    }
+  }
+  const before = text.charCodeAt(cut - 1);
+  if (before >= 55296 && before <= 56319) cut++;
+  if (cut <= 0 || cut >= text.length) return [text];
+  return [...splitToFit(text.slice(0, cut), fits), ...splitToFit(text.slice(cut), fits)];
+}
+function chunkFile(filePath, content, chunkSize = MAX_CHUNK_CHARS, overlap = 200, boundaries = [], countTokens) {
   const lines2 = content.split(/\r?\n/);
   if (lines2.length > 1 && lines2[lines2.length - 1] === "") lines2.pop();
   const totalLines = lines2.length;
   if (boundaries.length === 0) {
-    return splitRangeIntoChunks(filePath, lines2, 1, totalLines, chunkSize, overlap, "window");
+    return splitRangeIntoChunks(filePath, lines2, 1, totalLines, chunkSize, overlap, "window", countTokens);
   }
   const clipped = boundaries.map((b) => ({
     start: Math.max(1, Math.min(b.start, totalLines)),
@@ -907,7 +976,7 @@ function chunkFile(filePath, content, chunkSize = MAX_CHUNK_CHARS, overlap = 200
     kind: b.kind
   })).filter((b) => b.end >= b.start).sort((a, b) => a.start - b.start || b.end - a.end);
   if (clipped.length === 0) {
-    return splitRangeIntoChunks(filePath, lines2, 1, totalLines, chunkSize, overlap, "window");
+    return splitRangeIntoChunks(filePath, lines2, 1, totalLines, chunkSize, overlap, "window", countTokens);
   }
   const flattened = [];
   let openEnd = 0;
@@ -974,7 +1043,7 @@ function chunkFile(filePath, content, chunkSize = MAX_CHUNK_CHARS, overlap = 200
   }
   const chunks = [];
   for (const r of ranges) {
-    chunks.push(...splitRangeIntoChunks(filePath, lines2, r.start, r.end, chunkSize, overlap, r.kind));
+    chunks.push(...splitRangeIntoChunks(filePath, lines2, r.start, r.end, chunkSize, overlap, r.kind, countTokens));
   }
   return chunks;
 }
@@ -1225,8 +1294,12 @@ function mergeNearbyHits(hits, proximity = 20) {
   return merged;
 }
 async function indexFile(db, filePath, content, boundaries = []) {
-  const chunks = chunkFile(filePath, content, void 0, void 0, boundaries);
+  let chunks = chunkFile(filePath, content, void 0, void 0, boundaries);
   if (chunks.length > 0) {
+    if (embeddingsDepsAvailable(db)) {
+      const { countTokens } = await getExtractor(DEFAULT_MODEL);
+      if (countTokens !== void 0) chunks = chunkFile(filePath, content, void 0, void 0, boundaries, countTokens);
+    }
     return upsertChunks(db, chunks);
   }
   deleteFileEmbeddings(db, filePath);
@@ -1260,11 +1333,11 @@ function deleteFileEmbeddings(db, filePath) {
   }
   db.prepare(`DELETE FROM chunks WHERE ${pathEqClause("file_path")}`).run(folded);
 }
-function resetAllEmbeddings(db) {
+function resetAllEmbeddings(db, keepVectors = false) {
   const paths = db.prepare("SELECT DISTINCT file_path FROM chunks").pluck().all();
   const clearEmbedSha = db.prepare(`UPDATE files SET embed_sha = NULL WHERE ${pathEqClause("path")}`);
   const tx = db.transaction(() => {
-    for (const p of paths) deleteFileEmbeddings(db, p);
+    if (!keepVectors) for (const p of paths) deleteFileEmbeddings(db, p);
     for (const p of paths) clearEmbedSha.run(foldPath(p));
   });
   tx.immediate();
@@ -1282,17 +1355,22 @@ function majorMinor(version) {
   return parts.length >= 2 ? `${parts[0]}.${parts[1]}` : version;
 }
 var _provenanceChecked = /* @__PURE__ */ new WeakSet();
+function vectorSpaceOf(provenance) {
+  const at = provenance.lastIndexOf("/embed-");
+  return at === -1 ? provenance : provenance.slice(0, at);
+}
 function ensureEmbeddingProvenance(db, modelName = DEFAULT_MODEL) {
   if (_provenanceChecked.has(db)) return;
   _provenanceChecked.add(db);
   const current = embeddingProvenance(modelName);
   const stored = db.prepare("SELECT provenance FROM embedding_provenance WHERE id = 1").pluck().get();
   if (stored === current) return;
-  const cleared = resetAllEmbeddings(db);
+  const keepVectors = stored !== void 0 && vectorSpaceOf(stored) === vectorSpaceOf(current);
+  const cleared = resetAllEmbeddings(db, keepVectors);
   db.prepare(
     "INSERT INTO embedding_provenance (id, provenance) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET provenance = excluded.provenance"
   ).run(current);
-  if (cleared > 0) {
+  if (cleared > 0 && !keepVectors) {
     console.warn(
       `Embedding stack changed (${stored ?? "unrecorded"} -> ${current}); discarded ${cleared} file${cleared === 1 ? "" : "s"} worth of vectors because they no longer reliably describe this file's current chunks. Run \`token-goat index\` to rebuild them.`
     );
@@ -1335,7 +1413,7 @@ function _pathPriorityPenalty(filePath) {
 
 // src/parser_fingerprint.ts
 init_define_import_meta_env();
-var PARSER_FINGERPRINT = "ef41adb465538db6";
+var PARSER_FINGERPRINT = "0d80572d3abdfb19";
 
 // src/parser.ts
 init_define_import_meta_env();
@@ -1374,7 +1452,7 @@ function* eachUnfencedLine(lines2) {
 }
 
 // src/hints/markdown_hints.ts
-var MARKDOWN_SIZE_THRESHOLD = 8e3;
+var MARKDOWN_SIZE_THRESHOLD = 4e3;
 var MAX_HEADINGS = 40;
 var MAX_OUTPUT_LINES = 60;
 function extractMarkdownHeadings(content, limit = MAX_HEADINGS) {
@@ -1918,6 +1996,28 @@ function extractTsJsSymbols(root, filePath, lines2) {
         out.push(makeSymbol(filePath, fieldName.text, "method", node, lines2, "c"));
       }
     }
+    if (node.type === "method_definition" && node.parent?.type === "object") {
+      const name = nodeName(node);
+      if (name !== null && name !== "") {
+        out.push(makeSymbol(filePath, name, "method", node, lines2, "c"));
+      }
+    }
+    if (node.type === "pair" && node.parent?.type === "object") {
+      const key = node.childForFieldName("key");
+      const value = node.childForFieldName("value");
+      if (key !== null && value !== null) {
+        const keyName = key.type === "identifier" || key.type === "property_identifier" || key.type === "string" ? key.text.replace(/^['"]|['"]$/g, "") : null;
+        if (keyName !== null && keyName !== "") {
+          const isFn = value.type === "arrow_function" || value.type === "function_expression" || value.type === "function";
+          const isObjectOrArray = value.type === "object" || value.type === "array";
+          if (isFn) {
+            out.push(makeSymbol(filePath, keyName, "method", node, lines2, "c"));
+          } else if (isObjectOrArray && node.endPosition.row - node.startPosition.row >= 1) {
+            out.push(makeSymbol(filePath, keyName, "config", node, lines2, "c"));
+          }
+        }
+      }
+    }
     const childInside = insideFunction || TSJS_FN_SCOPE_TYPES.has(node.type);
     for (const child of node.namedChildren) {
       visit(child, childInside);
@@ -3594,7 +3694,7 @@ function parseSourceSymbolsTreeSitterOnly(content, filePath, language) {
 var REGEX_ADAPTERS_SLOT = /* @__PURE__ */ Symbol.for("token-goat.regex-adapters");
 var adapterSlot = globalThis;
 async function loadRegexExtractors() {
-  adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-GIIHUSZX.mjs");
+  adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import("./token-goat-chunk-GPCCYQSS.mjs");
 }
 function regexAdapters() {
   const loaded = adapterSlot[REGEX_ADAPTERS_SLOT];
@@ -4166,7 +4266,7 @@ function tableSectionEndIndex(headers, headerPos, totalLines) {
   }
   return totalLines;
 }
-function resolveHeaderPos(headers, base, ordinal) {
+function resolveHeaderPos(headers, base, ordinal, allowFuzzy = true) {
   const target = base.toLowerCase();
   const normalizedTarget = normalizeHeading(base).toLowerCase();
   const strippedTarget = normalizeHeadingStrip(base).toLowerCase();
@@ -4227,10 +4327,106 @@ function resolveHeaderPos(headers, base, ordinal) {
     const allWordsMatch = queryWords.every((qw) => headingWords.some((hw) => hw.includes(qw)));
     if (allWordsMatch) widenedMatches.push(i);
   }
-  if (widenedMatches.length !== 1) return null;
-  const widenedPos = widenedMatches[0];
-  if (widenedPos === void 0) return null;
-  return { headerPos: widenedPos, redirectedFrom: base, occurrences: null };
+  if (widenedMatches.length === 1) {
+    const widenedPos = widenedMatches[0];
+    if (widenedPos !== void 0) {
+      return { headerPos: widenedPos, redirectedFrom: base, occurrences: null };
+    }
+  }
+  function sectionLevenshtein(a, b) {
+    if (a === b) return 0;
+    const la = a.length;
+    const lb = b.length;
+    if (la === 0) return lb;
+    if (lb === 0) return la;
+    const v0 = new Array(lb + 1);
+    const v1 = new Array(lb + 1);
+    for (let i = 0; i <= lb; i++) v0[i] = i;
+    for (let i = 0; i < la; i++) {
+      v1[0] = i + 1;
+      for (let j = 0; j < lb; j++) {
+        const cost = a.charCodeAt(i) === b.charCodeAt(j) ? 0 : 1;
+        v1[j + 1] = Math.min(v1[j] + 1, v0[j + 1] + 1, v0[j] + cost);
+      }
+      for (let j = 0; j <= lb; j++) v0[j] = v1[j];
+    }
+    return v0[lb];
+  }
+  const STOP_WORDS = /* @__PURE__ */ new Set(["a", "an", "the", "and", "or", "in", "on", "at", "to", "for", "of", "with", "by", "is", "&"]);
+  const cleanTokens = (str) => str.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 0);
+  const rawQTokens = cleanTokens(base);
+  const qTokens = rawQTokens.filter((t) => !STOP_WORDS.has(t) || rawQTokens.length <= 2);
+  if (allowFuzzy && qTokens.length > 0) {
+    const candidates = [];
+    for (let i = 0; i < headers.length; i++) {
+      const h = headers[i];
+      if (h === void 0) continue;
+      const rawHTokens = cleanTokens(h.heading);
+      const hTokens = rawHTokens.filter((t) => !STOP_WORDS.has(t) || rawHTokens.length <= 2);
+      if (hTokens.length === 0) continue;
+      const normH = normalizeHeading(h.heading).toLowerCase();
+      const maxLen = Math.max(normalizedTarget.length, normH.length);
+      const editDist = sectionLevenshtein(normalizedTarget, normH);
+      const strSim = maxLen > 0 ? 1 - editDist / maxLen : 0;
+      let matchedQ = 0;
+      for (const qw of qTokens) {
+        let best = 0;
+        for (const hw of hTokens) {
+          if (qw === hw) {
+            best = 1;
+            break;
+          }
+          if (qw.length >= 3 && hw.length >= 3) {
+            if (hw.startsWith(qw) || qw.startsWith(hw)) {
+              best = Math.max(best, 0.9);
+            } else if (sectionLevenshtein(qw, hw) <= 1) {
+              best = Math.max(best, 0.85);
+            }
+          }
+        }
+        matchedQ += best;
+      }
+      let matchedH = 0;
+      for (const hw of hTokens) {
+        let best = 0;
+        for (const qw of qTokens) {
+          if (qw === hw) {
+            best = 1;
+            break;
+          }
+          if (qw.length >= 3 && hw.length >= 3) {
+            if (hw.startsWith(qw) || qw.startsWith(hw)) {
+              best = Math.max(best, 0.9);
+            } else if (sectionLevenshtein(qw, hw) <= 1) {
+              best = Math.max(best, 0.85);
+            }
+          }
+        }
+        matchedH += best;
+      }
+      const wordRecall = matchedQ / qTokens.length;
+      const wordPrecision = matchedH / hTokens.length;
+      const tokenDice = 2 * matchedQ / (qTokens.length + hTokens.length);
+      let score = 0;
+      if (strSim >= 0.8) {
+        score = strSim;
+      } else if (tokenDice >= 0.75 && wordRecall >= 0.7 && wordPrecision >= 0.7) {
+        score = (tokenDice + wordRecall + wordPrecision) / 3;
+      }
+      if (score >= 0.75) {
+        candidates.push({ index: i, score });
+      }
+    }
+    if (candidates.length > 0) {
+      candidates.sort((a, b) => b.score - a.score);
+      const best = candidates[0];
+      const runnerUp = candidates[1];
+      if (candidates.length === 1 || runnerUp !== void 0 && best.score - runnerUp.score >= 0.08) {
+        return { headerPos: best.index, redirectedFrom: base, occurrences: null };
+      }
+    }
+  }
+  return null;
 }
 function buildSectionResult(headers, kind, lines2, headerPos, redirectedFrom) {
   const header = headers[headerPos];
@@ -4249,12 +4445,12 @@ function buildSectionResult(headers, kind, lines2, headerPos, redirectedFrom) {
   };
   return redirectedFrom === null ? result : { ...result, redirectedFrom };
 }
-function resolveSectionFromText(text, headingSpec, language) {
+function resolveSectionFromText(text, headingSpec, language, allowFuzzy = true) {
   const { headers, kind } = findHeaders(text, language);
   const { base, ordinal } = parseHeadingSpec(headingSpec, headers);
   if (base.length === 0) return null;
   const lines2 = text.split("\n");
-  const resolved = resolveHeaderPos(headers, base, ordinal);
+  const resolved = resolveHeaderPos(headers, base, ordinal, allowFuzzy);
   if (resolved === null) return null;
   const built = buildSectionResult(headers, kind, lines2, resolved.headerPos, resolved.redirectedFrom);
   if (built === null || resolved.occurrences === null) return built;
@@ -4281,10 +4477,10 @@ function readTextForSections(filePath, readFn) {
   }
   return redactIfDotenv(filePath, text);
 }
-function readSection(filePath, headingSpec, readFn) {
+function readSection(filePath, headingSpec, readFn, allowFuzzy = true) {
   const text = readTextForSections(filePath, readFn);
   if (text === null) return null;
-  return resolveSectionFromText(text, headingSpec, refineLanguageByContent(filePath, detectLanguage(filePath), text));
+  return resolveSectionFromText(text, headingSpec, refineLanguageByContent(filePath, detectLanguage(filePath), text), allowFuzzy);
 }
 function findContainingSection(filePath, lineStart, lineEnd, readFn) {
   const text = readTextForSections(filePath, readFn);
@@ -4334,6 +4530,8 @@ export {
   isEmbeddableDocument,
   ipynbToVirtualSource,
   eachUnfencedLine,
+  modelFilesPresent,
+  ensureModelFiles,
   checkEmbeddingPreflight,
   OVER_FETCH_FACTOR,
   MAX_OVER_FETCH,

package/dist/{token-goat-chunk-ATIFTMRC.mjs → token-goat-chunk-7EGKXBWT.mjs}

@@ -3,7 +3,7 @@ const require = __cjsRequire(import.meta.url);
 import {
   deliveredOutputBytes,
   wrappedShell
-} from "./token-goat-chunk-YZX7EFG4.mjs";
+} from "./token-goat-chunk-GLOM5W5C.mjs";
 import {
   ToolFilter,
   capTokens,
@@ -14,18 +14,18 @@ import {
   selectFilter,
   shlexSplit,
   storeBashOutputSync
-} from "./token-goat-chunk-YQ7WI2CO.mjs";
-import "./token-goat-chunk-2X2EBBC6.mjs";
+} from "./token-goat-chunk-C26ZBGBC.mjs";
 import {
   recordStat
-} from "./token-goat-chunk-IVUQLQWN.mjs";
-import "./token-goat-chunk-K7F2BFIK.mjs";
+} from "./token-goat-chunk-26YUO7MP.mjs";
+import "./token-goat-chunk-BK3RPPTK.mjs";
+import "./token-goat-chunk-MWSVUKVV.mjs";
 import {
   loadConfig
-} from "./token-goat-chunk-S4XRY446.mjs";
+} from "./token-goat-chunk-2VASS5VR.mjs";
 import "./token-goat-chunk-EEIDFMEM.mjs";
 import "./token-goat-chunk-OSUFN2FV.mjs";
-import "./token-goat-chunk-PM76YS22.mjs";
+import "./token-goat-chunk-UBZCON6N.mjs";
 import "./token-goat-chunk-ERTXEKB6.mjs";
 import "./token-goat-chunk-GMOUBOX4.mjs";
 import {