sensemaking 0.21.0 → 0.22.0

package/README.md

@@ -93,7 +93,7 @@ ORDER BY bm25(content, 10.0, 5.0, 1.0) LIMIT 10
 |---|---|
 | `presets` | named bundles of `include`/`exclude` globs, `k` (result count), `signals` (which engines this scope searches with, `words`, `links`, `vectors`; every signal whose prerequisites hold, unless the preset lists them exhaustively), `where` (a standing SQL filter). A file is indexed if any preset includes it, embedded if a model is named and some covering preset's `signals` include `vectors`; `status` shows each preset's coverage. |
 | `embed` | the model vectors are built with. Naming one gives the tree vectors; omitting the block means none at all, whatever the presets say. `sense download` fetches it. |
-| `store` | backing store engine: `sqlite` (default, zero-dependency, Node's built-in SQLite), or the experimental `duckdb` and `turso`. The first command that opens such a tree installs that engine's package on its own (`@duckdb/node-api`, a one-time native download of ~110 MB; `@tursodatabase/database`, much smaller). The same commands and table names run on all three; what does not port is FTS5 syntax. Under `duckdb` and `turso`, `search` text and raw `MATCH` reject FTS5's prefix (`foo*`), boolean (`AND`/`OR`/`NOT`), `NEAR`, initial-token (`^`), and column-filter (`title:foo`) operators with a named error that says how to rephrase or set `store` to `sqlite`; bare words and quoted phrases work on all three. Raw `sql` is a per-store dialect: the tables are portable, but sqlite's FTS5 `MATCH`/`snippet()`/`bm25()` do not run under `duckdb` or `turso`, so saved queries written in FTS5 syntax are sqlite dialect, and `has`/`basename`/`segment` are registered on `sqlite` and `duckdb` but not `turso`, whose client cannot register SQL functions. `sense watch` requires `sqlite`; on the other two it errors at start naming the fix. Each store keeps its own cache file (`.sense/cache.db`, `.sense/cache.duckdb`, `.sense/cache.turso.db`); switching stores is a rebuild, not a migration. |
+| `store` | backing store engine: `sqlite` (default, zero-dependency, Node's built-in SQLite), or the experimental `duckdb` and `turso`. The first command that opens such a tree installs that engine's package on its own (`@duckdb/node-api`, a one-time native download of ~110 MB; `@tursodatabase/database`, much smaller). The same commands and table names run on all three; what does not port is FTS5 syntax. Under `duckdb` and `turso`, `search` text and raw `MATCH` reject FTS5's prefix (`foo*`), boolean (`AND`/`OR`/`NOT`), `NEAR`, initial-token (`^`), and column-filter (`title:foo`) operators with a named error that says how to rephrase or set `store` to `sqlite`; bare words and quoted phrases work on all three. Raw `sql` is a per-store dialect: the tables are portable, but sqlite's FTS5 `MATCH`/`snippet()`/`bm25()` do not run under `duckdb` or `turso`, so saved queries written in FTS5 syntax are sqlite dialect, and `has`/`basename`/`segment` are registered on `sqlite` and `duckdb` but not `turso`, whose client cannot register SQL functions. `sense watch` runs on all three; on `duckdb` and `turso`, which lock the cache file per connection, a concurrent command waits out the watcher's current cycle instead of failing. Each store keeps its own cache file (`.sense/cache.db`, `.sense/cache.duckdb`, `.sense/cache.turso.db`); switching stores is a rebuild, not a migration. |
 | `queries` | entries runnable as `sense <name>`, each naming the verb it runs: `{ sql }` for SQL (`?` binds positional args) or `{ search }` for a ranked search with its settings baked in, so `sense hot` needs no flags. Running an entry validates it: a typo'd column errors and exits nonzero, and a parameterised entry validates with any argument, since preparing precedes binding. |
 | `version` | schema version; older configs auto-migrate on load, noted on stderr. |
 

package/dist/cjs/chunk/group.d.cts

@@ -1,4 +1,2 @@
 import type { Block, Chunk, ChunkOptions } from './types.js';
-export declare const DEFAULT_TARGET_TOKENS = 500;
-export declare function estimateTokens(text: string): number;
 export declare function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[];

package/dist/cjs/chunk/group.d.ts

@@ -1,4 +1,2 @@
 import type { Block, Chunk, ChunkOptions } from './types.js';
-export declare const DEFAULT_TARGET_TOKENS = 500;
-export declare function estimateTokens(text: string): number;
 export declare function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[];

package/dist/cjs/chunk/group.js

@@ -2,26 +2,15 @@
 Object.defineProperty(exports, "__esModule", {
     value: true
 });
-function _export(target, all) {
-    for(var name in all)Object.defineProperty(target, name, {
-        enumerable: true,
-        get: Object.getOwnPropertyDescriptor(all, name).get
-    });
-}
-_export(exports, {
-    get DEFAULT_TARGET_TOKENS () {
-        return DEFAULT_TARGET_TOKENS;
-    },
-    get estimateTokens () {
-        return estimateTokens;
-    },
-    get group () {
+Object.defineProperty(exports, "group", {
+    enumerable: true,
+    get: function() {
         return group;
     }
 });
-var _segmentts = require("../text/segment.js");
 var _extractts = require("./extract.js");
 var _parsets = require("./parse.js");
+var _tokensts = require("./tokens.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -98,42 +87,12 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Map" || n === "Set") return Array.from(n);
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
-var DEFAULT_TARGET_TOKENS = 500;
 var PGC_GROUP_SIZE = 2;
 var OVERSIZE_TRIGGER_MULTIPLE = 2;
-// D5: segment.ts's unspaced-script set packs close to one token per character; Hangul is
-// spaced but still token-dense and sits in this set as a conservative bound (one Unicode set, one home).
-var DENSE_SCRIPT = new RegExp("[".concat(_segmentts.UNSPACED_SCRIPTS, "\\p{scx=Hangul}]"), 'u');
-function estimateTokens(text) {
-    var dense = 0;
-    var other = 0;
-    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-    try {
-        for(var _iterator = text[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-            var ch = _step.value;
-            if (DENSE_SCRIPT.test(ch)) dense++;
-            else other++;
-        }
-    } catch (err) {
-        _didIteratorError = true;
-        _iteratorError = err;
-    } finally{
-        try {
-            if (!_iteratorNormalCompletion && _iterator.return != null) {
-                _iterator.return();
-            }
-        } finally{
-            if (_didIteratorError) {
-                throw _iteratorError;
-            }
-        }
-    }
-    return dense + other / 4;
-}
 function resolveOptions(opts) {
     var _ref, _ref1;
     return {
-        targetTokens: (_ref = opts === null || opts === void 0 ? void 0 : opts.targetTokens) !== null && _ref !== void 0 ? _ref : DEFAULT_TARGET_TOKENS,
+        targetTokens: (_ref = opts === null || opts === void 0 ? void 0 : opts.targetTokens) !== null && _ref !== void 0 ? _ref : _tokensts.DEFAULT_TARGET_TOKENS,
         text: (_ref1 = opts === null || opts === void 0 ? void 0 : opts.text) !== null && _ref1 !== void 0 ? _ref1 : 'raw'
     };
 }
@@ -184,14 +143,18 @@ function finalize(parts) {
         })
     };
 }
-var NEWLINE_TOKENS = estimateTokens('\n');
-var SENTENCE_SEGMENTER = new Intl.Segmenter(undefined, {
-    granularity: 'sentence'
-});
-var WORD_SEGMENTER = new Intl.Segmenter(undefined, {
-    granularity: 'word'
-});
-function segmentsOf(text, segmenter) {
+var NEWLINE_TOKENS = (0, _tokensts.estimateTokens)('\n');
+// Built on first use and kept: each construction is ~3.5 ms, and only an oversize block is ever
+// split, so no command pays for a segmenter it never reaches.
+var SEGMENTERS = new Map();
+function segmentsOf(text, granularity) {
+    var segmenter = SEGMENTERS.get(granularity);
+    if (!segmenter) {
+        segmenter = new Intl.Segmenter(undefined, {
+            granularity: granularity
+        });
+        SEGMENTERS.set(granularity, segmenter);
+    }
     return Array.from(segmenter.segment(text), function(s) {
         return s.segment;
     });
@@ -206,7 +169,7 @@ function pack(segments, working) {
     try {
         for(var _iterator = segments[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
             var segment = _step.value;
-            var segmentTokens = estimateTokens(segment);
+            var segmentTokens = (0, _tokensts.estimateTokens)(segment);
             if (current.length > 0 && tokens + segmentTokens > working) {
                 groups.push(current);
                 current = '';
@@ -235,7 +198,7 @@ function pack(segments, working) {
 // Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then
 // word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.
 function splitLineText(text, working) {
-    var sentences = segmentsOf(text, SENTENCE_SEGMENTER);
+    var sentences = segmentsOf(text, 'sentence');
     var out = [];
     var current = '';
     var tokens = 0;
@@ -250,11 +213,11 @@ function splitLineText(text, working) {
     try {
         for(var _iterator = sentences[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
             var sentence = _step.value;
-            var sentenceTokens = estimateTokens(sentence);
+            var sentenceTokens = (0, _tokensts.estimateTokens)(sentence);
             if (sentenceTokens > working) {
                 var _out;
                 flush();
-                (_out = out).push.apply(_out, _to_consumable_array(pack(segmentsOf(sentence, WORD_SEGMENTER), working)));
+                (_out = out).push.apply(_out, _to_consumable_array(pack(segmentsOf(sentence, 'word'), working)));
                 continue;
             }
             if (current.length > 0 && tokens + sentenceTokens > working) flush();
@@ -299,7 +262,7 @@ function piece(pieceLines, startLine, endLine, blockType, textMode) {
 // splits at sentence/word boundaries instead; `final` stops group() re-deriving its text by extent (F5).
 function finalizePiece(pieceLines, startLine, endLine, working, blockType, textMode) {
     var p = piece(pieceLines, startLine, endLine, blockType, textMode);
-    if (pieceLines.length === 1 && estimateTokens(p.text) > working) {
+    if (pieceLines.length === 1 && (0, _tokensts.estimateTokens)(p.text) > working) {
         return splitLineText(p.text, working).map(function(text) {
             return {
                 startLine: startLine,
@@ -325,7 +288,7 @@ function splitOversizeBlock(lines, startLine, endLine, working, blockType, textM
     for(var line = startLine; line <= endLine; line++){
         var lineText = lines[line - 1];
         var sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;
-        var lineTokens = estimateTokens(lineText);
+        var lineTokens = (0, _tokensts.estimateTokens)(lineText);
         if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {
             var _pieces1;
             (_pieces1 = pieces).push.apply(_pieces1, _to_consumable_array(finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode)));
@@ -361,7 +324,7 @@ function groupScope(scopeBlocks, lines, resolved) {
     // tokens tracks the active text mode's own estimate (a newline between parts costs
     // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.
     function addPart(text, sizeText, startLine, endLine) {
-        tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);
+        tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + (0, _tokensts.estimateTokens)(sizeText);
         parts.push({
             startLine: startLine,
             endLine: endLine,
@@ -373,7 +336,7 @@ function groupScope(scopeBlocks, lines, resolved) {
         for(var _iterator = scopeBlocks[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
             var block = _step.value;
             var raw = lines.slice(block.startLine - 1, block.endLine).join('\n');
-            var blockTokens = estimateTokens(raw);
+            var blockTokens = (0, _tokensts.estimateTokens)(raw);
             if (blockTokens > trigger) {
                 var seed = parts.length > 0 ? tokens : 0;
                 var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;

package/dist/cjs/chunk/group.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/group.ts"],"sourcesContent":["import { UNSPACED_SCRIPTS } from '../text/segment.ts';\nimport { extractText } from './extract.ts';\nimport { parse } from './parse.ts';\nimport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\n\nexport const DEFAULT_TARGET_TOKENS = 500;\nconst PGC_GROUP_SIZE = 2;\nconst OVERSIZE_TRIGGER_MULTIPLE = 2;\n\n// D5: segment.ts's unspaced-script set packs close to one token per character; Hangul is\n// spaced but still token-dense and sits in this set as a conservative bound (one Unicode set, one home).\nconst DENSE_SCRIPT = new RegExp(`[${UNSPACED_SCRIPTS}\\\\p{scx=Hangul}]`, 'u');\n\n// D5's size estimate: dense-script graphemes 1:1, everything else at 4 chars/token.\nexport function estimateTokens(text: string): number {\n  let dense = 0;\n  let other = 0;\n  for (const ch of text) {\n    if (DENSE_SCRIPT.test(ch)) dense++;\n    else other++;\n  }\n  return dense + other / 4;\n}\n\ninterface ResolvedOptions {\n  targetTokens: number;\n  text: 'extracted' | 'raw';\n}\n\nfunction resolveOptions(opts?: ChunkOptions): ResolvedOptions {\n  return {\n    targetTokens: opts?.targetTokens ?? DEFAULT_TARGET_TOKENS,\n    text: opts?.text ?? 'raw',\n  };\n}\n\n// A heading of any depth ends the current scope and starts a new one (D1); the heading block\n// itself is carried into the new scope, where it joins that scope's first group (F7/F10).\nfunction splitScopes(blocks: Block[]): Block[][] {\n  const scopes: Block[][] = [];\n  let current: Block[] = [];\n  for (const block of blocks) {\n    if (block.type === 'heading' && current.length > 0) {\n      scopes.push(current);\n      current = [];\n    }\n    current.push(block);\n  }\n  if (current.length > 0) scopes.push(current);\n  return scopes;\n}\n\ninterface Part {\n  startLine: number;\n  endLine: number;\n  text: string;\n  // True for a sub-line split piece: its text is already the final slice, not the whole line --\n  // group()'s extent-based raw re-slice must not touch it (siblings share startLine === endLine).\n  final?: boolean;\n}\n\nfunction finalize(parts: Part[]): (Chunk & { final?: boolean }) | undefined {\n  if (parts.length === 0) return undefined;\n  const first = parts[0];\n  const last = parts[parts.length - 1];\n  return { startLine: first.startLine, endLine: last.endLine, text: parts.map((p) => p.text).join('\\n'), final: parts.some((p) => p.final) };\n}\n\nconst NEWLINE_TOKENS = estimateTokens('\\n');\nconst SENTENCE_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'sentence' });\nconst WORD_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'word' });\n\nfunction segmentsOf(text: string, segmenter: Intl.Segmenter): string[] {\n  return Array.from(segmenter.segment(text), (s) => s.segment);\n}\n\n// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups\n// of at most `working` estimated tokens; a lone segment over `working` still stands alone.\nfunction pack(segments: string[], working: number): string[] {\n  const groups: string[] = [];\n  let current = '';\n  let tokens = 0;\n  for (const segment of segments) {\n    const segmentTokens = estimateTokens(segment);\n    if (current.length > 0 && tokens + segmentTokens > working) {\n      groups.push(current);\n      current = '';\n      tokens = 0;\n    }\n    current += segment;\n    tokens += segmentTokens;\n  }\n  if (current.length > 0) groups.push(current);\n  return groups;\n}\n\n// Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then\n// word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.\nfunction splitLineText(text: string, working: number): string[] {\n  const sentences = segmentsOf(text, SENTENCE_SEGMENTER);\n  const out: string[] = [];\n  let current = '';\n  let tokens = 0;\n  const flush = () => {\n    if (current.length > 0) {\n      out.push(current);\n      current = '';\n      tokens = 0;\n    }\n  };\n  for (const sentence of sentences) {\n    const sentenceTokens = estimateTokens(sentence);\n    if (sentenceTokens > working) {\n      flush();\n      out.push(...pack(segmentsOf(sentence, WORD_SEGMENTER), working));\n      continue;\n    }\n    if (current.length > 0 && tokens + sentenceTokens > working) flush();\n    current += sentence;\n    tokens += sentenceTokens;\n  }\n  flush();\n  return out;\n}\n\nconst ATOMIC_TYPES: ReadonlySet<BlockType> = new Set(['code', 'table', 'list']);\n\n// Atomic (code/table/list) pieces are always a raw line slice: re-parsing a table's later pieces\n// without their header/delimiter rows would demote them to paragraph text.\nfunction piece(pieceLines: string[], startLine: number, endLine: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part {\n  const text =\n    ATOMIC_TYPES.has(blockType) || textMode === 'raw'\n      ? pieceLines.join('\\n')\n      : parse(pieceLines.join('\\n'))\n          .map((b) => extractText(b.node))\n          .join('\\n');\n  return { startLine, endLine, text };\n}\n\n// A one-line piece over working can't shrink via another line-boundary pass (rule 5's gap), so it\n// splits at sentence/word boundaries instead; `final` stops group() re-deriving its text by extent (F5).\nfunction finalizePiece(pieceLines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part[] {\n  const p = piece(pieceLines, startLine, endLine, blockType, textMode);\n  if (pieceLines.length === 1 && estimateTokens(p.text) > working) {\n    return splitLineText(p.text, working).map((text) => ({ startLine, endLine, text, final: true }));\n  }\n  return [p];\n}\n\n// A block over 2x working size splits at line boundaries into pieces each <= working size, never\n// mid-line. `seed`: pending tokens (e.g. a heading) the first piece must join, checked against the limit.\nfunction splitOversizeBlock(lines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw', seed = 0): Part[] {\n  const pieces: Part[] = [];\n  let pieceLines: string[] = [];\n  let pieceStart = startLine;\n  let tokens = seed;\n  for (let line = startLine; line <= endLine; line++) {\n    const lineText = lines[line - 1];\n    const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;\n    const lineTokens = estimateTokens(lineText);\n    if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {\n      pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));\n      pieceLines = [];\n      tokens = 0;\n      pieceStart = line;\n      pieceLines.push(lineText);\n      tokens += lineTokens;\n      continue;\n    }\n    pieceLines.push(lineText);\n    tokens += sep + lineTokens;\n  }\n  if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));\n  return pieces;\n}\n\n// One heading scope's groups (D1): a heading opens the first group, and an oversize block\n// (rule 5, including an oversize heading) splits into pieces that each close their own group.\nfunction groupScope(scopeBlocks: Block[], lines: string[], resolved: ResolvedOptions): (Chunk & { final?: boolean })[] {\n  const working = resolved.targetTokens;\n  const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;\n  const finished: (Chunk & { final?: boolean })[] = [];\n  let parts: Part[] = [];\n  let paragraphCount = 0;\n  let tokens = 0;\n\n  function close(): void {\n    const group = finalize(parts);\n    if (group) finished.push(group);\n    parts = [];\n    paragraphCount = 0;\n    tokens = 0;\n  }\n\n  // tokens tracks the active text mode's own estimate (a newline between parts costs\n  // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.\n  function addPart(text: string, sizeText: string, startLine: number, endLine: number): void {\n    tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);\n    parts.push({ startLine, endLine, text });\n  }\n\n  for (const block of scopeBlocks) {\n    const raw = lines.slice(block.startLine - 1, block.endLine).join('\\n');\n    const blockTokens = estimateTokens(raw);\n\n    if (blockTokens > trigger) {\n      const seed = parts.length > 0 ? tokens : 0;\n      for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)) {\n        parts.push(p);\n        close();\n      }\n      continue;\n    }\n\n    const extracted = extractText(block.node);\n    const sizeText = resolved.text === 'raw' ? raw : extracted;\n\n    if (block.type === 'heading') {\n      addPart(extracted, sizeText, block.startLine, block.endLine);\n      continue;\n    }\n\n    // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --\n    // close first if the pair about to form would cross it.\n    const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;\n    if (pairOversize) close();\n\n    addPart(extracted, sizeText, block.startLine, block.endLine);\n    paragraphCount++;\n\n    if (paragraphCount >= PGC_GROUP_SIZE) close();\n  }\n  close();\n\n  return finished;\n}\n\n// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed\n// from (line lookups for oversize splitting).\nexport function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  const resolved = resolveOptions(opts);\n  const lines = body.split('\\n');\n  const chunks: (Chunk & { final?: boolean })[] = [];\n  for (const scope of splitScopes(blocks)) chunks.push(...groupScope(scope, lines, resolved));\n  // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).\n  // A `final` chunk already carries its own slice's raw text; re-slicing by extent would return the whole shared line.\n  const texted =\n    resolved.text === 'raw'\n      ? chunks.map((c) =>\n          c.final\n            ? c\n            : {\n                ...c,\n                text: lines\n                  .slice(c.startLine - 1, c.endLine)\n                  .join('\\n')\n                  .trim(),\n              }\n        )\n      : chunks;\n  // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.\n  return texted.filter((c) => c.text.trim().length > 0).map((c) => ({ startLine: c.startLine, endLine: c.endLine, text: c.text }));\n}\n"],"names":["DEFAULT_TARGET_TOKENS","estimateTokens","group","PGC_GROUP_SIZE","OVERSIZE_TRIGGER_MULTIPLE","DENSE_SCRIPT","RegExp","UNSPACED_SCRIPTS","text","dense","other","ch","test","resolveOptions","opts","targetTokens","splitScopes","blocks","scopes","current","block","type","length","push","finalize","parts","undefined","first","last","startLine","endLine","map","p","join","final","some","NEWLINE_TOKENS","SENTENCE_SEGMENTER","Intl","Segmenter","granularity","WORD_SEGMENTER","segmentsOf","segmenter","Array","from","segment","s","pack","segments","working","groups","tokens","segmentTokens","splitLineText","sentences","out","flush","sentence","sentenceTokens","ATOMIC_TYPES","Set","piece","pieceLines","blockType","textMode","has","parse","b","extractText","node","finalizePiece","splitOversizeBlock","lines","seed","pieces","pieceStart","line","lineText","sep","lineTokens","groupScope","scopeBlocks","resolved","trigger","finished","paragraphCount","close","addPart","sizeText","raw","slice","blockTokens","extracted","pairOversize","body","chunks","split","scope","texted","c","trim","filter"],"mappings":";;;;;;;;;;;QAKaA;eAAAA;;QASGC;eAAAA;;QAiOAC;eAAAA;;;yBA/OiB;yBACL;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGf,IAAMF,wBAAwB;AACrC,IAAMG,iBAAiB;AACvB,IAAMC,4BAA4B;AAElC,yFAAyF;AACzF,yGAAyG;AACzG,IAAMC,eAAe,IAAIC,OAAO,AAAC,IAAoB,OAAjBC,2BAAgB,EAAC,qBAAmB;AAGjE,SAASN,eAAeO,IAAY;IACzC,IAAIC,QAAQ;IACZ,IAAIC,QAAQ;QACP,kCAAA,2BAAA;;QAAL,QAAK,YAAYF,yBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAkB;YAAlB,IAAMG,KAAN;YACH,IAAIN,aAAaO,IAAI,CAACD,KAAKF;iBACtBC;QACP;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAOD,QAAQC,QAAQ;AACzB;AAOA,SAASG,eAAeC,IAAmB;;IACzC,OAAO;QACLC,YAAY,UAAED,iBAAAA,2BAAAA,KAAMC,YAAY,uCAAIf;QACpCQ,IAAI,WAAEM,iBAAAA,2BAAAA,KAAMN,IAAI,yCAAI;IACtB;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASQ,YAAYC,MAAe;IAClC,IAAMC,SAAoB,EAAE;IAC5B,IAAIC,UAAmB,EAAE;QACpB,kCAAA,2BAAA;;QAAL,QAAK,YAAeF,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;YAAvB,IAAMG,QAAN;YACH,IAAIA,MAAMC,IAAI,KAAK,aAAaF,QAAQG,MAAM,GAAG,GAAG;gBAClDJ,OAAOK,IAAI,CAACJ;gBACZA,UAAU,EAAE;YACd;YACAA,QAAQI,IAAI,CAACH;QACf;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAOL,IAAID,QAAQG,MAAM,GAAG,GAAGJ,OAAOK,IAAI,CAACJ;IACpC,OAAOD;AACT;AAWA,SAASM,SAASC,KAAa;IAC7B,IAAIA,MAAMH,MAAM,KAAK,GAAG,OAAOI;IAC/B,IAAMC,QAAQF,KAAK,CAAC,EAAE;IACtB,IAAMG,OAAOH,KAAK,CAACA,MAAMH,MAAM,GAAG,EAAE;IACpC,OAAO;QAAEO,WAAWF,MAAME,SAAS;QAAEC,SAASF,KAAKE,OAAO;QAAEtB,MAAMiB,MAAMM,GAAG,CAAC,SAACC;mBAAMA,EAAExB,IAAI;WAAEyB,IAAI,CAAC;QAAOC,OAAOT,MAAMU,IAAI,CAAC,SAACH;mBAAMA,EAAEE,KAAK;;IAAE;AAC3I;AAEA,IAAME,iBAAiBnC,eAAe;AACtC,IAAMoC,qBAAqB,IAAIC,KAAKC,SAAS,CAACb,WAAW;IAAEc,aAAa;AAAW;AACnF,IAAMC,iBAAiB,IAAIH,KAAKC,SAAS,CAACb,WAAW;IAAEc,aAAa;AAAO;AAE3E,SAASE,WAAWlC,IAAY,EAAEmC,SAAyB;IACzD,OAAOC,MAAMC,IAAI,CAACF,UAAUG,OAAO,CAACtC,OAAO,SAACuC;eAAMA,EAAED,OAAO;;AAC7D;AAEA,gGAAgG;AAChG,2FAA2F;AAC3F,SAASE,KAAKC,QAAkB,EAAEC,OAAe;IAC/C,IAAMC,SAAmB,EAAE;IAC3B,IAAIhC,UAAU;IACd,IAAIiC,SAAS;QACR,kCAAA,2BAAA;;QAAL,QAAK,YAAiBH,6BAAjB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMH,UAAN;YACH,IAAMO,gBAAgBpD,eAAe6C;YACrC,IAAI3B,QAAQG,MAAM,GAAG,KAAK8B,SAASC,gBAAgBH,SAAS;gBAC1DC,OAAO5B,IAAI,CAACJ;gBACZA,UAAU;gBACViC,SAAS;YACX;YACAjC,WAAW2B;YACXM,UAAUC;QACZ;;QATK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,IAAIlC,QAAQG,MAAM,GAAG,GAAG6B,OAAO5B,IAAI,CAACJ;IACpC,OAAOgC;AACT;AAEA,8FAA8F;AAC9F,0GAA0G;AAC1G,SAASG,cAAc9C,IAAY,EAAE0C,OAAe;IAClD,IAAMK,YAAYb,WAAWlC,MAAM6B;IACnC,IAAMmB,MAAgB,EAAE;IACxB,IAAIrC,UAAU;IACd,IAAIiC,SAAS;IACb,IAAMK,QAAQ;QACZ,IAAItC,QAAQG,MAAM,GAAG,GAAG;YACtBkC,IAAIjC,IAAI,CAACJ;YACTA,UAAU;YACViC,SAAS;QACX;IACF;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAkBG,8BAAlB,SAAA,6BAAA,QAAA,yBAAA,iCAA6B;YAA7B,IAAMG,WAAN;YACH,IAAMC,iBAAiB1D,eAAeyD;YACtC,IAAIC,iBAAiBT,SAAS;oBAE5BM;gBADAC;gBACAD,CAAAA,OAAAA,KAAIjC,IAAI,OAARiC,MAAS,qBAAGR,KAAKN,WAAWgB,UAAUjB,iBAAiBS;gBACvD;YACF;YACA,IAAI/B,QAAQG,MAAM,GAAG,KAAK8B,SAASO,iBAAiBT,SAASO;YAC7DtC,WAAWuC;YACXN,UAAUO;QACZ;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAWLF;IACA,OAAOD;AACT;AAEA,IAAMI,eAAuC,IAAIC,IAAI;IAAC;IAAQ;IAAS;CAAO;AAE9E,iGAAiG;AACjG,2EAA2E;AAC3E,SAASC,MAAMC,UAAoB,EAAElC,SAAiB,EAAEC,OAAe,EAAEkC,SAAoB,EAAEC,QAA6B;IAC1H,IAAMzD,OACJoD,aAAaM,GAAG,CAACF,cAAcC,aAAa,QACxCF,WAAW9B,IAAI,CAAC,QAChBkC,IAAAA,cAAK,EAACJ,WAAW9B,IAAI,CAAC,OACnBF,GAAG,CAAC,SAACqC;eAAMC,IAAAA,sBAAW,EAACD,EAAEE,IAAI;OAC7BrC,IAAI,CAAC;IACd,OAAO;QAAEJ,WAAAA;QAAWC,SAAAA;QAAStB,MAAAA;IAAK;AACpC;AAEA,kGAAkG;AAClG,yGAAyG;AACzG,SAAS+D,cAAcR,UAAoB,EAAElC,SAAiB,EAAEC,OAAe,EAAEoB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;IACnJ,IAAMjC,IAAI8B,MAAMC,YAAYlC,WAAWC,SAASkC,WAAWC;IAC3D,IAAIF,WAAWzC,MAAM,KAAK,KAAKrB,eAAe+B,EAAExB,IAAI,IAAI0C,SAAS;QAC/D,OAAOI,cAActB,EAAExB,IAAI,EAAE0C,SAASnB,GAAG,CAAC,SAACvB;mBAAU;gBAAEqB,WAAAA;gBAAWC,SAAAA;gBAAStB,MAAAA;gBAAM0B,OAAO;YAAK;;IAC/F;IACA,OAAO;QAACF;KAAE;AACZ;AAEA,iGAAiG;AACjG,0GAA0G;AAC1G,SAASwC,mBAAmBC,KAAe,EAAE5C,SAAiB,EAAEC,OAAe,EAAEoB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;QAAES,OAAAA,iEAAO;QAqBjIC;IApB3B,IAAMA,SAAiB,EAAE;IACzB,IAAIZ,aAAuB,EAAE;IAC7B,IAAIa,aAAa/C;IACjB,IAAIuB,SAASsB;IACb,IAAK,IAAIG,OAAOhD,WAAWgD,QAAQ/C,SAAS+C,OAAQ;QAClD,IAAMC,WAAWL,KAAK,CAACI,OAAO,EAAE;QAChC,IAAME,MAAMhB,WAAWzC,MAAM,GAAG,KAAK8B,SAAS,IAAIhB,iBAAiB;QACnE,IAAM4C,aAAa/E,eAAe6E;QAClC,IAAIf,WAAWzC,MAAM,GAAG,KAAK8B,SAAS2B,MAAMC,aAAa9B,SAAS;gBAChEyB;YAAAA,CAAAA,WAAAA,QAAOpD,IAAI,OAAXoD,UAAY,qBAAGJ,cAAcR,YAAYa,YAAYC,OAAO,GAAG3B,SAASc,WAAWC;YACnFF,aAAa,EAAE;YACfX,SAAS;YACTwB,aAAaC;YACbd,WAAWxC,IAAI,CAACuD;YAChB1B,UAAU4B;YACV;QACF;QACAjB,WAAWxC,IAAI,CAACuD;QAChB1B,UAAU2B,MAAMC;IAClB;IACA,IAAIjB,WAAWzC,MAAM,GAAG,GAAGqD,CAAAA,UAAAA,QAAOpD,IAAI,OAAXoD,SAAY,qBAAGJ,cAAcR,YAAYa,YAAY9C,SAASoB,SAASc,WAAWC;IAC7G,OAAOU;AACT;AAEA,0FAA0F;AAC1F,8FAA8F;AAC9F,SAASM,WAAWC,WAAoB,EAAET,KAAe,EAAEU,QAAyB;IAClF,IAAMjC,UAAUiC,SAASpE,YAAY;IACrC,IAAMqE,UAAUlC,UAAU9C;IAC1B,IAAMiF,WAA4C,EAAE;IACpD,IAAI5D,QAAgB,EAAE;IACtB,IAAI6D,iBAAiB;IACrB,IAAIlC,SAAS;IAEb,SAASmC;QACP,IAAMrF,QAAQsB,SAASC;QACvB,IAAIvB,OAAOmF,SAAS9D,IAAI,CAACrB;QACzBuB,QAAQ,EAAE;QACV6D,iBAAiB;QACjBlC,SAAS;IACX;IAEA,mFAAmF;IACnF,2FAA2F;IAC3F,SAASoC,QAAQhF,IAAY,EAAEiF,QAAgB,EAAE5D,SAAiB,EAAEC,OAAe;QACjFsB,UAAU,AAAC3B,CAAAA,MAAMH,MAAM,GAAG,IAAIc,iBAAiB,CAAA,IAAKnC,eAAewF;QACnEhE,MAAMF,IAAI,CAAC;YAAEM,WAAAA;YAAWC,SAAAA;YAAStB,MAAAA;QAAK;IACxC;QAEK,kCAAA,2BAAA;;QAAL,QAAK,YAAe0E,gCAAf,SAAA,6BAAA,QAAA,yBAAA,iCAA4B;YAA5B,IAAM9D,QAAN;YACH,IAAMsE,MAAMjB,MAAMkB,KAAK,CAACvE,MAAMS,SAAS,GAAG,GAAGT,MAAMU,OAAO,EAAEG,IAAI,CAAC;YACjE,IAAM2D,cAAc3F,eAAeyF;YAEnC,IAAIE,cAAcR,SAAS;gBACzB,IAAMV,OAAOjD,MAAMH,MAAM,GAAG,IAAI8B,SAAS;oBACpC,mCAAA,4BAAA;;oBAAL,QAAK,aAAWoB,mBAAmBC,OAAOrD,MAAMS,SAAS,EAAET,MAAMU,OAAO,EAAEoB,SAAS9B,MAAMC,IAAI,EAAE8D,SAAS3E,IAAI,EAAEkE,0BAAzG,UAAA,8BAAA,SAAA,0BAAA,kCAAgH;wBAAhH,IAAM1C,IAAN;wBACHP,MAAMF,IAAI,CAACS;wBACXuD;oBACF;;oBAHK;oBAAA;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;gBAIL;YACF;YAEA,IAAMM,YAAYxB,IAAAA,sBAAW,EAACjD,MAAMkD,IAAI;YACxC,IAAMmB,WAAWN,SAAS3E,IAAI,KAAK,QAAQkF,MAAMG;YAEjD,IAAIzE,MAAMC,IAAI,KAAK,WAAW;gBAC5BmE,QAAQK,WAAWJ,UAAUrE,MAAMS,SAAS,EAAET,MAAMU,OAAO;gBAC3D;YACF;YAEA,wFAAwF;YACxF,wDAAwD;YACxD,IAAMgE,eAAerE,MAAMH,MAAM,GAAG,KAAK8B,SAAShB,iBAAiBwD,cAAcR;YACjF,IAAIU,cAAcP;YAElBC,QAAQK,WAAWJ,UAAUrE,MAAMS,SAAS,EAAET,MAAMU,OAAO;YAC3DwD;YAEA,IAAIA,kBAAkBnF,gBAAgBoF;QACxC;;QA9BK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IA+BLA;IAEA,OAAOF;AACT;AAIO,SAASnF,MAAMe,MAAe,EAAE8E,IAAY,EAAEjF,IAAmB;QAI7BkF;IAHzC,IAAMb,WAAWtE,eAAeC;IAChC,IAAM2D,QAAQsB,KAAKE,KAAK,CAAC;IACzB,IAAMD,SAA0C,EAAE;QAC7C,kCAAA,2BAAA;;QAAL,QAAK,YAAehF,YAAYC,4BAA3B,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMiF,QAAN;YAAoCF,CAAAA,UAAAA,QAAOzE,IAAI,OAAXyE,SAAY,qBAAGf,WAAWiB,OAAOzB,OAAOU;;;QAA5E;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,+FAA+F;IAC/F,qHAAqH;IACrH,IAAMgB,SACJhB,SAAS3E,IAAI,KAAK,QACdwF,OAAOjE,GAAG,CAAC,SAACqE;eACVA,EAAElE,KAAK,GACHkE,IACA,wCACKA;YACH5F,MAAMiE,MACHkB,KAAK,CAACS,EAAEvE,SAAS,GAAG,GAAGuE,EAAEtE,OAAO,EAChCG,IAAI,CAAC,MACLoE,IAAI;;SAGfL;IACN,mHAAmH;IACnH,OAAOG,OAAOG,MAAM,CAAC,SAACF;eAAMA,EAAE5F,IAAI,CAAC6F,IAAI,GAAG/E,MAAM,GAAG;OAAGS,GAAG,CAAC,SAACqE;eAAO;YAAEvE,WAAWuE,EAAEvE,SAAS;YAAEC,SAASsE,EAAEtE,OAAO;YAAEtB,MAAM4F,EAAE5F,IAAI;QAAC;;AAC/H"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/group.ts"],"sourcesContent":["import { extractText } from './extract.ts';\nimport { parse } from './parse.ts';\nimport { DEFAULT_TARGET_TOKENS, estimateTokens } from './tokens.ts';\nimport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\n\nconst PGC_GROUP_SIZE = 2;\nconst OVERSIZE_TRIGGER_MULTIPLE = 2;\n\ninterface ResolvedOptions {\n  targetTokens: number;\n  text: 'extracted' | 'raw';\n}\n\nfunction resolveOptions(opts?: ChunkOptions): ResolvedOptions {\n  return {\n    targetTokens: opts?.targetTokens ?? DEFAULT_TARGET_TOKENS,\n    text: opts?.text ?? 'raw',\n  };\n}\n\n// A heading of any depth ends the current scope and starts a new one (D1); the heading block\n// itself is carried into the new scope, where it joins that scope's first group (F7/F10).\nfunction splitScopes(blocks: Block[]): Block[][] {\n  const scopes: Block[][] = [];\n  let current: Block[] = [];\n  for (const block of blocks) {\n    if (block.type === 'heading' && current.length > 0) {\n      scopes.push(current);\n      current = [];\n    }\n    current.push(block);\n  }\n  if (current.length > 0) scopes.push(current);\n  return scopes;\n}\n\ninterface Part {\n  startLine: number;\n  endLine: number;\n  text: string;\n  // True for a sub-line split piece: its text is already the final slice, not the whole line --\n  // group()'s extent-based raw re-slice must not touch it (siblings share startLine === endLine).\n  final?: boolean;\n}\n\nfunction finalize(parts: Part[]): (Chunk & { final?: boolean }) | undefined {\n  if (parts.length === 0) return undefined;\n  const first = parts[0];\n  const last = parts[parts.length - 1];\n  return { startLine: first.startLine, endLine: last.endLine, text: parts.map((p) => p.text).join('\\n'), final: parts.some((p) => p.final) };\n}\n\nconst NEWLINE_TOKENS = estimateTokens('\\n');\n// Built on first use and kept: each construction is ~3.5 ms, and only an oversize block is ever\n// split, so no command pays for a segmenter it never reaches.\nconst SEGMENTERS = new Map<string, Intl.Segmenter>();\n\nfunction segmentsOf(text: string, granularity: 'sentence' | 'word'): string[] {\n  let segmenter = SEGMENTERS.get(granularity);\n  if (!segmenter) {\n    segmenter = new Intl.Segmenter(undefined, { granularity });\n    SEGMENTERS.set(granularity, segmenter);\n  }\n  return Array.from(segmenter.segment(text), (s) => s.segment);\n}\n\n// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups\n// of at most `working` estimated tokens; a lone segment over `working` still stands alone.\nfunction pack(segments: string[], working: number): string[] {\n  const groups: string[] = [];\n  let current = '';\n  let tokens = 0;\n  for (const segment of segments) {\n    const segmentTokens = estimateTokens(segment);\n    if (current.length > 0 && tokens + segmentTokens > working) {\n      groups.push(current);\n      current = '';\n      tokens = 0;\n    }\n    current += segment;\n    tokens += segmentTokens;\n  }\n  if (current.length > 0) groups.push(current);\n  return groups;\n}\n\n// Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then\n// word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.\nfunction splitLineText(text: string, working: number): string[] {\n  const sentences = segmentsOf(text, 'sentence');\n  const out: string[] = [];\n  let current = '';\n  let tokens = 0;\n  const flush = () => {\n    if (current.length > 0) {\n      out.push(current);\n      current = '';\n      tokens = 0;\n    }\n  };\n  for (const sentence of sentences) {\n    const sentenceTokens = estimateTokens(sentence);\n    if (sentenceTokens > working) {\n      flush();\n      out.push(...pack(segmentsOf(sentence, 'word'), working));\n      continue;\n    }\n    if (current.length > 0 && tokens + sentenceTokens > working) flush();\n    current += sentence;\n    tokens += sentenceTokens;\n  }\n  flush();\n  return out;\n}\n\nconst ATOMIC_TYPES: ReadonlySet<BlockType> = new Set(['code', 'table', 'list']);\n\n// Atomic (code/table/list) pieces are always a raw line slice: re-parsing a table's later pieces\n// without their header/delimiter rows would demote them to paragraph text.\nfunction piece(pieceLines: string[], startLine: number, endLine: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part {\n  const text =\n    ATOMIC_TYPES.has(blockType) || textMode === 'raw'\n      ? pieceLines.join('\\n')\n      : parse(pieceLines.join('\\n'))\n          .map((b) => extractText(b.node))\n          .join('\\n');\n  return { startLine, endLine, text };\n}\n\n// A one-line piece over working can't shrink via another line-boundary pass (rule 5's gap), so it\n// splits at sentence/word boundaries instead; `final` stops group() re-deriving its text by extent (F5).\nfunction finalizePiece(pieceLines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part[] {\n  const p = piece(pieceLines, startLine, endLine, blockType, textMode);\n  if (pieceLines.length === 1 && estimateTokens(p.text) > working) {\n    return splitLineText(p.text, working).map((text) => ({ startLine, endLine, text, final: true }));\n  }\n  return [p];\n}\n\n// A block over 2x working size splits at line boundaries into pieces each <= working size, never\n// mid-line. `seed`: pending tokens (e.g. a heading) the first piece must join, checked against the limit.\nfunction splitOversizeBlock(lines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw', seed = 0): Part[] {\n  const pieces: Part[] = [];\n  let pieceLines: string[] = [];\n  let pieceStart = startLine;\n  let tokens = seed;\n  for (let line = startLine; line <= endLine; line++) {\n    const lineText = lines[line - 1];\n    const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;\n    const lineTokens = estimateTokens(lineText);\n    if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {\n      pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));\n      pieceLines = [];\n      tokens = 0;\n      pieceStart = line;\n      pieceLines.push(lineText);\n      tokens += lineTokens;\n      continue;\n    }\n    pieceLines.push(lineText);\n    tokens += sep + lineTokens;\n  }\n  if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));\n  return pieces;\n}\n\n// One heading scope's groups (D1): a heading opens the first group, and an oversize block\n// (rule 5, including an oversize heading) splits into pieces that each close their own group.\nfunction groupScope(scopeBlocks: Block[], lines: string[], resolved: ResolvedOptions): (Chunk & { final?: boolean })[] {\n  const working = resolved.targetTokens;\n  const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;\n  const finished: (Chunk & { final?: boolean })[] = [];\n  let parts: Part[] = [];\n  let paragraphCount = 0;\n  let tokens = 0;\n\n  function close(): void {\n    const group = finalize(parts);\n    if (group) finished.push(group);\n    parts = [];\n    paragraphCount = 0;\n    tokens = 0;\n  }\n\n  // tokens tracks the active text mode's own estimate (a newline between parts costs\n  // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.\n  function addPart(text: string, sizeText: string, startLine: number, endLine: number): void {\n    tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);\n    parts.push({ startLine, endLine, text });\n  }\n\n  for (const block of scopeBlocks) {\n    const raw = lines.slice(block.startLine - 1, block.endLine).join('\\n');\n    const blockTokens = estimateTokens(raw);\n\n    if (blockTokens > trigger) {\n      const seed = parts.length > 0 ? tokens : 0;\n      for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)) {\n        parts.push(p);\n        close();\n      }\n      continue;\n    }\n\n    const extracted = extractText(block.node);\n    const sizeText = resolved.text === 'raw' ? raw : extracted;\n\n    if (block.type === 'heading') {\n      addPart(extracted, sizeText, block.startLine, block.endLine);\n      continue;\n    }\n\n    // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --\n    // close first if the pair about to form would cross it.\n    const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;\n    if (pairOversize) close();\n\n    addPart(extracted, sizeText, block.startLine, block.endLine);\n    paragraphCount++;\n\n    if (paragraphCount >= PGC_GROUP_SIZE) close();\n  }\n  close();\n\n  return finished;\n}\n\n// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed\n// from (line lookups for oversize splitting).\nexport function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  const resolved = resolveOptions(opts);\n  const lines = body.split('\\n');\n  const chunks: (Chunk & { final?: boolean })[] = [];\n  for (const scope of splitScopes(blocks)) chunks.push(...groupScope(scope, lines, resolved));\n  // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).\n  // A `final` chunk already carries its own slice's raw text; re-slicing by extent would return the whole shared line.\n  const texted =\n    resolved.text === 'raw'\n      ? chunks.map((c) =>\n          c.final\n            ? c\n            : {\n                ...c,\n                text: lines\n                  .slice(c.startLine - 1, c.endLine)\n                  .join('\\n')\n                  .trim(),\n              }\n        )\n      : chunks;\n  // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.\n  return texted.filter((c) => c.text.trim().length > 0).map((c) => ({ startLine: c.startLine, endLine: c.endLine, text: c.text }));\n}\n"],"names":["group","PGC_GROUP_SIZE","OVERSIZE_TRIGGER_MULTIPLE","resolveOptions","opts","targetTokens","DEFAULT_TARGET_TOKENS","text","splitScopes","blocks","scopes","current","block","type","length","push","finalize","parts","undefined","first","last","startLine","endLine","map","p","join","final","some","NEWLINE_TOKENS","estimateTokens","SEGMENTERS","Map","segmentsOf","granularity","segmenter","get","Intl","Segmenter","set","Array","from","segment","s","pack","segments","working","groups","tokens","segmentTokens","splitLineText","sentences","out","flush","sentence","sentenceTokens","ATOMIC_TYPES","Set","piece","pieceLines","blockType","textMode","has","parse","b","extractText","node","finalizePiece","splitOversizeBlock","lines","seed","pieces","pieceStart","line","lineText","sep","lineTokens","groupScope","scopeBlocks","resolved","trigger","finished","paragraphCount","close","addPart","sizeText","raw","slice","blockTokens","extracted","pairOversize","body","chunks","split","scope","texted","c","trim","filter"],"mappings":";;;;+BAqOgBA;;;eAAAA;;;yBArOY;uBACN;wBACgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGtD,IAAMC,iBAAiB;AACvB,IAAMC,4BAA4B;AAOlC,SAASC,eAAeC,IAAmB;;IACzC,OAAO;QACLC,YAAY,UAAED,iBAAAA,2BAAAA,KAAMC,YAAY,uCAAIC,+BAAqB;QACzDC,IAAI,WAAEH,iBAAAA,2BAAAA,KAAMG,IAAI,yCAAI;IACtB;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASC,YAAYC,MAAe;IAClC,IAAMC,SAAoB,EAAE;IAC5B,IAAIC,UAAmB,EAAE;QACpB,kCAAA,2BAAA;;QAAL,QAAK,YAAeF,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;YAAvB,IAAMG,QAAN;YACH,IAAIA,MAAMC,IAAI,KAAK,aAAaF,QAAQG,MAAM,GAAG,GAAG;gBAClDJ,OAAOK,IAAI,CAACJ;gBACZA,UAAU,EAAE;YACd;YACAA,QAAQI,IAAI,CAACH;QACf;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAOL,IAAID,QAAQG,MAAM,GAAG,GAAGJ,OAAOK,IAAI,CAACJ;IACpC,OAAOD;AACT;AAWA,SAASM,SAASC,KAAa;IAC7B,IAAIA,MAAMH,MAAM,KAAK,GAAG,OAAOI;IAC/B,IAAMC,QAAQF,KAAK,CAAC,EAAE;IACtB,IAAMG,OAAOH,KAAK,CAACA,MAAMH,MAAM,GAAG,EAAE;IACpC,OAAO;QAAEO,WAAWF,MAAME,SAAS;QAAEC,SAASF,KAAKE,OAAO;QAAEf,MAAMU,MAAMM,GAAG,CAAC,SAACC;mBAAMA,EAAEjB,IAAI;WAAEkB,IAAI,CAAC;QAAOC,OAAOT,MAAMU,IAAI,CAAC,SAACH;mBAAMA,EAAEE,KAAK;;IAAE;AAC3I;AAEA,IAAME,iBAAiBC,IAAAA,wBAAc,EAAC;AACtC,gGAAgG;AAChG,8DAA8D;AAC9D,IAAMC,aAAa,IAAIC;AAEvB,SAASC,WAAWzB,IAAY,EAAE0B,WAAgC;IAChE,IAAIC,YAAYJ,WAAWK,GAAG,CAACF;IAC/B,IAAI,CAACC,WAAW;QACdA,YAAY,IAAIE,KAAKC,SAAS,CAACnB,WAAW;YAAEe,aAAAA;QAAY;QACxDH,WAAWQ,GAAG,CAACL,aAAaC;IAC9B;IACA,OAAOK,MAAMC,IAAI,CAACN,UAAUO,OAAO,CAAClC,OAAO,SAACmC;eAAMA,EAAED,OAAO;;AAC7D;AAEA,gGAAgG;AAChG,2FAA2F;AAC3F,SAASE,KAAKC,QAAkB,EAAEC,OAAe;IAC/C,IAAMC,SAAmB,EAAE;IAC3B,IAAInC,UAAU;IACd,IAAIoC,SAAS;QACR,kCAAA,2BAAA;;QAAL,QAAK,YAAiBH,6BAAjB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMH,UAAN;YACH,IAAMO,gBAAgBnB,IAAAA,wBAAc,EAACY;YACrC,IAAI9B,QAAQG,MAAM,GAAG,KAAKiC,SAASC,gBAAgBH,SAAS;gBAC1DC,OAAO/B,IAAI,CAACJ;gBACZA,UAAU;gBACVoC,SAAS;YACX;YACApC,WAAW8B;YACXM,UAAUC;QACZ;;QATK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,IAAIrC,QAAQG,MAAM,GAAG,GAAGgC,OAAO/B,IAAI,CAACJ;IACpC,OAAOmC;AACT;AAEA,8FAA8F;AAC9F,0GAA0G;AAC1G,SAASG,cAAc1C,IAAY,EAAEsC,OAAe;IAClD,IAAMK,YAAYlB,WAAWzB,MAAM;IACnC,IAAM4C,MAAgB,EAAE;IACxB,IAAIxC,UAAU;IACd,IAAIoC,SAAS;IACb,IAAMK,QAAQ;QACZ,IAAIzC,QAAQG,MAAM,GAAG,GAAG;YACtBqC,IAAIpC,IAAI,CAACJ;YACTA,UAAU;YACVoC,SAAS;QACX;IACF;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAkBG,8BAAlB,SAAA,6BAAA,QAAA,yBAAA,iCAA6B;YAA7B,IAAMG,WAAN;YACH,IAAMC,iBAAiBzB,IAAAA,wBAAc,EAACwB;YACtC,IAAIC,iBAAiBT,SAAS;oBAE5BM;gBADAC;gBACAD,CAAAA,OAAAA,KAAIpC,IAAI,OAARoC,MAAS,qBAAGR,KAAKX,WAAWqB,UAAU,SAASR;gBAC/C;YACF;YACA,IAAIlC,QAAQG,MAAM,GAAG,KAAKiC,SAASO,iBAAiBT,SAASO;YAC7DzC,WAAW0C;YACXN,UAAUO;QACZ;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAWLF;IACA,OAAOD;AACT;AAEA,IAAMI,eAAuC,IAAIC,IAAI;IAAC;IAAQ;IAAS;CAAO;AAE9E,iGAAiG;AACjG,2EAA2E;AAC3E,SAASC,MAAMC,UAAoB,EAAErC,SAAiB,EAAEC,OAAe,EAAEqC,SAAoB,EAAEC,QAA6B;IAC1H,IAAMrD,OACJgD,aAAaM,GAAG,CAACF,cAAcC,aAAa,QACxCF,WAAWjC,IAAI,CAAC,QAChBqC,IAAAA,cAAK,EAACJ,WAAWjC,IAAI,CAAC,OACnBF,GAAG,CAAC,SAACwC;eAAMC,IAAAA,sBAAW,EAACD,EAAEE,IAAI;OAC7BxC,IAAI,CAAC;IACd,OAAO;QAAEJ,WAAAA;QAAWC,SAAAA;QAASf,MAAAA;IAAK;AACpC;AAEA,kGAAkG;AAClG,yGAAyG;AACzG,SAAS2D,cAAcR,UAAoB,EAAErC,SAAiB,EAAEC,OAAe,EAAEuB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;IACnJ,IAAMpC,IAAIiC,MAAMC,YAAYrC,WAAWC,SAASqC,WAAWC;IAC3D,IAAIF,WAAW5C,MAAM,KAAK,KAAKe,IAAAA,wBAAc,EAACL,EAAEjB,IAAI,IAAIsC,SAAS;QAC/D,OAAOI,cAAczB,EAAEjB,IAAI,EAAEsC,SAAStB,GAAG,CAAC,SAAChB;mBAAU;gBAAEc,WAAAA;gBAAWC,SAAAA;gBAASf,MAAAA;gBAAMmB,OAAO;YAAK;;IAC/F;IACA,OAAO;QAACF;KAAE;AACZ;AAEA,iGAAiG;AACjG,0GAA0G;AAC1G,SAAS2C,mBAAmBC,KAAe,EAAE/C,SAAiB,EAAEC,OAAe,EAAEuB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;QAAES,OAAAA,iEAAO;QAqBjIC;IApB3B,IAAMA,SAAiB,EAAE;IACzB,IAAIZ,aAAuB,EAAE;IAC7B,IAAIa,aAAalD;IACjB,IAAI0B,SAASsB;IACb,IAAK,IAAIG,OAAOnD,WAAWmD,QAAQlD,SAASkD,OAAQ;QAClD,IAAMC,WAAWL,KAAK,CAACI,OAAO,EAAE;QAChC,IAAME,MAAMhB,WAAW5C,MAAM,GAAG,KAAKiC,SAAS,IAAInB,iBAAiB;QACnE,IAAM+C,aAAa9C,IAAAA,wBAAc,EAAC4C;QAClC,IAAIf,WAAW5C,MAAM,GAAG,KAAKiC,SAAS2B,MAAMC,aAAa9B,SAAS;gBAChEyB;YAAAA,CAAAA,WAAAA,QAAOvD,IAAI,OAAXuD,UAAY,qBAAGJ,cAAcR,YAAYa,YAAYC,OAAO,GAAG3B,SAASc,WAAWC;YACnFF,aAAa,EAAE;YACfX,SAAS;YACTwB,aAAaC;YACbd,WAAW3C,IAAI,CAAC0D;YAChB1B,UAAU4B;YACV;QACF;QACAjB,WAAW3C,IAAI,CAAC0D;QAChB1B,UAAU2B,MAAMC;IAClB;IACA,IAAIjB,WAAW5C,MAAM,GAAG,GAAGwD,CAAAA,UAAAA,QAAOvD,IAAI,OAAXuD,SAAY,qBAAGJ,cAAcR,YAAYa,YAAYjD,SAASuB,SAASc,WAAWC;IAC7G,OAAOU;AACT;AAEA,0FAA0F;AAC1F,8FAA8F;AAC9F,SAASM,WAAWC,WAAoB,EAAET,KAAe,EAAEU,QAAyB;IAClF,IAAMjC,UAAUiC,SAASzE,YAAY;IACrC,IAAM0E,UAAUlC,UAAU3C;IAC1B,IAAM8E,WAA4C,EAAE;IACpD,IAAI/D,QAAgB,EAAE;IACtB,IAAIgE,iBAAiB;IACrB,IAAIlC,SAAS;IAEb,SAASmC;QACP,IAAMlF,QAAQgB,SAASC;QACvB,IAAIjB,OAAOgF,SAASjE,IAAI,CAACf;QACzBiB,QAAQ,EAAE;QACVgE,iBAAiB;QACjBlC,SAAS;IACX;IAEA,mFAAmF;IACnF,2FAA2F;IAC3F,SAASoC,QAAQ5E,IAAY,EAAE6E,QAAgB,EAAE/D,SAAiB,EAAEC,OAAe;QACjFyB,UAAU,AAAC9B,CAAAA,MAAMH,MAAM,GAAG,IAAIc,iBAAiB,CAAA,IAAKC,IAAAA,wBAAc,EAACuD;QACnEnE,MAAMF,IAAI,CAAC;YAAEM,WAAAA;YAAWC,SAAAA;YAASf,MAAAA;QAAK;IACxC;QAEK,kCAAA,2BAAA;;QAAL,QAAK,YAAesE,gCAAf,SAAA,6BAAA,QAAA,yBAAA,iCAA4B;YAA5B,IAAMjE,QAAN;YACH,IAAMyE,MAAMjB,MAAMkB,KAAK,CAAC1E,MAAMS,SAAS,GAAG,GAAGT,MAAMU,OAAO,EAAEG,IAAI,CAAC;YACjE,IAAM8D,cAAc1D,IAAAA,wBAAc,EAACwD;YAEnC,IAAIE,cAAcR,SAAS;gBACzB,IAAMV,OAAOpD,MAAMH,MAAM,GAAG,IAAIiC,SAAS;oBACpC,mCAAA,4BAAA;;oBAAL,QAAK,aAAWoB,mBAAmBC,OAAOxD,MAAMS,SAAS,EAAET,MAAMU,OAAO,EAAEuB,SAASjC,MAAMC,IAAI,EAAEiE,SAASvE,IAAI,EAAE8D,0BAAzG,UAAA,8BAAA,SAAA,0BAAA,kCAAgH;wBAAhH,IAAM7C,IAAN;wBACHP,MAAMF,IAAI,CAACS;wBACX0D;oBACF;;oBAHK;oBAAA;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;gBAIL;YACF;YAEA,IAAMM,YAAYxB,IAAAA,sBAAW,EAACpD,MAAMqD,IAAI;YACxC,IAAMmB,WAAWN,SAASvE,IAAI,KAAK,QAAQ8E,MAAMG;YAEjD,IAAI5E,MAAMC,IAAI,KAAK,WAAW;gBAC5BsE,QAAQK,WAAWJ,UAAUxE,MAAMS,SAAS,EAAET,MAAMU,OAAO;gBAC3D;YACF;YAEA,wFAAwF;YACxF,wDAAwD;YACxD,IAAMmE,eAAexE,MAAMH,MAAM,GAAG,KAAKiC,SAASnB,iBAAiB2D,cAAcR;YACjF,IAAIU,cAAcP;YAElBC,QAAQK,WAAWJ,UAAUxE,MAAMS,SAAS,EAAET,MAAMU,OAAO;YAC3D2D;YAEA,IAAIA,kBAAkBhF,gBAAgBiF;QACxC;;QA9BK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IA+BLA;IAEA,OAAOF;AACT;AAIO,SAAShF,MAAMS,MAAe,EAAEiF,IAAY,EAAEtF,IAAmB;QAI7BuF;IAHzC,IAAMb,WAAW3E,eAAeC;IAChC,IAAMgE,QAAQsB,KAAKE,KAAK,CAAC;IACzB,IAAMD,SAA0C,EAAE;QAC7C,kCAAA,2BAAA;;QAAL,QAAK,YAAenF,YAAYC,4BAA3B,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMoF,QAAN;YAAoCF,CAAAA,UAAAA,QAAO5E,IAAI,OAAX4E,SAAY,qBAAGf,WAAWiB,OAAOzB,OAAOU;;;QAA5E;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,+FAA+F;IAC/F,qHAAqH;IACrH,IAAMgB,SACJhB,SAASvE,IAAI,KAAK,QACdoF,OAAOpE,GAAG,CAAC,SAACwE;eACVA,EAAErE,KAAK,GACHqE,IACA,wCACKA;YACHxF,MAAM6D,MACHkB,KAAK,CAACS,EAAE1E,SAAS,GAAG,GAAG0E,EAAEzE,OAAO,EAChCG,IAAI,CAAC,MACLuE,IAAI;;SAGfL;IACN,mHAAmH;IACnH,OAAOG,OAAOG,MAAM,CAAC,SAACF;eAAMA,EAAExF,IAAI,CAACyF,IAAI,GAAGlF,MAAM,GAAG;OAAGS,GAAG,CAAC,SAACwE;eAAO;YAAE1E,WAAW0E,EAAE1E,SAAS;YAAEC,SAASyE,EAAEzE,OAAO;YAAEf,MAAMwF,EAAExF,IAAI;QAAC;;AAC/H"}

package/dist/cjs/chunk/index.d.cts

@@ -2,7 +2,7 @@ import type { Block, Chunk, ChunkOptions } from './types.js';
 export declare function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[];
 export declare function chunk(body: string, opts?: ChunkOptions): Chunk[];
 export { extractText, extractTexts } from './extract.js';
-export { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.js';
 export { parse } from './parse.js';
+export { DEFAULT_TARGET_TOKENS, estimateTokens } from './tokens.js';
 export type { Block, BlockType, Chunk, ChunkOptions } from './types.js';
 export { CHUNK_VERSION } from './version.js';

package/dist/cjs/chunk/index.d.ts

@@ -2,7 +2,7 @@ import type { Block, Chunk, ChunkOptions } from './types.js';
 export declare function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[];
 export declare function chunk(body: string, opts?: ChunkOptions): Chunk[];
 export { extractText, extractTexts } from './extract.js';
-export { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.js';
 export { parse } from './parse.js';
+export { DEFAULT_TARGET_TOKENS, estimateTokens } from './tokens.js';
 export type { Block, BlockType, Chunk, ChunkOptions } from './types.js';
 export { CHUNK_VERSION } from './version.js';

package/dist/cjs/chunk/index.js

@@ -13,7 +13,7 @@ _export(exports, {
         return _versionts.CHUNK_VERSION;
     },
     get DEFAULT_TARGET_TOKENS () {
-        return _groupts.DEFAULT_TARGET_TOKENS;
+        return _tokensts.DEFAULT_TARGET_TOKENS;
     },
     get chunk () {
         return chunk;
@@ -22,7 +22,7 @@ _export(exports, {
         return chunkFromBlocks;
     },
     get estimateTokens () {
-        return _groupts.estimateTokens;
+        return _tokensts.estimateTokens;
     },
     get extractText () {
         return _extractts.extractText;
@@ -37,6 +37,7 @@ _export(exports, {
 var _groupts = require("./group.js");
 var _parsets = require("./parse.js");
 var _extractts = require("./extract.js");
+var _tokensts = require("./tokens.js");
 var _versionts = require("./version.js");
 function chunkFromBlocks(blocks, body, opts) {
     return (0, _groupts.group)(blocks, body, opts);

package/dist/cjs/chunk/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/index.ts"],"sourcesContent":["import { group } from './group.ts';\nimport { parse } from './parse.ts';\nimport type { Block, Chunk, ChunkOptions } from './types.ts';\n\n// Groups blocks a caller already parsed (e.g. scan/index.ts, sharing one parse with the FTS\n// text path) against the same body they came from, per opts (D1/D3).\nexport function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  return group(blocks, body, opts);\n}\n\n// Pure, deterministic function of file content; chunk semantics are version-stamped via\n// CHUNK_VERSION (./version.ts). Algorithm and evidence: BENCHMARKING.md, \"The chunking algorithm\".\nexport function chunk(body: string, opts?: ChunkOptions): Chunk[] {\n  return chunkFromBlocks(parse(body), body, opts);\n}\n\nexport { extractText, extractTexts } from './extract.ts';\nexport { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.ts';\nexport { parse } from './parse.ts';\nexport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\nexport { CHUNK_VERSION } from './version.ts';\n"],"names":["CHUNK_VERSION","DEFAULT_TARGET_TOKENS","chunk","chunkFromBlocks","estimateTokens","extractText","extractTexts","parse","blocks","body","opts","group"],"mappings":";;;;;;;;;;;QAoBSA;eAAAA,wBAAa;;QAHbC;eAAAA,8BAAqB;;QALdC;eAAAA;;QANAC;eAAAA;;QAWgBC;eAAAA,uBAAc;;QADrCC;eAAAA,sBAAW;;QAAEC;eAAAA,uBAAY;;QAEzBC;eAAAA,cAAK;;;uBAlBQ;uBACA;yBAeoB;yBAIZ;AAdvB,SAASJ,gBAAgBK,MAAe,EAAEC,IAAY,EAAEC,IAAmB;IAChF,OAAOC,IAAAA,cAAK,EAACH,QAAQC,MAAMC;AAC7B;AAIO,SAASR,MAAMO,IAAY,EAAEC,IAAmB;IACrD,OAAOP,gBAAgBI,IAAAA,cAAK,EAACE,OAAOA,MAAMC;AAC5C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/index.ts"],"sourcesContent":["import { group } from './group.ts';\nimport { parse } from './parse.ts';\nimport type { Block, Chunk, ChunkOptions } from './types.ts';\n\n// Groups blocks a caller already parsed (e.g. scan/index.ts, sharing one parse with the FTS\n// text path) against the same body they came from, per opts (D1/D3).\nexport function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  return group(blocks, body, opts);\n}\n\n// Pure, deterministic function of file content; chunk semantics are version-stamped via\n// CHUNK_VERSION (./version.ts). Algorithm and evidence: BENCHMARKING.md, \"The chunking algorithm\".\nexport function chunk(body: string, opts?: ChunkOptions): Chunk[] {\n  return chunkFromBlocks(parse(body), body, opts);\n}\n\nexport { extractText, extractTexts } from './extract.ts';\nexport { parse } from './parse.ts';\nexport { DEFAULT_TARGET_TOKENS, estimateTokens } from './tokens.ts';\nexport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\nexport { CHUNK_VERSION } from './version.ts';\n"],"names":["CHUNK_VERSION","DEFAULT_TARGET_TOKENS","chunk","chunkFromBlocks","estimateTokens","extractText","extractTexts","parse","blocks","body","opts","group"],"mappings":";;;;;;;;;;;QAoBSA;eAAAA,wBAAa;;QAFbC;eAAAA,+BAAqB;;QANdC;eAAAA;;QANAC;eAAAA;;QAYgBC;eAAAA,wBAAc;;QAFrCC;eAAAA,sBAAW;;QAAEC;eAAAA,uBAAY;;QACzBC;eAAAA,cAAK;;;uBAjBQ;uBACA;yBAeoB;wBAEY;yBAExB;AAdvB,SAASJ,gBAAgBK,MAAe,EAAEC,IAAY,EAAEC,IAAmB;IAChF,OAAOC,IAAAA,cAAK,EAACH,QAAQC,MAAMC;AAC7B;AAIO,SAASR,MAAMO,IAAY,EAAEC,IAAmB;IACrD,OAAOP,gBAAgBI,IAAAA,cAAK,EAACE,OAAOA,MAAMC;AAC5C"}

package/dist/cjs/chunk/parse.js

@@ -8,18 +8,16 @@ Object.defineProperty(exports, "parse", {
         return parse;
     }
 });
-var _mdastutilfrommarkdown = require("mdast-util-from-markdown");
-var _mdastutilgfmautolinkliteral = require("mdast-util-gfm-autolink-literal");
-var _mdastutilgfmfootnote = require("mdast-util-gfm-footnote");
-var _mdastutilgfmstrikethrough = require("mdast-util-gfm-strikethrough");
-var _mdastutilgfmtable = require("mdast-util-gfm-table");
-var _mdastutilgfmtasklistitem = require("mdast-util-gfm-task-list-item");
-var _micromarkextensiongfmautolinkliteral = require("micromark-extension-gfm-autolink-literal");
-var _micromarkextensiongfmfootnote = require("micromark-extension-gfm-footnote");
-var _micromarkextensiongfmstrikethrough = require("micromark-extension-gfm-strikethrough");
-var _micromarkextensiongfmtable = require("micromark-extension-gfm-table");
-var _micromarkextensiongfmtasklistitem = require("micromark-extension-gfm-task-list-item");
+var _nodemodule = /*#__PURE__*/ _interop_require_default(require("node:module"));
 var _extractts = require("./extract.js");
+function _interop_require_default(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+// Tier-2, as embed/static.ts: the parser's packages cost ~19 ms to load and a warm tree never
+// parses, so every store-opening command paid for them until a file actually changed.
+var _require = typeof require === 'undefined' ? _nodemodule.default.createRequire(require("url").pathToFileURL(__filename).toString()) : require;
 var BLOCK_TYPES = {
     heading: 'heading',
     paragraph: 'paragraph',
@@ -28,27 +26,46 @@ var BLOCK_TYPES = {
     list: 'list',
     blockquote: 'blockquote'
 };
+var cached;
 // Imported individually, not via micromark-extension-gfm/mdast-util-gfm: those bundles also pull
 // in gfm-tagfilter, an HTML sanitizer this library never uses (no htmlExtensions call anywhere).
-var EXTENSIONS = [
-    (0, _micromarkextensiongfmautolinkliteral.gfmAutolinkLiteral)(),
-    (0, _micromarkextensiongfmfootnote.gfmFootnote)(),
-    (0, _micromarkextensiongfmstrikethrough.gfmStrikethrough)(),
-    (0, _micromarkextensiongfmtable.gfmTable)(),
-    (0, _micromarkextensiongfmtasklistitem.gfmTaskListItem)()
-];
-var MDAST_EXTENSIONS = [
-    (0, _mdastutilgfmautolinkliteral.gfmAutolinkLiteralFromMarkdown)(),
-    (0, _mdastutilgfmfootnote.gfmFootnoteFromMarkdown)(),
-    (0, _mdastutilgfmstrikethrough.gfmStrikethroughFromMarkdown)(),
-    (0, _mdastutilgfmtable.gfmTableFromMarkdown)(),
-    (0, _mdastutilgfmtasklistitem.gfmTaskListItemFromMarkdown)()
-];
+function parser() {
+    if (cached) return cached;
+    var fromMarkdown = _require('mdast-util-from-markdown').fromMarkdown;
+    var gfmAutolinkLiteralFromMarkdown = _require('mdast-util-gfm-autolink-literal').gfmAutolinkLiteralFromMarkdown;
+    var gfmFootnoteFromMarkdown = _require('mdast-util-gfm-footnote').gfmFootnoteFromMarkdown;
+    var gfmStrikethroughFromMarkdown = _require('mdast-util-gfm-strikethrough').gfmStrikethroughFromMarkdown;
+    var gfmTableFromMarkdown = _require('mdast-util-gfm-table').gfmTableFromMarkdown;
+    var gfmTaskListItemFromMarkdown = _require('mdast-util-gfm-task-list-item').gfmTaskListItemFromMarkdown;
+    var gfmAutolinkLiteral = _require('micromark-extension-gfm-autolink-literal').gfmAutolinkLiteral;
+    var gfmFootnote = _require('micromark-extension-gfm-footnote').gfmFootnote;
+    var gfmStrikethrough = _require('micromark-extension-gfm-strikethrough').gfmStrikethrough;
+    var gfmTable = _require('micromark-extension-gfm-table').gfmTable;
+    var gfmTaskListItem = _require('micromark-extension-gfm-task-list-item').gfmTaskListItem;
+    cached = {
+        fromMarkdown: fromMarkdown,
+        options: {
+            extensions: [
+                gfmAutolinkLiteral(),
+                gfmFootnote(),
+                gfmStrikethrough(),
+                gfmTable(),
+                gfmTaskListItem()
+            ],
+            mdastExtensions: [
+                gfmAutolinkLiteralFromMarkdown(),
+                gfmFootnoteFromMarkdown(),
+                gfmStrikethroughFromMarkdown(),
+                gfmTableFromMarkdown(),
+                gfmTaskListItemFromMarkdown()
+            ]
+        }
+    };
+    return cached;
+}
 function parse(body) {
-    var tree = (0, _mdastutilfrommarkdown.fromMarkdown)(body, {
-        extensions: EXTENSIONS,
-        mdastExtensions: MDAST_EXTENSIONS
-    });
+    var _parser = parser(), fromMarkdown = _parser.fromMarkdown, options = _parser.options;
+    var tree = fromMarkdown(body, options);
     return tree.children.map(function(node) {
         var _BLOCK_TYPES_node_type;
         var position = node.position;

package/dist/cjs/chunk/parse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parse.ts"],"sourcesContent":["import type { RootContent } from 'mdast';\nimport { fromMarkdown } from 'mdast-util-from-markdown';\nimport { gfmAutolinkLiteralFromMarkdown } from 'mdast-util-gfm-autolink-literal';\nimport { gfmFootnoteFromMarkdown } from 'mdast-util-gfm-footnote';\nimport { gfmStrikethroughFromMarkdown } from 'mdast-util-gfm-strikethrough';\nimport { gfmTableFromMarkdown } from 'mdast-util-gfm-table';\nimport { gfmTaskListItemFromMarkdown } from 'mdast-util-gfm-task-list-item';\nimport { gfmAutolinkLiteral } from 'micromark-extension-gfm-autolink-literal';\nimport { gfmFootnote } from 'micromark-extension-gfm-footnote';\nimport { gfmStrikethrough } from 'micromark-extension-gfm-strikethrough';\nimport { gfmTable } from 'micromark-extension-gfm-table';\nimport { gfmTaskListItem } from 'micromark-extension-gfm-task-list-item';\nimport { extractText } from './extract.ts';\nimport type { Block, BlockType } from './types.ts';\n\nconst BLOCK_TYPES: Partial<Record<RootContent['type'], BlockType>> = {\n  heading: 'heading',\n  paragraph: 'paragraph',\n  code: 'code',\n  table: 'table',\n  list: 'list',\n  blockquote: 'blockquote',\n};\n\n// Imported individually, not via micromark-extension-gfm/mdast-util-gfm: those bundles also pull\n// in gfm-tagfilter, an HTML sanitizer this library never uses (no htmlExtensions call anywhere).\nconst EXTENSIONS = [gfmAutolinkLiteral(), gfmFootnote(), gfmStrikethrough(), gfmTable(), gfmTaskListItem()];\nconst MDAST_EXTENSIONS = [gfmAutolinkLiteralFromMarkdown(), gfmFootnoteFromMarkdown(), gfmStrikethroughFromMarkdown(), gfmTableFromMarkdown(), gfmTaskListItemFromMarkdown()];\n\n// Top-level blocks of a markdown body, typed and line-extent bounded from mdast's own\n// node.position (never a regex guess). GFM extensions add tables, task lists, footnotes, strikethrough.\nexport function parse(body: string): Block[] {\n  const tree = fromMarkdown(body, { extensions: EXTENSIONS, mdastExtensions: MDAST_EXTENSIONS });\n  return tree.children.map((node) => {\n    const position = node.position;\n    const block: Block = {\n      type: BLOCK_TYPES[node.type] ?? 'other',\n      startLine: position ? position.start.line : 1,\n      endLine: position ? position.end.line : 1,\n      node,\n    };\n    if (node.type === 'heading') {\n      block.depth = node.depth;\n      block.text = extractText(node);\n    }\n    return block;\n  });\n}\n"],"names":["parse","BLOCK_TYPES","heading","paragraph","code","table","list","blockquote","EXTENSIONS","gfmAutolinkLiteral","gfmFootnote","gfmStrikethrough","gfmTable","gfmTaskListItem","MDAST_EXTENSIONS","gfmAutolinkLiteralFromMarkdown","gfmFootnoteFromMarkdown","gfmStrikethroughFromMarkdown","gfmTableFromMarkdown","gfmTaskListItemFromMarkdown","body","tree","fromMarkdown","extensions","mdastExtensions","children","map","node","position","block","type","startLine","start","line","endLine","end","depth","text","extractText"],"mappings":";;;;+BA+BgBA;;;eAAAA;;;qCA9Ba;2CACkB;oCACP;yCACK;iCACR;wCACO;oDACT;6CACP;kDACK;0CACR;iDACO;yBACJ;AAG5B,IAAMC,cAA+D;IACnEC,SAAS;IACTC,WAAW;IACXC,MAAM;IACNC,OAAO;IACPC,MAAM;IACNC,YAAY;AACd;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,IAAMC,aAAa;IAACC,IAAAA,wDAAkB;IAAIC,IAAAA,0CAAW;IAAIC,IAAAA,oDAAgB;IAAIC,IAAAA,oCAAQ;IAAIC,IAAAA,kDAAe;CAAG;AAC3G,IAAMC,mBAAmB;IAACC,IAAAA,2DAA8B;IAAIC,IAAAA,6CAAuB;IAAIC,IAAAA,uDAA4B;IAAIC,IAAAA,uCAAoB;IAAIC,IAAAA,qDAA2B;CAAG;AAItK,SAASnB,MAAMoB,IAAY;IAChC,IAAMC,OAAOC,IAAAA,mCAAY,EAACF,MAAM;QAAEG,YAAYf;QAAYgB,iBAAiBV;IAAiB;IAC5F,OAAOO,KAAKI,QAAQ,CAACC,GAAG,CAAC,SAACC;YAGhB1B;QAFR,IAAM2B,WAAWD,KAAKC,QAAQ;QAC9B,IAAMC,QAAe;YACnBC,IAAI,GAAE7B,yBAAAA,WAAW,CAAC0B,KAAKG,IAAI,CAAC,cAAtB7B,oCAAAA,yBAA0B;YAChC8B,WAAWH,WAAWA,SAASI,KAAK,CAACC,IAAI,GAAG;YAC5CC,SAASN,WAAWA,SAASO,GAAG,CAACF,IAAI,GAAG;YACxCN,MAAAA;QACF;QACA,IAAIA,KAAKG,IAAI,KAAK,WAAW;YAC3BD,MAAMO,KAAK,GAAGT,KAAKS,KAAK;YACxBP,MAAMQ,IAAI,GAAGC,IAAAA,sBAAW,EAACX;QAC3B;QACA,OAAOE;IACT;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parse.ts"],"sourcesContent":["import Module from 'node:module';\nimport type { RootContent } from 'mdast';\nimport { extractText } from './extract.ts';\nimport type { Block, BlockType } from './types.ts';\n\n// Tier-2, as embed/static.ts: the parser's packages cost ~19 ms to load and a warm tree never\n// parses, so every store-opening command paid for them until a file actually changed.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\nconst BLOCK_TYPES: Partial<Record<RootContent['type'], BlockType>> = {\n  heading: 'heading',\n  paragraph: 'paragraph',\n  code: 'code',\n  table: 'table',\n  list: 'list',\n  blockquote: 'blockquote',\n};\n\ntype FromMarkdown = typeof import('mdast-util-from-markdown').fromMarkdown;\ntype Parser = { fromMarkdown: FromMarkdown; options: NonNullable<Parameters<FromMarkdown>[1]> };\nlet cached: Parser | undefined;\n\n// Imported individually, not via micromark-extension-gfm/mdast-util-gfm: those bundles also pull\n// in gfm-tagfilter, an HTML sanitizer this library never uses (no htmlExtensions call anywhere).\nfunction parser(): Parser {\n  if (cached) return cached;\n  const { fromMarkdown } = _require('mdast-util-from-markdown') as typeof import('mdast-util-from-markdown');\n  const { gfmAutolinkLiteralFromMarkdown } = _require('mdast-util-gfm-autolink-literal') as typeof import('mdast-util-gfm-autolink-literal');\n  const { gfmFootnoteFromMarkdown } = _require('mdast-util-gfm-footnote') as typeof import('mdast-util-gfm-footnote');\n  const { gfmStrikethroughFromMarkdown } = _require('mdast-util-gfm-strikethrough') as typeof import('mdast-util-gfm-strikethrough');\n  const { gfmTableFromMarkdown } = _require('mdast-util-gfm-table') as typeof import('mdast-util-gfm-table');\n  const { gfmTaskListItemFromMarkdown } = _require('mdast-util-gfm-task-list-item') as typeof import('mdast-util-gfm-task-list-item');\n  const { gfmAutolinkLiteral } = _require('micromark-extension-gfm-autolink-literal') as typeof import('micromark-extension-gfm-autolink-literal');\n  const { gfmFootnote } = _require('micromark-extension-gfm-footnote') as typeof import('micromark-extension-gfm-footnote');\n  const { gfmStrikethrough } = _require('micromark-extension-gfm-strikethrough') as typeof import('micromark-extension-gfm-strikethrough');\n  const { gfmTable } = _require('micromark-extension-gfm-table') as typeof import('micromark-extension-gfm-table');\n  const { gfmTaskListItem } = _require('micromark-extension-gfm-task-list-item') as typeof import('micromark-extension-gfm-task-list-item');\n  cached = {\n    fromMarkdown,\n    options: {\n      extensions: [gfmAutolinkLiteral(), gfmFootnote(), gfmStrikethrough(), gfmTable(), gfmTaskListItem()],\n      mdastExtensions: [gfmAutolinkLiteralFromMarkdown(), gfmFootnoteFromMarkdown(), gfmStrikethroughFromMarkdown(), gfmTableFromMarkdown(), gfmTaskListItemFromMarkdown()],\n    },\n  };\n  return cached;\n}\n\n// Top-level blocks of a markdown body, typed and line-extent bounded from mdast's own\n// node.position (never a regex guess). GFM extensions add tables, task lists, footnotes, strikethrough.\nexport function parse(body: string): Block[] {\n  const { fromMarkdown, options } = parser();\n  const tree = fromMarkdown(body, options);\n  return tree.children.map((node) => {\n    const position = node.position;\n    const block: Block = {\n      type: BLOCK_TYPES[node.type] ?? 'other',\n      startLine: position ? position.start.line : 1,\n      endLine: position ? position.end.line : 1,\n      node,\n    };\n    if (node.type === 'heading') {\n      block.depth = node.depth;\n      block.text = extractText(node);\n    }\n    return block;\n  });\n}\n"],"names":["parse","_require","require","Module","createRequire","BLOCK_TYPES","heading","paragraph","code","table","list","blockquote","cached","parser","fromMarkdown","gfmAutolinkLiteralFromMarkdown","gfmFootnoteFromMarkdown","gfmStrikethroughFromMarkdown","gfmTableFromMarkdown","gfmTaskListItemFromMarkdown","gfmAutolinkLiteral","gfmFootnote","gfmStrikethrough","gfmTable","gfmTaskListItem","options","extensions","mdastExtensions","body","tree","children","map","node","position","block","type","startLine","start","line","endLine","end","depth","text","extractText"],"mappings":";;;;+BAiDgBA;;;eAAAA;;;iEAjDG;yBAES;;;;;;AAG5B,8FAA8F;AAC9F,sFAAsF;AACtF,IAAMC,WAAW,OAAOC,YAAY,cAAcC,mBAAM,CAACC,aAAa,CAAC,uDAAmBF;AAE1F,IAAMG,cAA+D;IACnEC,SAAS;IACTC,WAAW;IACXC,MAAM;IACNC,OAAO;IACPC,MAAM;IACNC,YAAY;AACd;AAIA,IAAIC;AAEJ,iGAAiG;AACjG,iGAAiG;AACjG,SAASC;IACP,IAAID,QAAQ,OAAOA;IACnB,IAAM,AAAEE,eAAiBb,SAAS,4BAA1Ba;IACR,IAAM,AAAEC,iCAAmCd,SAAS,mCAA5Cc;IACR,IAAM,AAAEC,0BAA4Bf,SAAS,2BAArCe;IACR,IAAM,AAAEC,+BAAiChB,SAAS,gCAA1CgB;IACR,IAAM,AAAEC,uBAAyBjB,SAAS,wBAAlCiB;IACR,IAAM,AAAEC,8BAAgClB,SAAS,iCAAzCkB;IACR,IAAM,AAAEC,qBAAuBnB,SAAS,4CAAhCmB;IACR,IAAM,AAAEC,cAAgBpB,SAAS,oCAAzBoB;IACR,IAAM,AAAEC,mBAAqBrB,SAAS,yCAA9BqB;IACR,IAAM,AAAEC,WAAatB,SAAS,iCAAtBsB;IACR,IAAM,AAAEC,kBAAoBvB,SAAS,0CAA7BuB;IACRZ,SAAS;QACPE,cAAAA;QACAW,SAAS;YACPC,YAAY;gBAACN;gBAAsBC;gBAAeC;gBAAoBC;gBAAYC;aAAkB;YACpGG,iBAAiB;gBAACZ;gBAAkCC;gBAA2BC;gBAAgCC;gBAAwBC;aAA8B;QACvK;IACF;IACA,OAAOP;AACT;AAIO,SAASZ,MAAM4B,IAAY;IAChC,IAAkCf,UAAAA,UAA1BC,eAA0BD,QAA1BC,cAAcW,UAAYZ,QAAZY;IACtB,IAAMI,OAAOf,aAAac,MAAMH;IAChC,OAAOI,KAAKC,QAAQ,CAACC,GAAG,CAAC,SAACC;YAGhB3B;QAFR,IAAM4B,WAAWD,KAAKC,QAAQ;QAC9B,IAAMC,QAAe;YACnBC,IAAI,GAAE9B,yBAAAA,WAAW,CAAC2B,KAAKG,IAAI,CAAC,cAAtB9B,oCAAAA,yBAA0B;YAChC+B,WAAWH,WAAWA,SAASI,KAAK,CAACC,IAAI,GAAG;YAC5CC,SAASN,WAAWA,SAASO,GAAG,CAACF,IAAI,GAAG;YACxCN,MAAAA;QACF;QACA,IAAIA,KAAKG,IAAI,KAAK,WAAW;YAC3BD,MAAMO,KAAK,GAAGT,KAAKS,KAAK;YACxBP,MAAMQ,IAAI,GAAGC,IAAAA,sBAAW,EAACX;QAC3B;QACA,OAAOE;IACT;AACF"}

package/dist/cjs/chunk/tokens.d.cts

@@ -0,0 +1,2 @@
+export declare const DEFAULT_TARGET_TOKENS = 500;
+export declare function estimateTokens(text: string): number;

package/dist/cjs/chunk/tokens.d.ts

@@ -0,0 +1,2 @@
+export declare const DEFAULT_TARGET_TOKENS = 500;
+export declare function estimateTokens(text: string): number;

package/dist/cjs/chunk/tokens.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get DEFAULT_TARGET_TOKENS () {
+        return DEFAULT_TARGET_TOKENS;
+    },
+    get estimateTokens () {
+        return estimateTokens;
+    }
+});
+var _segmentts = require("../text/segment.js");
+var DEFAULT_TARGET_TOKENS = 500;
+// D5: segment.ts's unspaced-script set packs close to one token per character; Hangul is
+// spaced but still token-dense and sits in this set as a conservative bound (one Unicode set, one home).
+var DENSE_SCRIPT = new RegExp("[".concat(_segmentts.UNSPACED_SCRIPTS, "\\p{scx=Hangul}]"), 'u');
+function estimateTokens(text) {
+    var dense = 0;
+    var other = 0;
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = text[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var ch = _step.value;
+            if (DENSE_SCRIPT.test(ch)) dense++;
+            else other++;
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    return dense + other / 4;
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/chunk/tokens.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/tokens.ts"],"sourcesContent":["import { UNSPACED_SCRIPTS } from '../text/segment.ts';\n\nexport const DEFAULT_TARGET_TOKENS = 500;\n\n// D5: segment.ts's unspaced-script set packs close to one token per character; Hangul is\n// spaced but still token-dense and sits in this set as a conservative bound (one Unicode set, one home).\nconst DENSE_SCRIPT = new RegExp(`[${UNSPACED_SCRIPTS}\\\\p{scx=Hangul}]`, 'u');\n\n// D5's size estimate: dense-script graphemes 1:1, everything else at 4 chars/token.\nexport function estimateTokens(text: string): number {\n  let dense = 0;\n  let other = 0;\n  for (const ch of text) {\n    if (DENSE_SCRIPT.test(ch)) dense++;\n    else other++;\n  }\n  return dense + other / 4;\n}\n"],"names":["DEFAULT_TARGET_TOKENS","estimateTokens","DENSE_SCRIPT","RegExp","UNSPACED_SCRIPTS","text","dense","other","ch","test"],"mappings":";;;;;;;;;;;QAEaA;eAAAA;;QAOGC;eAAAA;;;yBATiB;AAE1B,IAAMD,wBAAwB;AAErC,yFAAyF;AACzF,yGAAyG;AACzG,IAAME,eAAe,IAAIC,OAAO,AAAC,IAAoB,OAAjBC,2BAAgB,EAAC,qBAAmB;AAGjE,SAASH,eAAeI,IAAY;IACzC,IAAIC,QAAQ;IACZ,IAAIC,QAAQ;QACP,kCAAA,2BAAA;;QAAL,QAAK,YAAYF,yBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAkB;YAAlB,IAAMG,KAAN;YACH,IAAIN,aAAaO,IAAI,CAACD,KAAKF;iBACtBC;QACP;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAOD,QAAQC,QAAQ;AACzB"}

package/dist/cjs/commands/search.js

@@ -202,7 +202,7 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
 // snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc
-// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, rows get the JS excerpt.
+// (benchmark/reports/2026-08-23-0.13.2-hub-release-battery.md). Past this bound, rows get the JS excerpt.
 var EXCERPT_WINDOW = 160;
 // Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt
 // scan matches the same words the query matched on, not FTS5 syntax.