sensemaking 0.22.2 → 0.23.0

package/dist/cjs/chunk/group.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/group.ts"],"sourcesContent":["import { extractText } from './extract.ts';\nimport { parse } from './parse.ts';\nimport { DEFAULT_TARGET_TOKENS, estimateTokens } from './tokens.ts';\nimport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\n\nconst PGC_GROUP_SIZE = 2;\nconst OVERSIZE_TRIGGER_MULTIPLE = 2;\n\ninterface ResolvedOptions {\n  targetTokens: number;\n  text: 'extracted' | 'raw';\n}\n\nfunction resolveOptions(opts?: ChunkOptions): ResolvedOptions {\n  return {\n    targetTokens: opts?.targetTokens ?? DEFAULT_TARGET_TOKENS,\n    text: opts?.text ?? 'raw',\n  };\n}\n\n// A heading of any depth ends the current scope and starts a new one (D1); the heading block\n// itself is carried into the new scope, where it joins that scope's first group (F7/F10).\nfunction splitScopes(blocks: Block[]): Block[][] {\n  const scopes: Block[][] = [];\n  let current: Block[] = [];\n  for (const block of blocks) {\n    if (block.type === 'heading' && current.length > 0) {\n      scopes.push(current);\n      current = [];\n    }\n    current.push(block);\n  }\n  if (current.length > 0) scopes.push(current);\n  return scopes;\n}\n\ninterface Part {\n  startLine: number;\n  endLine: number;\n  text: string;\n  // True for a sub-line split piece: its text is already the final slice, not the whole line --\n  // group()'s extent-based raw re-slice must not touch it (siblings share startLine === endLine).\n  final?: boolean;\n}\n\nfunction finalize(parts: Part[]): (Chunk & { final?: boolean }) | undefined {\n  if (parts.length === 0) return undefined;\n  const first = parts[0];\n  const last = parts[parts.length - 1];\n  return { startLine: first.startLine, endLine: last.endLine, text: parts.map((p) => p.text).join('\\n'), final: parts.some((p) => p.final) };\n}\n\nconst NEWLINE_TOKENS = estimateTokens('\\n');\n// Built on first use and kept: each construction is ~3.5 ms, and only an oversize block is ever\n// split, so no command pays for a segmenter it never reaches.\nconst SEGMENTERS = new Map<string, Intl.Segmenter>();\n\nfunction segmentsOf(text: string, granularity: 'sentence' | 'word'): string[] {\n  let segmenter = SEGMENTERS.get(granularity);\n  if (!segmenter) {\n    segmenter = new Intl.Segmenter(undefined, { granularity });\n    SEGMENTERS.set(granularity, segmenter);\n  }\n  return Array.from(segmenter.segment(text), (s) => s.segment);\n}\n\n// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups\n// of at most `working` estimated tokens; a lone segment over `working` still stands alone.\nfunction pack(segments: string[], working: number): string[] {\n  const groups: string[] = [];\n  let current = '';\n  let tokens = 0;\n  for (const segment of segments) {\n    const segmentTokens = estimateTokens(segment);\n    if (current.length > 0 && tokens + segmentTokens > working) {\n      groups.push(current);\n      current = '';\n      tokens = 0;\n    }\n    current += segment;\n    tokens += segmentTokens;\n  }\n  if (current.length > 0) groups.push(current);\n  return groups;\n}\n\n// Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then\n// word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.\nfunction splitLineText(text: string, working: number): string[] {\n  const sentences = segmentsOf(text, 'sentence');\n  const out: string[] = [];\n  let current = '';\n  let tokens = 0;\n  const flush = () => {\n    if (current.length > 0) {\n      out.push(current);\n      current = '';\n      tokens = 0;\n    }\n  };\n  for (const sentence of sentences) {\n    const sentenceTokens = estimateTokens(sentence);\n    if (sentenceTokens > working) {\n      flush();\n      out.push(...pack(segmentsOf(sentence, 'word'), working));\n      continue;\n    }\n    if (current.length > 0 && tokens + sentenceTokens > working) flush();\n    current += sentence;\n    tokens += sentenceTokens;\n  }\n  flush();\n  return out;\n}\n\nconst ATOMIC_TYPES: ReadonlySet<BlockType> = new Set(['code', 'table', 'list']);\n\n// Atomic (code/table/list) pieces are always a raw line slice: re-parsing a table's later pieces\n// without their header/delimiter rows would demote them to paragraph text.\nfunction piece(pieceLines: string[], startLine: number, endLine: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part {\n  const text =\n    ATOMIC_TYPES.has(blockType) || textMode === 'raw'\n      ? pieceLines.join('\\n')\n      : parse(pieceLines.join('\\n'))\n          .map((b) => extractText(b.node))\n          .join('\\n');\n  return { startLine, endLine, text };\n}\n\n// A one-line piece over working can't shrink via another line-boundary pass (rule 5's gap), so it\n// splits at sentence/word boundaries instead; `final` stops group() re-deriving its text by extent (F5).\nfunction finalizePiece(pieceLines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part[] {\n  const p = piece(pieceLines, startLine, endLine, blockType, textMode);\n  if (pieceLines.length === 1 && estimateTokens(p.text) > working) {\n    return splitLineText(p.text, working).map((text) => ({ startLine, endLine, text, final: true }));\n  }\n  return [p];\n}\n\n// A block over 2x working size splits at line boundaries into pieces each <= working size, never\n// mid-line. `seed`: pending tokens (e.g. a heading) the first piece must join, checked against the limit.\nfunction splitOversizeBlock(lines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw', seed = 0): Part[] {\n  const pieces: Part[] = [];\n  let pieceLines: string[] = [];\n  let pieceStart = startLine;\n  let tokens = seed;\n  for (let line = startLine; line <= endLine; line++) {\n    const lineText = lines[line - 1];\n    const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;\n    const lineTokens = estimateTokens(lineText);\n    if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {\n      pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));\n      pieceLines = [];\n      tokens = 0;\n      pieceStart = line;\n      pieceLines.push(lineText);\n      tokens += lineTokens;\n      continue;\n    }\n    pieceLines.push(lineText);\n    tokens += sep + lineTokens;\n  }\n  if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));\n  return pieces;\n}\n\n// One heading scope's groups (D1): a heading opens the first group, and an oversize block\n// (rule 5, including an oversize heading) splits into pieces that each close their own group.\nfunction groupScope(scopeBlocks: Block[], lines: string[], resolved: ResolvedOptions): (Chunk & { final?: boolean })[] {\n  const working = resolved.targetTokens;\n  const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;\n  const finished: (Chunk & { final?: boolean })[] = [];\n  let parts: Part[] = [];\n  let paragraphCount = 0;\n  let tokens = 0;\n\n  function close(): void {\n    const group = finalize(parts);\n    if (group) finished.push(group);\n    parts = [];\n    paragraphCount = 0;\n    tokens = 0;\n  }\n\n  // tokens tracks the active text mode's own estimate (a newline between parts costs\n  // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.\n  function addPart(text: string, sizeText: string, startLine: number, endLine: number): void {\n    tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);\n    parts.push({ startLine, endLine, text });\n  }\n\n  for (const block of scopeBlocks) {\n    const raw = lines.slice(block.startLine - 1, block.endLine).join('\\n');\n    const blockTokens = estimateTokens(raw);\n\n    if (blockTokens > trigger) {\n      const seed = parts.length > 0 ? tokens : 0;\n      for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)) {\n        parts.push(p);\n        close();\n      }\n      continue;\n    }\n\n    const extracted = extractText(block.node);\n    const sizeText = resolved.text === 'raw' ? raw : extracted;\n\n    if (block.type === 'heading') {\n      addPart(extracted, sizeText, block.startLine, block.endLine);\n      continue;\n    }\n\n    // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --\n    // close first if the pair about to form would cross it.\n    const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;\n    if (pairOversize) close();\n\n    addPart(extracted, sizeText, block.startLine, block.endLine);\n    paragraphCount++;\n\n    if (paragraphCount >= PGC_GROUP_SIZE) close();\n  }\n  close();\n\n  return finished;\n}\n\n// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed\n// from (line lookups for oversize splitting).\nexport function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  const resolved = resolveOptions(opts);\n  const lines = body.split('\\n');\n  const chunks: (Chunk & { final?: boolean })[] = [];\n  for (const scope of splitScopes(blocks)) chunks.push(...groupScope(scope, lines, resolved));\n  // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).\n  // A `final` chunk already carries its own slice's raw text; re-slicing by extent would return the whole shared line.\n  const texted =\n    resolved.text === 'raw'\n      ? chunks.map((c) =>\n          c.final\n            ? c\n            : {\n                ...c,\n                text: lines\n                  .slice(c.startLine - 1, c.endLine)\n                  .join('\\n')\n                  .trim(),\n              }\n        )\n      : chunks;\n  // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.\n  return texted.filter((c) => c.text.trim().length > 0).map((c) => ({ startLine: c.startLine, endLine: c.endLine, text: c.text }));\n}\n"],"names":["group","PGC_GROUP_SIZE","OVERSIZE_TRIGGER_MULTIPLE","resolveOptions","opts","targetTokens","DEFAULT_TARGET_TOKENS","text","splitScopes","blocks","scopes","current","block","type","length","push","finalize","parts","undefined","first","last","startLine","endLine","map","p","join","final","some","NEWLINE_TOKENS","estimateTokens","SEGMENTERS","Map","segmentsOf","granularity","segmenter","get","Intl","Segmenter","set","Array","from","segment","s","pack","segments","working","groups","tokens","segmentTokens","splitLineText","sentences","out","flush","sentence","sentenceTokens","ATOMIC_TYPES","Set","piece","pieceLines","blockType","textMode","has","parse","b","extractText","node","finalizePiece","splitOversizeBlock","lines","seed","pieces","pieceStart","line","lineText","sep","lineTokens","groupScope","scopeBlocks","resolved","trigger","finished","paragraphCount","close","addPart","sizeText","raw","slice","blockTokens","extracted","pairOversize","body","chunks","split","scope","texted","c","trim","filter"],"mappings":";;;;+BAqOgBA;;;eAAAA;;;yBArOY;uBACN;wBACgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGtD,IAAMC,iBAAiB;AACvB,IAAMC,4BAA4B;AAOlC,SAASC,eAAeC,IAAmB;;IACzC,OAAO;QACLC,YAAY,UAAED,iBAAAA,2BAAAA,KAAMC,YAAY,uCAAIC,+BAAqB;QACzDC,IAAI,WAAEH,iBAAAA,2BAAAA,KAAMG,IAAI,yCAAI;IACtB;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASC,YAAYC,MAAe;IAClC,IAAMC,SAAoB,EAAE;IAC5B,IAAIC,UAAmB,EAAE;QACpB,kCAAA,2BAAA;;QAAL,QAAK,YAAeF,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;YAAvB,IAAMG,QAAN;YACH,IAAIA,MAAMC,IAAI,KAAK,aAAaF,QAAQG,MAAM,GAAG,GAAG;gBAClDJ,OAAOK,IAAI,CAACJ;gBACZA,UAAU,EAAE;YACd;YACAA,QAAQI,IAAI,CAACH;QACf;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAOL,IAAID,QAAQG,MAAM,GAAG,GAAGJ,OAAOK,IAAI,CAACJ;IACpC,OAAOD;AACT;AAWA,SAASM,SAASC,KAAa;IAC7B,IAAIA,MAAMH,MAAM,KAAK,GAAG,OAAOI;IAC/B,IAAMC,QAAQF,KAAK,CAAC,EAAE;IACtB,IAAMG,OAAOH,KAAK,CAACA,MAAMH,MAAM,GAAG,EAAE;IACpC,OAAO;QAAEO,WAAWF,MAAME,SAAS;QAAEC,SAASF,KAAKE,OAAO;QAAEf,MAAMU,MAAMM,GAAG,CAAC,SAACC;mBAAMA,EAAEjB,IAAI;WAAEkB,IAAI,CAAC;QAAOC,OAAOT,MAAMU,IAAI,CAAC,SAACH;mBAAMA,EAAEE,KAAK;;IAAE;AAC3I;AAEA,IAAME,iBAAiBC,IAAAA,wBAAc,EAAC;AACtC,gGAAgG;AAChG,8DAA8D;AAC9D,IAAMC,aAAa,IAAIC;AAEvB,SAASC,WAAWzB,IAAY,EAAE0B,WAAgC;IAChE,IAAIC,YAAYJ,WAAWK,GAAG,CAACF;IAC/B,IAAI,CAACC,WAAW;QACdA,YAAY,IAAIE,KAAKC,SAAS,CAACnB,WAAW;YAAEe,aAAAA;QAAY;QACxDH,WAAWQ,GAAG,CAACL,aAAaC;IAC9B;IACA,OAAOK,MAAMC,IAAI,CAACN,UAAUO,OAAO,CAAClC,OAAO,SAACmC;eAAMA,EAAED,OAAO;;AAC7D;AAEA,gGAAgG;AAChG,2FAA2F;AAC3F,SAASE,KAAKC,QAAkB,EAAEC,OAAe;IAC/C,IAAMC,SAAmB,EAAE;IAC3B,IAAInC,UAAU;IACd,IAAIoC,SAAS;QACR,kCAAA,2BAAA;;QAAL,QAAK,YAAiBH,6BAAjB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMH,UAAN;YACH,IAAMO,gBAAgBnB,IAAAA,wBAAc,EAACY;YACrC,IAAI9B,QAAQG,MAAM,GAAG,KAAKiC,SAASC,gBAAgBH,SAAS;gBAC1DC,OAAO/B,IAAI,CAACJ;gBACZA,UAAU;gBACVoC,SAAS;YACX;YACApC,WAAW8B;YACXM,UAAUC;QACZ;;QATK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,IAAIrC,QAAQG,MAAM,GAAG,GAAGgC,OAAO/B,IAAI,CAACJ;IACpC,OAAOmC;AACT;AAEA,8FAA8F;AAC9F,0GAA0G;AAC1G,SAASG,cAAc1C,IAAY,EAAEsC,OAAe;IAClD,IAAMK,YAAYlB,WAAWzB,MAAM;IACnC,IAAM4C,MAAgB,EAAE;IACxB,IAAIxC,UAAU;IACd,IAAIoC,SAAS;IACb,IAAMK,QAAQ;QACZ,IAAIzC,QAAQG,MAAM,GAAG,GAAG;YACtBqC,IAAIpC,IAAI,CAACJ;YACTA,UAAU;YACVoC,SAAS;QACX;IACF;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAkBG,8BAAlB,SAAA,6BAAA,QAAA,yBAAA,iCAA6B;YAA7B,IAAMG,WAAN;YACH,IAAMC,iBAAiBzB,IAAAA,wBAAc,EAACwB;YACtC,IAAIC,iBAAiBT,SAAS;oBAE5BM;gBADAC;gBACAD,CAAAA,OAAAA,KAAIpC,IAAI,OAARoC,MAAS,qBAAGR,KAAKX,WAAWqB,UAAU,SAASR;gBAC/C;YACF;YACA,IAAIlC,QAAQG,MAAM,GAAG,KAAKiC,SAASO,iBAAiBT,SAASO;YAC7DzC,WAAW0C;YACXN,UAAUO;QACZ;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAWLF;IACA,OAAOD;AACT;AAEA,IAAMI,eAAuC,IAAIC,IAAI;IAAC;IAAQ;IAAS;CAAO;AAE9E,iGAAiG;AACjG,2EAA2E;AAC3E,SAASC,MAAMC,UAAoB,EAAErC,SAAiB,EAAEC,OAAe,EAAEqC,SAAoB,EAAEC,QAA6B;IAC1H,IAAMrD,OACJgD,aAAaM,GAAG,CAACF,cAAcC,aAAa,QACxCF,WAAWjC,IAAI,CAAC,QAChBqC,IAAAA,cAAK,EAACJ,WAAWjC,IAAI,CAAC,OACnBF,GAAG,CAAC,SAACwC;eAAMC,IAAAA,sBAAW,EAACD,EAAEE,IAAI;OAC7BxC,IAAI,CAAC;IACd,OAAO;QAAEJ,WAAAA;QAAWC,SAAAA;QAASf,MAAAA;IAAK;AACpC;AAEA,kGAAkG;AAClG,yGAAyG;AACzG,SAAS2D,cAAcR,UAAoB,EAAErC,SAAiB,EAAEC,OAAe,EAAEuB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;IACnJ,IAAMpC,IAAIiC,MAAMC,YAAYrC,WAAWC,SAASqC,WAAWC;IAC3D,IAAIF,WAAW5C,MAAM,KAAK,KAAKe,IAAAA,wBAAc,EAACL,EAAEjB,IAAI,IAAIsC,SAAS;QAC/D,OAAOI,cAAczB,EAAEjB,IAAI,EAAEsC,SAAStB,GAAG,CAAC,SAAChB;mBAAU;gBAAEc,WAAAA;gBAAWC,SAAAA;gBAASf,MAAAA;gBAAMmB,OAAO;YAAK;;IAC/F;IACA,OAAO;QAACF;KAAE;AACZ;AAEA,iGAAiG;AACjG,0GAA0G;AAC1G,SAAS2C,mBAAmBC,KAAe,EAAE/C,SAAiB,EAAEC,OAAe,EAAEuB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;QAAES,OAAAA,iEAAO;QAqBjIC;IApB3B,IAAMA,SAAiB,EAAE;IACzB,IAAIZ,aAAuB,EAAE;IAC7B,IAAIa,aAAalD;IACjB,IAAI0B,SAASsB;IACb,IAAK,IAAIG,OAAOnD,WAAWmD,QAAQlD,SAASkD,OAAQ;QAClD,IAAMC,WAAWL,KAAK,CAACI,OAAO,EAAE;QAChC,IAAME,MAAMhB,WAAW5C,MAAM,GAAG,KAAKiC,SAAS,IAAInB,iBAAiB;QACnE,IAAM+C,aAAa9C,IAAAA,wBAAc,EAAC4C;QAClC,IAAIf,WAAW5C,MAAM,GAAG,KAAKiC,SAAS2B,MAAMC,aAAa9B,SAAS;gBAChEyB;YAAAA,CAAAA,WAAAA,QAAOvD,IAAI,OAAXuD,UAAY,qBAAGJ,cAAcR,YAAYa,YAAYC,OAAO,GAAG3B,SAASc,WAAWC;YACnFF,aAAa,EAAE;YACfX,SAAS;YACTwB,aAAaC;YACbd,WAAW3C,IAAI,CAAC0D;YAChB1B,UAAU4B;YACV;QACF;QACAjB,WAAW3C,IAAI,CAAC0D;QAChB1B,UAAU2B,MAAMC;IAClB;IACA,IAAIjB,WAAW5C,MAAM,GAAG,GAAGwD,CAAAA,UAAAA,QAAOvD,IAAI,OAAXuD,SAAY,qBAAGJ,cAAcR,YAAYa,YAAYjD,SAASuB,SAASc,WAAWC;IAC7G,OAAOU;AACT;AAEA,0FAA0F;AAC1F,8FAA8F;AAC9F,SAASM,WAAWC,WAAoB,EAAET,KAAe,EAAEU,QAAyB;IAClF,IAAMjC,UAAUiC,SAASzE,YAAY;IACrC,IAAM0E,UAAUlC,UAAU3C;IAC1B,IAAM8E,WAA4C,EAAE;IACpD,IAAI/D,QAAgB,EAAE;IACtB,IAAIgE,iBAAiB;IACrB,IAAIlC,SAAS;IAEb,SAASmC;QACP,IAAMlF,QAAQgB,SAASC;QACvB,IAAIjB,OAAOgF,SAASjE,IAAI,CAACf;QACzBiB,QAAQ,EAAE;QACVgE,iBAAiB;QACjBlC,SAAS;IACX;IAEA,mFAAmF;IACnF,2FAA2F;IAC3F,SAASoC,QAAQ5E,IAAY,EAAE6E,QAAgB,EAAE/D,SAAiB,EAAEC,OAAe;QACjFyB,UAAU,AAAC9B,CAAAA,MAAMH,MAAM,GAAG,IAAIc,iBAAiB,CAAA,IAAKC,IAAAA,wBAAc,EAACuD;QACnEnE,MAAMF,IAAI,CAAC;YAAEM,WAAAA;YAAWC,SAAAA;YAASf,MAAAA;QAAK;IACxC;QAEK,kCAAA,2BAAA;;QAAL,QAAK,YAAesE,gCAAf,SAAA,6BAAA,QAAA,yBAAA,iCAA4B;YAA5B,IAAMjE,QAAN;YACH,IAAMyE,MAAMjB,MAAMkB,KAAK,CAAC1E,MAAMS,SAAS,GAAG,GAAGT,MAAMU,OAAO,EAAEG,IAAI,CAAC;YACjE,IAAM8D,cAAc1D,IAAAA,wBAAc,EAACwD;YAEnC,IAAIE,cAAcR,SAAS;gBACzB,IAAMV,OAAOpD,MAAMH,MAAM,GAAG,IAAIiC,SAAS;oBACpC,mCAAA,4BAAA;;oBAAL,QAAK,aAAWoB,mBAAmBC,OAAOxD,MAAMS,SAAS,EAAET,MAAMU,OAAO,EAAEuB,SAASjC,MAAMC,IAAI,EAAEiE,SAASvE,IAAI,EAAE8D,0BAAzG,UAAA,8BAAA,SAAA,0BAAA,kCAAgH;wBAAhH,IAAM7C,IAAN;wBACHP,MAAMF,IAAI,CAACS;wBACX0D;oBACF;;oBAHK;oBAAA;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;gBAIL;YACF;YAEA,IAAMM,YAAYxB,IAAAA,sBAAW,EAACpD,MAAMqD,IAAI;YACxC,IAAMmB,WAAWN,SAASvE,IAAI,KAAK,QAAQ8E,MAAMG;YAEjD,IAAI5E,MAAMC,IAAI,KAAK,WAAW;gBAC5BsE,QAAQK,WAAWJ,UAAUxE,MAAMS,SAAS,EAAET,MAAMU,OAAO;gBAC3D;YACF;YAEA,wFAAwF;YACxF,wDAAwD;YACxD,IAAMmE,eAAexE,MAAMH,MAAM,GAAG,KAAKiC,SAASnB,iBAAiB2D,cAAcR;YACjF,IAAIU,cAAcP;YAElBC,QAAQK,WAAWJ,UAAUxE,MAAMS,SAAS,EAAET,MAAMU,OAAO;YAC3D2D;YAEA,IAAIA,kBAAkBhF,gBAAgBiF;QACxC;;QA9BK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IA+BLA;IAEA,OAAOF;AACT;AAIO,SAAShF,MAAMS,MAAe,EAAEiF,IAAY,EAAEtF,IAAmB;QAI7BuF;IAHzC,IAAMb,WAAW3E,eAAeC;IAChC,IAAMgE,QAAQsB,KAAKE,KAAK,CAAC;IACzB,IAAMD,SAA0C,EAAE;QAC7C,kCAAA,2BAAA;;QAAL,QAAK,YAAenF,YAAYC,4BAA3B,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMoF,QAAN;YAAoCF,CAAAA,UAAAA,QAAO5E,IAAI,OAAX4E,SAAY,qBAAGf,WAAWiB,OAAOzB,OAAOU;;;QAA5E;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,+FAA+F;IAC/F,qHAAqH;IACrH,IAAMgB,SACJhB,SAASvE,IAAI,KAAK,QACdoF,OAAOpE,GAAG,CAAC,SAACwE;eACVA,EAAErE,KAAK,GACHqE,IACA,wCACKA;YACHxF,MAAM6D,MACHkB,KAAK,CAACS,EAAE1E,SAAS,GAAG,GAAG0E,EAAEzE,OAAO,EAChCG,IAAI,CAAC,MACLuE,IAAI;;SAGfL;IACN,mHAAmH;IACnH,OAAOG,OAAOG,MAAM,CAAC,SAACF;eAAMA,EAAExF,IAAI,CAACyF,IAAI,GAAGlF,MAAM,GAAG;OAAGS,GAAG,CAAC,SAACwE;eAAO;YAAE1E,WAAW0E,EAAE1E,SAAS;YAAEC,SAASyE,EAAEzE,OAAO;YAAEf,MAAMwF,EAAExF,IAAI;QAAC;;AAC/H"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/group.ts"],"sourcesContent":["import { extractText } from './extract.ts';\nimport { parse } from './parse.ts';\nimport { DEFAULT_TARGET_TOKENS, estimateTokens } from './tokens.ts';\nimport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\n\nconst PGC_GROUP_SIZE = 2;\nconst OVERSIZE_TRIGGER_MULTIPLE = 2;\n\ninterface ResolvedOptions {\n  targetTokens: number;\n  text: 'extracted' | 'raw';\n}\n\nfunction resolveOptions(opts?: ChunkOptions): ResolvedOptions {\n  return {\n    targetTokens: opts?.targetTokens ?? DEFAULT_TARGET_TOKENS,\n    text: opts?.text ?? 'raw',\n  };\n}\n\n// A heading of any depth ends the current scope and starts a new one (D1); the heading block\n// itself is carried into the new scope, where it joins that scope's first group (F7/F10).\nfunction splitScopes(blocks: Block[]): Block[][] {\n  const scopes: Block[][] = [];\n  let current: Block[] = [];\n  for (const block of blocks) {\n    if (block.type === 'heading' && current.length > 0) {\n      scopes.push(current);\n      current = [];\n    }\n    current.push(block);\n  }\n  if (current.length > 0) scopes.push(current);\n  return scopes;\n}\n\ninterface Part {\n  startLine: number;\n  endLine: number;\n  text: string;\n  // True for a sub-line split piece: its text is already the final slice, not the whole line --\n  // group()'s extent-based raw re-slice must not touch it (siblings share startLine === endLine).\n  final?: boolean;\n}\n\nfunction finalize(parts: Part[]): (Chunk & { final?: boolean }) | undefined {\n  if (parts.length === 0) return undefined;\n  const first = parts[0];\n  const last = parts[parts.length - 1];\n  return { startLine: first.startLine, endLine: last.endLine, text: parts.map((p) => p.text).join('\\n'), final: parts.some((p) => p.final) };\n}\n\nconst NEWLINE_TOKENS = estimateTokens('\\n');\n// Built on first use and kept: each construction is ~3.5 ms, and only an oversize block is ever\n// split, so no command pays for a segmenter it never reaches.\nconst SEGMENTERS = new Map<string, Intl.Segmenter>();\n\nfunction segmentsOf(text: string, granularity: 'sentence' | 'word'): string[] {\n  let segmenter = SEGMENTERS.get(granularity);\n  if (!segmenter) {\n    segmenter = new Intl.Segmenter(undefined, { granularity });\n    SEGMENTERS.set(granularity, segmenter);\n  }\n  return Array.from(segmenter.segment(text), (s) => s.segment);\n}\n\n// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups\n// of at most `working` estimated tokens; a lone segment over `working` still stands alone.\nfunction pack(segments: string[], working: number): string[] {\n  const groups: string[] = [];\n  let current = '';\n  let tokens = 0;\n  for (const segment of segments) {\n    const segmentTokens = estimateTokens(segment);\n    if (current.length > 0 && tokens + segmentTokens > working) {\n      groups.push(current);\n      current = '';\n      tokens = 0;\n    }\n    current += segment;\n    tokens += segmentTokens;\n  }\n  if (current.length > 0) groups.push(current);\n  return groups;\n}\n\n// Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then\n// word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.\nfunction splitLineText(text: string, working: number): string[] {\n  const sentences = segmentsOf(text, 'sentence');\n  const out: string[] = [];\n  let current = '';\n  let tokens = 0;\n  const flush = () => {\n    if (current.length > 0) {\n      out.push(current);\n      current = '';\n      tokens = 0;\n    }\n  };\n  for (const sentence of sentences) {\n    const sentenceTokens = estimateTokens(sentence);\n    if (sentenceTokens > working) {\n      flush();\n      out.push(...pack(segmentsOf(sentence, 'word'), working));\n      continue;\n    }\n    if (current.length > 0 && tokens + sentenceTokens > working) flush();\n    current += sentence;\n    tokens += sentenceTokens;\n  }\n  flush();\n  return out;\n}\n\nconst ATOMIC_TYPES: ReadonlySet<BlockType> = new Set(['code', 'table', 'list']);\n\n// Atomic (code/table/list) pieces are always a raw line slice: re-parsing a table's later pieces\n// without their header/delimiter rows would demote them to paragraph text.\nfunction piece(pieceLines: string[], startLine: number, endLine: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part {\n  const text =\n    ATOMIC_TYPES.has(blockType) || textMode === 'raw'\n      ? pieceLines.join('\\n')\n      : parse(pieceLines.join('\\n'))\n          .map((b) => extractText(b))\n          .join('\\n');\n  return { startLine, endLine, text };\n}\n\n// A one-line piece over working can't shrink via another line-boundary pass (rule 5's gap), so it\n// splits at sentence/word boundaries instead; `final` stops group() re-deriving its text by extent (F5).\nfunction finalizePiece(pieceLines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part[] {\n  const p = piece(pieceLines, startLine, endLine, blockType, textMode);\n  if (pieceLines.length === 1 && estimateTokens(p.text) > working) {\n    return splitLineText(p.text, working).map((text) => ({ startLine, endLine, text, final: true }));\n  }\n  return [p];\n}\n\n// A block over 2x working size splits at line boundaries into pieces each <= working size, never\n// mid-line. `seed`: pending tokens (e.g. a heading) the first piece must join, checked against the limit.\nfunction splitOversizeBlock(lines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw', seed = 0): Part[] {\n  const pieces: Part[] = [];\n  let pieceLines: string[] = [];\n  let pieceStart = startLine;\n  let tokens = seed;\n  for (let line = startLine; line <= endLine; line++) {\n    const lineText = lines[line - 1];\n    const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;\n    const lineTokens = estimateTokens(lineText);\n    if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {\n      pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));\n      pieceLines = [];\n      tokens = 0;\n      pieceStart = line;\n      pieceLines.push(lineText);\n      tokens += lineTokens;\n      continue;\n    }\n    pieceLines.push(lineText);\n    tokens += sep + lineTokens;\n  }\n  if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));\n  return pieces;\n}\n\n// One heading scope's groups (D1): a heading opens the first group, and an oversize block\n// (rule 5, including an oversize heading) splits into pieces that each close their own group.\nfunction groupScope(scopeBlocks: Block[], lines: string[], resolved: ResolvedOptions): (Chunk & { final?: boolean })[] {\n  const working = resolved.targetTokens;\n  const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;\n  const finished: (Chunk & { final?: boolean })[] = [];\n  let parts: Part[] = [];\n  let paragraphCount = 0;\n  let tokens = 0;\n\n  function close(): void {\n    const group = finalize(parts);\n    if (group) finished.push(group);\n    parts = [];\n    paragraphCount = 0;\n    tokens = 0;\n  }\n\n  // tokens tracks the active text mode's own estimate (a newline between parts costs\n  // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.\n  function addPart(text: string, sizeText: string, startLine: number, endLine: number): void {\n    tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);\n    parts.push({ startLine, endLine, text });\n  }\n\n  for (const block of scopeBlocks) {\n    const raw = lines.slice(block.startLine - 1, block.endLine).join('\\n');\n    const blockTokens = estimateTokens(raw);\n\n    if (blockTokens > trigger) {\n      const seed = parts.length > 0 ? tokens : 0;\n      for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)) {\n        parts.push(p);\n        close();\n      }\n      continue;\n    }\n\n    const extracted = extractText(block);\n    const sizeText = resolved.text === 'raw' ? raw : extracted;\n\n    if (block.type === 'heading') {\n      addPart(extracted, sizeText, block.startLine, block.endLine);\n      continue;\n    }\n\n    // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --\n    // close first if the pair about to form would cross it.\n    const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;\n    if (pairOversize) close();\n\n    addPart(extracted, sizeText, block.startLine, block.endLine);\n    paragraphCount++;\n\n    if (paragraphCount >= PGC_GROUP_SIZE) close();\n  }\n  close();\n\n  return finished;\n}\n\n// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed\n// from (line lookups for oversize splitting).\nexport function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  const resolved = resolveOptions(opts);\n  const lines = body.split('\\n');\n  const chunks: (Chunk & { final?: boolean })[] = [];\n  for (const scope of splitScopes(blocks)) chunks.push(...groupScope(scope, lines, resolved));\n  // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).\n  // A `final` chunk already carries its own slice's raw text; re-slicing by extent would return the whole shared line.\n  const texted =\n    resolved.text === 'raw'\n      ? chunks.map((c) =>\n          c.final\n            ? c\n            : {\n                ...c,\n                text: lines\n                  .slice(c.startLine - 1, c.endLine)\n                  .join('\\n')\n                  .trim(),\n              }\n        )\n      : chunks;\n  // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.\n  return texted.filter((c) => c.text.trim().length > 0).map((c) => ({ startLine: c.startLine, endLine: c.endLine, text: c.text }));\n}\n"],"names":["group","PGC_GROUP_SIZE","OVERSIZE_TRIGGER_MULTIPLE","resolveOptions","opts","targetTokens","DEFAULT_TARGET_TOKENS","text","splitScopes","blocks","scopes","current","block","type","length","push","finalize","parts","undefined","first","last","startLine","endLine","map","p","join","final","some","NEWLINE_TOKENS","estimateTokens","SEGMENTERS","Map","segmentsOf","granularity","segmenter","get","Intl","Segmenter","set","Array","from","segment","s","pack","segments","working","groups","tokens","segmentTokens","splitLineText","sentences","out","flush","sentence","sentenceTokens","ATOMIC_TYPES","Set","piece","pieceLines","blockType","textMode","has","parse","b","extractText","finalizePiece","splitOversizeBlock","lines","seed","pieces","pieceStart","line","lineText","sep","lineTokens","groupScope","scopeBlocks","resolved","trigger","finished","paragraphCount","close","addPart","sizeText","raw","slice","blockTokens","extracted","pairOversize","body","chunks","split","scope","texted","c","trim","filter"],"mappings":";;;;+BAqOgBA;;;eAAAA;;;yBArOY;uBACN;wBACgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGtD,IAAMC,iBAAiB;AACvB,IAAMC,4BAA4B;AAOlC,SAASC,eAAeC,IAAmB;;IACzC,OAAO;QACLC,YAAY,UAAED,iBAAAA,2BAAAA,KAAMC,YAAY,uCAAIC,+BAAqB;QACzDC,IAAI,WAAEH,iBAAAA,2BAAAA,KAAMG,IAAI,yCAAI;IACtB;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASC,YAAYC,MAAe;IAClC,IAAMC,SAAoB,EAAE;IAC5B,IAAIC,UAAmB,EAAE;QACpB,kCAAA,2BAAA;;QAAL,QAAK,YAAeF,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;YAAvB,IAAMG,QAAN;YACH,IAAIA,MAAMC,IAAI,KAAK,aAAaF,QAAQG,MAAM,GAAG,GAAG;gBAClDJ,OAAOK,IAAI,CAACJ;gBACZA,UAAU,EAAE;YACd;YACAA,QAAQI,IAAI,CAACH;QACf;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAOL,IAAID,QAAQG,MAAM,GAAG,GAAGJ,OAAOK,IAAI,CAACJ;IACpC,OAAOD;AACT;AAWA,SAASM,SAASC,KAAa;IAC7B,IAAIA,MAAMH,MAAM,KAAK,GAAG,OAAOI;IAC/B,IAAMC,QAAQF,KAAK,CAAC,EAAE;IACtB,IAAMG,OAAOH,KAAK,CAACA,MAAMH,MAAM,GAAG,EAAE;IACpC,OAAO;QAAEO,WAAWF,MAAME,SAAS;QAAEC,SAASF,KAAKE,OAAO;QAAEf,MAAMU,MAAMM,GAAG,CAAC,SAACC;mBAAMA,EAAEjB,IAAI;WAAEkB,IAAI,CAAC;QAAOC,OAAOT,MAAMU,IAAI,CAAC,SAACH;mBAAMA,EAAEE,KAAK;;IAAE;AAC3I;AAEA,IAAME,iBAAiBC,IAAAA,wBAAc,EAAC;AACtC,gGAAgG;AAChG,8DAA8D;AAC9D,IAAMC,aAAa,IAAIC;AAEvB,SAASC,WAAWzB,IAAY,EAAE0B,WAAgC;IAChE,IAAIC,YAAYJ,WAAWK,GAAG,CAACF;IAC/B,IAAI,CAACC,WAAW;QACdA,YAAY,IAAIE,KAAKC,SAAS,CAACnB,WAAW;YAAEe,aAAAA;QAAY;QACxDH,WAAWQ,GAAG,CAACL,aAAaC;IAC9B;IACA,OAAOK,MAAMC,IAAI,CAACN,UAAUO,OAAO,CAAClC,OAAO,SAACmC;eAAMA,EAAED,OAAO;;AAC7D;AAEA,gGAAgG;AAChG,2FAA2F;AAC3F,SAASE,KAAKC,QAAkB,EAAEC,OAAe;IAC/C,IAAMC,SAAmB,EAAE;IAC3B,IAAInC,UAAU;IACd,IAAIoC,SAAS;QACR,kCAAA,2BAAA;;QAAL,QAAK,YAAiBH,6BAAjB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMH,UAAN;YACH,IAAMO,gBAAgBnB,IAAAA,wBAAc,EAACY;YACrC,IAAI9B,QAAQG,MAAM,GAAG,KAAKiC,SAASC,gBAAgBH,SAAS;gBAC1DC,OAAO/B,IAAI,CAACJ;gBACZA,UAAU;gBACVoC,SAAS;YACX;YACApC,WAAW8B;YACXM,UAAUC;QACZ;;QATK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,IAAIrC,QAAQG,MAAM,GAAG,GAAGgC,OAAO/B,IAAI,CAACJ;IACpC,OAAOmC;AACT;AAEA,8FAA8F;AAC9F,0GAA0G;AAC1G,SAASG,cAAc1C,IAAY,EAAEsC,OAAe;IAClD,IAAMK,YAAYlB,WAAWzB,MAAM;IACnC,IAAM4C,MAAgB,EAAE;IACxB,IAAIxC,UAAU;IACd,IAAIoC,SAAS;IACb,IAAMK,QAAQ;QACZ,IAAIzC,QAAQG,MAAM,GAAG,GAAG;YACtBqC,IAAIpC,IAAI,CAACJ;YACTA,UAAU;YACVoC,SAAS;QACX;IACF;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAkBG,8BAAlB,SAAA,6BAAA,QAAA,yBAAA,iCAA6B;YAA7B,IAAMG,WAAN;YACH,IAAMC,iBAAiBzB,IAAAA,wBAAc,EAACwB;YACtC,IAAIC,iBAAiBT,SAAS;oBAE5BM;gBADAC;gBACAD,CAAAA,OAAAA,KAAIpC,IAAI,OAARoC,MAAS,qBAAGR,KAAKX,WAAWqB,UAAU,SAASR;gBAC/C;YACF;YACA,IAAIlC,QAAQG,MAAM,GAAG,KAAKiC,SAASO,iBAAiBT,SAASO;YAC7DzC,WAAW0C;YACXN,UAAUO;QACZ;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAWLF;IACA,OAAOD;AACT;AAEA,IAAMI,eAAuC,IAAIC,IAAI;IAAC;IAAQ;IAAS;CAAO;AAE9E,iGAAiG;AACjG,2EAA2E;AAC3E,SAASC,MAAMC,UAAoB,EAAErC,SAAiB,EAAEC,OAAe,EAAEqC,SAAoB,EAAEC,QAA6B;IAC1H,IAAMrD,OACJgD,aAAaM,GAAG,CAACF,cAAcC,aAAa,QACxCF,WAAWjC,IAAI,CAAC,QAChBqC,IAAAA,cAAK,EAACJ,WAAWjC,IAAI,CAAC,OACnBF,GAAG,CAAC,SAACwC;eAAMC,IAAAA,sBAAW,EAACD;OACvBtC,IAAI,CAAC;IACd,OAAO;QAAEJ,WAAAA;QAAWC,SAAAA;QAASf,MAAAA;IAAK;AACpC;AAEA,kGAAkG;AAClG,yGAAyG;AACzG,SAAS0D,cAAcP,UAAoB,EAAErC,SAAiB,EAAEC,OAAe,EAAEuB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;IACnJ,IAAMpC,IAAIiC,MAAMC,YAAYrC,WAAWC,SAASqC,WAAWC;IAC3D,IAAIF,WAAW5C,MAAM,KAAK,KAAKe,IAAAA,wBAAc,EAACL,EAAEjB,IAAI,IAAIsC,SAAS;QAC/D,OAAOI,cAAczB,EAAEjB,IAAI,EAAEsC,SAAStB,GAAG,CAAC,SAAChB;mBAAU;gBAAEc,WAAAA;gBAAWC,SAAAA;gBAASf,MAAAA;gBAAMmB,OAAO;YAAK;;IAC/F;IACA,OAAO;QAACF;KAAE;AACZ;AAEA,iGAAiG;AACjG,0GAA0G;AAC1G,SAAS0C,mBAAmBC,KAAe,EAAE9C,SAAiB,EAAEC,OAAe,EAAEuB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;QAAEQ,OAAAA,iEAAO;QAqBjIC;IApB3B,IAAMA,SAAiB,EAAE;IACzB,IAAIX,aAAuB,EAAE;IAC7B,IAAIY,aAAajD;IACjB,IAAI0B,SAASqB;IACb,IAAK,IAAIG,OAAOlD,WAAWkD,QAAQjD,SAASiD,OAAQ;QAClD,IAAMC,WAAWL,KAAK,CAACI,OAAO,EAAE;QAChC,IAAME,MAAMf,WAAW5C,MAAM,GAAG,KAAKiC,SAAS,IAAInB,iBAAiB;QACnE,IAAM8C,aAAa7C,IAAAA,wBAAc,EAAC2C;QAClC,IAAId,WAAW5C,MAAM,GAAG,KAAKiC,SAAS0B,MAAMC,aAAa7B,SAAS;gBAChEwB;YAAAA,CAAAA,WAAAA,QAAOtD,IAAI,OAAXsD,UAAY,qBAAGJ,cAAcP,YAAYY,YAAYC,OAAO,GAAG1B,SAASc,WAAWC;YACnFF,aAAa,EAAE;YACfX,SAAS;YACTuB,aAAaC;YACbb,WAAW3C,IAAI,CAACyD;YAChBzB,UAAU2B;YACV;QACF;QACAhB,WAAW3C,IAAI,CAACyD;QAChBzB,UAAU0B,MAAMC;IAClB;IACA,IAAIhB,WAAW5C,MAAM,GAAG,GAAGuD,CAAAA,UAAAA,QAAOtD,IAAI,OAAXsD,SAAY,qBAAGJ,cAAcP,YAAYY,YAAYhD,SAASuB,SAASc,WAAWC;IAC7G,OAAOS;AACT;AAEA,0FAA0F;AAC1F,8FAA8F;AAC9F,SAASM,WAAWC,WAAoB,EAAET,KAAe,EAAEU,QAAyB;IAClF,IAAMhC,UAAUgC,SAASxE,YAAY;IACrC,IAAMyE,UAAUjC,UAAU3C;IAC1B,IAAM6E,WAA4C,EAAE;IACpD,IAAI9D,QAAgB,EAAE;IACtB,IAAI+D,iBAAiB;IACrB,IAAIjC,SAAS;IAEb,SAASkC;QACP,IAAMjF,QAAQgB,SAASC;QACvB,IAAIjB,OAAO+E,SAAShE,IAAI,CAACf;QACzBiB,QAAQ,EAAE;QACV+D,iBAAiB;QACjBjC,SAAS;IACX;IAEA,mFAAmF;IACnF,2FAA2F;IAC3F,SAASmC,QAAQ3E,IAAY,EAAE4E,QAAgB,EAAE9D,SAAiB,EAAEC,OAAe;QACjFyB,UAAU,AAAC9B,CAAAA,MAAMH,MAAM,GAAG,IAAIc,iBAAiB,CAAA,IAAKC,IAAAA,wBAAc,EAACsD;QACnElE,MAAMF,IAAI,CAAC;YAAEM,WAAAA;YAAWC,SAAAA;YAASf,MAAAA;QAAK;IACxC;QAEK,kCAAA,2BAAA;;QAAL,QAAK,YAAeqE,gCAAf,SAAA,6BAAA,QAAA,yBAAA,iCAA4B;YAA5B,IAAMhE,QAAN;YACH,IAAMwE,MAAMjB,MAAMkB,KAAK,CAACzE,MAAMS,SAAS,GAAG,GAAGT,MAAMU,OAAO,EAAEG,IAAI,CAAC;YACjE,IAAM6D,cAAczD,IAAAA,wBAAc,EAACuD;YAEnC,IAAIE,cAAcR,SAAS;gBACzB,IAAMV,OAAOnD,MAAMH,MAAM,GAAG,IAAIiC,SAAS;oBACpC,mCAAA,4BAAA;;oBAAL,QAAK,aAAWmB,mBAAmBC,OAAOvD,MAAMS,SAAS,EAAET,MAAMU,OAAO,EAAEuB,SAASjC,MAAMC,IAAI,EAAEgE,SAAStE,IAAI,EAAE6D,0BAAzG,UAAA,8BAAA,SAAA,0BAAA,kCAAgH;wBAAhH,IAAM5C,IAAN;wBACHP,MAAMF,IAAI,CAACS;wBACXyD;oBACF;;oBAHK;oBAAA;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;gBAIL;YACF;YAEA,IAAMM,YAAYvB,IAAAA,sBAAW,EAACpD;YAC9B,IAAMuE,WAAWN,SAAStE,IAAI,KAAK,QAAQ6E,MAAMG;YAEjD,IAAI3E,MAAMC,IAAI,KAAK,WAAW;gBAC5BqE,QAAQK,WAAWJ,UAAUvE,MAAMS,SAAS,EAAET,MAAMU,OAAO;gBAC3D;YACF;YAEA,wFAAwF;YACxF,wDAAwD;YACxD,IAAMkE,eAAevE,MAAMH,MAAM,GAAG,KAAKiC,SAASnB,iBAAiB0D,cAAcR;YACjF,IAAIU,cAAcP;YAElBC,QAAQK,WAAWJ,UAAUvE,MAAMS,SAAS,EAAET,MAAMU,OAAO;YAC3D0D;YAEA,IAAIA,kBAAkB/E,gBAAgBgF;QACxC;;QA9BK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IA+BLA;IAEA,OAAOF;AACT;AAIO,SAAS/E,MAAMS,MAAe,EAAEgF,IAAY,EAAErF,IAAmB;QAI7BsF;IAHzC,IAAMb,WAAW1E,eAAeC;IAChC,IAAM+D,QAAQsB,KAAKE,KAAK,CAAC;IACzB,IAAMD,SAA0C,EAAE;QAC7C,kCAAA,2BAAA;;QAAL,QAAK,YAAelF,YAAYC,4BAA3B,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMmF,QAAN;YAAoCF,CAAAA,UAAAA,QAAO3E,IAAI,OAAX2E,SAAY,qBAAGf,WAAWiB,OAAOzB,OAAOU;;;QAA5E;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,+FAA+F;IAC/F,qHAAqH;IACrH,IAAMgB,SACJhB,SAAStE,IAAI,KAAK,QACdmF,OAAOnE,GAAG,CAAC,SAACuE;eACVA,EAAEpE,KAAK,GACHoE,IACA,wCACKA;YACHvF,MAAM4D,MACHkB,KAAK,CAACS,EAAEzE,SAAS,GAAG,GAAGyE,EAAExE,OAAO,EAChCG,IAAI,CAAC,MACLsE,IAAI;;SAGfL;IACN,mHAAmH;IACnH,OAAOG,OAAOG,MAAM,CAAC,SAACF;eAAMA,EAAEvF,IAAI,CAACwF,IAAI,GAAGjF,MAAM,GAAG;OAAGS,GAAG,CAAC,SAACuE;eAAO;YAAEzE,WAAWyE,EAAEzE,SAAS;YAAEC,SAASwE,EAAExE,OAAO;YAAEf,MAAMuF,EAAEvF,IAAI;QAAC;;AAC/H"}

package/dist/cjs/chunk/parse.js

@@ -8,78 +8,74 @@ Object.defineProperty(exports, "parse", {
         return parse;
     }
 });
-var _nodemodule = /*#__PURE__*/ _interop_require_default(require("node:module"));
 var _extractts = require("./extract.js");
-function _interop_require_default(obj) {
-    return obj && obj.__esModule ? obj : {
-        default: obj
-    };
-}
-// Tier-2, as embed/static.ts: the parser's packages cost ~19 ms to load and a warm tree never
-// parses, so every store-opening command paid for them until a file actually changed.
-var _require = typeof require === 'undefined' ? _nodemodule.default.createRequire(require("url").pathToFileURL(__filename).toString()) : require;
+var _parserts = require("./parser.js");
+// Opening token type to Block type. Everything else (rules, raw html, footnote definitions) is
+// 'other', as mdast's BLOCK_TYPES mapped no entry for those nodes.
 var BLOCK_TYPES = {
-    heading: 'heading',
-    paragraph: 'paragraph',
-    code: 'code',
-    table: 'table',
-    list: 'list',
-    blockquote: 'blockquote'
+    heading_open: 'heading',
+    paragraph_open: 'paragraph',
+    fence: 'code',
+    code_block: 'code',
+    table_open: 'table',
+    ordered_list_open: 'list',
+    bullet_list_open: 'list',
+    blockquote_open: 'blockquote'
 };
-var cached;
-// Imported individually, not via micromark-extension-gfm/mdast-util-gfm: those bundles also pull
-// in gfm-tagfilter, an HTML sanitizer this library never uses (no htmlExtensions call anywhere).
-function parser() {
-    if (cached) return cached;
-    var fromMarkdown = _require('mdast-util-from-markdown').fromMarkdown;
-    var gfmAutolinkLiteralFromMarkdown = _require('mdast-util-gfm-autolink-literal').gfmAutolinkLiteralFromMarkdown;
-    var gfmFootnoteFromMarkdown = _require('mdast-util-gfm-footnote').gfmFootnoteFromMarkdown;
-    var gfmStrikethroughFromMarkdown = _require('mdast-util-gfm-strikethrough').gfmStrikethroughFromMarkdown;
-    var gfmTableFromMarkdown = _require('mdast-util-gfm-table').gfmTableFromMarkdown;
-    var gfmTaskListItemFromMarkdown = _require('mdast-util-gfm-task-list-item').gfmTaskListItemFromMarkdown;
-    var gfmAutolinkLiteral = _require('micromark-extension-gfm-autolink-literal').gfmAutolinkLiteral;
-    var gfmFootnote = _require('micromark-extension-gfm-footnote').gfmFootnote;
-    var gfmStrikethrough = _require('micromark-extension-gfm-strikethrough').gfmStrikethrough;
-    var gfmTable = _require('micromark-extension-gfm-table').gfmTable;
-    var gfmTaskListItem = _require('micromark-extension-gfm-task-list-item').gfmTaskListItem;
-    cached = {
-        fromMarkdown: fromMarkdown,
-        options: {
-            extensions: [
-                gfmAutolinkLiteral(),
-                gfmFootnote(),
-                gfmStrikethrough(),
-                gfmTable(),
-                gfmTaskListItem()
-            ],
-            mdastExtensions: [
-                gfmAutolinkLiteralFromMarkdown(),
-                gfmFootnoteFromMarkdown(),
-                gfmStrikethroughFromMarkdown(),
-                gfmTableFromMarkdown(),
-                gfmTaskListItemFromMarkdown()
-            ]
+function parse(body) {
+    var tokens = (0, _parserts.parser)().parse(body, {});
+    var lines = body.split('\n');
+    var blocks = [];
+    var i = 0;
+    while(i < tokens.length){
+        var token = tokens[i];
+        if (token.nesting === 0) {
+            var _BLOCK_TYPES_token_type;
+            blocks.push(makeBlock((_BLOCK_TYPES_token_type = BLOCK_TYPES[token.type]) !== null && _BLOCK_TYPES_token_type !== void 0 ? _BLOCK_TYPES_token_type : 'other', tokens, i, i + 1, lines));
+            i += 1;
+        } else if (token.nesting === 1) {
+            var _BLOCK_TYPES_token_type1;
+            var depth = 1;
+            var j = i + 1;
+            while(j < tokens.length && depth > 0){
+                depth += tokens[j].nesting === 1 ? 1 : tokens[j].nesting === -1 ? -1 : 0;
+                j += 1;
+            }
+            blocks.push(makeBlock((_BLOCK_TYPES_token_type1 = BLOCK_TYPES[token.type]) !== null && _BLOCK_TYPES_token_type1 !== void 0 ? _BLOCK_TYPES_token_type1 : 'other', tokens, i, j, lines));
+            i = j;
+        } else {
+            // Defensive: the balanced scan above consumes every close belonging to an open.
+            i += 1;
         }
-    };
-    return cached;
+    }
+    return blocks;
 }
-function parse(body) {
-    var _parser = parser(), fromMarkdown = _parser.fromMarkdown, options = _parser.options;
-    var tree = fromMarkdown(body, options);
-    return tree.children.map(function(node) {
-        var _BLOCK_TYPES_node_type;
-        var position = node.position;
-        var block = {
-            type: (_BLOCK_TYPES_node_type = BLOCK_TYPES[node.type]) !== null && _BLOCK_TYPES_node_type !== void 0 ? _BLOCK_TYPES_node_type : 'other',
-            startLine: position ? position.start.line : 1,
-            endLine: position ? position.end.line : 1,
-            node: node
-        };
-        if (node.type === 'heading') {
-            block.depth = node.depth;
-            block.text = (0, _extractts.extractText)(node);
+// 1-based inclusive extent from the min/max of the tokens' maps (0-based half-open); a block
+// whose tokens carry no map (an empty footnote definition) falls back to line 1. Trailing
+// blank lines trim so a list's endLine lands where mdast's position ended it.
+function makeBlock(type, tokens, i, j, lines) {
+    var start = Infinity;
+    var end = 0;
+    for(var k = i; k < j; k++){
+        var map = tokens[k].map;
+        if (map) {
+            start = Math.min(start, map[0]);
+            end = Math.max(end, map[1]);
         }
-        return block;
-    });
+    }
+    var startLine = Number.isFinite(start) ? start + 1 : 1;
+    var endLine = end > start ? end : startLine;
+    while(endLine > startLine && lines[endLine - 1].trim() === '')endLine--;
+    var block = {
+        type: type,
+        startLine: startLine,
+        endLine: endLine,
+        node: tokens.slice(i, j)
+    };
+    if (type === 'heading') {
+        block.depth = Number(tokens[i].tag.slice(1));
+        block.text = (0, _extractts.extractText)(block);
+    }
+    return block;
 }
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/chunk/parse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parse.ts"],"sourcesContent":["import Module from 'node:module';\nimport type { RootContent } from 'mdast';\nimport { extractText } from './extract.ts';\nimport type { Block, BlockType } from './types.ts';\n\n// Tier-2, as embed/static.ts: the parser's packages cost ~19 ms to load and a warm tree never\n// parses, so every store-opening command paid for them until a file actually changed.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\nconst BLOCK_TYPES: Partial<Record<RootContent['type'], BlockType>> = {\n  heading: 'heading',\n  paragraph: 'paragraph',\n  code: 'code',\n  table: 'table',\n  list: 'list',\n  blockquote: 'blockquote',\n};\n\ntype FromMarkdown = typeof import('mdast-util-from-markdown').fromMarkdown;\ntype Parser = { fromMarkdown: FromMarkdown; options: NonNullable<Parameters<FromMarkdown>[1]> };\nlet cached: Parser | undefined;\n\n// Imported individually, not via micromark-extension-gfm/mdast-util-gfm: those bundles also pull\n// in gfm-tagfilter, an HTML sanitizer this library never uses (no htmlExtensions call anywhere).\nfunction parser(): Parser {\n  if (cached) return cached;\n  const { fromMarkdown } = _require('mdast-util-from-markdown') as typeof import('mdast-util-from-markdown');\n  const { gfmAutolinkLiteralFromMarkdown } = _require('mdast-util-gfm-autolink-literal') as typeof import('mdast-util-gfm-autolink-literal');\n  const { gfmFootnoteFromMarkdown } = _require('mdast-util-gfm-footnote') as typeof import('mdast-util-gfm-footnote');\n  const { gfmStrikethroughFromMarkdown } = _require('mdast-util-gfm-strikethrough') as typeof import('mdast-util-gfm-strikethrough');\n  const { gfmTableFromMarkdown } = _require('mdast-util-gfm-table') as typeof import('mdast-util-gfm-table');\n  const { gfmTaskListItemFromMarkdown } = _require('mdast-util-gfm-task-list-item') as typeof import('mdast-util-gfm-task-list-item');\n  const { gfmAutolinkLiteral } = _require('micromark-extension-gfm-autolink-literal') as typeof import('micromark-extension-gfm-autolink-literal');\n  const { gfmFootnote } = _require('micromark-extension-gfm-footnote') as typeof import('micromark-extension-gfm-footnote');\n  const { gfmStrikethrough } = _require('micromark-extension-gfm-strikethrough') as typeof import('micromark-extension-gfm-strikethrough');\n  const { gfmTable } = _require('micromark-extension-gfm-table') as typeof import('micromark-extension-gfm-table');\n  const { gfmTaskListItem } = _require('micromark-extension-gfm-task-list-item') as typeof import('micromark-extension-gfm-task-list-item');\n  cached = {\n    fromMarkdown,\n    options: {\n      extensions: [gfmAutolinkLiteral(), gfmFootnote(), gfmStrikethrough(), gfmTable(), gfmTaskListItem()],\n      mdastExtensions: [gfmAutolinkLiteralFromMarkdown(), gfmFootnoteFromMarkdown(), gfmStrikethroughFromMarkdown(), gfmTableFromMarkdown(), gfmTaskListItemFromMarkdown()],\n    },\n  };\n  return cached;\n}\n\n// Top-level blocks of a markdown body, typed and line-extent bounded from mdast's own\n// node.position (never a regex guess). GFM extensions add tables, task lists, footnotes, strikethrough.\nexport function parse(body: string): Block[] {\n  const { fromMarkdown, options } = parser();\n  const tree = fromMarkdown(body, options);\n  return tree.children.map((node) => {\n    const position = node.position;\n    const block: Block = {\n      type: BLOCK_TYPES[node.type] ?? 'other',\n      startLine: position ? position.start.line : 1,\n      endLine: position ? position.end.line : 1,\n      node,\n    };\n    if (node.type === 'heading') {\n      block.depth = node.depth;\n      block.text = extractText(node);\n    }\n    return block;\n  });\n}\n"],"names":["parse","_require","require","Module","createRequire","BLOCK_TYPES","heading","paragraph","code","table","list","blockquote","cached","parser","fromMarkdown","gfmAutolinkLiteralFromMarkdown","gfmFootnoteFromMarkdown","gfmStrikethroughFromMarkdown","gfmTableFromMarkdown","gfmTaskListItemFromMarkdown","gfmAutolinkLiteral","gfmFootnote","gfmStrikethrough","gfmTable","gfmTaskListItem","options","extensions","mdastExtensions","body","tree","children","map","node","position","block","type","startLine","start","line","endLine","end","depth","text","extractText"],"mappings":";;;;+BAiDgBA;;;eAAAA;;;iEAjDG;yBAES;;;;;;AAG5B,8FAA8F;AAC9F,sFAAsF;AACtF,IAAMC,WAAW,OAAOC,YAAY,cAAcC,mBAAM,CAACC,aAAa,CAAC,uDAAmBF;AAE1F,IAAMG,cAA+D;IACnEC,SAAS;IACTC,WAAW;IACXC,MAAM;IACNC,OAAO;IACPC,MAAM;IACNC,YAAY;AACd;AAIA,IAAIC;AAEJ,iGAAiG;AACjG,iGAAiG;AACjG,SAASC;IACP,IAAID,QAAQ,OAAOA;IACnB,IAAM,AAAEE,eAAiBb,SAAS,4BAA1Ba;IACR,IAAM,AAAEC,iCAAmCd,SAAS,mCAA5Cc;IACR,IAAM,AAAEC,0BAA4Bf,SAAS,2BAArCe;IACR,IAAM,AAAEC,+BAAiChB,SAAS,gCAA1CgB;IACR,IAAM,AAAEC,uBAAyBjB,SAAS,wBAAlCiB;IACR,IAAM,AAAEC,8BAAgClB,SAAS,iCAAzCkB;IACR,IAAM,AAAEC,qBAAuBnB,SAAS,4CAAhCmB;IACR,IAAM,AAAEC,cAAgBpB,SAAS,oCAAzBoB;IACR,IAAM,AAAEC,mBAAqBrB,SAAS,yCAA9BqB;IACR,IAAM,AAAEC,WAAatB,SAAS,iCAAtBsB;IACR,IAAM,AAAEC,kBAAoBvB,SAAS,0CAA7BuB;IACRZ,SAAS;QACPE,cAAAA;QACAW,SAAS;YACPC,YAAY;gBAACN;gBAAsBC;gBAAeC;gBAAoBC;gBAAYC;aAAkB;YACpGG,iBAAiB;gBAACZ;gBAAkCC;gBAA2BC;gBAAgCC;gBAAwBC;aAA8B;QACvK;IACF;IACA,OAAOP;AACT;AAIO,SAASZ,MAAM4B,IAAY;IAChC,IAAkCf,UAAAA,UAA1BC,eAA0BD,QAA1BC,cAAcW,UAAYZ,QAAZY;IACtB,IAAMI,OAAOf,aAAac,MAAMH;IAChC,OAAOI,KAAKC,QAAQ,CAACC,GAAG,CAAC,SAACC;YAGhB3B;QAFR,IAAM4B,WAAWD,KAAKC,QAAQ;QAC9B,IAAMC,QAAe;YACnBC,IAAI,GAAE9B,yBAAAA,WAAW,CAAC2B,KAAKG,IAAI,CAAC,cAAtB9B,oCAAAA,yBAA0B;YAChC+B,WAAWH,WAAWA,SAASI,KAAK,CAACC,IAAI,GAAG;YAC5CC,SAASN,WAAWA,SAASO,GAAG,CAACF,IAAI,GAAG;YACxCN,MAAAA;QACF;QACA,IAAIA,KAAKG,IAAI,KAAK,WAAW;YAC3BD,MAAMO,KAAK,GAAGT,KAAKS,KAAK;YACxBP,MAAMQ,IAAI,GAAGC,IAAAA,sBAAW,EAACX;QAC3B;QACA,OAAOE;IACT;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parse.ts"],"sourcesContent":["import type { Token } from 'markdown-it';\nimport { extractText } from './extract.ts';\nimport { parser } from './parser.ts';\nimport type { Block, BlockType } from './types.ts';\n\n// Opening token type to Block type. Everything else (rules, raw html, footnote definitions) is\n// 'other', as mdast's BLOCK_TYPES mapped no entry for those nodes.\nconst BLOCK_TYPES: Record<string, BlockType> = {\n  heading_open: 'heading',\n  paragraph_open: 'paragraph',\n  fence: 'code',\n  code_block: 'code',\n  table_open: 'table',\n  ordered_list_open: 'list',\n  bullet_list_open: 'list',\n  blockquote_open: 'blockquote',\n};\n\n// Top-level blocks of a markdown body, typed and line-extent bounded from markdown-it's own\n// token maps (never a regex guess). html and linkify on, with the footnote and task-list plugins.\nexport function parse(body: string): Block[] {\n  const tokens = parser().parse(body, {});\n  const lines = body.split('\\n');\n  const blocks: Block[] = [];\n  let i = 0;\n  while (i < tokens.length) {\n    const token = tokens[i];\n    if (token.nesting === 0) {\n      blocks.push(makeBlock(BLOCK_TYPES[token.type] ?? 'other', tokens, i, i + 1, lines));\n      i += 1;\n    } else if (token.nesting === 1) {\n      let depth = 1;\n      let j = i + 1;\n      while (j < tokens.length && depth > 0) {\n        depth += tokens[j].nesting === 1 ? 1 : tokens[j].nesting === -1 ? -1 : 0;\n        j += 1;\n      }\n      blocks.push(makeBlock(BLOCK_TYPES[token.type] ?? 'other', tokens, i, j, lines));\n      i = j;\n    } else {\n      // Defensive: the balanced scan above consumes every close belonging to an open.\n      i += 1;\n    }\n  }\n  return blocks;\n}\n\n// 1-based inclusive extent from the min/max of the tokens' maps (0-based half-open); a block\n// whose tokens carry no map (an empty footnote definition) falls back to line 1. Trailing\n// blank lines trim so a list's endLine lands where mdast's position ended it.\nfunction makeBlock(type: BlockType, tokens: Token[], i: number, j: number, lines: string[]): Block {\n  let start = Infinity;\n  let end = 0;\n  for (let k = i; k < j; k++) {\n    const map = tokens[k].map;\n    if (map) {\n      start = Math.min(start, map[0]);\n      end = Math.max(end, map[1]);\n    }\n  }\n  const startLine = Number.isFinite(start) ? start + 1 : 1;\n  let endLine = end > start ? end : startLine;\n  while (endLine > startLine && lines[endLine - 1].trim() === '') endLine--;\n  const block: Block = { type, startLine, endLine, node: tokens.slice(i, j) };\n  if (type === 'heading') {\n    block.depth = Number(tokens[i].tag.slice(1));\n    block.text = extractText(block);\n  }\n  return block;\n}\n"],"names":["parse","BLOCK_TYPES","heading_open","paragraph_open","fence","code_block","table_open","ordered_list_open","bullet_list_open","blockquote_open","body","tokens","parser","lines","split","blocks","i","length","token","nesting","push","makeBlock","type","depth","j","start","Infinity","end","k","map","Math","min","max","startLine","Number","isFinite","endLine","trim","block","node","slice","tag","text","extractText"],"mappings":";;;;+BAoBgBA;;;eAAAA;;;yBAnBY;wBACL;AAGvB,+FAA+F;AAC/F,mEAAmE;AACnE,IAAMC,cAAyC;IAC7CC,cAAc;IACdC,gBAAgB;IAChBC,OAAO;IACPC,YAAY;IACZC,YAAY;IACZC,mBAAmB;IACnBC,kBAAkB;IAClBC,iBAAiB;AACnB;AAIO,SAAST,MAAMU,IAAY;IAChC,IAAMC,SAASC,IAAAA,gBAAM,IAAGZ,KAAK,CAACU,MAAM,CAAC;IACrC,IAAMG,QAAQH,KAAKI,KAAK,CAAC;IACzB,IAAMC,SAAkB,EAAE;IAC1B,IAAIC,IAAI;IACR,MAAOA,IAAIL,OAAOM,MAAM,CAAE;QACxB,IAAMC,QAAQP,MAAM,CAACK,EAAE;QACvB,IAAIE,MAAMC,OAAO,KAAK,GAAG;gBACDlB;YAAtBc,OAAOK,IAAI,CAACC,WAAUpB,0BAAAA,WAAW,CAACiB,MAAMI,IAAI,CAAC,cAAvBrB,qCAAAA,0BAA2B,SAASU,QAAQK,GAAGA,IAAI,GAAGH;YAC5EG,KAAK;QACP,OAAO,IAAIE,MAAMC,OAAO,KAAK,GAAG;gBAORlB;YANtB,IAAIsB,QAAQ;YACZ,IAAIC,IAAIR,IAAI;YACZ,MAAOQ,IAAIb,OAAOM,MAAM,IAAIM,QAAQ,EAAG;gBACrCA,SAASZ,MAAM,CAACa,EAAE,CAACL,OAAO,KAAK,IAAI,IAAIR,MAAM,CAACa,EAAE,CAACL,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI;gBACvEK,KAAK;YACP;YACAT,OAAOK,IAAI,CAACC,WAAUpB,2BAAAA,WAAW,CAACiB,MAAMI,IAAI,CAAC,cAAvBrB,sCAAAA,2BAA2B,SAASU,QAAQK,GAAGQ,GAAGX;YACxEG,IAAIQ;QACN,OAAO;YACL,gFAAgF;YAChFR,KAAK;QACP;IACF;IACA,OAAOD;AACT;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,8EAA8E;AAC9E,SAASM,UAAUC,IAAe,EAAEX,MAAe,EAAEK,CAAS,EAAEQ,CAAS,EAAEX,KAAe;IACxF,IAAIY,QAAQC;IACZ,IAAIC,MAAM;IACV,IAAK,IAAIC,IAAIZ,GAAGY,IAAIJ,GAAGI,IAAK;QAC1B,IAAMC,MAAMlB,MAAM,CAACiB,EAAE,CAACC,GAAG;QACzB,IAAIA,KAAK;YACPJ,QAAQK,KAAKC,GAAG,CAACN,OAAOI,GAAG,CAAC,EAAE;YAC9BF,MAAMG,KAAKE,GAAG,CAACL,KAAKE,GAAG,CAAC,EAAE;QAC5B;IACF;IACA,IAAMI,YAAYC,OAAOC,QAAQ,CAACV,SAASA,QAAQ,IAAI;IACvD,IAAIW,UAAUT,MAAMF,QAAQE,MAAMM;IAClC,MAAOG,UAAUH,aAAapB,KAAK,CAACuB,UAAU,EAAE,CAACC,IAAI,OAAO,GAAID;IAChE,IAAME,QAAe;QAAEhB,MAAAA;QAAMW,WAAAA;QAAWG,SAAAA;QAASG,MAAM5B,OAAO6B,KAAK,CAACxB,GAAGQ;IAAG;IAC1E,IAAIF,SAAS,WAAW;QACtBgB,MAAMf,KAAK,GAAGW,OAAOvB,MAAM,CAACK,EAAE,CAACyB,GAAG,CAACD,KAAK,CAAC;QACzCF,MAAMI,IAAI,GAAGC,IAAAA,sBAAW,EAACL;IAC3B;IACA,OAAOA;AACT"}

package/dist/cjs/chunk/parser.d.cts

@@ -0,0 +1,2 @@
+import type { MarkdownIt } from 'markdown-it';
+export declare function parser(): MarkdownIt;

package/dist/cjs/chunk/parser.d.ts

@@ -0,0 +1,2 @@
+import type { MarkdownIt } from 'markdown-it';
+export declare function parser(): MarkdownIt;

package/dist/cjs/chunk/parser.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "parser", {
+    enumerable: true,
+    get: function() {
+        return parser;
+    }
+});
+var _nodemodule = /*#__PURE__*/ _interop_require_default(require("node:module"));
+function _interop_require_default(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+// Tier-2, as embed/static.ts: the parser's packages cost ~19 ms to load and a warm tree never
+// parses, so every store-opening command paid for them until a file actually changed.
+var _require = typeof require === 'undefined' ? _nodemodule.default.createRequire(require("url").pathToFileURL(__filename).toString()) : require;
+var cached;
+function parser() {
+    if (cached) return cached;
+    var Ctor = _require('markdown-it');
+    var footnote = _require('markdown-it-footnote');
+    var taskLists = _require('markdown-it-task-lists');
+    var md = new Ctor({
+        html: true,
+        linkify: true
+    }).use(footnote).use(taskLists);
+    // Fuzzy linking is what links www. domains and emails (the scheme matcher does neither); its
+    // bare-domain over-matching vs GFM is reined in at extraction, where the text is kept.
+    md.linkify.set({
+        fuzzyLink: true
+    });
+    md.core.ruler.disable('footnote_tail');
+    md.inline.ruler.disable('footnote_inline');
+    cached = md;
+    return cached;
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/chunk/parser.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parser.ts"],"sourcesContent":["import Module from 'node:module';\nimport type { MarkdownIt } from 'markdown-it';\nimport type footnotePlugin from 'markdown-it-footnote';\nimport type taskListsPlugin from 'markdown-it-task-lists';\n\n// Tier-2, as embed/static.ts: the parser's packages cost ~19 ms to load and a warm tree never\n// parses, so every store-opening command paid for them until a file actually changed.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// The CJS build resolves markdown-it's export= types, so the constructor is named locally.\ntype Ctor = new (options?: { html?: boolean; linkify?: boolean }) => MarkdownIt;\n\nlet cached: MarkdownIt | undefined;\n\n// The one shared parser: html and linkify on, fuzzy links, footnotes and task lists, with the\n// footnote rules disabled for GFM parity (definitions stay spans; ^[...] stays inert text).\nexport function parser(): MarkdownIt {\n  if (cached) return cached;\n  const Ctor = _require('markdown-it') as Ctor;\n  const footnote = _require('markdown-it-footnote') as typeof footnotePlugin;\n  const taskLists = _require('markdown-it-task-lists') as typeof taskListsPlugin;\n  const md = new Ctor({ html: true, linkify: true }).use(footnote).use(taskLists);\n  // Fuzzy linking is what links www. domains and emails (the scheme matcher does neither); its\n  // bare-domain over-matching vs GFM is reined in at extraction, where the text is kept.\n  md.linkify.set({ fuzzyLink: true });\n  md.core.ruler.disable('footnote_tail');\n  md.inline.ruler.disable('footnote_inline');\n  cached = md;\n  return cached;\n}\n"],"names":["parser","_require","require","Module","createRequire","cached","Ctor","footnote","taskLists","md","html","linkify","use","set","fuzzyLink","core","ruler","disable","inline"],"mappings":";;;;+BAgBgBA;;;eAAAA;;;iEAhBG;;;;;;AAKnB,8FAA8F;AAC9F,sFAAsF;AACtF,IAAMC,WAAW,OAAOC,YAAY,cAAcC,mBAAM,CAACC,aAAa,CAAC,uDAAmBF;AAK1F,IAAIG;AAIG,SAASL;IACd,IAAIK,QAAQ,OAAOA;IACnB,IAAMC,OAAOL,SAAS;IACtB,IAAMM,WAAWN,SAAS;IAC1B,IAAMO,YAAYP,SAAS;IAC3B,IAAMQ,KAAK,IAAIH,KAAK;QAAEI,MAAM;QAAMC,SAAS;IAAK,GAAGC,GAAG,CAACL,UAAUK,GAAG,CAACJ;IACrE,6FAA6F;IAC7F,uFAAuF;IACvFC,GAAGE,OAAO,CAACE,GAAG,CAAC;QAAEC,WAAW;IAAK;IACjCL,GAAGM,IAAI,CAACC,KAAK,CAACC,OAAO,CAAC;IACtBR,GAAGS,MAAM,CAACF,KAAK,CAACC,OAAO,CAAC;IACxBZ,SAASI;IACT,OAAOJ;AACT"}

package/dist/cjs/chunk/types.d.cts

@@ -1,4 +1,4 @@
-import type { RootContent } from 'mdast';
+import type { Token } from 'markdown-it';
 export type BlockType = 'heading' | 'paragraph' | 'code' | 'table' | 'list' | 'blockquote' | 'other';
 export interface Block {
     type: BlockType;
@@ -6,7 +6,7 @@ export interface Block {
     endLine: number;
     depth?: number;
     text?: string;
-    node: RootContent;
+    node: Token[];
 }
 export interface ChunkOptions {
     targetTokens?: number;

package/dist/cjs/chunk/types.d.ts

@@ -1,4 +1,4 @@
-import type { RootContent } from 'mdast';
+import type { Token } from 'markdown-it';
 export type BlockType = 'heading' | 'paragraph' | 'code' | 'table' | 'list' | 'blockquote' | 'other';
 export interface Block {
     type: BlockType;
@@ -6,7 +6,7 @@ export interface Block {
     endLine: number;
     depth?: number;
     text?: string;
-    node: RootContent;
+    node: Token[];
 }
 export interface ChunkOptions {
     targetTokens?: number;

package/dist/cjs/chunk/version.d.cts

@@ -1 +1 @@
-export declare const CHUNK_VERSION = "chunk:v5";
+export declare const CHUNK_VERSION = "chunk:v6";

package/dist/cjs/chunk/version.d.ts

@@ -1 +1 @@
-export declare const CHUNK_VERSION = "chunk:v5";
+export declare const CHUNK_VERSION = "chunk:v6";

package/dist/cjs/chunk/version.js

@@ -10,5 +10,5 @@ Object.defineProperty(exports, "CHUNK_VERSION", {
         return CHUNK_VERSION;
     }
 });
-var CHUNK_VERSION = 'chunk:v5';
+var CHUNK_VERSION = 'chunk:v6';
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/chunk/version.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/version.ts"],"sourcesContent":["// D8: bump whenever chunk semantics change (grouping, sizing, splitting); the digest in\n// test/unit/chunk/version.test.ts tracks EXTRACTION changes only, so the two move independently.\nexport const CHUNK_VERSION = 'chunk:v5';\n"],"names":["CHUNK_VERSION"],"mappings":"AAAA,wFAAwF;AACxF,iGAAiG;;;;;+BACpFA;;;eAAAA;;;AAAN,IAAMA,gBAAgB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/version.ts"],"sourcesContent":["// D8: bump whenever chunk semantics change (grouping, sizing, splitting); the digest in\n// test/unit/chunk/version.test.ts tracks EXTRACTION changes only, so the two move independently.\nexport const CHUNK_VERSION = 'chunk:v6';\n"],"names":["CHUNK_VERSION"],"mappings":"AAAA,wFAAwF;AACxF,iGAAiG;;;;;+BACpFA;;;eAAAA;;;AAAN,IAAMA,gBAAgB"}

package/dist/cjs/features/sections.js

@@ -136,7 +136,7 @@ function _ts_generator(thisArg, body) {
         };
     }
 }
-// Heading blocks parse() already found (mdast/CommonMark fences), offset back onto the raw file;
+// Heading blocks parse() already found (markdown-it/CommonMark fences), offset back onto the raw file;
 // a section runs to just before the next heading, or EOF.
 function sectionsFromBlocks(blocks, raw, body) {
     var rawLines = raw.split('\n');

package/dist/cjs/features/sections.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/sections.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { estimateTokens, parse } from '../chunk/index.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// sections(path, idx, level, heading, start_line, end_line, tokens): the heading outline,\n// 1-indexed over the raw file so a row is a direct Read range; tokens is estimateTokens (D5), CJK-aware.\n\nexport interface Section {\n  level: number;\n  heading: string;\n  startLine: number;\n  endLine: number;\n  tokens: number;\n}\n\n// Heading blocks parse() already found (mdast/CommonMark fences), offset back onto the raw file;\n// a section runs to just before the next heading, or EOF.\nfunction sectionsFromBlocks(blocks: Block[], raw: string, body: string): Section[] {\n  const rawLines = raw.split('\\n');\n  const offset = rawLines.length - countLines(body);\n  const found: Section[] = blocks.filter((b) => b.type === 'heading').map((b) => ({ level: b.depth ?? 1, heading: (b.text ?? '').trim(), startLine: b.startLine + offset, endLine: rawLines.length, tokens: 0 }));\n  for (let s = 0; s < found.length; s++) {\n    if (s + 1 < found.length) found[s].endLine = found[s + 1].startLine - 1;\n    const text = rawLines.slice(found[s].startLine - 1, found[s].endLine).join('\\n');\n    found[s].tokens = Math.ceil(estimateTokens(text));\n  }\n  return found;\n}\n\nconst SECTION_COLUMNS = ['path', 'idx', 'level', 'heading', 'start_line', 'end_line', 'tokens'];\n\nexport const sections: Feature = {\n  name: 'sections',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS sections (\"path\" TEXT, idx INTEGER, level INTEGER, heading TEXT, start_line INTEGER, end_line INTEGER, tokens INTEGER, PRIMARY KEY (\"path\", idx))`);\n  },\n  extract(raw, body, _search, _data, _cfg, blocks) {\n    return sectionsFromBlocks(blocks ?? parse(body), raw, body);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM sections WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) (extracted as Section[]).forEach((s, idx) => rows.push([path, idx, s.level, s.heading, s.startLine, s.endLine, s.tokens]));\n    // DO NOTHING, not a bare INSERT: reconcile's added/touched split is decided before this\n    // write's lock, so a path called \"added\" here can already have this row from a concurrent\n    // reconcile that committed first -- same file, same parse, same row. A store with an append\n    // path has no second writer, and remove() cleared every touched path above, so it skips the guard.\n    await appendRows(db, 'sections', SECTION_COLUMNS, 'INSERT INTO sections (\"path\", idx, level, heading, start_line, end_line, tokens) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(\"path\", idx) DO NOTHING', rows);\n  },\n};\n"],"names":["sections","sectionsFromBlocks","blocks","raw","body","rawLines","split","offset","length","countLines","found","filter","b","type","map","level","depth","heading","text","trim","startLine","endLine","tokens","s","slice","join","Math","ceil","estimateTokens","SECTION_COLUMNS","name","schema","db","exec","extract","_search","_data","_cfg","parse","remove","paths","runBatch","p","store","docs","rows","path","extracted","forEach","idx","push","appendRows"],"mappings":";;;;+BAiCaA;;;eAAAA;;;uBAhCyB;6BACX;wBACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAc3B,iGAAiG;AACjG,0DAA0D;AAC1D,SAASC,mBAAmBC,MAAe,EAAEC,GAAW,EAAEC,IAAY;IACpE,IAAMC,WAAWF,IAAIG,KAAK,CAAC;IAC3B,IAAMC,SAASF,SAASG,MAAM,GAAGC,IAAAA,yBAAU,EAACL;IAC5C,IAAMM,QAAmBR,OAAOS,MAAM,CAAC,SAACC;eAAMA,EAAEC,IAAI,KAAK;OAAWC,GAAG,CAAC,SAACF;YAAgBA,UAAwBA;eAAjC;YAAEG,KAAK,GAAEH,WAAAA,EAAEI,KAAK,cAAPJ,sBAAAA,WAAW;YAAGK,SAAS,EAACL,UAAAA,EAAEM,IAAI,cAANN,qBAAAA,UAAU,IAAIO,IAAI;YAAIC,WAAWR,EAAEQ,SAAS,GAAGb;YAAQc,SAAShB,SAASG,MAAM;YAAEc,QAAQ;QAAE;;IAC5M,IAAK,IAAIC,IAAI,GAAGA,IAAIb,MAAMF,MAAM,EAAEe,IAAK;QACrC,IAAIA,IAAI,IAAIb,MAAMF,MAAM,EAAEE,KAAK,CAACa,EAAE,CAACF,OAAO,GAAGX,KAAK,CAACa,IAAI,EAAE,CAACH,SAAS,GAAG;QACtE,IAAMF,OAAOb,SAASmB,KAAK,CAACd,KAAK,CAACa,EAAE,CAACH,SAAS,GAAG,GAAGV,KAAK,CAACa,EAAE,CAACF,OAAO,EAAEI,IAAI,CAAC;QAC3Ef,KAAK,CAACa,EAAE,CAACD,MAAM,GAAGI,KAAKC,IAAI,CAACC,IAAAA,uBAAc,EAACV;IAC7C;IACA,OAAOR;AACT;AAEA,IAAMmB,kBAAkB;IAAC;IAAQ;IAAO;IAAS;IAAW;IAAc;IAAY;CAAS;AAExF,IAAM7B,WAAoB;IAC/B8B,MAAM;IACAC,QAAN,SAAMA,OAAOC,EAAE;;;;;wBACb;;4BAAMA,GAAGC,IAAI,CAAC;;;wBAAd;;;;;;QACF;;IACAC,SAAAA,SAAAA,QAAQ/B,GAAG,EAAEC,IAAI,EAAE+B,OAAO,EAAEC,KAAK,EAAEC,IAAI,EAAEnC,MAAM;QAC7C,OAAOD,mBAAmBC,mBAAAA,oBAAAA,SAAUoC,IAAAA,cAAK,EAAClC,OAAOD,KAAKC;IACxD;IACMmC,QAAN,SAAMA,OAAOP,EAAE,EAAEQ,KAAK;;;;;wBACpB,IAAIA,MAAMhC,MAAM,KAAK,GAAG;;;wBACxB;;4BAAMwB,GAAGS,QAAQ,CACf,yCACAD,MAAM1B,GAAG,CAAC,SAAC4B;uCAAM;oCAACA;iCAAE;;;;wBAFtB;;;;;;QAIF;;IACMC,OAAN,SAAMA,MAAMX,EAAE,EAAEY,IAAI;;gBACZC,MACD,2BAAA,mBAAA,uBAAA,WAAA;;;;wBADCA;wBACD,kCAAA,2BAAA;;;gCAAA,kBAAA,aAAQC,mBAAAA,MAAMC,wBAAAA;gCAAsBA,UAAwBC,OAAO,CAAC,SAACzB,GAAG0B;2CAAQJ,KAAKK,IAAI,CAAC;wCAACJ;wCAAMG;wCAAK1B,EAAER,KAAK;wCAAEQ,EAAEN,OAAO;wCAAEM,EAAEH,SAAS;wCAAEG,EAAEF,OAAO;wCAAEE,EAAED,MAAM;qCAAC;;;4BAAhK,IAAK,YAA6BsB,2BAA7B,6BAAA,QAAA,yBAAA;;4BAAA;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;wBACL,wFAAwF;wBACxF,0FAA0F;wBAC1F,4FAA4F;wBAC5F,mGAAmG;wBACnG;;4BAAMO,IAAAA,oBAAU,EAACnB,IAAI,YAAYH,iBAAiB,qJAAqJgB;;;wBAAvM;;;;;;QACF;;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/sections.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { estimateTokens, parse } from '../chunk/index.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// sections(path, idx, level, heading, start_line, end_line, tokens): the heading outline,\n// 1-indexed over the raw file so a row is a direct Read range; tokens is estimateTokens (D5), CJK-aware.\n\nexport interface Section {\n  level: number;\n  heading: string;\n  startLine: number;\n  endLine: number;\n  tokens: number;\n}\n\n// Heading blocks parse() already found (markdown-it/CommonMark fences), offset back onto the raw file;\n// a section runs to just before the next heading, or EOF.\nfunction sectionsFromBlocks(blocks: Block[], raw: string, body: string): Section[] {\n  const rawLines = raw.split('\\n');\n  const offset = rawLines.length - countLines(body);\n  const found: Section[] = blocks.filter((b) => b.type === 'heading').map((b) => ({ level: b.depth ?? 1, heading: (b.text ?? '').trim(), startLine: b.startLine + offset, endLine: rawLines.length, tokens: 0 }));\n  for (let s = 0; s < found.length; s++) {\n    if (s + 1 < found.length) found[s].endLine = found[s + 1].startLine - 1;\n    const text = rawLines.slice(found[s].startLine - 1, found[s].endLine).join('\\n');\n    found[s].tokens = Math.ceil(estimateTokens(text));\n  }\n  return found;\n}\n\nconst SECTION_COLUMNS = ['path', 'idx', 'level', 'heading', 'start_line', 'end_line', 'tokens'];\n\nexport const sections: Feature = {\n  name: 'sections',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS sections (\"path\" TEXT, idx INTEGER, level INTEGER, heading TEXT, start_line INTEGER, end_line INTEGER, tokens INTEGER, PRIMARY KEY (\"path\", idx))`);\n  },\n  extract(raw, body, _search, _data, _cfg, blocks) {\n    return sectionsFromBlocks(blocks ?? parse(body), raw, body);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM sections WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) (extracted as Section[]).forEach((s, idx) => rows.push([path, idx, s.level, s.heading, s.startLine, s.endLine, s.tokens]));\n    // DO NOTHING, not a bare INSERT: reconcile's added/touched split is decided before this\n    // write's lock, so a path called \"added\" here can already have this row from a concurrent\n    // reconcile that committed first -- same file, same parse, same row. A store with an append\n    // path has no second writer, and remove() cleared every touched path above, so it skips the guard.\n    await appendRows(db, 'sections', SECTION_COLUMNS, 'INSERT INTO sections (\"path\", idx, level, heading, start_line, end_line, tokens) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(\"path\", idx) DO NOTHING', rows);\n  },\n};\n"],"names":["sections","sectionsFromBlocks","blocks","raw","body","rawLines","split","offset","length","countLines","found","filter","b","type","map","level","depth","heading","text","trim","startLine","endLine","tokens","s","slice","join","Math","ceil","estimateTokens","SECTION_COLUMNS","name","schema","db","exec","extract","_search","_data","_cfg","parse","remove","paths","runBatch","p","store","docs","rows","path","extracted","forEach","idx","push","appendRows"],"mappings":";;;;+BAiCaA;;;eAAAA;;;uBAhCyB;6BACX;wBACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAc3B,uGAAuG;AACvG,0DAA0D;AAC1D,SAASC,mBAAmBC,MAAe,EAAEC,GAAW,EAAEC,IAAY;IACpE,IAAMC,WAAWF,IAAIG,KAAK,CAAC;IAC3B,IAAMC,SAASF,SAASG,MAAM,GAAGC,IAAAA,yBAAU,EAACL;IAC5C,IAAMM,QAAmBR,OAAOS,MAAM,CAAC,SAACC;eAAMA,EAAEC,IAAI,KAAK;OAAWC,GAAG,CAAC,SAACF;YAAgBA,UAAwBA;eAAjC;YAAEG,KAAK,GAAEH,WAAAA,EAAEI,KAAK,cAAPJ,sBAAAA,WAAW;YAAGK,SAAS,EAACL,UAAAA,EAAEM,IAAI,cAANN,qBAAAA,UAAU,IAAIO,IAAI;YAAIC,WAAWR,EAAEQ,SAAS,GAAGb;YAAQc,SAAShB,SAASG,MAAM;YAAEc,QAAQ;QAAE;;IAC5M,IAAK,IAAIC,IAAI,GAAGA,IAAIb,MAAMF,MAAM,EAAEe,IAAK;QACrC,IAAIA,IAAI,IAAIb,MAAMF,MAAM,EAAEE,KAAK,CAACa,EAAE,CAACF,OAAO,GAAGX,KAAK,CAACa,IAAI,EAAE,CAACH,SAAS,GAAG;QACtE,IAAMF,OAAOb,SAASmB,KAAK,CAACd,KAAK,CAACa,EAAE,CAACH,SAAS,GAAG,GAAGV,KAAK,CAACa,EAAE,CAACF,OAAO,EAAEI,IAAI,CAAC;QAC3Ef,KAAK,CAACa,EAAE,CAACD,MAAM,GAAGI,KAAKC,IAAI,CAACC,IAAAA,uBAAc,EAACV;IAC7C;IACA,OAAOR;AACT;AAEA,IAAMmB,kBAAkB;IAAC;IAAQ;IAAO;IAAS;IAAW;IAAc;IAAY;CAAS;AAExF,IAAM7B,WAAoB;IAC/B8B,MAAM;IACAC,QAAN,SAAMA,OAAOC,EAAE;;;;;wBACb;;4BAAMA,GAAGC,IAAI,CAAC;;;wBAAd;;;;;;QACF;;IACAC,SAAAA,SAAAA,QAAQ/B,GAAG,EAAEC,IAAI,EAAE+B,OAAO,EAAEC,KAAK,EAAEC,IAAI,EAAEnC,MAAM;QAC7C,OAAOD,mBAAmBC,mBAAAA,oBAAAA,SAAUoC,IAAAA,cAAK,EAAClC,OAAOD,KAAKC;IACxD;IACMmC,QAAN,SAAMA,OAAOP,EAAE,EAAEQ,KAAK;;;;;wBACpB,IAAIA,MAAMhC,MAAM,KAAK,GAAG;;;wBACxB;;4BAAMwB,GAAGS,QAAQ,CACf,yCACAD,MAAM1B,GAAG,CAAC,SAAC4B;uCAAM;oCAACA;iCAAE;;;;wBAFtB;;;;;;QAIF;;IACMC,OAAN,SAAMA,MAAMX,EAAE,EAAEY,IAAI;;gBACZC,MACD,2BAAA,mBAAA,uBAAA,WAAA;;;;wBADCA;wBACD,kCAAA,2BAAA;;;gCAAA,kBAAA,aAAQC,mBAAAA,MAAMC,wBAAAA;gCAAsBA,UAAwBC,OAAO,CAAC,SAACzB,GAAG0B;2CAAQJ,KAAKK,IAAI,CAAC;wCAACJ;wCAAMG;wCAAK1B,EAAER,KAAK;wCAAEQ,EAAEN,OAAO;wCAAEM,EAAEH,SAAS;wCAAEG,EAAEF,OAAO;wCAAEE,EAAED,MAAM;qCAAC;;;4BAAhK,IAAK,YAA6BsB,2BAA7B,6BAAA,QAAA,yBAAA;;4BAAA;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;wBACL,wFAAwF;wBACxF,0FAA0F;wBAC1F,4FAA4F;wBAC5F,mGAAmG;wBACnG;;4BAAMO,IAAAA,oBAAU,EAACnB,IAAI,YAAYH,iBAAiB,qJAAqJgB;;;wBAAvM;;;;;;QACF;;AACF"}

package/dist/cjs/scan/pool.js

@@ -185,7 +185,7 @@ var ParsePool = /*#__PURE__*/ function() {
         return this.pool;
     };
     // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --
-    // extracted text and feature values, never the mdast tree.
+    // extracted text and feature values, never the token tree.
     _proto.run = function run(files, features, cfg, onParsed, maxWorkers) {
         return _async_to_generator(function() {
             var workerData, pool, done;

package/dist/cjs/scan/pool.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\n// parseMs is worker-side only (see workers/parse.ts); absent on the serial path.\nexport type FileResult = { doc: ParsedDoc; warnings: string[]; parseMs?: number };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the mdast tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings, parseMs: result.parseMs };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["ParsePool","_require","require","createRequire","workerFile","resolveWorkerFile","load","rel","name","join","dirname","resolve","Error","poolsCreated","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","run","files","features","cfg","onParsed","done","featureNames","map","feature","Promise","all","file","result","ok","reviveError","error","doc","warnings","parseMs","close","undefined","destroy"],"mappings":";;;;+BAqCaA;;;eAAAA;;;0BArCiB;wBACA;6BASF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,8GAA8G;AAC9G,IAAMC,WAAW,OAAOC,YAAY,cAAcC,IAAAA,yBAAa,EAAC,uDAAmBD;AAEnF,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIE;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,IAAME,OAAOH,IAAAA,yBAAa,EAAC;IAC3B,gBAAkB,QAAA;QAAC;QAAM;QAAS;KAAW,OAA3B,mBAA6B;YAApCI,MAAO;QAChB,IAAI;YACF,IAAI,AAACD,KAAK,AAAC,GAAM,OAAJC,KAAI,kBAAsCC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaK,IAAAA,cAAI,EAACC,IAAAA,iBAAO,EAACJ,KAAKK,OAAO,CAAC,AAAC,GAAM,OAAJJ,KAAI,oBAAkB,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,eAAM,CAAC;IACX;IACA,MAAM,IAAIQ,MAAM;AAClB;AAOO,IAAA,AAAMZ,0BAAN;;aAAMA;gCAAAA;QAEX,2FAA2F;QAC3F,uGAAuG;aACvGa,eAAe;;iBAJJb;IAMX,OAAQc,MAOP,GAPD,SAAQA,OAAOC,UAA2B,EAAEC,UAAkB;QAC5D,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,IAAM,AAAEC,WAAajB,SAAS,YAAtBiB;YACR,IAAI,CAACD,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUd;gBAAqBe,YAAYJ;gBAAYK,YAAYL;gBAAYD,YAAAA;YAAW;YACrH,IAAI,CAACF,YAAY;QACnB;QACA,OAAO,IAAI,CAACI,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,OAAMK,GAgBL,GAhBD,SAAMA,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEV,UAAkB;;gBAGzHD,YACAE,MACFU;;gBAJJ,6FAA6F;gBAC7F,uFAAuF;gBACjFZ,aAA8B;oBAAEU,KAAAA;oBAAKG,cAAcJ,SAASK,GAAG,CAAC,SAACC;+BAAYA,QAAQtB,IAAI;;gBAAE;gBAC3FS,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;gBACjCW,OAAO;gBACX,0FAA0F;gBAC1F,4FAA4F;gBAC5F;;oBAAOI,QAAQC,GAAG,CAChBT,MAAMM,GAAG,CAAC,SAAOI;;gCACTC;;;;wCAAU;;4CAAMjB,KAAKK,GAAG,CAACW;;;wCAAzBC,SAAU;wCAChB,IAAI,CAACA,OAAOC,EAAE,EAAE,MAAMC,IAAAA,0BAAW,EAACF,OAAOG,KAAK;wCAC9CX,qBAAAA,+BAAAA,SAAW,EAAEC;wCACb;;4CAAO;gDAAEW,KAAKJ,OAAOI,GAAG;gDAAEC,UAAUL,OAAOK,QAAQ;gDAAEC,SAASN,OAAOM,OAAO;4CAAC;;;;wBAC/E;;;;QAEJ;;IAEA,iDAAiD;IACjD,OAAMC,KAKL,GALD,SAAMA;;gBAEExB;;;;wBADN,IAAI,CAAC,IAAI,CAACA,IAAI,EAAE;;;wBACVA,OAAO,IAAI,CAACA,IAAI;wBACtB,IAAI,CAACA,IAAI,GAAGyB;wBACZ;;4BAAMzB,KAAK0B,OAAO;;;wBAAlB;;;;;;QACF;;WAzCW3C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\n// parseMs is worker-side only (see workers/parse.ts); absent on the serial path.\nexport type FileResult = { doc: ParsedDoc; warnings: string[]; parseMs?: number };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the token tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings, parseMs: result.parseMs };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["ParsePool","_require","require","createRequire","workerFile","resolveWorkerFile","load","rel","name","join","dirname","resolve","Error","poolsCreated","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","run","files","features","cfg","onParsed","done","featureNames","map","feature","Promise","all","file","result","ok","reviveError","error","doc","warnings","parseMs","close","undefined","destroy"],"mappings":";;;;+BAqCaA;;;eAAAA;;;0BArCiB;wBACA;6BASF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,8GAA8G;AAC9G,IAAMC,WAAW,OAAOC,YAAY,cAAcC,IAAAA,yBAAa,EAAC,uDAAmBD;AAEnF,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIE;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,IAAME,OAAOH,IAAAA,yBAAa,EAAC;IAC3B,gBAAkB,QAAA;QAAC;QAAM;QAAS;KAAW,OAA3B,mBAA6B;YAApCI,MAAO;QAChB,IAAI;YACF,IAAI,AAACD,KAAK,AAAC,GAAM,OAAJC,KAAI,kBAAsCC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaK,IAAAA,cAAI,EAACC,IAAAA,iBAAO,EAACJ,KAAKK,OAAO,CAAC,AAAC,GAAM,OAAJJ,KAAI,oBAAkB,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,eAAM,CAAC;IACX;IACA,MAAM,IAAIQ,MAAM;AAClB;AAOO,IAAA,AAAMZ,0BAAN;;aAAMA;gCAAAA;QAEX,2FAA2F;QAC3F,uGAAuG;aACvGa,eAAe;;iBAJJb;IAMX,OAAQc,MAOP,GAPD,SAAQA,OAAOC,UAA2B,EAAEC,UAAkB;QAC5D,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,IAAM,AAAEC,WAAajB,SAAS,YAAtBiB;YACR,IAAI,CAACD,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUd;gBAAqBe,YAAYJ;gBAAYK,YAAYL;gBAAYD,YAAAA;YAAW;YACrH,IAAI,CAACF,YAAY;QACnB;QACA,OAAO,IAAI,CAACI,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,OAAMK,GAgBL,GAhBD,SAAMA,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEV,UAAkB;;gBAGzHD,YACAE,MACFU;;gBAJJ,6FAA6F;gBAC7F,uFAAuF;gBACjFZ,aAA8B;oBAAEU,KAAAA;oBAAKG,cAAcJ,SAASK,GAAG,CAAC,SAACC;+BAAYA,QAAQtB,IAAI;;gBAAE;gBAC3FS,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;gBACjCW,OAAO;gBACX,0FAA0F;gBAC1F,4FAA4F;gBAC5F;;oBAAOI,QAAQC,GAAG,CAChBT,MAAMM,GAAG,CAAC,SAAOI;;gCACTC;;;;wCAAU;;4CAAMjB,KAAKK,GAAG,CAACW;;;wCAAzBC,SAAU;wCAChB,IAAI,CAACA,OAAOC,EAAE,EAAE,MAAMC,IAAAA,0BAAW,EAACF,OAAOG,KAAK;wCAC9CX,qBAAAA,+BAAAA,SAAW,EAAEC;wCACb;;4CAAO;gDAAEW,KAAKJ,OAAOI,GAAG;gDAAEC,UAAUL,OAAOK,QAAQ;gDAAEC,SAASN,OAAOM,OAAO;4CAAC;;;;wBAC/E;;;;QAEJ;;IAEA,iDAAiD;IACjD,OAAMC,KAKL,GALD,SAAMA;;gBAEExB;;;;wBADN,IAAI,CAAC,IAAI,CAACA,IAAI,EAAE;;;wBACVA,OAAO,IAAI,CAACA,IAAI;wBACtB,IAAI,CAACA,IAAI,GAAGyB;wBACZ;;4BAAMzB,KAAK0B,OAAO;;;wBAAlB;;;;;;QACF;;WAzCW3C"}

package/dist/cjs/store/duckdb/open.d.cts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.duckdb";
-export declare const SCHEMA_VERSION = "3";
+export declare const SCHEMA_VERSION = "4";
 export type { OpenResult };
 interface DuckdbHandle {
     instance: DuckDBInstance;

package/dist/cjs/store/duckdb/open.d.ts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.duckdb";
-export declare const SCHEMA_VERSION = "3";
+export declare const SCHEMA_VERSION = "4";
 export type { OpenResult };
 interface DuckdbHandle {
     instance: DuckDBInstance;

package/dist/cjs/store/duckdb/open.js

@@ -165,7 +165,7 @@ function _ts_generator(thisArg, body) {
     }
 }
 var DB_FILENAME = 'cache.duckdb';
-var SCHEMA_VERSION = '3';
+var SCHEMA_VERSION = '4';
 // `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
 // No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
 function ensureSchema(_handle, conn, cfg) {

package/dist/cjs/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '3';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","_handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAGAC;eAAAA;;QAwEAC;eAAAA;;QAcSC;eAAAA;;;uBAvGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,gGAAgG;AAChG,SAAeG,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;;YAKzE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMa,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMC,MAAoB;;;YACvCA,OAAOC,MAAM,CAACC,cAAc;YAC5BF,OAAOG,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLR,QACAE,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BJ,SAAS;;;;;;oBACFQ;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACf;;;oBAAxB;oBACMd,OAAO8B,IAAAA,8BAAgB,EAAChB;oBAC9B;;wBAAO;4BAAED,QAAQ;gCAAEG,UAAAA;gCAAUF,QAAAA;4BAAO;4BAAGd,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMV,MAAM;4BAAEI,UAAAA;4BAAUF,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMQ;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAN,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFuB,UAAU,SAAVA,SAAWb;eAAQ,2DAA2Dc,IAAI,CAACd,IAAII,OAAO;;IAC9F5B,cAAAA;IACAuC,aAAa,SAAbA,YAAcxB,QAAQb;eAASqC,IAAAA,oBAAW,EAACxB,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAEd;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKL;;;IAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","_handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAGAC;eAAAA;;QAwEAC;eAAAA;;QAcSC;eAAAA;;;uBAvGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,gGAAgG;AAChG,SAAeG,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;;YAKzE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMa,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMC,MAAoB;;;YACvCA,OAAOC,MAAM,CAACC,cAAc;YAC5BF,OAAOG,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLR,QACAE,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BJ,SAAS;;;;;;oBACFQ;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACf;;;oBAAxB;oBACMd,OAAO8B,IAAAA,8BAAgB,EAAChB;oBAC9B;;wBAAO;4BAAED,QAAQ;gCAAEG,UAAAA;gCAAUF,QAAAA;4BAAO;4BAAGd,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMV,MAAM;4BAAEI,UAAAA;4BAAUF,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMQ;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAN,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFuB,UAAU,SAAVA,SAAWb;eAAQ,2DAA2Dc,IAAI,CAACd,IAAII,OAAO;;IAC9F5B,cAAAA;IACAuC,aAAa,SAAbA,YAAcxB,QAAQb;eAASqC,IAAAA,oBAAW,EAACxB,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAEd;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKL;;;IAC9B"}

package/dist/cjs/store/sqlite/open.d.cts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.db";
-export declare const SCHEMA_VERSION = "19";
+export declare const SCHEMA_VERSION = "20";
 export type { OpenResult };
 interface SqliteHandle {
     db: DatabaseSync;

package/dist/cjs/store/sqlite/open.d.ts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.db";
-export declare const SCHEMA_VERSION = "19";
+export declare const SCHEMA_VERSION = "20";
 export type { OpenResult };
 interface SqliteHandle {
     db: DatabaseSync;