sensemaking 0.22.2 → 0.23.1

package/dist/esm/chunk/parse.js

@@ -1,71 +1,72 @@
-import Module from 'node:module';
 import { extractText } from './extract.js';
-// Tier-2, as embed/static.ts: the parser's packages cost ~19 ms to load and a warm tree never
-// parses, so every store-opening command paid for them until a file actually changed.
-const _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;
+import { parser } from './parser.js';
+// Opening token type to Block type. Everything else (rules, raw html, footnote definitions) is
+// 'other', as mdast's BLOCK_TYPES mapped no entry for those nodes.
 const BLOCK_TYPES = {
-    heading: 'heading',
-    paragraph: 'paragraph',
-    code: 'code',
-    table: 'table',
-    list: 'list',
-    blockquote: 'blockquote'
+    heading_open: 'heading',
+    paragraph_open: 'paragraph',
+    fence: 'code',
+    code_block: 'code',
+    table_open: 'table',
+    ordered_list_open: 'list',
+    bullet_list_open: 'list',
+    blockquote_open: 'blockquote'
 };
-let cached;
-// Imported individually, not via micromark-extension-gfm/mdast-util-gfm: those bundles also pull
-// in gfm-tagfilter, an HTML sanitizer this library never uses (no htmlExtensions call anywhere).
-function parser() {
-    if (cached) return cached;
-    const { fromMarkdown } = _require('mdast-util-from-markdown');
-    const { gfmAutolinkLiteralFromMarkdown } = _require('mdast-util-gfm-autolink-literal');
-    const { gfmFootnoteFromMarkdown } = _require('mdast-util-gfm-footnote');
-    const { gfmStrikethroughFromMarkdown } = _require('mdast-util-gfm-strikethrough');
-    const { gfmTableFromMarkdown } = _require('mdast-util-gfm-table');
-    const { gfmTaskListItemFromMarkdown } = _require('mdast-util-gfm-task-list-item');
-    const { gfmAutolinkLiteral } = _require('micromark-extension-gfm-autolink-literal');
-    const { gfmFootnote } = _require('micromark-extension-gfm-footnote');
-    const { gfmStrikethrough } = _require('micromark-extension-gfm-strikethrough');
-    const { gfmTable } = _require('micromark-extension-gfm-table');
-    const { gfmTaskListItem } = _require('micromark-extension-gfm-task-list-item');
-    cached = {
-        fromMarkdown,
-        options: {
-            extensions: [
-                gfmAutolinkLiteral(),
-                gfmFootnote(),
-                gfmStrikethrough(),
-                gfmTable(),
-                gfmTaskListItem()
-            ],
-            mdastExtensions: [
-                gfmAutolinkLiteralFromMarkdown(),
-                gfmFootnoteFromMarkdown(),
-                gfmStrikethroughFromMarkdown(),
-                gfmTableFromMarkdown(),
-                gfmTaskListItemFromMarkdown()
-            ]
+// Top-level blocks of a markdown body, typed and line-extent bounded from markdown-it's own
+// token maps (never a regex guess). html and linkify on, with the footnote and task-list plugins.
+export function parse(body) {
+    const tokens = parser().parse(body, {});
+    const lines = body.split('\n');
+    const blocks = [];
+    let i = 0;
+    while(i < tokens.length){
+        const token = tokens[i];
+        if (token.nesting === 0) {
+            var _BLOCK_TYPES_token_type;
+            blocks.push(makeBlock((_BLOCK_TYPES_token_type = BLOCK_TYPES[token.type]) !== null && _BLOCK_TYPES_token_type !== void 0 ? _BLOCK_TYPES_token_type : 'other', tokens, i, i + 1, lines));
+            i += 1;
+        } else if (token.nesting === 1) {
+            var _BLOCK_TYPES_token_type1;
+            let depth = 1;
+            let j = i + 1;
+            while(j < tokens.length && depth > 0){
+                depth += tokens[j].nesting === 1 ? 1 : tokens[j].nesting === -1 ? -1 : 0;
+                j += 1;
+            }
+            blocks.push(makeBlock((_BLOCK_TYPES_token_type1 = BLOCK_TYPES[token.type]) !== null && _BLOCK_TYPES_token_type1 !== void 0 ? _BLOCK_TYPES_token_type1 : 'other', tokens, i, j, lines));
+            i = j;
+        } else {
+            // Defensive: the balanced scan above consumes every close belonging to an open.
+            i += 1;
         }
-    };
-    return cached;
+    }
+    return blocks;
 }
-// Top-level blocks of a markdown body, typed and line-extent bounded from mdast's own
-// node.position (never a regex guess). GFM extensions add tables, task lists, footnotes, strikethrough.
-export function parse(body) {
-    const { fromMarkdown, options } = parser();
-    const tree = fromMarkdown(body, options);
-    return tree.children.map((node)=>{
-        var _BLOCK_TYPES_node_type;
-        const position = node.position;
-        const block = {
-            type: (_BLOCK_TYPES_node_type = BLOCK_TYPES[node.type]) !== null && _BLOCK_TYPES_node_type !== void 0 ? _BLOCK_TYPES_node_type : 'other',
-            startLine: position ? position.start.line : 1,
-            endLine: position ? position.end.line : 1,
-            node
-        };
-        if (node.type === 'heading') {
-            block.depth = node.depth;
-            block.text = extractText(node);
+// 1-based inclusive extent from the min/max of the tokens' maps (0-based half-open); a block
+// whose tokens carry no map (an empty footnote definition) falls back to line 1. Trailing
+// blank lines trim so a list's endLine lands where mdast's position ended it.
+function makeBlock(type, tokens, i, j, lines) {
+    let start = Infinity;
+    let end = 0;
+    for(let k = i; k < j; k++){
+        const map = tokens[k].map;
+        if (map) {
+            start = Math.min(start, map[0]);
+            end = Math.max(end, map[1]);
         }
-        return block;
-    });
+    }
+    const startLine = Number.isFinite(start) ? start + 1 : 1;
+    let endLine = end > start ? end : startLine;
+    while(endLine > startLine && lines[endLine - 1].trim() === '')endLine--;
+    const block = {
+        type,
+        startLine,
+        endLine,
+        node: tokens.slice(i, j)
+    };
+    if (type === 'heading') {
+        block.depth = Number(tokens[i].tag.slice(1));
+        block.text = extractText(block);
+    }
+    return block;
 }

package/dist/esm/chunk/parse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parse.ts"],"sourcesContent":["import Module from 'node:module';\nimport type { RootContent } from 'mdast';\nimport { extractText } from './extract.ts';\nimport type { Block, BlockType } from './types.ts';\n\n// Tier-2, as embed/static.ts: the parser's packages cost ~19 ms to load and a warm tree never\n// parses, so every store-opening command paid for them until a file actually changed.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\nconst BLOCK_TYPES: Partial<Record<RootContent['type'], BlockType>> = {\n  heading: 'heading',\n  paragraph: 'paragraph',\n  code: 'code',\n  table: 'table',\n  list: 'list',\n  blockquote: 'blockquote',\n};\n\ntype FromMarkdown = typeof import('mdast-util-from-markdown').fromMarkdown;\ntype Parser = { fromMarkdown: FromMarkdown; options: NonNullable<Parameters<FromMarkdown>[1]> };\nlet cached: Parser | undefined;\n\n// Imported individually, not via micromark-extension-gfm/mdast-util-gfm: those bundles also pull\n// in gfm-tagfilter, an HTML sanitizer this library never uses (no htmlExtensions call anywhere).\nfunction parser(): Parser {\n  if (cached) return cached;\n  const { fromMarkdown } = _require('mdast-util-from-markdown') as typeof import('mdast-util-from-markdown');\n  const { gfmAutolinkLiteralFromMarkdown } = _require('mdast-util-gfm-autolink-literal') as typeof import('mdast-util-gfm-autolink-literal');\n  const { gfmFootnoteFromMarkdown } = _require('mdast-util-gfm-footnote') as typeof import('mdast-util-gfm-footnote');\n  const { gfmStrikethroughFromMarkdown } = _require('mdast-util-gfm-strikethrough') as typeof import('mdast-util-gfm-strikethrough');\n  const { gfmTableFromMarkdown } = _require('mdast-util-gfm-table') as typeof import('mdast-util-gfm-table');\n  const { gfmTaskListItemFromMarkdown } = _require('mdast-util-gfm-task-list-item') as typeof import('mdast-util-gfm-task-list-item');\n  const { gfmAutolinkLiteral } = _require('micromark-extension-gfm-autolink-literal') as typeof import('micromark-extension-gfm-autolink-literal');\n  const { gfmFootnote } = _require('micromark-extension-gfm-footnote') as typeof import('micromark-extension-gfm-footnote');\n  const { gfmStrikethrough } = _require('micromark-extension-gfm-strikethrough') as typeof import('micromark-extension-gfm-strikethrough');\n  const { gfmTable } = _require('micromark-extension-gfm-table') as typeof import('micromark-extension-gfm-table');\n  const { gfmTaskListItem } = _require('micromark-extension-gfm-task-list-item') as typeof import('micromark-extension-gfm-task-list-item');\n  cached = {\n    fromMarkdown,\n    options: {\n      extensions: [gfmAutolinkLiteral(), gfmFootnote(), gfmStrikethrough(), gfmTable(), gfmTaskListItem()],\n      mdastExtensions: [gfmAutolinkLiteralFromMarkdown(), gfmFootnoteFromMarkdown(), gfmStrikethroughFromMarkdown(), gfmTableFromMarkdown(), gfmTaskListItemFromMarkdown()],\n    },\n  };\n  return cached;\n}\n\n// Top-level blocks of a markdown body, typed and line-extent bounded from mdast's own\n// node.position (never a regex guess). GFM extensions add tables, task lists, footnotes, strikethrough.\nexport function parse(body: string): Block[] {\n  const { fromMarkdown, options } = parser();\n  const tree = fromMarkdown(body, options);\n  return tree.children.map((node) => {\n    const position = node.position;\n    const block: Block = {\n      type: BLOCK_TYPES[node.type] ?? 'other',\n      startLine: position ? position.start.line : 1,\n      endLine: position ? position.end.line : 1,\n      node,\n    };\n    if (node.type === 'heading') {\n      block.depth = node.depth;\n      block.text = extractText(node);\n    }\n    return block;\n  });\n}\n"],"names":["Module","extractText","_require","require","createRequire","url","BLOCK_TYPES","heading","paragraph","code","table","list","blockquote","cached","parser","fromMarkdown","gfmAutolinkLiteralFromMarkdown","gfmFootnoteFromMarkdown","gfmStrikethroughFromMarkdown","gfmTableFromMarkdown","gfmTaskListItemFromMarkdown","gfmAutolinkLiteral","gfmFootnote","gfmStrikethrough","gfmTable","gfmTaskListItem","options","extensions","mdastExtensions","parse","body","tree","children","map","node","position","block","type","startLine","start","line","endLine","end","depth","text"],"mappings":"AAAA,OAAOA,YAAY,cAAc;AAEjC,SAASC,WAAW,QAAQ,eAAe;AAG3C,8FAA8F;AAC9F,sFAAsF;AACtF,MAAMC,WAAW,OAAOC,YAAY,cAAcH,OAAOI,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAE1F,MAAMG,cAA+D;IACnEC,SAAS;IACTC,WAAW;IACXC,MAAM;IACNC,OAAO;IACPC,MAAM;IACNC,YAAY;AACd;AAIA,IAAIC;AAEJ,iGAAiG;AACjG,iGAAiG;AACjG,SAASC;IACP,IAAID,QAAQ,OAAOA;IACnB,MAAM,EAAEE,YAAY,EAAE,GAAGb,SAAS;IAClC,MAAM,EAAEc,8BAA8B,EAAE,GAAGd,SAAS;IACpD,MAAM,EAAEe,uBAAuB,EAAE,GAAGf,SAAS;IAC7C,MAAM,EAAEgB,4BAA4B,EAAE,GAAGhB,SAAS;IAClD,MAAM,EAAEiB,oBAAoB,EAAE,GAAGjB,SAAS;IAC1C,MAAM,EAAEkB,2BAA2B,EAAE,GAAGlB,SAAS;IACjD,MAAM,EAAEmB,kBAAkB,EAAE,GAAGnB,SAAS;IACxC,MAAM,EAAEoB,WAAW,EAAE,GAAGpB,SAAS;IACjC,MAAM,EAAEqB,gBAAgB,EAAE,GAAGrB,SAAS;IACtC,MAAM,EAAEsB,QAAQ,EAAE,GAAGtB,SAAS;IAC9B,MAAM,EAAEuB,eAAe,EAAE,GAAGvB,SAAS;IACrCW,SAAS;QACPE;QACAW,SAAS;YACPC,YAAY;gBAACN;gBAAsBC;gBAAeC;gBAAoBC;gBAAYC;aAAkB;YACpGG,iBAAiB;gBAACZ;gBAAkCC;gBAA2BC;gBAAgCC;gBAAwBC;aAA8B;QACvK;IACF;IACA,OAAOP;AACT;AAEA,sFAAsF;AACtF,wGAAwG;AACxG,OAAO,SAASgB,MAAMC,IAAY;IAChC,MAAM,EAAEf,YAAY,EAAEW,OAAO,EAAE,GAAGZ;IAClC,MAAMiB,OAAOhB,aAAae,MAAMJ;IAChC,OAAOK,KAAKC,QAAQ,CAACC,GAAG,CAAC,CAACC;YAGhB5B;QAFR,MAAM6B,WAAWD,KAAKC,QAAQ;QAC9B,MAAMC,QAAe;YACnBC,IAAI,GAAE/B,yBAAAA,WAAW,CAAC4B,KAAKG,IAAI,CAAC,cAAtB/B,oCAAAA,yBAA0B;YAChCgC,WAAWH,WAAWA,SAASI,KAAK,CAACC,IAAI,GAAG;YAC5CC,SAASN,WAAWA,SAASO,GAAG,CAACF,IAAI,GAAG;YACxCN;QACF;QACA,IAAIA,KAAKG,IAAI,KAAK,WAAW;YAC3BD,MAAMO,KAAK,GAAGT,KAAKS,KAAK;YACxBP,MAAMQ,IAAI,GAAG3C,YAAYiC;QAC3B;QACA,OAAOE;IACT;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parse.ts"],"sourcesContent":["import type { Token } from 'markdown-it';\nimport { extractText } from './extract.ts';\nimport { parser } from './parser.ts';\nimport type { Block, BlockType } from './types.ts';\n\n// Opening token type to Block type. Everything else (rules, raw html, footnote definitions) is\n// 'other', as mdast's BLOCK_TYPES mapped no entry for those nodes.\nconst BLOCK_TYPES: Record<string, BlockType> = {\n  heading_open: 'heading',\n  paragraph_open: 'paragraph',\n  fence: 'code',\n  code_block: 'code',\n  table_open: 'table',\n  ordered_list_open: 'list',\n  bullet_list_open: 'list',\n  blockquote_open: 'blockquote',\n};\n\n// Top-level blocks of a markdown body, typed and line-extent bounded from markdown-it's own\n// token maps (never a regex guess). html and linkify on, with the footnote and task-list plugins.\nexport function parse(body: string): Block[] {\n  const tokens = parser().parse(body, {});\n  const lines = body.split('\\n');\n  const blocks: Block[] = [];\n  let i = 0;\n  while (i < tokens.length) {\n    const token = tokens[i];\n    if (token.nesting === 0) {\n      blocks.push(makeBlock(BLOCK_TYPES[token.type] ?? 'other', tokens, i, i + 1, lines));\n      i += 1;\n    } else if (token.nesting === 1) {\n      let depth = 1;\n      let j = i + 1;\n      while (j < tokens.length && depth > 0) {\n        depth += tokens[j].nesting === 1 ? 1 : tokens[j].nesting === -1 ? -1 : 0;\n        j += 1;\n      }\n      blocks.push(makeBlock(BLOCK_TYPES[token.type] ?? 'other', tokens, i, j, lines));\n      i = j;\n    } else {\n      // Defensive: the balanced scan above consumes every close belonging to an open.\n      i += 1;\n    }\n  }\n  return blocks;\n}\n\n// 1-based inclusive extent from the min/max of the tokens' maps (0-based half-open); a block\n// whose tokens carry no map (an empty footnote definition) falls back to line 1. Trailing\n// blank lines trim so a list's endLine lands where mdast's position ended it.\nfunction makeBlock(type: BlockType, tokens: Token[], i: number, j: number, lines: string[]): Block {\n  let start = Infinity;\n  let end = 0;\n  for (let k = i; k < j; k++) {\n    const map = tokens[k].map;\n    if (map) {\n      start = Math.min(start, map[0]);\n      end = Math.max(end, map[1]);\n    }\n  }\n  const startLine = Number.isFinite(start) ? start + 1 : 1;\n  let endLine = end > start ? end : startLine;\n  while (endLine > startLine && lines[endLine - 1].trim() === '') endLine--;\n  const block: Block = { type, startLine, endLine, node: tokens.slice(i, j) };\n  if (type === 'heading') {\n    block.depth = Number(tokens[i].tag.slice(1));\n    block.text = extractText(block);\n  }\n  return block;\n}\n"],"names":["extractText","parser","BLOCK_TYPES","heading_open","paragraph_open","fence","code_block","table_open","ordered_list_open","bullet_list_open","blockquote_open","parse","body","tokens","lines","split","blocks","i","length","token","nesting","push","makeBlock","type","depth","j","start","Infinity","end","k","map","Math","min","max","startLine","Number","isFinite","endLine","trim","block","node","slice","tag","text"],"mappings":"AACA,SAASA,WAAW,QAAQ,eAAe;AAC3C,SAASC,MAAM,QAAQ,cAAc;AAGrC,+FAA+F;AAC/F,mEAAmE;AACnE,MAAMC,cAAyC;IAC7CC,cAAc;IACdC,gBAAgB;IAChBC,OAAO;IACPC,YAAY;IACZC,YAAY;IACZC,mBAAmB;IACnBC,kBAAkB;IAClBC,iBAAiB;AACnB;AAEA,4FAA4F;AAC5F,kGAAkG;AAClG,OAAO,SAASC,MAAMC,IAAY;IAChC,MAAMC,SAASZ,SAASU,KAAK,CAACC,MAAM,CAAC;IACrC,MAAME,QAAQF,KAAKG,KAAK,CAAC;IACzB,MAAMC,SAAkB,EAAE;IAC1B,IAAIC,IAAI;IACR,MAAOA,IAAIJ,OAAOK,MAAM,CAAE;QACxB,MAAMC,QAAQN,MAAM,CAACI,EAAE;QACvB,IAAIE,MAAMC,OAAO,KAAK,GAAG;gBACDlB;YAAtBc,OAAOK,IAAI,CAACC,WAAUpB,0BAAAA,WAAW,CAACiB,MAAMI,IAAI,CAAC,cAAvBrB,qCAAAA,0BAA2B,SAASW,QAAQI,GAAGA,IAAI,GAAGH;YAC5EG,KAAK;QACP,OAAO,IAAIE,MAAMC,OAAO,KAAK,GAAG;gBAORlB;YANtB,IAAIsB,QAAQ;YACZ,IAAIC,IAAIR,IAAI;YACZ,MAAOQ,IAAIZ,OAAOK,MAAM,IAAIM,QAAQ,EAAG;gBACrCA,SAASX,MAAM,CAACY,EAAE,CAACL,OAAO,KAAK,IAAI,IAAIP,MAAM,CAACY,EAAE,CAACL,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI;gBACvEK,KAAK;YACP;YACAT,OAAOK,IAAI,CAACC,WAAUpB,2BAAAA,WAAW,CAACiB,MAAMI,IAAI,CAAC,cAAvBrB,sCAAAA,2BAA2B,SAASW,QAAQI,GAAGQ,GAAGX;YACxEG,IAAIQ;QACN,OAAO;YACL,gFAAgF;YAChFR,KAAK;QACP;IACF;IACA,OAAOD;AACT;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,8EAA8E;AAC9E,SAASM,UAAUC,IAAe,EAAEV,MAAe,EAAEI,CAAS,EAAEQ,CAAS,EAAEX,KAAe;IACxF,IAAIY,QAAQC;IACZ,IAAIC,MAAM;IACV,IAAK,IAAIC,IAAIZ,GAAGY,IAAIJ,GAAGI,IAAK;QAC1B,MAAMC,MAAMjB,MAAM,CAACgB,EAAE,CAACC,GAAG;QACzB,IAAIA,KAAK;YACPJ,QAAQK,KAAKC,GAAG,CAACN,OAAOI,GAAG,CAAC,EAAE;YAC9BF,MAAMG,KAAKE,GAAG,CAACL,KAAKE,GAAG,CAAC,EAAE;QAC5B;IACF;IACA,MAAMI,YAAYC,OAAOC,QAAQ,CAACV,SAASA,QAAQ,IAAI;IACvD,IAAIW,UAAUT,MAAMF,QAAQE,MAAMM;IAClC,MAAOG,UAAUH,aAAapB,KAAK,CAACuB,UAAU,EAAE,CAACC,IAAI,OAAO,GAAID;IAChE,MAAME,QAAe;QAAEhB;QAAMW;QAAWG;QAASG,MAAM3B,OAAO4B,KAAK,CAACxB,GAAGQ;IAAG;IAC1E,IAAIF,SAAS,WAAW;QACtBgB,MAAMf,KAAK,GAAGW,OAAOtB,MAAM,CAACI,EAAE,CAACyB,GAAG,CAACD,KAAK,CAAC;QACzCF,MAAMI,IAAI,GAAG3C,YAAYuC;IAC3B;IACA,OAAOA;AACT"}

package/dist/esm/chunk/parser.d.ts

@@ -0,0 +1,2 @@
+import type { MarkdownIt } from 'markdown-it';
+export declare function parser(): MarkdownIt;

package/dist/esm/chunk/parser.js

@@ -0,0 +1,26 @@
+import Module from 'node:module';
+// Tier-2, as embed/static.ts: the parser's packages cost ~19 ms to load and a warm tree never
+// parses, so every store-opening command paid for them until a file actually changed.
+const _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;
+let cached;
+// The one shared parser: html and linkify on, fuzzy links, footnotes and task lists, with the
+// footnote rules disabled for GFM parity (definitions stay spans; ^[...] stays inert text).
+export function parser() {
+    if (cached) return cached;
+    const Ctor = _require('markdown-it');
+    const footnote = _require('markdown-it-footnote');
+    const taskLists = _require('markdown-it-task-lists');
+    const md = new Ctor({
+        html: true,
+        linkify: true
+    }).use(footnote).use(taskLists);
+    // Fuzzy linking is what links www. domains and emails (the scheme matcher does neither); its
+    // bare-domain over-matching vs GFM is reined in at extraction, where the text is kept.
+    md.linkify.set({
+        fuzzyLink: true
+    });
+    md.core.ruler.disable('footnote_tail');
+    md.inline.ruler.disable('footnote_inline');
+    cached = md;
+    return cached;
+}

package/dist/esm/chunk/parser.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parser.ts"],"sourcesContent":["import Module from 'node:module';\nimport type { MarkdownIt } from 'markdown-it';\nimport type footnotePlugin from 'markdown-it-footnote';\nimport type taskListsPlugin from 'markdown-it-task-lists';\n\n// Tier-2, as embed/static.ts: the parser's packages cost ~19 ms to load and a warm tree never\n// parses, so every store-opening command paid for them until a file actually changed.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// The CJS build resolves markdown-it's export= types, so the constructor is named locally.\ntype Ctor = new (options?: { html?: boolean; linkify?: boolean }) => MarkdownIt;\n\nlet cached: MarkdownIt | undefined;\n\n// The one shared parser: html and linkify on, fuzzy links, footnotes and task lists, with the\n// footnote rules disabled for GFM parity (definitions stay spans; ^[...] stays inert text).\nexport function parser(): MarkdownIt {\n  if (cached) return cached;\n  const Ctor = _require('markdown-it') as Ctor;\n  const footnote = _require('markdown-it-footnote') as typeof footnotePlugin;\n  const taskLists = _require('markdown-it-task-lists') as typeof taskListsPlugin;\n  const md = new Ctor({ html: true, linkify: true }).use(footnote).use(taskLists);\n  // Fuzzy linking is what links www. domains and emails (the scheme matcher does neither); its\n  // bare-domain over-matching vs GFM is reined in at extraction, where the text is kept.\n  md.linkify.set({ fuzzyLink: true });\n  md.core.ruler.disable('footnote_tail');\n  md.inline.ruler.disable('footnote_inline');\n  cached = md;\n  return cached;\n}\n"],"names":["Module","_require","require","createRequire","url","cached","parser","Ctor","footnote","taskLists","md","html","linkify","use","set","fuzzyLink","core","ruler","disable","inline"],"mappings":"AAAA,OAAOA,YAAY,cAAc;AAKjC,8FAA8F;AAC9F,sFAAsF;AACtF,MAAMC,WAAW,OAAOC,YAAY,cAAcF,OAAOG,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAK1F,IAAIG;AAEJ,8FAA8F;AAC9F,4FAA4F;AAC5F,OAAO,SAASC;IACd,IAAID,QAAQ,OAAOA;IACnB,MAAME,OAAON,SAAS;IACtB,MAAMO,WAAWP,SAAS;IAC1B,MAAMQ,YAAYR,SAAS;IAC3B,MAAMS,KAAK,IAAIH,KAAK;QAAEI,MAAM;QAAMC,SAAS;IAAK,GAAGC,GAAG,CAACL,UAAUK,GAAG,CAACJ;IACrE,6FAA6F;IAC7F,uFAAuF;IACvFC,GAAGE,OAAO,CAACE,GAAG,CAAC;QAAEC,WAAW;IAAK;IACjCL,GAAGM,IAAI,CAACC,KAAK,CAACC,OAAO,CAAC;IACtBR,GAAGS,MAAM,CAACF,KAAK,CAACC,OAAO,CAAC;IACxBb,SAASK;IACT,OAAOL;AACT"}

package/dist/esm/chunk/types.d.ts

@@ -1,4 +1,4 @@
-import type { RootContent } from 'mdast';
+import type { Token } from 'markdown-it';
 export type BlockType = 'heading' | 'paragraph' | 'code' | 'table' | 'list' | 'blockquote' | 'other';
 export interface Block {
     type: BlockType;
@@ -6,7 +6,7 @@ export interface Block {
     endLine: number;
     depth?: number;
     text?: string;
-    node: RootContent;
+    node: Token[];
 }
 export interface ChunkOptions {
     targetTokens?: number;

package/dist/esm/chunk/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/types.ts"],"sourcesContent":["import type { RootContent } from 'mdast';\n\nexport type BlockType = 'heading' | 'paragraph' | 'code' | 'table' | 'list' | 'blockquote' | 'other';\n\n// One top-level mdast node plus its 1-indexed source line extent, heading depth/text if any.\n// `node` is the mdast node itself, so extract.ts and group.ts (W2) read the same parse.\nexport interface Block {\n  type: BlockType;\n  startLine: number;\n  endLine: number;\n  depth?: number;\n  text?: string;\n  node: RootContent;\n}\n\nexport interface ChunkOptions {\n  // D3/W3b: the pgc pairing cap and the sole owner lever (embed.chunkTokens); default 500.\n  targetTokens?: number;\n  // D9: 'raw' (source lines verbatim) is the shipped default; 'extracted' (flavor-resolved)\n  // stays available for a future measured revisit, not currently shipped.\n  text?: 'extracted' | 'raw';\n}\n\n// One grouped, extracted unit: 1-indexed line extent over the body, flavor-resolved text.\nexport interface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n"],"names":[],"mappings":"AAuBA,0FAA0F;AAC1F,WAIC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/types.ts"],"sourcesContent":["import type { Token } from 'markdown-it';\n\nexport type BlockType = 'heading' | 'paragraph' | 'code' | 'table' | 'list' | 'blockquote' | 'other';\n\n// One top-level markdown-it block plus its 1-indexed source line extent, heading depth/text if any.\n// `node` is the block's token slice, so extract.ts and group.ts (W2) read the same parse.\nexport interface Block {\n  type: BlockType;\n  startLine: number;\n  endLine: number;\n  depth?: number;\n  text?: string;\n  node: Token[];\n}\n\nexport interface ChunkOptions {\n  // D3/W3b: the pgc pairing cap and the sole owner lever (embed.chunkTokens); default 500.\n  targetTokens?: number;\n  // D9: 'raw' (source lines verbatim) is the shipped default; 'extracted' (flavor-resolved)\n  // stays available for a future measured revisit, not currently shipped.\n  text?: 'extracted' | 'raw';\n}\n\n// One grouped, extracted unit: 1-indexed line extent over the body, flavor-resolved text.\nexport interface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n"],"names":[],"mappings":"AAuBA,0FAA0F;AAC1F,WAIC"}

package/dist/esm/chunk/version.d.ts

@@ -1 +1 @@
-export declare const CHUNK_VERSION = "chunk:v5";
+export declare const CHUNK_VERSION = "chunk:v6";

package/dist/esm/chunk/version.js

@@ -1,3 +1,3 @@
 // D8: bump whenever chunk semantics change (grouping, sizing, splitting); the digest in
 // test/unit/chunk/version.test.ts tracks EXTRACTION changes only, so the two move independently.
-export const CHUNK_VERSION = 'chunk:v5';
+export const CHUNK_VERSION = 'chunk:v6';

package/dist/esm/chunk/version.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/version.ts"],"sourcesContent":["// D8: bump whenever chunk semantics change (grouping, sizing, splitting); the digest in\n// test/unit/chunk/version.test.ts tracks EXTRACTION changes only, so the two move independently.\nexport const CHUNK_VERSION = 'chunk:v5';\n"],"names":["CHUNK_VERSION"],"mappings":"AAAA,wFAAwF;AACxF,iGAAiG;AACjG,OAAO,MAAMA,gBAAgB,WAAW"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/version.ts"],"sourcesContent":["// D8: bump whenever chunk semantics change (grouping, sizing, splitting); the digest in\n// test/unit/chunk/version.test.ts tracks EXTRACTION changes only, so the two move independently.\nexport const CHUNK_VERSION = 'chunk:v6';\n"],"names":["CHUNK_VERSION"],"mappings":"AAAA,wFAAwF;AACxF,iGAAiG;AACjG,OAAO,MAAMA,gBAAgB,WAAW"}

package/dist/esm/features/sections.js

@@ -1,7 +1,7 @@
 import { estimateTokens, parse } from '../chunk/index.js';
 import { countLines } from '../scan/frontmatter.js';
 import { appendRows } from '../store/shared.js';
-// Heading blocks parse() already found (mdast/CommonMark fences), offset back onto the raw file;
+// Heading blocks parse() already found (markdown-it/CommonMark fences), offset back onto the raw file;
 // a section runs to just before the next heading, or EOF.
 function sectionsFromBlocks(blocks, raw, body) {
     const rawLines = raw.split('\n');

package/dist/esm/features/sections.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/sections.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { estimateTokens, parse } from '../chunk/index.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// sections(path, idx, level, heading, start_line, end_line, tokens): the heading outline,\n// 1-indexed over the raw file so a row is a direct Read range; tokens is estimateTokens (D5), CJK-aware.\n\nexport interface Section {\n  level: number;\n  heading: string;\n  startLine: number;\n  endLine: number;\n  tokens: number;\n}\n\n// Heading blocks parse() already found (mdast/CommonMark fences), offset back onto the raw file;\n// a section runs to just before the next heading, or EOF.\nfunction sectionsFromBlocks(blocks: Block[], raw: string, body: string): Section[] {\n  const rawLines = raw.split('\\n');\n  const offset = rawLines.length - countLines(body);\n  const found: Section[] = blocks.filter((b) => b.type === 'heading').map((b) => ({ level: b.depth ?? 1, heading: (b.text ?? '').trim(), startLine: b.startLine + offset, endLine: rawLines.length, tokens: 0 }));\n  for (let s = 0; s < found.length; s++) {\n    if (s + 1 < found.length) found[s].endLine = found[s + 1].startLine - 1;\n    const text = rawLines.slice(found[s].startLine - 1, found[s].endLine).join('\\n');\n    found[s].tokens = Math.ceil(estimateTokens(text));\n  }\n  return found;\n}\n\nconst SECTION_COLUMNS = ['path', 'idx', 'level', 'heading', 'start_line', 'end_line', 'tokens'];\n\nexport const sections: Feature = {\n  name: 'sections',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS sections (\"path\" TEXT, idx INTEGER, level INTEGER, heading TEXT, start_line INTEGER, end_line INTEGER, tokens INTEGER, PRIMARY KEY (\"path\", idx))`);\n  },\n  extract(raw, body, _search, _data, _cfg, blocks) {\n    return sectionsFromBlocks(blocks ?? parse(body), raw, body);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM sections WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) (extracted as Section[]).forEach((s, idx) => rows.push([path, idx, s.level, s.heading, s.startLine, s.endLine, s.tokens]));\n    // DO NOTHING, not a bare INSERT: reconcile's added/touched split is decided before this\n    // write's lock, so a path called \"added\" here can already have this row from a concurrent\n    // reconcile that committed first -- same file, same parse, same row. A store with an append\n    // path has no second writer, and remove() cleared every touched path above, so it skips the guard.\n    await appendRows(db, 'sections', SECTION_COLUMNS, 'INSERT INTO sections (\"path\", idx, level, heading, start_line, end_line, tokens) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(\"path\", idx) DO NOTHING', rows);\n  },\n};\n"],"names":["estimateTokens","parse","countLines","appendRows","sectionsFromBlocks","blocks","raw","body","rawLines","split","offset","length","found","filter","b","type","map","level","depth","heading","text","trim","startLine","endLine","tokens","s","slice","join","Math","ceil","SECTION_COLUMNS","sections","name","schema","db","exec","extract","_search","_data","_cfg","remove","paths","runBatch","p","store","docs","rows","path","extracted","forEach","idx","push"],"mappings":"AACA,SAASA,cAAc,EAAEC,KAAK,QAAQ,oBAAoB;AAC1D,SAASC,UAAU,QAAQ,yBAAyB;AACpD,SAASC,UAAU,QAAQ,qBAAqB;AAchD,iGAAiG;AACjG,0DAA0D;AAC1D,SAASC,mBAAmBC,MAAe,EAAEC,GAAW,EAAEC,IAAY;IACpE,MAAMC,WAAWF,IAAIG,KAAK,CAAC;IAC3B,MAAMC,SAASF,SAASG,MAAM,GAAGT,WAAWK;IAC5C,MAAMK,QAAmBP,OAAOQ,MAAM,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,WAAWC,GAAG,CAAC,CAACF;YAAgBA,UAAwBA;eAAjC;YAAEG,KAAK,GAAEH,WAAAA,EAAEI,KAAK,cAAPJ,sBAAAA,WAAW;YAAGK,SAAS,EAACL,UAAAA,EAAEM,IAAI,cAANN,qBAAAA,UAAU,IAAIO,IAAI;YAAIC,WAAWR,EAAEQ,SAAS,GAAGZ;YAAQa,SAASf,SAASG,MAAM;YAAEa,QAAQ;QAAE;;IAC5M,IAAK,IAAIC,IAAI,GAAGA,IAAIb,MAAMD,MAAM,EAAEc,IAAK;QACrC,IAAIA,IAAI,IAAIb,MAAMD,MAAM,EAAEC,KAAK,CAACa,EAAE,CAACF,OAAO,GAAGX,KAAK,CAACa,IAAI,EAAE,CAACH,SAAS,GAAG;QACtE,MAAMF,OAAOZ,SAASkB,KAAK,CAACd,KAAK,CAACa,EAAE,CAACH,SAAS,GAAG,GAAGV,KAAK,CAACa,EAAE,CAACF,OAAO,EAAEI,IAAI,CAAC;QAC3Ef,KAAK,CAACa,EAAE,CAACD,MAAM,GAAGI,KAAKC,IAAI,CAAC7B,eAAeoB;IAC7C;IACA,OAAOR;AACT;AAEA,MAAMkB,kBAAkB;IAAC;IAAQ;IAAO;IAAS;IAAW;IAAc;IAAY;CAAS;AAE/F,OAAO,MAAMC,WAAoB;IAC/BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,4KAA4K,CAAC;IAC9L;IACAC,SAAQ9B,GAAG,EAAEC,IAAI,EAAE8B,OAAO,EAAEC,KAAK,EAAEC,IAAI,EAAElC,MAAM;QAC7C,OAAOD,mBAAmBC,mBAAAA,oBAAAA,SAAUJ,MAAMM,OAAOD,KAAKC;IACxD;IACA,MAAMiC,QAAON,EAAE,EAAEO,KAAK;QACpB,IAAIA,MAAM9B,MAAM,KAAK,GAAG;QACxB,MAAMuB,GAAGQ,QAAQ,CACf,yCACAD,MAAMzB,GAAG,CAAC,CAAC2B,IAAM;gBAACA;aAAE;IAExB;IACA,MAAMC,OAAMV,EAAE,EAAEW,IAAI;QAClB,MAAMC,OAAoB,EAAE;QAC5B,KAAK,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE,IAAIH,KAAM,AAACG,UAAwBC,OAAO,CAAC,CAACxB,GAAGyB,MAAQJ,KAAKK,IAAI,CAAC;gBAACJ;gBAAMG;gBAAKzB,EAAER,KAAK;gBAAEQ,EAAEN,OAAO;gBAAEM,EAAEH,SAAS;gBAAEG,EAAEF,OAAO;gBAAEE,EAAED,MAAM;aAAC;QAChK,wFAAwF;QACxF,0FAA0F;QAC1F,4FAA4F;QAC5F,mGAAmG;QACnG,MAAMrB,WAAW+B,IAAI,YAAYJ,iBAAiB,qJAAqJgB;IACzM;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/sections.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { estimateTokens, parse } from '../chunk/index.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// sections(path, idx, level, heading, start_line, end_line, tokens): the heading outline,\n// 1-indexed over the raw file so a row is a direct Read range; tokens is estimateTokens (D5), CJK-aware.\n\nexport interface Section {\n  level: number;\n  heading: string;\n  startLine: number;\n  endLine: number;\n  tokens: number;\n}\n\n// Heading blocks parse() already found (markdown-it/CommonMark fences), offset back onto the raw file;\n// a section runs to just before the next heading, or EOF.\nfunction sectionsFromBlocks(blocks: Block[], raw: string, body: string): Section[] {\n  const rawLines = raw.split('\\n');\n  const offset = rawLines.length - countLines(body);\n  const found: Section[] = blocks.filter((b) => b.type === 'heading').map((b) => ({ level: b.depth ?? 1, heading: (b.text ?? '').trim(), startLine: b.startLine + offset, endLine: rawLines.length, tokens: 0 }));\n  for (let s = 0; s < found.length; s++) {\n    if (s + 1 < found.length) found[s].endLine = found[s + 1].startLine - 1;\n    const text = rawLines.slice(found[s].startLine - 1, found[s].endLine).join('\\n');\n    found[s].tokens = Math.ceil(estimateTokens(text));\n  }\n  return found;\n}\n\nconst SECTION_COLUMNS = ['path', 'idx', 'level', 'heading', 'start_line', 'end_line', 'tokens'];\n\nexport const sections: Feature = {\n  name: 'sections',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS sections (\"path\" TEXT, idx INTEGER, level INTEGER, heading TEXT, start_line INTEGER, end_line INTEGER, tokens INTEGER, PRIMARY KEY (\"path\", idx))`);\n  },\n  extract(raw, body, _search, _data, _cfg, blocks) {\n    return sectionsFromBlocks(blocks ?? parse(body), raw, body);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM sections WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) (extracted as Section[]).forEach((s, idx) => rows.push([path, idx, s.level, s.heading, s.startLine, s.endLine, s.tokens]));\n    // DO NOTHING, not a bare INSERT: reconcile's added/touched split is decided before this\n    // write's lock, so a path called \"added\" here can already have this row from a concurrent\n    // reconcile that committed first -- same file, same parse, same row. A store with an append\n    // path has no second writer, and remove() cleared every touched path above, so it skips the guard.\n    await appendRows(db, 'sections', SECTION_COLUMNS, 'INSERT INTO sections (\"path\", idx, level, heading, start_line, end_line, tokens) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(\"path\", idx) DO NOTHING', rows);\n  },\n};\n"],"names":["estimateTokens","parse","countLines","appendRows","sectionsFromBlocks","blocks","raw","body","rawLines","split","offset","length","found","filter","b","type","map","level","depth","heading","text","trim","startLine","endLine","tokens","s","slice","join","Math","ceil","SECTION_COLUMNS","sections","name","schema","db","exec","extract","_search","_data","_cfg","remove","paths","runBatch","p","store","docs","rows","path","extracted","forEach","idx","push"],"mappings":"AACA,SAASA,cAAc,EAAEC,KAAK,QAAQ,oBAAoB;AAC1D,SAASC,UAAU,QAAQ,yBAAyB;AACpD,SAASC,UAAU,QAAQ,qBAAqB;AAchD,uGAAuG;AACvG,0DAA0D;AAC1D,SAASC,mBAAmBC,MAAe,EAAEC,GAAW,EAAEC,IAAY;IACpE,MAAMC,WAAWF,IAAIG,KAAK,CAAC;IAC3B,MAAMC,SAASF,SAASG,MAAM,GAAGT,WAAWK;IAC5C,MAAMK,QAAmBP,OAAOQ,MAAM,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,WAAWC,GAAG,CAAC,CAACF;YAAgBA,UAAwBA;eAAjC;YAAEG,KAAK,GAAEH,WAAAA,EAAEI,KAAK,cAAPJ,sBAAAA,WAAW;YAAGK,SAAS,EAACL,UAAAA,EAAEM,IAAI,cAANN,qBAAAA,UAAU,IAAIO,IAAI;YAAIC,WAAWR,EAAEQ,SAAS,GAAGZ;YAAQa,SAASf,SAASG,MAAM;YAAEa,QAAQ;QAAE;;IAC5M,IAAK,IAAIC,IAAI,GAAGA,IAAIb,MAAMD,MAAM,EAAEc,IAAK;QACrC,IAAIA,IAAI,IAAIb,MAAMD,MAAM,EAAEC,KAAK,CAACa,EAAE,CAACF,OAAO,GAAGX,KAAK,CAACa,IAAI,EAAE,CAACH,SAAS,GAAG;QACtE,MAAMF,OAAOZ,SAASkB,KAAK,CAACd,KAAK,CAACa,EAAE,CAACH,SAAS,GAAG,GAAGV,KAAK,CAACa,EAAE,CAACF,OAAO,EAAEI,IAAI,CAAC;QAC3Ef,KAAK,CAACa,EAAE,CAACD,MAAM,GAAGI,KAAKC,IAAI,CAAC7B,eAAeoB;IAC7C;IACA,OAAOR;AACT;AAEA,MAAMkB,kBAAkB;IAAC;IAAQ;IAAO;IAAS;IAAW;IAAc;IAAY;CAAS;AAE/F,OAAO,MAAMC,WAAoB;IAC/BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,4KAA4K,CAAC;IAC9L;IACAC,SAAQ9B,GAAG,EAAEC,IAAI,EAAE8B,OAAO,EAAEC,KAAK,EAAEC,IAAI,EAAElC,MAAM;QAC7C,OAAOD,mBAAmBC,mBAAAA,oBAAAA,SAAUJ,MAAMM,OAAOD,KAAKC;IACxD;IACA,MAAMiC,QAAON,EAAE,EAAEO,KAAK;QACpB,IAAIA,MAAM9B,MAAM,KAAK,GAAG;QACxB,MAAMuB,GAAGQ,QAAQ,CACf,yCACAD,MAAMzB,GAAG,CAAC,CAAC2B,IAAM;gBAACA;aAAE;IAExB;IACA,MAAMC,OAAMV,EAAE,EAAEW,IAAI;QAClB,MAAMC,OAAoB,EAAE;QAC5B,KAAK,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE,IAAIH,KAAM,AAACG,UAAwBC,OAAO,CAAC,CAACxB,GAAGyB,MAAQJ,KAAKK,IAAI,CAAC;gBAACJ;gBAAMG;gBAAKzB,EAAER,KAAK;gBAAEQ,EAAEN,OAAO;gBAAEM,EAAEH,SAAS;gBAAEG,EAAEF,OAAO;gBAAEE,EAAED,MAAM;aAAC;QAChK,wFAAwF;QACxF,0FAA0F;QAC1F,4FAA4F;QAC5F,mGAAmG;QACnG,MAAMrB,WAAW+B,IAAI,YAAYJ,iBAAiB,qJAAqJgB;IACzM;AACF,EAAE"}

package/dist/esm/scan/pool.js

@@ -40,7 +40,7 @@ export class ParsePool {
         return this.pool;
     }
     // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --
-    // extracted text and feature values, never the mdast tree.
+    // extracted text and feature values, never the token tree.
     async run(files, features, cfg, onParsed, maxWorkers) {
         // cfg and the feature selection are constant for the dispatch, so they cross once per worker
         // as workerData. Features carry closures and cannot cross at all; only their names do.

package/dist/esm/scan/pool.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\n// parseMs is worker-side only (see workers/parse.ts); absent on the serial path.\nexport type FileResult = { doc: ParsedDoc; warnings: string[]; parseMs?: number };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the mdast tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings, parseMs: result.parseMs };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["createRequire","dirname","join","reviveError","_require","require","url","workerFile","resolveWorkerFile","load","rel","name","resolve","Error","ParsePool","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","poolsCreated","run","files","features","cfg","onParsed","featureNames","map","feature","done","Promise","all","file","result","ok","error","doc","warnings","parseMs","close","undefined","destroy"],"mappings":"AAAA,SAASA,aAAa,QAAQ,cAAc;AAC5C,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAS1C,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,8GAA8G;AAC9G,MAAMC,WAAW,OAAOC,YAAY,cAAcL,cAAc,YAAYM,GAAG,IAAID;AAEnF,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIE;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,MAAME,OAAOT,cAAc,YAAYM,GAAG;IAC1C,KAAK,MAAMI,OAAO;QAAC;QAAM;QAAS;KAAW,CAAE;QAC7C,IAAI;YACF,IAAI,AAACD,KAAK,GAAGC,IAAI,aAAa,CAAC,EAAwBC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaL,KAAKD,QAAQQ,KAAKG,OAAO,CAAC,GAAGF,IAAI,aAAa,CAAC,IAAI,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,OAAM,CAAC;IACX;IACA,MAAM,IAAIM,MAAM;AAClB;AAKA,8FAA8F;AAC9F,mGAAmG;AACnG,OAAO,MAAMC;IAMHC,OAAOC,UAA2B,EAAEC,UAAkB,EAAuB;QACnF,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,MAAM,EAAEC,QAAQ,EAAE,GAAGf,SAAS;YAC9B,IAAI,CAACc,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUZ;gBAAqBa,YAAYJ;gBAAYK,YAAYL;gBAAYD;YAAW;YACrH,IAAI,CAACO,YAAY;QACnB;QACA,OAAO,IAAI,CAACL,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,MAAMM,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEX,UAAkB,EAAyB;QACxJ,6FAA6F;QAC7F,uFAAuF;QACvF,MAAMD,aAA8B;YAAEW;YAAKE,cAAcH,SAASI,GAAG,CAAC,CAACC,UAAYA,QAAQpB,IAAI;QAAE;QACjG,MAAMO,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;QACrC,IAAIe,OAAO;QACX,0FAA0F;QAC1F,4FAA4F;QAC5F,OAAOC,QAAQC,GAAG,CAChBT,MAAMK,GAAG,CAAC,OAAOK;YACf,MAAMC,SAAU,MAAMlB,KAAKM,GAAG,CAACW;YAC/B,IAAI,CAACC,OAAOC,EAAE,EAAE,MAAMlC,YAAYiC,OAAOE,KAAK;YAC9CV,qBAAAA,+BAAAA,SAAW,EAAEI;YACb,OAAO;gBAAEO,KAAKH,OAAOG,GAAG;gBAAEC,UAAUJ,OAAOI,QAAQ;gBAAEC,SAASL,OAAOK,OAAO;YAAC;QAC/E;IAEJ;IAEA,iDAAiD;IACjD,MAAMC,QAAuB;QAC3B,IAAI,CAAC,IAAI,CAACxB,IAAI,EAAE;QAChB,MAAMA,OAAO,IAAI,CAACA,IAAI;QACtB,IAAI,CAACA,IAAI,GAAGyB;QACZ,MAAMzB,KAAK0B,OAAO;IACpB;;QAvCA,2FAA2F;QAC3F,uGAAuG;aACvGrB,eAAe;;AAsCjB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\n// parseMs is worker-side only (see workers/parse.ts); absent on the serial path.\nexport type FileResult = { doc: ParsedDoc; warnings: string[]; parseMs?: number };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the token tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings, parseMs: result.parseMs };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["createRequire","dirname","join","reviveError","_require","require","url","workerFile","resolveWorkerFile","load","rel","name","resolve","Error","ParsePool","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","poolsCreated","run","files","features","cfg","onParsed","featureNames","map","feature","done","Promise","all","file","result","ok","error","doc","warnings","parseMs","close","undefined","destroy"],"mappings":"AAAA,SAASA,aAAa,QAAQ,cAAc;AAC5C,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAS1C,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,8GAA8G;AAC9G,MAAMC,WAAW,OAAOC,YAAY,cAAcL,cAAc,YAAYM,GAAG,IAAID;AAEnF,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIE;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,MAAME,OAAOT,cAAc,YAAYM,GAAG;IAC1C,KAAK,MAAMI,OAAO;QAAC;QAAM;QAAS;KAAW,CAAE;QAC7C,IAAI;YACF,IAAI,AAACD,KAAK,GAAGC,IAAI,aAAa,CAAC,EAAwBC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaL,KAAKD,QAAQQ,KAAKG,OAAO,CAAC,GAAGF,IAAI,aAAa,CAAC,IAAI,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,OAAM,CAAC;IACX;IACA,MAAM,IAAIM,MAAM;AAClB;AAKA,8FAA8F;AAC9F,mGAAmG;AACnG,OAAO,MAAMC;IAMHC,OAAOC,UAA2B,EAAEC,UAAkB,EAAuB;QACnF,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,MAAM,EAAEC,QAAQ,EAAE,GAAGf,SAAS;YAC9B,IAAI,CAACc,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUZ;gBAAqBa,YAAYJ;gBAAYK,YAAYL;gBAAYD;YAAW;YACrH,IAAI,CAACO,YAAY;QACnB;QACA,OAAO,IAAI,CAACL,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,MAAMM,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEX,UAAkB,EAAyB;QACxJ,6FAA6F;QAC7F,uFAAuF;QACvF,MAAMD,aAA8B;YAAEW;YAAKE,cAAcH,SAASI,GAAG,CAAC,CAACC,UAAYA,QAAQpB,IAAI;QAAE;QACjG,MAAMO,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;QACrC,IAAIe,OAAO;QACX,0FAA0F;QAC1F,4FAA4F;QAC5F,OAAOC,QAAQC,GAAG,CAChBT,MAAMK,GAAG,CAAC,OAAOK;YACf,MAAMC,SAAU,MAAMlB,KAAKM,GAAG,CAACW;YAC/B,IAAI,CAACC,OAAOC,EAAE,EAAE,MAAMlC,YAAYiC,OAAOE,KAAK;YAC9CV,qBAAAA,+BAAAA,SAAW,EAAEI;YACb,OAAO;gBAAEO,KAAKH,OAAOG,GAAG;gBAAEC,UAAUJ,OAAOI,QAAQ;gBAAEC,SAASL,OAAOK,OAAO;YAAC;QAC/E;IAEJ;IAEA,iDAAiD;IACjD,MAAMC,QAAuB;QAC3B,IAAI,CAAC,IAAI,CAACxB,IAAI,EAAE;QAChB,MAAMA,OAAO,IAAI,CAACA,IAAI;QACtB,IAAI,CAACA,IAAI,GAAGyB;QACZ,MAAMzB,KAAK0B,OAAO;IACpB;;QAvCA,2FAA2F;QAC3F,uGAAuG;aACvGrB,eAAe;;AAsCjB"}

package/dist/esm/store/batch.d.ts

@@ -0,0 +1,4 @@
+export declare function rewriteInsert(sql: string, rowCount: number): {
+    sql: string;
+    width: number;
+} | null;

package/dist/esm/store/batch.js

@@ -0,0 +1,21 @@
+// Rewrites a single-row parameterized INSERT into one multi-row statement. Engine-neutral string
+// work, shared by duckdb (batch.ts's rewriteBatch) and turso (turso/connection.ts's runBatch).
+const INSERT_RE = /^(INSERT\s+(?:OR\s+\w+\s+)?INTO\s+\S+\s*\([^)]*\)\s*VALUES\s*)\(([^()]*)\)(.*)$/is;
+function placeholderCount(tuple) {
+    var _tuple_match;
+    return ((_tuple_match = tuple.match(/\?/g)) !== null && _tuple_match !== void 0 ? _tuple_match : []).length;
+}
+export function rewriteInsert(sql, rowCount) {
+    const m = INSERT_RE.exec(sql.trim());
+    if (!m) return null;
+    const [, head, tuple, tail] = m;
+    const width = placeholderCount(tuple);
+    if (width === 0) return null;
+    const rows = Array.from({
+        length: rowCount
+    }, ()=>`(${tuple})`).join(', ');
+    return {
+        sql: `${head}${rows}${tail}`,
+        width
+    };
+}

package/dist/esm/store/batch.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/batch.ts"],"sourcesContent":["// Rewrites a single-row parameterized INSERT into one multi-row statement. Engine-neutral string\n// work, shared by duckdb (batch.ts's rewriteBatch) and turso (turso/connection.ts's runBatch).\n\nconst INSERT_RE = /^(INSERT\\s+(?:OR\\s+\\w+\\s+)?INTO\\s+\\S+\\s*\\([^)]*\\)\\s*VALUES\\s*)\\(([^()]*)\\)(.*)$/is;\n\nfunction placeholderCount(tuple: string): number {\n  return (tuple.match(/\\?/g) ?? []).length;\n}\n\nexport function rewriteInsert(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = INSERT_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, head, tuple, tail] = m;\n  const width = placeholderCount(tuple);\n  if (width === 0) return null;\n  const rows = Array.from({ length: rowCount }, () => `(${tuple})`).join(', ');\n  return { sql: `${head}${rows}${tail}`, width };\n}\n"],"names":["INSERT_RE","placeholderCount","tuple","match","length","rewriteInsert","sql","rowCount","m","exec","trim","head","tail","width","rows","Array","from","join"],"mappings":"AAAA,iGAAiG;AACjG,+FAA+F;AAE/F,MAAMA,YAAY;AAElB,SAASC,iBAAiBC,KAAa;QAC7BA;IAAR,OAAO,EAACA,eAAAA,MAAMC,KAAK,CAAC,oBAAZD,0BAAAA,eAAsB,EAAE,EAAEE,MAAM;AAC1C;AAEA,OAAO,SAASC,cAAcC,GAAW,EAAEC,QAAgB;IACzD,MAAMC,IAAIR,UAAUS,IAAI,CAACH,IAAII,IAAI;IACjC,IAAI,CAACF,GAAG,OAAO;IACf,MAAM,GAAGG,MAAMT,OAAOU,KAAK,GAAGJ;IAC9B,MAAMK,QAAQZ,iBAAiBC;IAC/B,IAAIW,UAAU,GAAG,OAAO;IACxB,MAAMC,OAAOC,MAAMC,IAAI,CAAC;QAAEZ,QAAQG;IAAS,GAAG,IAAM,CAAC,CAAC,EAAEL,MAAM,CAAC,CAAC,EAAEe,IAAI,CAAC;IACvE,OAAO;QAAEX,KAAK,GAAGK,OAAOG,OAAOF,MAAM;QAAEC;IAAM;AAC/C"}

package/dist/esm/store/duckdb/batch.d.ts

@@ -1,7 +1,3 @@
-export declare function rewriteInsert(sql: string, rowCount: number): {
-    sql: string;
-    width: number;
-} | null;
 export declare function rewriteUpdate(sql: string, rowCount: number): {
     sql: string;
     width: number;

package/dist/esm/store/duckdb/batch.js

@@ -1,12 +1,9 @@
 // Rewrites the store's single-row parameterized INSERT/UPDATE/DELETE into one multi-row statement --
 // DuckDB's columnar engine pays for row-at-a-time writes, not just the round trip (PRINCIPLES: documented-means-tested).
-const INSERT_RE = /^(INSERT\s+(?:OR\s+\w+\s+)?INTO\s+\S+\s*\([^)]*\)\s*VALUES\s*)\(([^()]*)\)(.*)$/is;
+// rewriteInsert itself lives in the shared ../batch.ts: turso folds INSERTs the same way (3.36 B).
+import { rewriteInsert } from '../batch.js';
 const UPDATE_RE = /^UPDATE\s+(\S+)\s+SET\s+(.+?)\s+WHERE\s+(.+)$/is;
 const DELETE_RE = /^DELETE\s+FROM\s+(\S+)\s+WHERE\s+(.+)$/is;
-function placeholderCount(tuple) {
-    var _tuple_match;
-    return ((_tuple_match = tuple.match(/\?/g)) !== null && _tuple_match !== void 0 ? _tuple_match : []).length;
-}
 // "a = ?, b = ?" or "a = ? AND b = ?" -> ["a", "b"]. A clause that doesn't fit yields a shorter
 // list, which rewriteUpdate/rewriteDelete reject as a width mismatch.
 function equalityColumns(clause, sep) {
@@ -27,20 +24,6 @@ function tuples(rowCount, width) {
         length: rowCount
     }, ()=>row).join(', ');
 }
-export function rewriteInsert(sql, rowCount) {
-    const m = INSERT_RE.exec(sql.trim());
-    if (!m) return null;
-    const [, head, tuple, tail] = m;
-    const width = placeholderCount(tuple);
-    if (width === 0) return null;
-    const rows = Array.from({
-        length: rowCount
-    }, ()=>`(${tuple})`).join(', ');
-    return {
-        sql: `${head}${rows}${tail}`,
-        width
-    };
-}
 export function rewriteUpdate(sql, rowCount) {
     const m = UPDATE_RE.exec(sql.trim());
     if (!m) return null;

package/dist/esm/store/duckdb/batch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/batch.ts"],"sourcesContent":["// Rewrites the store's single-row parameterized INSERT/UPDATE/DELETE into one multi-row statement --\n// DuckDB's columnar engine pays for row-at-a-time writes, not just the round trip (PRINCIPLES: documented-means-tested).\n\nconst INSERT_RE = /^(INSERT\\s+(?:OR\\s+\\w+\\s+)?INTO\\s+\\S+\\s*\\([^)]*\\)\\s*VALUES\\s*)\\(([^()]*)\\)(.*)$/is;\nconst UPDATE_RE = /^UPDATE\\s+(\\S+)\\s+SET\\s+(.+?)\\s+WHERE\\s+(.+)$/is;\nconst DELETE_RE = /^DELETE\\s+FROM\\s+(\\S+)\\s+WHERE\\s+(.+)$/is;\n\nfunction placeholderCount(tuple: string): number {\n  return (tuple.match(/\\?/g) ?? []).length;\n}\n\n// \"a = ?, b = ?\" or \"a = ? AND b = ?\" -> [\"a\", \"b\"]. A clause that doesn't fit yields a shorter\n// list, which rewriteUpdate/rewriteDelete reject as a width mismatch.\nfunction equalityColumns(clause: string, sep: RegExp): string[] | null {\n  const parts = clause.split(sep).map((p) => p.trim());\n  const cols: string[] = [];\n  for (const part of parts) {\n    const m = /^(.+?)=\\s*\\?$/.exec(part);\n    if (!m) return null;\n    cols.push(m[1].trim());\n  }\n  return cols;\n}\n\nfunction tuples(rowCount: number, width: number): string {\n  const row = `(${Array.from({ length: width }, () => '?').join(', ')})`;\n  return Array.from({ length: rowCount }, () => row).join(', ');\n}\n\nexport function rewriteInsert(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = INSERT_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, head, tuple, tail] = m;\n  const width = placeholderCount(tuple);\n  if (width === 0) return null;\n  const rows = Array.from({ length: rowCount }, () => `(${tuple})`).join(', ');\n  return { sql: `${head}${rows}${tail}`, width };\n}\n\nexport function rewriteUpdate(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = UPDATE_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, table, setClause, whereClause] = m;\n  const setCols = equalityColumns(setClause, /,/);\n  const whereCols = equalityColumns(whereClause, /\\bAND\\b/i);\n  if (!setCols || !whereCols) return null;\n  const dataCols = [...setCols, ...whereCols];\n  const setAssign = setCols.map((c) => `${c} = data.${c}`).join(', ');\n  const joinCond = whereCols.map((c) => `${table}.${c} = data.${c}`).join(' AND ');\n  return { sql: `UPDATE ${table} SET ${setAssign} FROM (VALUES ${tuples(rowCount, dataCols.length)}) AS data(${dataCols.join(', ')}) WHERE ${joinCond}`, width: dataCols.length };\n}\n\nexport function rewriteDelete(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = DELETE_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, table, whereClause] = m;\n  const cols = equalityColumns(whereClause, /\\bAND\\b/i);\n  if (!cols) return null;\n  if (cols.length === 1) {\n    return { sql: `DELETE FROM ${table} WHERE ${cols[0]} IN (${Array.from({ length: rowCount }, () => '?').join(', ')})`, width: 1 };\n  }\n  return { sql: `DELETE FROM ${table} WHERE (${cols.join(', ')}) IN (VALUES ${tuples(rowCount, cols.length)})`, width: cols.length };\n}\n\n// Tries each shape in turn; null means the caller should fall back to a per-row loop.\nexport function rewriteBatch(sql: string, rowCount: number): { sql: string; width: number } | null {\n  return rewriteInsert(sql, rowCount) ?? rewriteUpdate(sql, rowCount) ?? rewriteDelete(sql, rowCount);\n}\n"],"names":["INSERT_RE","UPDATE_RE","DELETE_RE","placeholderCount","tuple","match","length","equalityColumns","clause","sep","parts","split","map","p","trim","cols","part","m","exec","push","tuples","rowCount","width","row","Array","from","join","rewriteInsert","sql","head","tail","rows","rewriteUpdate","table","setClause","whereClause","setCols","whereCols","dataCols","setAssign","c","joinCond","rewriteDelete","rewriteBatch"],"mappings":"AAAA,qGAAqG;AACrG,yHAAyH;AAEzH,MAAMA,YAAY;AAClB,MAAMC,YAAY;AAClB,MAAMC,YAAY;AAElB,SAASC,iBAAiBC,KAAa;QAC7BA;IAAR,OAAO,EAACA,eAAAA,MAAMC,KAAK,CAAC,oBAAZD,0BAAAA,eAAsB,EAAE,EAAEE,MAAM;AAC1C;AAEA,gGAAgG;AAChG,sEAAsE;AACtE,SAASC,gBAAgBC,MAAc,EAAEC,GAAW;IAClD,MAAMC,QAAQF,OAAOG,KAAK,CAACF,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACjD,MAAMC,OAAiB,EAAE;IACzB,KAAK,MAAMC,QAAQN,MAAO;QACxB,MAAMO,IAAI,gBAAgBC,IAAI,CAACF;QAC/B,IAAI,CAACC,GAAG,OAAO;QACfF,KAAKI,IAAI,CAACF,CAAC,CAAC,EAAE,CAACH,IAAI;IACrB;IACA,OAAOC;AACT;AAEA,SAASK,OAAOC,QAAgB,EAAEC,KAAa;IAC7C,MAAMC,MAAM,CAAC,CAAC,EAAEC,MAAMC,IAAI,CAAC;QAAEnB,QAAQgB;IAAM,GAAG,IAAM,KAAKI,IAAI,CAAC,MAAM,CAAC,CAAC;IACtE,OAAOF,MAAMC,IAAI,CAAC;QAAEnB,QAAQe;IAAS,GAAG,IAAME,KAAKG,IAAI,CAAC;AAC1D;AAEA,OAAO,SAASC,cAAcC,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIjB,UAAUkB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGY,MAAMzB,OAAO0B,KAAK,GAAGb;IAC9B,MAAMK,QAAQnB,iBAAiBC;IAC/B,IAAIkB,UAAU,GAAG,OAAO;IACxB,MAAMS,OAAOP,MAAMC,IAAI,CAAC;QAAEnB,QAAQe;IAAS,GAAG,IAAM,CAAC,CAAC,EAAEjB,MAAM,CAAC,CAAC,EAAEsB,IAAI,CAAC;IACvE,OAAO;QAAEE,KAAK,GAAGC,OAAOE,OAAOD,MAAM;QAAER;IAAM;AAC/C;AAEA,OAAO,SAASU,cAAcJ,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIhB,UAAUiB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGgB,OAAOC,WAAWC,YAAY,GAAGlB;IAC1C,MAAMmB,UAAU7B,gBAAgB2B,WAAW;IAC3C,MAAMG,YAAY9B,gBAAgB4B,aAAa;IAC/C,IAAI,CAACC,WAAW,CAACC,WAAW,OAAO;IACnC,MAAMC,WAAW;WAAIF;WAAYC;KAAU;IAC3C,MAAME,YAAYH,QAAQxB,GAAG,CAAC,CAAC4B,IAAM,GAAGA,EAAE,QAAQ,EAAEA,GAAG,EAAEd,IAAI,CAAC;IAC9D,MAAMe,WAAWJ,UAAUzB,GAAG,CAAC,CAAC4B,IAAM,GAAGP,MAAM,CAAC,EAAEO,EAAE,QAAQ,EAAEA,GAAG,EAAEd,IAAI,CAAC;IACxE,OAAO;QAAEE,KAAK,CAAC,OAAO,EAAEK,MAAM,KAAK,EAAEM,UAAU,cAAc,EAAEnB,OAAOC,UAAUiB,SAAShC,MAAM,EAAE,UAAU,EAAEgC,SAASZ,IAAI,CAAC,MAAM,QAAQ,EAAEe,UAAU;QAAEnB,OAAOgB,SAAShC,MAAM;IAAC;AAChL;AAEA,OAAO,SAASoC,cAAcd,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIf,UAAUgB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGgB,OAAOE,YAAY,GAAGlB;IAC/B,MAAMF,OAAOR,gBAAgB4B,aAAa;IAC1C,IAAI,CAACpB,MAAM,OAAO;IAClB,IAAIA,KAAKT,MAAM,KAAK,GAAG;QACrB,OAAO;YAAEsB,KAAK,CAAC,YAAY,EAAEK,MAAM,OAAO,EAAElB,IAAI,CAAC,EAAE,CAAC,KAAK,EAAES,MAAMC,IAAI,CAAC;gBAAEnB,QAAQe;YAAS,GAAG,IAAM,KAAKK,IAAI,CAAC,MAAM,CAAC,CAAC;YAAEJ,OAAO;QAAE;IACjI;IACA,OAAO;QAAEM,KAAK,CAAC,YAAY,EAAEK,MAAM,QAAQ,EAAElB,KAAKW,IAAI,CAAC,MAAM,aAAa,EAAEN,OAAOC,UAAUN,KAAKT,MAAM,EAAE,CAAC,CAAC;QAAEgB,OAAOP,KAAKT,MAAM;IAAC;AACnI;AAEA,sFAAsF;AACtF,OAAO,SAASqC,aAAaf,GAAW,EAAEP,QAAgB;QACjDM,MAAAA;IAAP,QAAOA,QAAAA,iBAAAA,cAAcC,KAAKP,uBAAnBM,4BAAAA,iBAAgCK,cAAcJ,KAAKP,uBAAnDM,kBAAAA,OAAgEe,cAAcd,KAAKP;AAC5F"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/batch.ts"],"sourcesContent":["// Rewrites the store's single-row parameterized INSERT/UPDATE/DELETE into one multi-row statement --\n// DuckDB's columnar engine pays for row-at-a-time writes, not just the round trip (PRINCIPLES: documented-means-tested).\n// rewriteInsert itself lives in the shared ../batch.ts: turso folds INSERTs the same way (3.36 B).\n\nimport { rewriteInsert } from '../batch.ts';\n\nconst UPDATE_RE = /^UPDATE\\s+(\\S+)\\s+SET\\s+(.+?)\\s+WHERE\\s+(.+)$/is;\nconst DELETE_RE = /^DELETE\\s+FROM\\s+(\\S+)\\s+WHERE\\s+(.+)$/is;\n\n// \"a = ?, b = ?\" or \"a = ? AND b = ?\" -> [\"a\", \"b\"]. A clause that doesn't fit yields a shorter\n// list, which rewriteUpdate/rewriteDelete reject as a width mismatch.\nfunction equalityColumns(clause: string, sep: RegExp): string[] | null {\n  const parts = clause.split(sep).map((p) => p.trim());\n  const cols: string[] = [];\n  for (const part of parts) {\n    const m = /^(.+?)=\\s*\\?$/.exec(part);\n    if (!m) return null;\n    cols.push(m[1].trim());\n  }\n  return cols;\n}\n\nfunction tuples(rowCount: number, width: number): string {\n  const row = `(${Array.from({ length: width }, () => '?').join(', ')})`;\n  return Array.from({ length: rowCount }, () => row).join(', ');\n}\n\nexport function rewriteUpdate(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = UPDATE_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, table, setClause, whereClause] = m;\n  const setCols = equalityColumns(setClause, /,/);\n  const whereCols = equalityColumns(whereClause, /\\bAND\\b/i);\n  if (!setCols || !whereCols) return null;\n  const dataCols = [...setCols, ...whereCols];\n  const setAssign = setCols.map((c) => `${c} = data.${c}`).join(', ');\n  const joinCond = whereCols.map((c) => `${table}.${c} = data.${c}`).join(' AND ');\n  return { sql: `UPDATE ${table} SET ${setAssign} FROM (VALUES ${tuples(rowCount, dataCols.length)}) AS data(${dataCols.join(', ')}) WHERE ${joinCond}`, width: dataCols.length };\n}\n\nexport function rewriteDelete(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = DELETE_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, table, whereClause] = m;\n  const cols = equalityColumns(whereClause, /\\bAND\\b/i);\n  if (!cols) return null;\n  if (cols.length === 1) {\n    return { sql: `DELETE FROM ${table} WHERE ${cols[0]} IN (${Array.from({ length: rowCount }, () => '?').join(', ')})`, width: 1 };\n  }\n  return { sql: `DELETE FROM ${table} WHERE (${cols.join(', ')}) IN (VALUES ${tuples(rowCount, cols.length)})`, width: cols.length };\n}\n\n// Tries each shape in turn; null means the caller should fall back to a per-row loop.\nexport function rewriteBatch(sql: string, rowCount: number): { sql: string; width: number } | null {\n  return rewriteInsert(sql, rowCount) ?? rewriteUpdate(sql, rowCount) ?? rewriteDelete(sql, rowCount);\n}\n"],"names":["rewriteInsert","UPDATE_RE","DELETE_RE","equalityColumns","clause","sep","parts","split","map","p","trim","cols","part","m","exec","push","tuples","rowCount","width","row","Array","from","length","join","rewriteUpdate","sql","table","setClause","whereClause","setCols","whereCols","dataCols","setAssign","c","joinCond","rewriteDelete","rewriteBatch"],"mappings":"AAAA,qGAAqG;AACrG,yHAAyH;AACzH,mGAAmG;AAEnG,SAASA,aAAa,QAAQ,cAAc;AAE5C,MAAMC,YAAY;AAClB,MAAMC,YAAY;AAElB,gGAAgG;AAChG,sEAAsE;AACtE,SAASC,gBAAgBC,MAAc,EAAEC,GAAW;IAClD,MAAMC,QAAQF,OAAOG,KAAK,CAACF,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACjD,MAAMC,OAAiB,EAAE;IACzB,KAAK,MAAMC,QAAQN,MAAO;QACxB,MAAMO,IAAI,gBAAgBC,IAAI,CAACF;QAC/B,IAAI,CAACC,GAAG,OAAO;QACfF,KAAKI,IAAI,CAACF,CAAC,CAAC,EAAE,CAACH,IAAI;IACrB;IACA,OAAOC;AACT;AAEA,SAASK,OAAOC,QAAgB,EAAEC,KAAa;IAC7C,MAAMC,MAAM,CAAC,CAAC,EAAEC,MAAMC,IAAI,CAAC;QAAEC,QAAQJ;IAAM,GAAG,IAAM,KAAKK,IAAI,CAAC,MAAM,CAAC,CAAC;IACtE,OAAOH,MAAMC,IAAI,CAAC;QAAEC,QAAQL;IAAS,GAAG,IAAME,KAAKI,IAAI,CAAC;AAC1D;AAEA,OAAO,SAASC,cAAcC,GAAW,EAAER,QAAgB;IACzD,MAAMJ,IAAIZ,UAAUa,IAAI,CAACW,IAAIf,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGa,OAAOC,WAAWC,YAAY,GAAGf;IAC1C,MAAMgB,UAAU1B,gBAAgBwB,WAAW;IAC3C,MAAMG,YAAY3B,gBAAgByB,aAAa;IAC/C,IAAI,CAACC,WAAW,CAACC,WAAW,OAAO;IACnC,MAAMC,WAAW;WAAIF;WAAYC;KAAU;IAC3C,MAAME,YAAYH,QAAQrB,GAAG,CAAC,CAACyB,IAAM,GAAGA,EAAE,QAAQ,EAAEA,GAAG,EAAEV,IAAI,CAAC;IAC9D,MAAMW,WAAWJ,UAAUtB,GAAG,CAAC,CAACyB,IAAM,GAAGP,MAAM,CAAC,EAAEO,EAAE,QAAQ,EAAEA,GAAG,EAAEV,IAAI,CAAC;IACxE,OAAO;QAAEE,KAAK,CAAC,OAAO,EAAEC,MAAM,KAAK,EAAEM,UAAU,cAAc,EAAEhB,OAAOC,UAAUc,SAAST,MAAM,EAAE,UAAU,EAAES,SAASR,IAAI,CAAC,MAAM,QAAQ,EAAEW,UAAU;QAAEhB,OAAOa,SAAST,MAAM;IAAC;AAChL;AAEA,OAAO,SAASa,cAAcV,GAAW,EAAER,QAAgB;IACzD,MAAMJ,IAAIX,UAAUY,IAAI,CAACW,IAAIf,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGa,OAAOE,YAAY,GAAGf;IAC/B,MAAMF,OAAOR,gBAAgByB,aAAa;IAC1C,IAAI,CAACjB,MAAM,OAAO;IAClB,IAAIA,KAAKW,MAAM,KAAK,GAAG;QACrB,OAAO;YAAEG,KAAK,CAAC,YAAY,EAAEC,MAAM,OAAO,EAAEf,IAAI,CAAC,EAAE,CAAC,KAAK,EAAES,MAAMC,IAAI,CAAC;gBAAEC,QAAQL;YAAS,GAAG,IAAM,KAAKM,IAAI,CAAC,MAAM,CAAC,CAAC;YAAEL,OAAO;QAAE;IACjI;IACA,OAAO;QAAEO,KAAK,CAAC,YAAY,EAAEC,MAAM,QAAQ,EAAEf,KAAKY,IAAI,CAAC,MAAM,aAAa,EAAEP,OAAOC,UAAUN,KAAKW,MAAM,EAAE,CAAC,CAAC;QAAEJ,OAAOP,KAAKW,MAAM;IAAC;AACnI;AAEA,sFAAsF;AACtF,OAAO,SAASc,aAAaX,GAAW,EAAER,QAAgB;QACjDjB,MAAAA;IAAP,QAAOA,QAAAA,iBAAAA,cAAcyB,KAAKR,uBAAnBjB,4BAAAA,iBAAgCwB,cAAcC,KAAKR,uBAAnDjB,kBAAAA,OAAgEmC,cAAcV,KAAKR;AAC5F"}

package/dist/esm/store/duckdb/open.d.ts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.duckdb";
-export declare const SCHEMA_VERSION = "3";
+export declare const SCHEMA_VERSION = "4";
 export type { OpenResult };
 interface DuckdbHandle {
     instance: DuckDBInstance;

package/dist/esm/store/duckdb/open.js

@@ -12,7 +12,7 @@ import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.duckdb';
 // Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).
 // The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.
-export const SCHEMA_VERSION = '3';
+export const SCHEMA_VERSION = '4';
 // `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
 // No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
 async function ensureSchema(_handle, conn, cfg) {

package/dist/esm/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '3';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","feature","name","schema","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","message","create","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,8LAA8L;AAC9L,gGAAgG;AAChG,eAAeC,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;IAC9E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWlB,eAAegB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEnB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMoB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKf;AACxG;AAEA,iFAAiF;AACjF,eAAeoB,MAAMC,MAAoB;IACvCA,OAAOC,MAAM,CAACC,cAAc;IAC5BF,OAAOG,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMvB,WAAU;IACxC,EAAE,OAAOwB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACyB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIT;IACJ,IAAIE;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQ;YAAEM,+BAA+B;QAAS;QACzFX,SAAS,MAAME,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMvB,kBAAkBc;QACxB,MAAMR,OAAOV,iBAAiBkB;QAC9B,OAAO;YAAED,QAAQ;gBAAEG;gBAAUF;YAAO;YAAGR;QAAK;IAC9C,EAAE,OAAOgB,KAAK;QACZ,MAAMV,MAAM;YAAEI;YAAUF;QAAO;QAC/B,MAAMQ;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMI,oBAA+C;IAC1DC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB9B;IAClBmB;IACAN;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFkB,UAAU,CAACR,MAAQ,2DAA2DS,IAAI,CAACT,IAAIC,OAAO;IAC9FnB;IACAH,aAAa,CAACY,QAAQP,OAASL,YAAYY,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAER;AAC7E,EAAE;AAEF,OAAO,eAAe0B,WAAWzB,GAAmB;IAClD,OAAOd,gBAAgBc,KAAKmB;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","feature","name","schema","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","message","create","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,8LAA8L;AAC9L,gGAAgG;AAChG,eAAeC,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;IAC9E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWlB,eAAegB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEnB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMoB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKf;AACxG;AAEA,iFAAiF;AACjF,eAAeoB,MAAMC,MAAoB;IACvCA,OAAOC,MAAM,CAACC,cAAc;IAC5BF,OAAOG,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMvB,WAAU;IACxC,EAAE,OAAOwB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACyB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIT;IACJ,IAAIE;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQ;YAAEM,+BAA+B;QAAS;QACzFX,SAAS,MAAME,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMvB,kBAAkBc;QACxB,MAAMR,OAAOV,iBAAiBkB;QAC9B,OAAO;YAAED,QAAQ;gBAAEG;gBAAUF;YAAO;YAAGR;QAAK;IAC9C,EAAE,OAAOgB,KAAK;QACZ,MAAMV,MAAM;YAAEI;YAAUF;QAAO;QAC/B,MAAMQ;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMI,oBAA+C;IAC1DC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB9B;IAClBmB;IACAN;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFkB,UAAU,CAACR,MAAQ,2DAA2DS,IAAI,CAACT,IAAIC,OAAO;IAC9FnB;IACAH,aAAa,CAACY,QAAQP,OAASL,YAAYY,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAER;AAC7E,EAAE;AAEF,OAAO,eAAe0B,WAAWzB,GAAmB;IAClD,OAAOd,gBAAgBc,KAAKmB;AAC9B"}

package/dist/esm/store/sqlite/open.d.ts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.db";
-export declare const SCHEMA_VERSION = "19";
+export declare const SCHEMA_VERSION = "20";
 export type { OpenResult };
 interface SqliteHandle {
     db: DatabaseSync;

package/dist/esm/store/sqlite/open.js

@@ -13,7 +13,7 @@ import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.db';
 // Cache shape version, independent of the config's own `version`. Bumping it rebuilds
 // existing trees on first query.
-export const SCHEMA_VERSION = '19';
+export const SCHEMA_VERSION = '20';
 // unicode61 splits on spaces, so a language written without them indexes a whole run as one
 // token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.
 const TOKENIZE = 'porter unicode61';