sensemaking 0.19.1 → 0.20.0

package/dist/cjs/chunk/extract.d.cts

@@ -1,2 +1,3 @@
 import type { Nodes } from 'mdast';
 export declare function extractText(node: Nodes): string;
+export declare function extractTexts(nodes: Nodes[]): string;

package/dist/cjs/chunk/extract.d.ts

@@ -1,2 +1,3 @@
 import type { Nodes } from 'mdast';
 export declare function extractText(node: Nodes): string;
+export declare function extractTexts(nodes: Nodes[]): string;

package/dist/cjs/chunk/extract.js

@@ -2,10 +2,18 @@
 Object.defineProperty(exports, "__esModule", {
     value: true
 });
-Object.defineProperty(exports, "extractText", {
-    enumerable: true,
-    get: function() {
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get extractText () {
         return extractText;
+    },
+    get extractTexts () {
+        return extractTexts;
     }
 });
 // mdast has no wikilink, embed, or %%comment%% node -- literal text resolved by regex passes
@@ -46,7 +54,7 @@ function embedText(inner) {
     return anchorText(pipeIdx === -1 ? inner : inner.slice(0, pipeIdx));
 }
 function resolveFlavor(text) {
-    return text.replace(COMMENT_PAIR, '').replace(EMBED, function(_, inner) {
+    return text.replace(HTML_COMMENT, '').replace(COMMENT_PAIR, '').replace(EMBED, function(_, inner) {
         return embedText(inner);
     }).replace(WIKILINK, function(_, inner) {
         return wikilinkText(inner);
@@ -130,8 +138,18 @@ function extractNode(node, code) {
 }
 var CODE_PLACEHOLDER = /\uE000(\d+)\uE000/g;
 function extractText(node) {
+    return extractTexts([
+        node
+    ]);
+}
+function extractTexts(nodes) {
     var code = [];
-    var resolved = resolveFlavor(extractNode(node, code));
+    var joined = nodes.map(function(n) {
+        return extractNode(n, code);
+    }).filter(function(s) {
+        return s.length > 0;
+    }).join('\n');
+    var resolved = resolveFlavor(joined);
     return code.length === 0 ? resolved : resolved.replace(CODE_PLACEHOLDER, function(_, i) {
         return code[Number(i)];
     });

package/dist/cjs/chunk/extract.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/extract.ts"],"sourcesContent":["import type { Nodes } from 'mdast';\n\n// mdast has no wikilink, embed, or %%comment%% node -- literal text resolved by regex passes\n// over non-code text (code/inlineCode is placeholder-held and spliced back verbatim, below).\nconst COMMENT_PAIR = /%%[\\s\\S]*?%%/g;\n// An embed always yields its target, whatever follows a pipe (a resize suffix, never an alias);\n// must run before the wikilink regex below, bang included, or that regex matches first.\nconst EMBED = /!\\[\\[([^\\]]+)\\]\\]/g;\nconst WIKILINK = /\\[\\[([^\\]]+)\\]\\]/g;\n// A trailing \" ^id\" is an Obsidian block anchor, valid only at a line's end; a caret elsewhere\n// (mid-line) is ordinary text and is left alone.\nconst BLOCK_ID = / \\^[A-Za-z0-9-]+$/gm;\n// A callout marker is only ever the first thing in a blockquote's text (Obsidian's grammar),\n// so this anchors to blockquote output alone -- never applied to prose in general.\nconst CALLOUT_MARKER = /^\\[!\\w[\\w-]*\\][+-]?[ \\t]?/;\n// An html node's value is raw HTML, block or inline: a full <!-- --> comment is dropped, any\n// remaining tags are stripped, and the text a browser would still render survives.\nconst HTML_COMMENT = /<!--[\\s\\S]*?-->/g;\nconst HTML_TAG = /<[^>]+>/g;\n\nfunction stripHtml(value: string): string {\n  return value.replace(HTML_COMMENT, '').replace(HTML_TAG, '');\n}\n\n// A `#anchor` keeps its text (hash dropped), unlike links.ts's parseWikilinkInner (the resolver\n// authority), which discards it entirely -- replicated here since that return value doesn't fit.\nfunction anchorText(base: string): string {\n  const hashIdx = base.indexOf('#');\n  if (hashIdx === -1) return base.trim();\n  const target = base.slice(0, hashIdx).trim();\n  const anchor = base.slice(hashIdx + 1).trim();\n  return target ? `${target} ${anchor}` : anchor;\n}\n\nfunction wikilinkText(inner: string): string {\n  const pipeIdx = inner.indexOf('|');\n  return pipeIdx === -1 ? anchorText(inner) : inner.slice(pipeIdx + 1).trim();\n}\n\nfunction embedText(inner: string): string {\n  const pipeIdx = inner.indexOf('|');\n  return anchorText(pipeIdx === -1 ? inner : inner.slice(0, pipeIdx));\n}\n\nfunction resolveFlavor(text: string): string {\n  return text\n    .replace(COMMENT_PAIR, '')\n    .replace(EMBED, (_, inner: string) => embedText(inner))\n    .replace(WIKILINK, (_, inner: string) => wikilinkText(inner))\n    .replace(BLOCK_ID, '');\n}\n\n// Sibling blocks (list items, blockquote content, footnote bodies) joined one per line;\n// empties dropped so a blank child never widens the gap between its neighbors.\nfunction joinBlocks(nodes: Nodes[], code: string[]): string {\n  return nodes\n    .map((n) => extractNode(n, code))\n    .filter((s) => s.length > 0)\n    .join('\\n');\n}\n\n// Inline content (headings, paragraphs, emphasis, links) concatenated with no separator: the\n// source text nodes already carry whatever spacing separates them.\nfunction joinInline(nodes: Nodes[], code: string[]): string {\n  return nodes.map((n) => extractNode(n, code)).join('');\n}\n\n// U+E000 (Private Use Area) never appears in real markdown text, so it is a collision-free\n// placeholder delimiter, spliced back to the literal code value after flavor resolution.\nfunction codePlaceholder(value: string, code: string[]): string {\n  const idx = code.push(value) - 1;\n  return `\\uE000${idx}\\uE000`;\n}\n\n// A GFM autolink's display text is its own target, missing only the scheme\n// mdast-util-gfm-autolink-literal fills into node.url (http:// for www., mailto: for an email).\nconst SCHEME_PREFIX = /^[a-z][a-z0-9+.-]*:\\/{0,2}/i;\nfunction isAutolink(url: string, text: string): boolean {\n  return url === text || url.replace(SCHEME_PREFIX, '') === text;\n}\n\nfunction extractNode(node: Nodes, code: string[]): string {\n  switch (node.type) {\n    case 'root':\n    case 'list':\n    case 'listItem':\n    case 'footnoteDefinition':\n      return joinBlocks(node.children, code);\n    case 'blockquote':\n      return joinBlocks(node.children, code).replace(CALLOUT_MARKER, '');\n    case 'link': {\n      const text = joinInline(node.children, code);\n      return isAutolink(node.url, text) ? '' : text;\n    }\n    case 'heading':\n    case 'paragraph':\n    case 'linkReference':\n    case 'emphasis':\n    case 'strong':\n    case 'delete':\n      return joinInline(node.children, code);\n    case 'table':\n      return node.children.map((n) => extractNode(n, code)).join('\\n');\n    case 'tableRow':\n      return node.children.map((n) => extractNode(n, code)).join(' ');\n    case 'tableCell':\n      return joinInline(node.children, code);\n    case 'image':\n    case 'imageReference':\n      return node.alt ?? '';\n    case 'text':\n      return node.value;\n    case 'inlineCode':\n    case 'code':\n      return codePlaceholder(node.value, code);\n    case 'html':\n      return stripHtml(node.value);\n    case 'break':\n      return ' ';\n    default:\n      return '';\n  }\n}\n\nconst CODE_PLACEHOLDER = /\\uE000(\\d+)\\uE000/g;\n\n// Plain text of one mdast node (or a whole tree): heading/list/table structure is kept as text,\n// markup (emphasis, link targets, task and callout markers) is dropped. Pure, synchronous.\nexport function extractText(node: Nodes): string {\n  const code: string[] = [];\n  const resolved = resolveFlavor(extractNode(node, code));\n  return code.length === 0 ? resolved : resolved.replace(CODE_PLACEHOLDER, (_, i: string) => code[Number(i)]);\n}\n"],"names":["extractText","COMMENT_PAIR","EMBED","WIKILINK","BLOCK_ID","CALLOUT_MARKER","HTML_COMMENT","HTML_TAG","stripHtml","value","replace","anchorText","base","hashIdx","indexOf","trim","target","slice","anchor","wikilinkText","inner","pipeIdx","embedText","resolveFlavor","text","_","joinBlocks","nodes","code","map","n","extractNode","filter","s","length","join","joinInline","codePlaceholder","idx","push","SCHEME_PREFIX","isAutolink","url","node","type","children","alt","CODE_PLACEHOLDER","resolved","i","Number"],"mappings":";;;;+BAgIgBA;;;eAAAA;;;AA9HhB,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,eAAe;AACrB,gGAAgG;AAChG,wFAAwF;AACxF,IAAMC,QAAQ;AACd,IAAMC,WAAW;AACjB,+FAA+F;AAC/F,iDAAiD;AACjD,IAAMC,WAAW;AACjB,6FAA6F;AAC7F,mFAAmF;AACnF,IAAMC,iBAAiB;AACvB,6FAA6F;AAC7F,mFAAmF;AACnF,IAAMC,eAAe;AACrB,IAAMC,WAAW;AAEjB,SAASC,UAAUC,KAAa;IAC9B,OAAOA,MAAMC,OAAO,CAACJ,cAAc,IAAII,OAAO,CAACH,UAAU;AAC3D;AAEA,gGAAgG;AAChG,iGAAiG;AACjG,SAASI,WAAWC,IAAY;IAC9B,IAAMC,UAAUD,KAAKE,OAAO,CAAC;IAC7B,IAAID,YAAY,CAAC,GAAG,OAAOD,KAAKG,IAAI;IACpC,IAAMC,SAASJ,KAAKK,KAAK,CAAC,GAAGJ,SAASE,IAAI;IAC1C,IAAMG,SAASN,KAAKK,KAAK,CAACJ,UAAU,GAAGE,IAAI;IAC3C,OAAOC,SAAS,AAAC,GAAYE,OAAVF,QAAO,KAAU,OAAPE,UAAWA;AAC1C;AAEA,SAASC,aAAaC,KAAa;IACjC,IAAMC,UAAUD,MAAMN,OAAO,CAAC;IAC9B,OAAOO,YAAY,CAAC,IAAIV,WAAWS,SAASA,MAAMH,KAAK,CAACI,UAAU,GAAGN,IAAI;AAC3E;AAEA,SAASO,UAAUF,KAAa;IAC9B,IAAMC,UAAUD,MAAMN,OAAO,CAAC;IAC9B,OAAOH,WAAWU,YAAY,CAAC,IAAID,QAAQA,MAAMH,KAAK,CAAC,GAAGI;AAC5D;AAEA,SAASE,cAAcC,IAAY;IACjC,OAAOA,KACJd,OAAO,CAACT,cAAc,IACtBS,OAAO,CAACR,OAAO,SAACuB,GAAGL;eAAkBE,UAAUF;OAC/CV,OAAO,CAACP,UAAU,SAACsB,GAAGL;eAAkBD,aAAaC;OACrDV,OAAO,CAACN,UAAU;AACvB;AAEA,wFAAwF;AACxF,+EAA+E;AAC/E,SAASsB,WAAWC,KAAc,EAAEC,IAAc;IAChD,OAAOD,MACJE,GAAG,CAAC,SAACC;eAAMC,YAAYD,GAAGF;OAC1BI,MAAM,CAAC,SAACC;eAAMA,EAAEC,MAAM,GAAG;OACzBC,IAAI,CAAC;AACV;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,SAASC,WAAWT,KAAc,EAAEC,IAAc;IAChD,OAAOD,MAAME,GAAG,CAAC,SAACC;eAAMC,YAAYD,GAAGF;OAAOO,IAAI,CAAC;AACrD;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASE,gBAAgB5B,KAAa,EAAEmB,IAAc;IACpD,IAAMU,MAAMV,KAAKW,IAAI,CAAC9B,SAAS;IAC/B,OAAO,AAAC,IAAY,OAAJ6B,KAAI;AACtB;AAEA,2EAA2E;AAC3E,gGAAgG;AAChG,IAAME,gBAAgB;AACtB,SAASC,WAAWC,GAAW,EAAElB,IAAY;IAC3C,OAAOkB,QAAQlB,QAAQkB,IAAIhC,OAAO,CAAC8B,eAAe,QAAQhB;AAC5D;AAEA,SAASO,YAAYY,IAAW,EAAEf,IAAc;IAC9C,OAAQe,KAAKC,IAAI;QACf,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAOlB,WAAWiB,KAAKE,QAAQ,EAAEjB;QACnC,KAAK;YACH,OAAOF,WAAWiB,KAAKE,QAAQ,EAAEjB,MAAMlB,OAAO,CAACL,gBAAgB;QACjE,KAAK;YAAQ;gBACX,IAAMmB,OAAOY,WAAWO,KAAKE,QAAQ,EAAEjB;gBACvC,OAAOa,WAAWE,KAAKD,GAAG,EAAElB,QAAQ,KAAKA;YAC3C;QACA,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAOY,WAAWO,KAAKE,QAAQ,EAAEjB;QACnC,KAAK;YACH,OAAOe,KAAKE,QAAQ,CAAChB,GAAG,CAAC,SAACC;uBAAMC,YAAYD,GAAGF;eAAOO,IAAI,CAAC;QAC7D,KAAK;YACH,OAAOQ,KAAKE,QAAQ,CAAChB,GAAG,CAAC,SAACC;uBAAMC,YAAYD,GAAGF;eAAOO,IAAI,CAAC;QAC7D,KAAK;YACH,OAAOC,WAAWO,KAAKE,QAAQ,EAAEjB;QACnC,KAAK;QACL,KAAK;gBACIe;YAAP,QAAOA,YAAAA,KAAKG,GAAG,cAARH,uBAAAA,YAAY;QACrB,KAAK;YACH,OAAOA,KAAKlC,KAAK;QACnB,KAAK;QACL,KAAK;YACH,OAAO4B,gBAAgBM,KAAKlC,KAAK,EAAEmB;QACrC,KAAK;YACH,OAAOpB,UAAUmC,KAAKlC,KAAK;QAC7B,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,IAAMsC,mBAAmB;AAIlB,SAAS/C,YAAY2C,IAAW;IACrC,IAAMf,OAAiB,EAAE;IACzB,IAAMoB,WAAWzB,cAAcQ,YAAYY,MAAMf;IACjD,OAAOA,KAAKM,MAAM,KAAK,IAAIc,WAAWA,SAAStC,OAAO,CAACqC,kBAAkB,SAACtB,GAAGwB;eAAcrB,IAAI,CAACsB,OAAOD,GAAG;;AAC5G"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/extract.ts"],"sourcesContent":["import type { Nodes } from 'mdast';\n\n// mdast has no wikilink, embed, or %%comment%% node -- literal text resolved by regex passes\n// over non-code text (code/inlineCode is placeholder-held and spliced back verbatim, below).\nconst COMMENT_PAIR = /%%[\\s\\S]*?%%/g;\n// An embed always yields its target, whatever follows a pipe (a resize suffix, never an alias);\n// must run before the wikilink regex below, bang included, or that regex matches first.\nconst EMBED = /!\\[\\[([^\\]]+)\\]\\]/g;\nconst WIKILINK = /\\[\\[([^\\]]+)\\]\\]/g;\n// A trailing \" ^id\" is an Obsidian block anchor, valid only at a line's end; a caret elsewhere\n// (mid-line) is ordinary text and is left alone.\nconst BLOCK_ID = / \\^[A-Za-z0-9-]+$/gm;\n// A callout marker is only ever the first thing in a blockquote's text (Obsidian's grammar),\n// so this anchors to blockquote output alone -- never applied to prose in general.\nconst CALLOUT_MARKER = /^\\[!\\w[\\w-]*\\][+-]?[ \\t]?/;\n// An html node's value is raw HTML, block or inline: a full <!-- --> comment is dropped, any\n// remaining tags are stripped, and the text a browser would still render survives.\nconst HTML_COMMENT = /<!--[\\s\\S]*?-->/g;\nconst HTML_TAG = /<[^>]+>/g;\n\nfunction stripHtml(value: string): string {\n  return value.replace(HTML_COMMENT, '').replace(HTML_TAG, '');\n}\n\n// A `#anchor` keeps its text (hash dropped), unlike links.ts's parseWikilinkInner (the resolver\n// authority), which discards it entirely -- replicated here since that return value doesn't fit.\nfunction anchorText(base: string): string {\n  const hashIdx = base.indexOf('#');\n  if (hashIdx === -1) return base.trim();\n  const target = base.slice(0, hashIdx).trim();\n  const anchor = base.slice(hashIdx + 1).trim();\n  return target ? `${target} ${anchor}` : anchor;\n}\n\nfunction wikilinkText(inner: string): string {\n  const pipeIdx = inner.indexOf('|');\n  return pipeIdx === -1 ? anchorText(inner) : inner.slice(pipeIdx + 1).trim();\n}\n\nfunction embedText(inner: string): string {\n  const pipeIdx = inner.indexOf('|');\n  return anchorText(pipeIdx === -1 ? inner : inner.slice(0, pipeIdx));\n}\n\nfunction resolveFlavor(text: string): string {\n  return text\n    .replace(HTML_COMMENT, '')\n    .replace(COMMENT_PAIR, '')\n    .replace(EMBED, (_, inner: string) => embedText(inner))\n    .replace(WIKILINK, (_, inner: string) => wikilinkText(inner))\n    .replace(BLOCK_ID, '');\n}\n\n// Sibling blocks (list items, blockquote content, footnote bodies) joined one per line;\n// empties dropped so a blank child never widens the gap between its neighbors.\nfunction joinBlocks(nodes: Nodes[], code: string[]): string {\n  return nodes\n    .map((n) => extractNode(n, code))\n    .filter((s) => s.length > 0)\n    .join('\\n');\n}\n\n// Inline content (headings, paragraphs, emphasis, links) concatenated with no separator: the\n// source text nodes already carry whatever spacing separates them.\nfunction joinInline(nodes: Nodes[], code: string[]): string {\n  return nodes.map((n) => extractNode(n, code)).join('');\n}\n\n// U+E000 (Private Use Area) never appears in real markdown text, so it is a collision-free\n// placeholder delimiter, spliced back to the literal code value after flavor resolution.\nfunction codePlaceholder(value: string, code: string[]): string {\n  const idx = code.push(value) - 1;\n  return `\\uE000${idx}\\uE000`;\n}\n\n// A GFM autolink's display text is its own target, missing only the scheme\n// mdast-util-gfm-autolink-literal fills into node.url (http:// for www., mailto: for an email).\nconst SCHEME_PREFIX = /^[a-z][a-z0-9+.-]*:\\/{0,2}/i;\nfunction isAutolink(url: string, text: string): boolean {\n  return url === text || url.replace(SCHEME_PREFIX, '') === text;\n}\n\nfunction extractNode(node: Nodes, code: string[]): string {\n  switch (node.type) {\n    case 'root':\n    case 'list':\n    case 'listItem':\n    case 'footnoteDefinition':\n      return joinBlocks(node.children, code);\n    case 'blockquote':\n      return joinBlocks(node.children, code).replace(CALLOUT_MARKER, '');\n    case 'link': {\n      const text = joinInline(node.children, code);\n      return isAutolink(node.url, text) ? '' : text;\n    }\n    case 'heading':\n    case 'paragraph':\n    case 'linkReference':\n    case 'emphasis':\n    case 'strong':\n    case 'delete':\n      return joinInline(node.children, code);\n    case 'table':\n      return node.children.map((n) => extractNode(n, code)).join('\\n');\n    case 'tableRow':\n      return node.children.map((n) => extractNode(n, code)).join(' ');\n    case 'tableCell':\n      return joinInline(node.children, code);\n    case 'image':\n    case 'imageReference':\n      return node.alt ?? '';\n    case 'text':\n      return node.value;\n    case 'inlineCode':\n    case 'code':\n      return codePlaceholder(node.value, code);\n    case 'html':\n      return stripHtml(node.value);\n    case 'break':\n      return ' ';\n    default:\n      return '';\n  }\n}\n\nconst CODE_PLACEHOLDER = /\\uE000(\\d+)\\uE000/g;\n\n// Plain text of one mdast node (or a whole tree): heading/list/table structure is kept as text,\n// markup (emphasis, link targets, task and callout markers) is dropped. Pure, synchronous.\nexport function extractText(node: Nodes): string {\n  return extractTexts([node]);\n}\n\n// Blocks resolved together: a blank line inside %%...%% splits it across blocks, and a per-block\n// strip then never sees the closing %%. Code stays placeholder-held across all of them.\nexport function extractTexts(nodes: Nodes[]): string {\n  const code: string[] = [];\n  const joined = nodes\n    .map((n) => extractNode(n, code))\n    .filter((s) => s.length > 0)\n    .join('\\n');\n  const resolved = resolveFlavor(joined);\n  return code.length === 0 ? resolved : resolved.replace(CODE_PLACEHOLDER, (_, i: string) => code[Number(i)]);\n}\n"],"names":["extractText","extractTexts","COMMENT_PAIR","EMBED","WIKILINK","BLOCK_ID","CALLOUT_MARKER","HTML_COMMENT","HTML_TAG","stripHtml","value","replace","anchorText","base","hashIdx","indexOf","trim","target","slice","anchor","wikilinkText","inner","pipeIdx","embedText","resolveFlavor","text","_","joinBlocks","nodes","code","map","n","extractNode","filter","s","length","join","joinInline","codePlaceholder","idx","push","SCHEME_PREFIX","isAutolink","url","node","type","children","alt","CODE_PLACEHOLDER","joined","resolved","i","Number"],"mappings":";;;;;;;;;;;QAiIgBA;eAAAA;;QAMAC;eAAAA;;;AArIhB,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,eAAe;AACrB,gGAAgG;AAChG,wFAAwF;AACxF,IAAMC,QAAQ;AACd,IAAMC,WAAW;AACjB,+FAA+F;AAC/F,iDAAiD;AACjD,IAAMC,WAAW;AACjB,6FAA6F;AAC7F,mFAAmF;AACnF,IAAMC,iBAAiB;AACvB,6FAA6F;AAC7F,mFAAmF;AACnF,IAAMC,eAAe;AACrB,IAAMC,WAAW;AAEjB,SAASC,UAAUC,KAAa;IAC9B,OAAOA,MAAMC,OAAO,CAACJ,cAAc,IAAII,OAAO,CAACH,UAAU;AAC3D;AAEA,gGAAgG;AAChG,iGAAiG;AACjG,SAASI,WAAWC,IAAY;IAC9B,IAAMC,UAAUD,KAAKE,OAAO,CAAC;IAC7B,IAAID,YAAY,CAAC,GAAG,OAAOD,KAAKG,IAAI;IACpC,IAAMC,SAASJ,KAAKK,KAAK,CAAC,GAAGJ,SAASE,IAAI;IAC1C,IAAMG,SAASN,KAAKK,KAAK,CAACJ,UAAU,GAAGE,IAAI;IAC3C,OAAOC,SAAS,AAAC,GAAYE,OAAVF,QAAO,KAAU,OAAPE,UAAWA;AAC1C;AAEA,SAASC,aAAaC,KAAa;IACjC,IAAMC,UAAUD,MAAMN,OAAO,CAAC;IAC9B,OAAOO,YAAY,CAAC,IAAIV,WAAWS,SAASA,MAAMH,KAAK,CAACI,UAAU,GAAGN,IAAI;AAC3E;AAEA,SAASO,UAAUF,KAAa;IAC9B,IAAMC,UAAUD,MAAMN,OAAO,CAAC;IAC9B,OAAOH,WAAWU,YAAY,CAAC,IAAID,QAAQA,MAAMH,KAAK,CAAC,GAAGI;AAC5D;AAEA,SAASE,cAAcC,IAAY;IACjC,OAAOA,KACJd,OAAO,CAACJ,cAAc,IACtBI,OAAO,CAACT,cAAc,IACtBS,OAAO,CAACR,OAAO,SAACuB,GAAGL;eAAkBE,UAAUF;OAC/CV,OAAO,CAACP,UAAU,SAACsB,GAAGL;eAAkBD,aAAaC;OACrDV,OAAO,CAACN,UAAU;AACvB;AAEA,wFAAwF;AACxF,+EAA+E;AAC/E,SAASsB,WAAWC,KAAc,EAAEC,IAAc;IAChD,OAAOD,MACJE,GAAG,CAAC,SAACC;eAAMC,YAAYD,GAAGF;OAC1BI,MAAM,CAAC,SAACC;eAAMA,EAAEC,MAAM,GAAG;OACzBC,IAAI,CAAC;AACV;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,SAASC,WAAWT,KAAc,EAAEC,IAAc;IAChD,OAAOD,MAAME,GAAG,CAAC,SAACC;eAAMC,YAAYD,GAAGF;OAAOO,IAAI,CAAC;AACrD;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASE,gBAAgB5B,KAAa,EAAEmB,IAAc;IACpD,IAAMU,MAAMV,KAAKW,IAAI,CAAC9B,SAAS;IAC/B,OAAO,AAAC,IAAY,OAAJ6B,KAAI;AACtB;AAEA,2EAA2E;AAC3E,gGAAgG;AAChG,IAAME,gBAAgB;AACtB,SAASC,WAAWC,GAAW,EAAElB,IAAY;IAC3C,OAAOkB,QAAQlB,QAAQkB,IAAIhC,OAAO,CAAC8B,eAAe,QAAQhB;AAC5D;AAEA,SAASO,YAAYY,IAAW,EAAEf,IAAc;IAC9C,OAAQe,KAAKC,IAAI;QACf,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAOlB,WAAWiB,KAAKE,QAAQ,EAAEjB;QACnC,KAAK;YACH,OAAOF,WAAWiB,KAAKE,QAAQ,EAAEjB,MAAMlB,OAAO,CAACL,gBAAgB;QACjE,KAAK;YAAQ;gBACX,IAAMmB,OAAOY,WAAWO,KAAKE,QAAQ,EAAEjB;gBACvC,OAAOa,WAAWE,KAAKD,GAAG,EAAElB,QAAQ,KAAKA;YAC3C;QACA,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAOY,WAAWO,KAAKE,QAAQ,EAAEjB;QACnC,KAAK;YACH,OAAOe,KAAKE,QAAQ,CAAChB,GAAG,CAAC,SAACC;uBAAMC,YAAYD,GAAGF;eAAOO,IAAI,CAAC;QAC7D,KAAK;YACH,OAAOQ,KAAKE,QAAQ,CAAChB,GAAG,CAAC,SAACC;uBAAMC,YAAYD,GAAGF;eAAOO,IAAI,CAAC;QAC7D,KAAK;YACH,OAAOC,WAAWO,KAAKE,QAAQ,EAAEjB;QACnC,KAAK;QACL,KAAK;gBACIe;YAAP,QAAOA,YAAAA,KAAKG,GAAG,cAARH,uBAAAA,YAAY;QACrB,KAAK;YACH,OAAOA,KAAKlC,KAAK;QACnB,KAAK;QACL,KAAK;YACH,OAAO4B,gBAAgBM,KAAKlC,KAAK,EAAEmB;QACrC,KAAK;YACH,OAAOpB,UAAUmC,KAAKlC,KAAK;QAC7B,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,IAAMsC,mBAAmB;AAIlB,SAAShD,YAAY4C,IAAW;IACrC,OAAO3C,aAAa;QAAC2C;KAAK;AAC5B;AAIO,SAAS3C,aAAa2B,KAAc;IACzC,IAAMC,OAAiB,EAAE;IACzB,IAAMoB,SAASrB,MACZE,GAAG,CAAC,SAACC;eAAMC,YAAYD,GAAGF;OAC1BI,MAAM,CAAC,SAACC;eAAMA,EAAEC,MAAM,GAAG;OACzBC,IAAI,CAAC;IACR,IAAMc,WAAW1B,cAAcyB;IAC/B,OAAOpB,KAAKM,MAAM,KAAK,IAAIe,WAAWA,SAASvC,OAAO,CAACqC,kBAAkB,SAACtB,GAAGyB;eAActB,IAAI,CAACuB,OAAOD,GAAG;;AAC5G"}

package/dist/cjs/chunk/index.d.cts

@@ -1,7 +1,7 @@
 import type { Block, Chunk, ChunkOptions } from './types.js';
 export declare function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[];
 export declare function chunk(body: string, opts?: ChunkOptions): Chunk[];
-export { extractText } from './extract.js';
+export { extractText, extractTexts } from './extract.js';
 export { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.js';
 export { parse } from './parse.js';
 export type { Block, BlockType, Chunk, ChunkOptions } from './types.js';

package/dist/cjs/chunk/index.d.ts

@@ -1,7 +1,7 @@
 import type { Block, Chunk, ChunkOptions } from './types.js';
 export declare function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[];
 export declare function chunk(body: string, opts?: ChunkOptions): Chunk[];
-export { extractText } from './extract.js';
+export { extractText, extractTexts } from './extract.js';
 export { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.js';
 export { parse } from './parse.js';
 export type { Block, BlockType, Chunk, ChunkOptions } from './types.js';

package/dist/cjs/chunk/index.js

@@ -27,6 +27,9 @@ _export(exports, {
     get extractText () {
         return _extractts.extractText;
     },
+    get extractTexts () {
+        return _extractts.extractTexts;
+    },
     get parse () {
         return _parsets.parse;
     }

package/dist/cjs/chunk/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/index.ts"],"sourcesContent":["import { group } from './group.ts';\nimport { parse } from './parse.ts';\nimport type { Block, Chunk, ChunkOptions } from './types.ts';\n\n// Groups blocks a caller already parsed (e.g. scan/index.ts, sharing one parse with the FTS\n// text path) against the same body they came from, per opts (D1/D3).\nexport function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  return group(blocks, body, opts);\n}\n\n// Pure, deterministic function of file content; chunk semantics are version-stamped via\n// CHUNK_VERSION (./version.ts). Algorithm and evidence: BENCHMARKING.md, \"The chunking algorithm\".\nexport function chunk(body: string, opts?: ChunkOptions): Chunk[] {\n  return chunkFromBlocks(parse(body), body, opts);\n}\n\nexport { extractText } from './extract.ts';\nexport { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.ts';\nexport { parse } from './parse.ts';\nexport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\nexport { CHUNK_VERSION } from './version.ts';\n"],"names":["CHUNK_VERSION","DEFAULT_TARGET_TOKENS","chunk","chunkFromBlocks","estimateTokens","extractText","parse","blocks","body","opts","group"],"mappings":";;;;;;;;;;;QAoBSA;eAAAA,wBAAa;;QAHbC;eAAAA,8BAAqB;;QALdC;eAAAA;;QANAC;eAAAA;;QAWgBC;eAAAA,uBAAc;;QADrCC;eAAAA,sBAAW;;QAEXC;eAAAA,cAAK;;;uBAlBQ;uBACA;yBAeM;yBAIE;AAdvB,SAASH,gBAAgBI,MAAe,EAAEC,IAAY,EAAEC,IAAmB;IAChF,OAAOC,IAAAA,cAAK,EAACH,QAAQC,MAAMC;AAC7B;AAIO,SAASP,MAAMM,IAAY,EAAEC,IAAmB;IACrD,OAAON,gBAAgBG,IAAAA,cAAK,EAACE,OAAOA,MAAMC;AAC5C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/index.ts"],"sourcesContent":["import { group } from './group.ts';\nimport { parse } from './parse.ts';\nimport type { Block, Chunk, ChunkOptions } from './types.ts';\n\n// Groups blocks a caller already parsed (e.g. scan/index.ts, sharing one parse with the FTS\n// text path) against the same body they came from, per opts (D1/D3).\nexport function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  return group(blocks, body, opts);\n}\n\n// Pure, deterministic function of file content; chunk semantics are version-stamped via\n// CHUNK_VERSION (./version.ts). Algorithm and evidence: BENCHMARKING.md, \"The chunking algorithm\".\nexport function chunk(body: string, opts?: ChunkOptions): Chunk[] {\n  return chunkFromBlocks(parse(body), body, opts);\n}\n\nexport { extractText, extractTexts } from './extract.ts';\nexport { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.ts';\nexport { parse } from './parse.ts';\nexport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\nexport { CHUNK_VERSION } from './version.ts';\n"],"names":["CHUNK_VERSION","DEFAULT_TARGET_TOKENS","chunk","chunkFromBlocks","estimateTokens","extractText","extractTexts","parse","blocks","body","opts","group"],"mappings":";;;;;;;;;;;QAoBSA;eAAAA,wBAAa;;QAHbC;eAAAA,8BAAqB;;QALdC;eAAAA;;QANAC;eAAAA;;QAWgBC;eAAAA,uBAAc;;QADrCC;eAAAA,sBAAW;;QAAEC;eAAAA,uBAAY;;QAEzBC;eAAAA,cAAK;;;uBAlBQ;uBACA;yBAeoB;yBAIZ;AAdvB,SAASJ,gBAAgBK,MAAe,EAAEC,IAAY,EAAEC,IAAmB;IAChF,OAAOC,IAAAA,cAAK,EAACH,QAAQC,MAAMC;AAC7B;AAIO,SAASR,MAAMO,IAAY,EAAEC,IAAmB;IACrD,OAAOP,gBAAgBI,IAAAA,cAAK,EAACE,OAAOA,MAAMC;AAC5C"}

package/dist/cjs/index.d.cts

@@ -7,7 +7,7 @@ export { SenseError } from './errors.js';
 export type { Format, Row, RowFormat } from './output/output.js';
 export { printRows } from './output/output.js';
 export type { OpenResult } from './store/index.js';
-export { clearCache, openStore as open } from './store/index.js';
+export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.js';
 export type { WatchEvent, WatchOptions } from './watch.js';
 export { runWatch } from './watch.js';

package/dist/cjs/index.d.ts

@@ -7,7 +7,7 @@ export { SenseError } from './errors.js';
 export type { Format, Row, RowFormat } from './output/output.js';
 export { printRows } from './output/output.js';
 export type { OpenResult } from './store/index.js';
-export { clearCache, openStore as open } from './store/index.js';
+export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.js';
 export type { WatchEvent, WatchOptions } from './watch.js';
 export { runWatch } from './watch.js';

package/dist/cjs/index.js

@@ -11,6 +11,9 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get CAPABILITY_NAMES () {
+        return _indexts2.CAPABILITY_NAMES;
+    },
     get CONFIG_FILENAME () {
         return _indexts1.CONFIG_FILENAME;
     },

package/dist/cjs/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult } from './store/index.ts';\nexport { clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CONFIG_FILENAME","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","clearCache","initConfig","loadConfig","mapTree","migrateConfig","open","openStore","peek","presetCoverage","printRows","runWatch","search"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;;;;;;;;;;;;QAK9DA;eAAAA,yBAAe;;QAAyCC;eAAAA,sBAAY;;QAAEC;eAAAA,mBAAS;;QAAEC;eAAAA,qBAAW;;QAAEC;eAAAA,kCAAwB;;QAEtHC;eAAAA,oBAAU;;QAIVC;eAAAA,oBAAU;;QANOC;eAAAA,oBAAU;;QAAEC;eAAAA,oBAAU;;QAFvCC;eAAAA,gBAAO;;QAEkCC;eAAAA,uBAAa;;QAM7BC;eAAbC,mBAAS;;QARZC;eAAAA,aAAI;;QAAEC;eAAAA,uBAAc;;QAM7BC;eAAAA,mBAAS;;QAMTC;eAAAA,iBAAQ;;QAZuBC;eAAAA,eAAM;;;uBAAQ;wBAEiF;wBAE5G;wBAED;wBAEoB;uBAIrB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult } from './store/index.ts';\nexport { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CAPABILITY_NAMES","CONFIG_FILENAME","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","clearCache","initConfig","loadConfig","mapTree","migrateConfig","open","openStore","peek","presetCoverage","printRows","runWatch","search"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;;;;;;;;;;;;QAW9DA;eAAAA,0BAAgB;;QANhBC;eAAAA,yBAAe;;QAAyCC;eAAAA,sBAAY;;QAAEC;eAAAA,mBAAS;;QAAEC;eAAAA,qBAAW;;QAAEC;eAAAA,kCAAwB;;QAEtHC;eAAAA,oBAAU;;QAIQC;eAAAA,oBAAU;;QANXC;eAAAA,oBAAU;;QAAEC;eAAAA,oBAAU;;QAFvCC;eAAAA,gBAAO;;QAEkCC;eAAAA,uBAAa;;QAMXC;eAAbC,mBAAS;;QAR9BC;eAAAA,aAAI;;QAAEC;eAAAA,uBAAc;;QAM7BC;eAAAA,mBAAS;;QAMTC;eAAAA,iBAAQ;;QAZuBC;eAAAA,eAAM;;;uBAAQ;wBAEiF;wBAE5G;wBAED;wBAEsC;uBAIvC"}

package/dist/cjs/scan/pool.d.cts

@@ -0,0 +1,15 @@
+import type { Config } from '../config/index.js';
+import type { Feature } from '../features/types.js';
+import type { ParsedDoc } from '../scan/index.js';
+import type { FileStat } from './list.js';
+export type FileResult = {
+    doc: ParsedDoc;
+    warnings: string[];
+};
+export declare class ParsePool {
+    private pool;
+    poolsCreated: number;
+    private ensure;
+    run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]>;
+    close(): Promise<void>;
+}

package/dist/cjs/scan/pool.d.ts

@@ -0,0 +1,15 @@
+import type { Config } from '../config/index.js';
+import type { Feature } from '../features/types.js';
+import type { ParsedDoc } from '../scan/index.js';
+import type { FileStat } from './list.js';
+export type FileResult = {
+    doc: ParsedDoc;
+    warnings: string[];
+};
+export declare class ParsePool {
+    private pool;
+    poolsCreated: number;
+    private ensure;
+    run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]>;
+    close(): Promise<void>;
+}

package/dist/cjs/scan/pool.js

@@ -0,0 +1,263 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "ParsePool", {
+    enumerable: true,
+    get: function() {
+        return ParsePool;
+    }
+});
+var _nodemodule = require("node:module");
+var _nodepath = require("node:path");
+var _workererrorts = require("./worker-error.js");
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _class_call_check(instance, Constructor) {
+    if (!(instance instanceof Constructor)) throw new TypeError("Cannot call a class as a function");
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.
+var _require = typeof require === 'undefined' ? (0, _nodemodule.createRequire)(require("url").pathToFileURL(__filename).toString()) : require;
+// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited
+// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.
+var workerFile;
+function resolveWorkerFile() {
+    if (workerFile) return workerFile;
+    var load = (0, _nodemodule.createRequire)(require("url").pathToFileURL(__filename).toString());
+    for(var _i = 0, _iter = [
+        '..',
+        '../..',
+        '../../..'
+    ]; _i < _iter.length; _i++){
+        var rel = _iter[_i];
+        try {
+            if (load("".concat(rel, "/package.json")).name === 'sensemaking') {
+                workerFile = (0, _nodepath.join)((0, _nodepath.dirname)(load.resolve("".concat(rel, "/package.json"))), 'dist', 'cjs', 'workers', 'parse.js');
+                return workerFile;
+            }
+        } catch (unused) {}
+    }
+    throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');
+}
+var ParsePool = /*#__PURE__*/ function() {
+    "use strict";
+    function ParsePool() {
+        _class_call_check(this, ParsePool);
+        // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;
+        // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.
+        this.poolsCreated = 0;
+    }
+    var _proto = ParsePool.prototype;
+    _proto.ensure = function ensure(workerData, maxWorkers) {
+        if (!this.pool) {
+            var Tinypool = _require('tinypool').Tinypool;
+            this.pool = new Tinypool({
+                filename: resolveWorkerFile(),
+                minThreads: maxWorkers,
+                maxThreads: maxWorkers,
+                workerData: workerData
+            });
+            this.poolsCreated++;
+        }
+        return this.pool;
+    };
+    // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --
+    // extracted text and feature values, never the mdast tree.
+    _proto.run = function run(files, features, cfg, onParsed, maxWorkers) {
+        return _async_to_generator(function() {
+            var workerData, pool, done;
+            return _ts_generator(this, function(_state) {
+                // cfg and the feature selection are constant for the dispatch, so they cross once per worker
+                // as workerData. Features carry closures and cannot cross at all; only their names do.
+                workerData = {
+                    cfg: cfg,
+                    featureNames: features.map(function(feature) {
+                        return feature.name;
+                    })
+                };
+                pool = this.ensure(workerData, maxWorkers);
+                done = 0;
+                // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order
+                // regardless of task completion order, which first-seen column order depends on downstream.
+                return [
+                    2,
+                    Promise.all(files.map(function(file) {
+                        return _async_to_generator(function() {
+                            var result;
+                            return _ts_generator(this, function(_state) {
+                                switch(_state.label){
+                                    case 0:
+                                        return [
+                                            4,
+                                            pool.run(file)
+                                        ];
+                                    case 1:
+                                        result = _state.sent();
+                                        if (!result.ok) throw (0, _workererrorts.reviveError)(result.error);
+                                        onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
+                                        return [
+                                            2,
+                                            {
+                                                doc: result.doc,
+                                                warnings: result.warnings
+                                            }
+                                        ];
+                                }
+                            });
+                        })();
+                    }))
+                ];
+            });
+        }).call(this);
+    };
+    // A pool never created costs nothing to destroy.
+    _proto.close = function close() {
+        return _async_to_generator(function() {
+            var pool;
+            return _ts_generator(this, function(_state) {
+                switch(_state.label){
+                    case 0:
+                        if (!this.pool) return [
+                            2
+                        ];
+                        pool = this.pool;
+                        this.pool = undefined;
+                        return [
+                            4,
+                            pool.destroy()
+                        ];
+                    case 1:
+                        _state.sent();
+                        return [
+                            2
+                        ];
+                }
+            });
+        }).call(this);
+    };
+    return ParsePool;
+}();
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/scan/pool.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\nexport type FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the mdast tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["ParsePool","_require","require","createRequire","workerFile","resolveWorkerFile","load","rel","name","join","dirname","resolve","Error","poolsCreated","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","run","files","features","cfg","onParsed","done","featureNames","map","feature","Promise","all","file","result","ok","reviveError","error","doc","warnings","close","undefined","destroy"],"mappings":";;;;+BAoCaA;;;eAAAA;;;0BApCiB;wBACA;6BASF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,8GAA8G;AAC9G,IAAMC,WAAW,OAAOC,YAAY,cAAcC,IAAAA,yBAAa,EAAC,uDAAmBD;AAEnF,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIE;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,IAAME,OAAOH,IAAAA,yBAAa,EAAC;IAC3B,gBAAkB,QAAA;QAAC;QAAM;QAAS;KAAW,OAA3B,mBAA6B;YAApCI,MAAO;QAChB,IAAI;YACF,IAAI,AAACD,KAAK,AAAC,GAAM,OAAJC,KAAI,kBAAsCC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaK,IAAAA,cAAI,EAACC,IAAAA,iBAAO,EAACJ,KAAKK,OAAO,CAAC,AAAC,GAAM,OAAJJ,KAAI,oBAAkB,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,eAAM,CAAC;IACX;IACA,MAAM,IAAIQ,MAAM;AAClB;AAMO,IAAA,AAAMZ,0BAAN;;aAAMA;gCAAAA;QAEX,2FAA2F;QAC3F,uGAAuG;aACvGa,eAAe;;iBAJJb;IAMX,OAAQc,MAOP,GAPD,SAAQA,OAAOC,UAA2B,EAAEC,UAAkB;QAC5D,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,IAAM,AAAEC,WAAajB,SAAS,YAAtBiB;YACR,IAAI,CAACD,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUd;gBAAqBe,YAAYJ;gBAAYK,YAAYL;gBAAYD,YAAAA;YAAW;YACrH,IAAI,CAACF,YAAY;QACnB;QACA,OAAO,IAAI,CAACI,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,OAAMK,GAgBL,GAhBD,SAAMA,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEV,UAAkB;;gBAGzHD,YACAE,MACFU;;gBAJJ,6FAA6F;gBAC7F,uFAAuF;gBACjFZ,aAA8B;oBAAEU,KAAAA;oBAAKG,cAAcJ,SAASK,GAAG,CAAC,SAACC;+BAAYA,QAAQtB,IAAI;;gBAAE;gBAC3FS,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;gBACjCW,OAAO;gBACX,0FAA0F;gBAC1F,4FAA4F;gBAC5F;;oBAAOI,QAAQC,GAAG,CAChBT,MAAMM,GAAG,CAAC,SAAOI;;gCACTC;;;;wCAAU;;4CAAMjB,KAAKK,GAAG,CAACW;;;wCAAzBC,SAAU;wCAChB,IAAI,CAACA,OAAOC,EAAE,EAAE,MAAMC,IAAAA,0BAAW,EAACF,OAAOG,KAAK;wCAC9CX,qBAAAA,+BAAAA,SAAW,EAAEC;wCACb;;4CAAO;gDAAEW,KAAKJ,OAAOI,GAAG;gDAAEC,UAAUL,OAAOK,QAAQ;4CAAC;;;;wBACtD;;;;QAEJ;;IAEA,iDAAiD;IACjD,OAAMC,KAKL,GALD,SAAMA;;gBAEEvB;;;;wBADN,IAAI,CAAC,IAAI,CAACA,IAAI,EAAE;;;wBACVA,OAAO,IAAI,CAACA,IAAI;wBACtB,IAAI,CAACA,IAAI,GAAGwB;wBACZ;;4BAAMxB,KAAKyB,OAAO;;;wBAAlB;;;;;;QACF;;WAzCW1C"}

package/dist/cjs/scan/reparse.d.cts

@@ -2,6 +2,7 @@ import type { Config } from '../config/index.js';
 import type { Feature } from '../features/types.js';
 import type { ParsedDoc } from './index.js';
 import type { FileStat } from './list.js';
+import { ParsePool } from './pool.js';
 export interface ReparseResult {
     docs: ParsedDoc[];
     warnings: string[];
@@ -10,6 +11,7 @@ export interface ReparseResult {
 export interface ReparseOptions {
     threshold?: number;
     maxWorkers?: number;
+    pool?: ParsePool;
 }
 export declare function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[];
 export declare function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options?: ReparseOptions): Promise<ReparseResult>;

package/dist/cjs/scan/reparse.d.ts

@@ -2,6 +2,7 @@ import type { Config } from '../config/index.js';
 import type { Feature } from '../features/types.js';
 import type { ParsedDoc } from './index.js';
 import type { FileStat } from './list.js';
+import { ParsePool } from './pool.js';
 export interface ReparseResult {
     docs: ParsedDoc[];
     warnings: string[];
@@ -10,6 +11,7 @@ export interface ReparseResult {
 export interface ReparseOptions {
     threshold?: number;
     maxWorkers?: number;
+    pool?: ParsePool;
 }
 export declare function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[];
 export declare function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options?: ReparseOptions): Promise<ReparseResult>;