sensemaking 0.16.0 → 0.17.0

package/README.md

@@ -164,7 +164,7 @@ Two skills: `sense` for querying a tree (what each command is for, FTS5 syntax,
 - **RAG / vector stores:** similarity can't express `WHERE status = 'active'`. Here vectors are one signal inside `search`: same SQLite file, filters compose, every row labels its evidence (`via`), and a preset turns vectors off per layer of the tree. No second store, no daemon, no native builds.
 - **Document-OS apps (Anytype, Logseq, SilverBullet, Capacities):** full applications with their own UI and storage. `sense` is headless: your files stay files, there's no app to run.
 
-Dependencies: [yaml](https://github.com/eemeli/yaml), [remove-markdown](https://github.com/zuchka/remove-markdown), [@huggingface/tokenizers](https://github.com/huggingface/tokenizers.js), [franc-min](https://github.com/wooorm/franc) (all pure JS), and Node's built-in SQLite. No native builds.
+Dependencies: [yaml](https://github.com/eemeli/yaml), [@huggingface/tokenizers](https://github.com/huggingface/tokenizers.js), [franc-min](https://github.com/wooorm/franc), [mdast-util-from-markdown](https://github.com/syntax-tree/mdast-util-from-markdown), and the five GFM extensions it needs (not the full gfm/tagfilter bundle): [micromark-extension-gfm-autolink-literal](https://github.com/micromark/micromark-extension-gfm-autolink-literal), [micromark-extension-gfm-footnote](https://github.com/micromark/micromark-extension-gfm-footnote), [micromark-extension-gfm-strikethrough](https://github.com/micromark/micromark-extension-gfm-strikethrough), [micromark-extension-gfm-table](https://github.com/micromark/micromark-extension-gfm-table), [micromark-extension-gfm-task-list-item](https://github.com/micromark/micromark-extension-gfm-task-list-item), and their mdast counterparts [mdast-util-gfm-autolink-literal](https://github.com/syntax-tree/mdast-util-gfm-autolink-literal), [mdast-util-gfm-footnote](https://github.com/syntax-tree/mdast-util-gfm-footnote), [mdast-util-gfm-strikethrough](https://github.com/syntax-tree/mdast-util-gfm-strikethrough), [mdast-util-gfm-table](https://github.com/syntax-tree/mdast-util-gfm-table), [mdast-util-gfm-task-list-item](https://github.com/syntax-tree/mdast-util-gfm-task-list-item) (all pure JS), and Node's built-in SQLite. No native builds.
 
 ## License
 

package/dist/cjs/chunk/extract.d.cts

@@ -0,0 +1,2 @@
+import type { Nodes } from 'mdast';
+export declare function extractText(node: Nodes): string;

package/dist/cjs/chunk/extract.d.ts

@@ -0,0 +1,2 @@
+import type { Nodes } from 'mdast';
+export declare function extractText(node: Nodes): string;

package/dist/cjs/chunk/extract.js

@@ -0,0 +1,139 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "extractText", {
+    enumerable: true,
+    get: function() {
+        return extractText;
+    }
+});
+// mdast has no wikilink, embed, or %%comment%% node -- literal text resolved by regex passes
+// over non-code text (code/inlineCode is placeholder-held and spliced back verbatim, below).
+var COMMENT_PAIR = /%%[\s\S]*?%%/g;
+// An embed always yields its target, whatever follows a pipe (a resize suffix, never an alias);
+// must run before the wikilink regex below, bang included, or that regex matches first.
+var EMBED = /!\[\[([^\]]+)\]\]/g;
+var WIKILINK = /\[\[([^\]]+)\]\]/g;
+// A trailing " ^id" is an Obsidian block anchor, valid only at a line's end; a caret elsewhere
+// (mid-line) is ordinary text and is left alone.
+var BLOCK_ID = / \^[A-Za-z0-9-]+$/gm;
+// A callout marker is only ever the first thing in a blockquote's text (Obsidian's grammar),
+// so this anchors to blockquote output alone -- never applied to prose in general.
+var CALLOUT_MARKER = /^\[!\w[\w-]*\][+-]?[ \t]?/;
+// An html node's value is raw HTML, block or inline: a full <!-- --> comment is dropped, any
+// remaining tags are stripped, and the text a browser would still render survives.
+var HTML_COMMENT = /<!--[\s\S]*?-->/g;
+var HTML_TAG = /<[^>]+>/g;
+function stripHtml(value) {
+    return value.replace(HTML_COMMENT, '').replace(HTML_TAG, '');
+}
+// A `#anchor` keeps its text (hash dropped), unlike links.ts's parseWikilinkInner (the resolver
+// authority), which discards it entirely -- replicated here since that return value doesn't fit.
+function anchorText(base) {
+    var hashIdx = base.indexOf('#');
+    if (hashIdx === -1) return base.trim();
+    var target = base.slice(0, hashIdx).trim();
+    var anchor = base.slice(hashIdx + 1).trim();
+    return target ? "".concat(target, " ").concat(anchor) : anchor;
+}
+function wikilinkText(inner) {
+    var pipeIdx = inner.indexOf('|');
+    return pipeIdx === -1 ? anchorText(inner) : inner.slice(pipeIdx + 1).trim();
+}
+function embedText(inner) {
+    var pipeIdx = inner.indexOf('|');
+    return anchorText(pipeIdx === -1 ? inner : inner.slice(0, pipeIdx));
+}
+function resolveFlavor(text) {
+    return text.replace(COMMENT_PAIR, '').replace(EMBED, function(_, inner) {
+        return embedText(inner);
+    }).replace(WIKILINK, function(_, inner) {
+        return wikilinkText(inner);
+    }).replace(BLOCK_ID, '');
+}
+// Sibling blocks (list items, blockquote content, footnote bodies) joined one per line;
+// empties dropped so a blank child never widens the gap between its neighbors.
+function joinBlocks(nodes, code) {
+    return nodes.map(function(n) {
+        return extractNode(n, code);
+    }).filter(function(s) {
+        return s.length > 0;
+    }).join('\n');
+}
+// Inline content (headings, paragraphs, emphasis, links) concatenated with no separator: the
+// source text nodes already carry whatever spacing separates them.
+function joinInline(nodes, code) {
+    return nodes.map(function(n) {
+        return extractNode(n, code);
+    }).join('');
+}
+// U+E000 (Private Use Area) never appears in real markdown text, so it is a collision-free
+// placeholder delimiter, spliced back to the literal code value after flavor resolution.
+function codePlaceholder(value, code) {
+    var idx = code.push(value) - 1;
+    return "".concat(idx, "");
+}
+// A GFM autolink's display text is its own target, missing only the scheme
+// mdast-util-gfm-autolink-literal fills into node.url (http:// for www., mailto: for an email).
+var SCHEME_PREFIX = /^[a-z][a-z0-9+.-]*:\/{0,2}/i;
+function isAutolink(url, text) {
+    return url === text || url.replace(SCHEME_PREFIX, '') === text;
+}
+function extractNode(node, code) {
+    switch(node.type){
+        case 'root':
+        case 'list':
+        case 'listItem':
+        case 'footnoteDefinition':
+            return joinBlocks(node.children, code);
+        case 'blockquote':
+            return joinBlocks(node.children, code).replace(CALLOUT_MARKER, '');
+        case 'link':
+            {
+                var text = joinInline(node.children, code);
+                return isAutolink(node.url, text) ? '' : text;
+            }
+        case 'heading':
+        case 'paragraph':
+        case 'linkReference':
+        case 'emphasis':
+        case 'strong':
+        case 'delete':
+            return joinInline(node.children, code);
+        case 'table':
+            return node.children.map(function(n) {
+                return extractNode(n, code);
+            }).join('\n');
+        case 'tableRow':
+            return node.children.map(function(n) {
+                return extractNode(n, code);
+            }).join(' ');
+        case 'tableCell':
+            return joinInline(node.children, code);
+        case 'image':
+        case 'imageReference':
+            var _node_alt;
+            return (_node_alt = node.alt) !== null && _node_alt !== void 0 ? _node_alt : '';
+        case 'text':
+            return node.value;
+        case 'inlineCode':
+        case 'code':
+            return codePlaceholder(node.value, code);
+        case 'html':
+            return stripHtml(node.value);
+        case 'break':
+            return ' ';
+        default:
+            return '';
+    }
+}
+var CODE_PLACEHOLDER = /\uE000(\d+)\uE000/g;
+function extractText(node) {
+    var code = [];
+    var resolved = resolveFlavor(extractNode(node, code));
+    return code.length === 0 ? resolved : resolved.replace(CODE_PLACEHOLDER, function(_, i) {
+        return code[Number(i)];
+    });
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/chunk/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/extract.ts"],"sourcesContent":["import type { Nodes } from 'mdast';\n\n// mdast has no wikilink, embed, or %%comment%% node -- literal text resolved by regex passes\n// over non-code text (code/inlineCode is placeholder-held and spliced back verbatim, below).\nconst COMMENT_PAIR = /%%[\\s\\S]*?%%/g;\n// An embed always yields its target, whatever follows a pipe (a resize suffix, never an alias);\n// must run before the wikilink regex below, bang included, or that regex matches first.\nconst EMBED = /!\\[\\[([^\\]]+)\\]\\]/g;\nconst WIKILINK = /\\[\\[([^\\]]+)\\]\\]/g;\n// A trailing \" ^id\" is an Obsidian block anchor, valid only at a line's end; a caret elsewhere\n// (mid-line) is ordinary text and is left alone.\nconst BLOCK_ID = / \\^[A-Za-z0-9-]+$/gm;\n// A callout marker is only ever the first thing in a blockquote's text (Obsidian's grammar),\n// so this anchors to blockquote output alone -- never applied to prose in general.\nconst CALLOUT_MARKER = /^\\[!\\w[\\w-]*\\][+-]?[ \\t]?/;\n// An html node's value is raw HTML, block or inline: a full <!-- --> comment is dropped, any\n// remaining tags are stripped, and the text a browser would still render survives.\nconst HTML_COMMENT = /<!--[\\s\\S]*?-->/g;\nconst HTML_TAG = /<[^>]+>/g;\n\nfunction stripHtml(value: string): string {\n  return value.replace(HTML_COMMENT, '').replace(HTML_TAG, '');\n}\n\n// A `#anchor` keeps its text (hash dropped), unlike links.ts's parseWikilinkInner (the resolver\n// authority), which discards it entirely -- replicated here since that return value doesn't fit.\nfunction anchorText(base: string): string {\n  const hashIdx = base.indexOf('#');\n  if (hashIdx === -1) return base.trim();\n  const target = base.slice(0, hashIdx).trim();\n  const anchor = base.slice(hashIdx + 1).trim();\n  return target ? `${target} ${anchor}` : anchor;\n}\n\nfunction wikilinkText(inner: string): string {\n  const pipeIdx = inner.indexOf('|');\n  return pipeIdx === -1 ? anchorText(inner) : inner.slice(pipeIdx + 1).trim();\n}\n\nfunction embedText(inner: string): string {\n  const pipeIdx = inner.indexOf('|');\n  return anchorText(pipeIdx === -1 ? inner : inner.slice(0, pipeIdx));\n}\n\nfunction resolveFlavor(text: string): string {\n  return text\n    .replace(COMMENT_PAIR, '')\n    .replace(EMBED, (_, inner: string) => embedText(inner))\n    .replace(WIKILINK, (_, inner: string) => wikilinkText(inner))\n    .replace(BLOCK_ID, '');\n}\n\n// Sibling blocks (list items, blockquote content, footnote bodies) joined one per line;\n// empties dropped so a blank child never widens the gap between its neighbors.\nfunction joinBlocks(nodes: Nodes[], code: string[]): string {\n  return nodes\n    .map((n) => extractNode(n, code))\n    .filter((s) => s.length > 0)\n    .join('\\n');\n}\n\n// Inline content (headings, paragraphs, emphasis, links) concatenated with no separator: the\n// source text nodes already carry whatever spacing separates them.\nfunction joinInline(nodes: Nodes[], code: string[]): string {\n  return nodes.map((n) => extractNode(n, code)).join('');\n}\n\n// U+E000 (Private Use Area) never appears in real markdown text, so it is a collision-free\n// placeholder delimiter, spliced back to the literal code value after flavor resolution.\nfunction codePlaceholder(value: string, code: string[]): string {\n  const idx = code.push(value) - 1;\n  return `\\uE000${idx}\\uE000`;\n}\n\n// A GFM autolink's display text is its own target, missing only the scheme\n// mdast-util-gfm-autolink-literal fills into node.url (http:// for www., mailto: for an email).\nconst SCHEME_PREFIX = /^[a-z][a-z0-9+.-]*:\\/{0,2}/i;\nfunction isAutolink(url: string, text: string): boolean {\n  return url === text || url.replace(SCHEME_PREFIX, '') === text;\n}\n\nfunction extractNode(node: Nodes, code: string[]): string {\n  switch (node.type) {\n    case 'root':\n    case 'list':\n    case 'listItem':\n    case 'footnoteDefinition':\n      return joinBlocks(node.children, code);\n    case 'blockquote':\n      return joinBlocks(node.children, code).replace(CALLOUT_MARKER, '');\n    case 'link': {\n      const text = joinInline(node.children, code);\n      return isAutolink(node.url, text) ? '' : text;\n    }\n    case 'heading':\n    case 'paragraph':\n    case 'linkReference':\n    case 'emphasis':\n    case 'strong':\n    case 'delete':\n      return joinInline(node.children, code);\n    case 'table':\n      return node.children.map((n) => extractNode(n, code)).join('\\n');\n    case 'tableRow':\n      return node.children.map((n) => extractNode(n, code)).join(' ');\n    case 'tableCell':\n      return joinInline(node.children, code);\n    case 'image':\n    case 'imageReference':\n      return node.alt ?? '';\n    case 'text':\n      return node.value;\n    case 'inlineCode':\n    case 'code':\n      return codePlaceholder(node.value, code);\n    case 'html':\n      return stripHtml(node.value);\n    case 'break':\n      return ' ';\n    default:\n      return '';\n  }\n}\n\nconst CODE_PLACEHOLDER = /\\uE000(\\d+)\\uE000/g;\n\n// Plain text of one mdast node (or a whole tree): heading/list/table structure is kept as text,\n// markup (emphasis, link targets, task and callout markers) is dropped. Pure, synchronous.\nexport function extractText(node: Nodes): string {\n  const code: string[] = [];\n  const resolved = resolveFlavor(extractNode(node, code));\n  return code.length === 0 ? resolved : resolved.replace(CODE_PLACEHOLDER, (_, i: string) => code[Number(i)]);\n}\n"],"names":["extractText","COMMENT_PAIR","EMBED","WIKILINK","BLOCK_ID","CALLOUT_MARKER","HTML_COMMENT","HTML_TAG","stripHtml","value","replace","anchorText","base","hashIdx","indexOf","trim","target","slice","anchor","wikilinkText","inner","pipeIdx","embedText","resolveFlavor","text","_","joinBlocks","nodes","code","map","n","extractNode","filter","s","length","join","joinInline","codePlaceholder","idx","push","SCHEME_PREFIX","isAutolink","url","node","type","children","alt","CODE_PLACEHOLDER","resolved","i","Number"],"mappings":";;;;+BAgIgBA;;;eAAAA;;;AA9HhB,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,eAAe;AACrB,gGAAgG;AAChG,wFAAwF;AACxF,IAAMC,QAAQ;AACd,IAAMC,WAAW;AACjB,+FAA+F;AAC/F,iDAAiD;AACjD,IAAMC,WAAW;AACjB,6FAA6F;AAC7F,mFAAmF;AACnF,IAAMC,iBAAiB;AACvB,6FAA6F;AAC7F,mFAAmF;AACnF,IAAMC,eAAe;AACrB,IAAMC,WAAW;AAEjB,SAASC,UAAUC,KAAa;IAC9B,OAAOA,MAAMC,OAAO,CAACJ,cAAc,IAAII,OAAO,CAACH,UAAU;AAC3D;AAEA,gGAAgG;AAChG,iGAAiG;AACjG,SAASI,WAAWC,IAAY;IAC9B,IAAMC,UAAUD,KAAKE,OAAO,CAAC;IAC7B,IAAID,YAAY,CAAC,GAAG,OAAOD,KAAKG,IAAI;IACpC,IAAMC,SAASJ,KAAKK,KAAK,CAAC,GAAGJ,SAASE,IAAI;IAC1C,IAAMG,SAASN,KAAKK,KAAK,CAACJ,UAAU,GAAGE,IAAI;IAC3C,OAAOC,SAAS,AAAC,GAAYE,OAAVF,QAAO,KAAU,OAAPE,UAAWA;AAC1C;AAEA,SAASC,aAAaC,KAAa;IACjC,IAAMC,UAAUD,MAAMN,OAAO,CAAC;IAC9B,OAAOO,YAAY,CAAC,IAAIV,WAAWS,SAASA,MAAMH,KAAK,CAACI,UAAU,GAAGN,IAAI;AAC3E;AAEA,SAASO,UAAUF,KAAa;IAC9B,IAAMC,UAAUD,MAAMN,OAAO,CAAC;IAC9B,OAAOH,WAAWU,YAAY,CAAC,IAAID,QAAQA,MAAMH,KAAK,CAAC,GAAGI;AAC5D;AAEA,SAASE,cAAcC,IAAY;IACjC,OAAOA,KACJd,OAAO,CAACT,cAAc,IACtBS,OAAO,CAACR,OAAO,SAACuB,GAAGL;eAAkBE,UAAUF;OAC/CV,OAAO,CAACP,UAAU,SAACsB,GAAGL;eAAkBD,aAAaC;OACrDV,OAAO,CAACN,UAAU;AACvB;AAEA,wFAAwF;AACxF,+EAA+E;AAC/E,SAASsB,WAAWC,KAAc,EAAEC,IAAc;IAChD,OAAOD,MACJE,GAAG,CAAC,SAACC;eAAMC,YAAYD,GAAGF;OAC1BI,MAAM,CAAC,SAACC;eAAMA,EAAEC,MAAM,GAAG;OACzBC,IAAI,CAAC;AACV;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,SAASC,WAAWT,KAAc,EAAEC,IAAc;IAChD,OAAOD,MAAME,GAAG,CAAC,SAACC;eAAMC,YAAYD,GAAGF;OAAOO,IAAI,CAAC;AACrD;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASE,gBAAgB5B,KAAa,EAAEmB,IAAc;IACpD,IAAMU,MAAMV,KAAKW,IAAI,CAAC9B,SAAS;IAC/B,OAAO,AAAC,IAAY,OAAJ6B,KAAI;AACtB;AAEA,2EAA2E;AAC3E,gGAAgG;AAChG,IAAME,gBAAgB;AACtB,SAASC,WAAWC,GAAW,EAAElB,IAAY;IAC3C,OAAOkB,QAAQlB,QAAQkB,IAAIhC,OAAO,CAAC8B,eAAe,QAAQhB;AAC5D;AAEA,SAASO,YAAYY,IAAW,EAAEf,IAAc;IAC9C,OAAQe,KAAKC,IAAI;QACf,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAOlB,WAAWiB,KAAKE,QAAQ,EAAEjB;QACnC,KAAK;YACH,OAAOF,WAAWiB,KAAKE,QAAQ,EAAEjB,MAAMlB,OAAO,CAACL,gBAAgB;QACjE,KAAK;YAAQ;gBACX,IAAMmB,OAAOY,WAAWO,KAAKE,QAAQ,EAAEjB;gBACvC,OAAOa,WAAWE,KAAKD,GAAG,EAAElB,QAAQ,KAAKA;YAC3C;QACA,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAOY,WAAWO,KAAKE,QAAQ,EAAEjB;QACnC,KAAK;YACH,OAAOe,KAAKE,QAAQ,CAAChB,GAAG,CAAC,SAACC;uBAAMC,YAAYD,GAAGF;eAAOO,IAAI,CAAC;QAC7D,KAAK;YACH,OAAOQ,KAAKE,QAAQ,CAAChB,GAAG,CAAC,SAACC;uBAAMC,YAAYD,GAAGF;eAAOO,IAAI,CAAC;QAC7D,KAAK;YACH,OAAOC,WAAWO,KAAKE,QAAQ,EAAEjB;QACnC,KAAK;QACL,KAAK;gBACIe;YAAP,QAAOA,YAAAA,KAAKG,GAAG,cAARH,uBAAAA,YAAY;QACrB,KAAK;YACH,OAAOA,KAAKlC,KAAK;QACnB,KAAK;QACL,KAAK;YACH,OAAO4B,gBAAgBM,KAAKlC,KAAK,EAAEmB;QACrC,KAAK;YACH,OAAOpB,UAAUmC,KAAKlC,KAAK;QAC7B,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,IAAMsC,mBAAmB;AAIlB,SAAS/C,YAAY2C,IAAW;IACrC,IAAMf,OAAiB,EAAE;IACzB,IAAMoB,WAAWzB,cAAcQ,YAAYY,MAAMf;IACjD,OAAOA,KAAKM,MAAM,KAAK,IAAIc,WAAWA,SAAStC,OAAO,CAACqC,kBAAkB,SAACtB,GAAGwB;eAAcrB,IAAI,CAACsB,OAAOD,GAAG;;AAC5G"}

package/dist/cjs/chunk/group.d.cts

@@ -0,0 +1,4 @@
+import type { Block, Chunk, ChunkOptions } from './types.js';
+export declare const DEFAULT_TARGET_TOKENS = 500;
+export declare function estimateTokens(text: string): number;
+export declare function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[];

package/dist/cjs/chunk/group.d.ts

@@ -0,0 +1,4 @@
+import type { Block, Chunk, ChunkOptions } from './types.js';
+export declare const DEFAULT_TARGET_TOKENS = 500;
+export declare function estimateTokens(text: string): number;
+export declare function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[];