sensemaking 0.22.2 → 0.23.0

package/README.md

@@ -170,20 +170,18 @@ Dependencies, all pure JS. No native builds by default.
 | | |
 |---|---|
 | [yaml](https://github.com/eemeli/yaml) | frontmatter |
-| [mdast-util-from-markdown](https://github.com/syntax-tree/mdast-util-from-markdown) | markdown parsing |
+| [markdown-it](https://github.com/markdown-it/markdown-it) | markdown parsing |
 | [@huggingface/tokenizers](https://github.com/huggingface/tokenizers.js) | chunking |
 | [franc-min](https://github.com/wooorm/franc) | language detection |
 | [tinypool](https://github.com/tinylibs/tinypool) | worker pool for parallel parsing on large trees |
 | [install-module-linked](https://github.com/kmalakoff/install-module-linked) | installs the optional `duckdb` and `turso` bindings on first use, instead of shipping them to every install |
 | Node's built-in SQLite | the default store |
 
-Plus the five GFM extensions `mdast-util-from-markdown` needs and their mdast counterparts, not the full gfm/tagfilter bundle:
+Plus two plugins for the GFM constructs the default preset lacks; tables, strikethrough, and
+autolinks are built in:
 
-- [micromark-extension-gfm-autolink-literal](https://github.com/micromark/micromark-extension-gfm-autolink-literal) / [mdast-util-gfm-autolink-literal](https://github.com/syntax-tree/mdast-util-gfm-autolink-literal)
-- [micromark-extension-gfm-footnote](https://github.com/micromark/micromark-extension-gfm-footnote) / [mdast-util-gfm-footnote](https://github.com/syntax-tree/mdast-util-gfm-footnote)
-- [micromark-extension-gfm-strikethrough](https://github.com/micromark/micromark-extension-gfm-strikethrough) / [mdast-util-gfm-strikethrough](https://github.com/syntax-tree/mdast-util-gfm-strikethrough)
-- [micromark-extension-gfm-table](https://github.com/micromark/micromark-extension-gfm-table) / [mdast-util-gfm-table](https://github.com/syntax-tree/mdast-util-gfm-table)
-- [micromark-extension-gfm-task-list-item](https://github.com/micromark/micromark-extension-gfm-task-list-item) / [mdast-util-gfm-task-list-item](https://github.com/syntax-tree/mdast-util-gfm-task-list-item)
+- [markdown-it-footnote](https://github.com/markdown-it/markdown-it-footnote)
+- [markdown-it-task-lists](https://github.com/revin/markdown-it-task-lists)
 
 ## License
 

package/dist/cjs/chunk/extract.d.cts

@@ -1,3 +1,3 @@
-import type { Nodes } from 'mdast';
-export declare function extractText(node: Nodes): string;
-export declare function extractTexts(nodes: Nodes[]): string;
+import type { Block } from './types.js';
+export declare function extractText(block: Block): string;
+export declare function extractTexts(blocks: Block[]): string;

package/dist/cjs/chunk/extract.d.ts

@@ -1,3 +1,3 @@
-import type { Nodes } from 'mdast';
-export declare function extractText(node: Nodes): string;
-export declare function extractTexts(nodes: Nodes[]): string;
+import type { Block } from './types.js';
+export declare function extractText(block: Block): string;
+export declare function extractTexts(blocks: Block[]): string;

package/dist/cjs/chunk/extract.js

@@ -16,8 +16,58 @@ _export(exports, {
         return extractTexts;
     }
 });
-// mdast has no wikilink, embed, or %%comment%% node -- literal text resolved by regex passes
-// over non-code text (code/inlineCode is placeholder-held and spliced back verbatim, below).
+var _parserts = require("./parser.js");
+function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else obj[key] = value;
+    return obj;
+}
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = arguments[i] != null ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if (typeof Object.getOwnPropertySymbols === "function") {
+            ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+            }));
+        }
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
+function ownKeys(object, enumerableOnly) {
+    var keys = Object.keys(object);
+    if (Object.getOwnPropertySymbols) {
+        var symbols = Object.getOwnPropertySymbols(object);
+        if (enumerableOnly) {
+            symbols = symbols.filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(object, sym).enumerable;
+            });
+        }
+        keys.push.apply(keys, symbols);
+    }
+    return keys;
+}
+function _object_spread_props(target, source) {
+    source = source != null ? source : {};
+    if (Object.getOwnPropertyDescriptors) Object.defineProperties(target, Object.getOwnPropertyDescriptors(source));
+    else {
+        ownKeys(Object(source)).forEach(function(key) {
+            Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key));
+        });
+    }
+    return target;
+}
+// markdown-it, like mdast, has no wikilink, embed, or %%comment%% token -- literal text resolved
+// by regex passes over non-code text (code/inline code is placeholder-held and spliced back verbatim).
 var COMMENT_PAIR = /%%[\s\S]*?%%/g;
 // An embed always yields its target, whatever follows a pipe (a resize suffix, never an alias);
 // must run before the wikilink regex below, bang included, or that regex matches first.
@@ -29,7 +79,7 @@ var BLOCK_ID = / \^[A-Za-z0-9-]+$/gm;
 // A callout marker is only ever the first thing in a blockquote's text (Obsidian's grammar),
 // so this anchors to blockquote output alone -- never applied to prose in general.
 var CALLOUT_MARKER = /^\[!\w[\w-]*\][+-]?[ \t]?/;
-// An html node's value is raw HTML, block or inline: a full <!-- --> comment is dropped, any
+// An html token's content is raw HTML, block or inline: a full <!-- --> comment is dropped, any
 // remaining tags are stripped, and the text a browser would still render survives.
 var HTML_COMMENT = /<!--[\s\S]*?-->/g;
 var HTML_TAG = /<[^>]+>/g;
@@ -60,92 +110,339 @@ function resolveFlavor(text) {
         return wikilinkText(inner);
     }).replace(BLOCK_ID, '');
 }
-// Sibling blocks (list items, blockquote content, footnote bodies) joined one per line;
-// empties dropped so a blank child never widens the gap between its neighbors.
-function joinBlocks(nodes, code) {
-    return nodes.map(function(n) {
-        return extractNode(n, code);
-    }).filter(function(s) {
-        return s.length > 0;
-    }).join('\n');
-}
-// Inline content (headings, paragraphs, emphasis, links) concatenated with no separator: the
-// source text nodes already carry whatever spacing separates them.
-function joinInline(nodes, code) {
-    return nodes.map(function(n) {
-        return extractNode(n, code);
-    }).join('');
-}
 // U+E000 (Private Use Area) never appears in real markdown text, so it is a collision-free
 // placeholder delimiter, spliced back to the literal code value after flavor resolution.
 function codePlaceholder(value, code) {
     var idx = code.push(value) - 1;
     return "".concat(idx, "");
 }
-// A GFM autolink's display text is its own target, missing only the scheme
-// mdast-util-gfm-autolink-literal fills into node.url (http:// for www., mailto: for an email).
-var SCHEME_PREFIX = /^[a-z][a-z0-9+.-]*:\/{0,2}/i;
-function isAutolink(url, text) {
-    return url === text || url.replace(SCHEME_PREFIX, '') === text;
-}
-function extractNode(node, code) {
-    switch(node.type){
-        case 'root':
-        case 'list':
-        case 'listItem':
-        case 'footnoteDefinition':
-            return joinBlocks(node.children, code);
-        case 'blockquote':
-            return joinBlocks(node.children, code).replace(CALLOUT_MARKER, '');
-        case 'link':
+// GFM's autolink literal ends where its path machine stops: a trail of punctuation is a genuine
+// end (link stops before it) only when trailEndsAt accepts it, else the marks are part of the link.
+var PUNCT = new Set([
+    '!',
+    '"',
+    '&',
+    "'",
+    ')',
+    '*',
+    ',',
+    '.',
+    ';',
+    ':',
+    '<',
+    '?',
+    '_',
+    '~',
+    ']'
+]);
+var TRAIL_PUNCT = new Set([
+    '!',
+    '"',
+    "'",
+    ')',
+    '*',
+    ',',
+    '.',
+    ';',
+    ':',
+    '?',
+    '_',
+    '~'
+]);
+var isAlpha = function isAlpha(c) {
+    return c !== undefined && (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z');
+};
+function trailEndsAt(s, j) {
+    for(;;){
+        var c = s[j];
+        if (c === undefined) return true;
+        if (TRAIL_PUNCT.has(c)) {
+            j += 1;
+            continue;
+        }
+        if (c === '&') {
+            j += 1;
+            if (!isAlpha(s[j])) return false;
+            j += 1;
+            while(isAlpha(s[j]))j += 1;
+            if (s[j] !== ';') return false;
+            j += 1;
+            continue;
+        }
+        if (c === ']') {
+            j += 1;
+            var d = s[j];
+            return d === undefined || d === '(' || d === '[' || RegExp("\\s", "u").test(d !== null && d !== void 0 ? d : ' ');
+        }
+        if (c === '<') return true;
+        if (c !== undefined && RegExp("^\\s$", "u").test(c)) return true;
+        return false;
+    }
+}
+// Where the GFM autolink ends in S (the linkifier's text plus the following text), old-style:
+// punctuation that fails the trail test, or a ) with closes <= opens, extends the link.
+function gfmAutolinkEnd(s) {
+    var open = 0;
+    var close = 0;
+    for(var i = 0; i < s.length; i++){
+        var c = s[i];
+        if (c === '(') {
+            open += 1;
+            continue;
+        }
+        if (c === ')' && close < open) {
+            close += 1;
+            continue;
+        }
+        if (PUNCT.has(c) && trailEndsAt(s, i)) return i;
+        if (c !== undefined && RegExp("^\\s$", "u").test(c)) return i;
+    }
+    return s.length;
+}
+// linkify-it links more than GFM's autolink literal (bare domains, ftp://, //host); only the
+// GFM forms (http(s)://, www., email) drop their text.
+function isGfmUrl(text) {
+    return /^https?:\/\//i.test(text) || /^www\./i.test(text);
+}
+// Only a bare address is GFM's email autolink: a url keeps its @, since a path can hold one.
+function isGfmEmail(text) {
+    return !isGfmUrl(text) && text.includes('@');
+}
+// The task-lists plugin injects its checkbox as the item's first inline child.
+function isTaskCheckbox(token) {
+    return (token === null || token === void 0 ? void 0 : token.type) === 'html_inline' && /^<input\b/i.test(token.content);
+}
+// A linkify span can swallow the & of a following entity (span ...c& + text amp; z); the
+// re-emitted lead is then a reference the entity rule never saw, so decode it old-style.
+var LEAD_ENTITY = /^&[a-z#][a-z0-9]{1,31};/i;
+// Split a flat token range into its top-level blocks: nesting 1 opens, nesting 0 is one block.
+function topLevelBlocks(tokens) {
+    var out = [];
+    var i = 0;
+    while(i < tokens.length){
+        var token = tokens[i];
+        if (token.nesting === 1) {
+            var depth = 1;
+            var j = i + 1;
+            while(j < tokens.length && depth > 0){
+                depth += tokens[j].nesting === 1 ? 1 : tokens[j].nesting === -1 ? -1 : 0;
+                j += 1;
+            }
+            out.push(tokens.slice(i, j));
+            i = j;
+        } else if (token.nesting === 0) {
+            out.push(tokens.slice(i, i + 1));
+            i += 1;
+        } else {
+            i += 1;
+        }
+    }
+    return out;
+}
+// A first line with no source content (a bare task checkbox, whitespace) contributes no line to
+// the value, so its softbreak is dropped; a first line whose content extraction removes (an
+// autolink, an image) still leaves its line ending behind.
+function paragraphText(children, code) {
+    var _children_k;
+    var k = 0;
+    var first = children[0];
+    if (isTaskCheckbox(first)) k = 1;
+    while(k < children.length && children[k].type === 'text' && children[k].content.trim() === '')k += 1;
+    if (k > 0 && ((_children_k = children[k]) === null || _children_k === void 0 ? void 0 : _children_k.type) === 'softbreak') return inlineText(children.slice(k + 1), code);
+    return inlineText(children, code);
+}
+function blockText(tokens, code) {
+    var first = tokens[0];
+    if (!first) return '';
+    switch(first.type){
+        case 'heading_open':
+        case 'paragraph_open':
             {
-                var text = joinInline(node.children, code);
-                return isAutolink(node.url, text) ? '' : text;
+                var _inline_children;
+                var inline = tokens.find(function(t) {
+                    return t.type === 'inline';
+                });
+                return inline ? paragraphText((_inline_children = inline.children) !== null && _inline_children !== void 0 ? _inline_children : [], code) : '';
             }
-        case 'heading':
-        case 'paragraph':
-        case 'linkReference':
-        case 'emphasis':
-        case 'strong':
-        case 'delete':
-            return joinInline(node.children, code);
-        case 'table':
-            return node.children.map(function(n) {
-                return extractNode(n, code);
-            }).join('\n');
-        case 'tableRow':
-            return node.children.map(function(n) {
-                return extractNode(n, code);
-            }).join(' ');
-        case 'tableCell':
-            return joinInline(node.children, code);
-        case 'image':
-        case 'imageReference':
-            var _node_alt;
-            return (_node_alt = node.alt) !== null && _node_alt !== void 0 ? _node_alt : '';
-        case 'text':
-            return node.value;
-        case 'inlineCode':
-        case 'code':
-            return codePlaceholder(node.value, code);
-        case 'html':
-            return stripHtml(node.value);
-        case 'break':
-            return ' ';
+        case 'fence':
+        case 'code_block':
+            // Fence and indented code tokens carry the closing newline their lines were joined with.
+            return codePlaceholder(first.content.replace(/\n$/, ''), code);
+        case 'table_open':
+            return tableText(tokens, code);
+        case 'ordered_list_open':
+        case 'bullet_list_open':
+            return listItemsText(tokens, code);
+        case 'blockquote_open':
+            return blocksIn(tokens.slice(1, -1), code).replace(CALLOUT_MARKER, '');
+        case 'html_block':
+            // A non-final-line html block carries its last line's terminator, which the mdast value lacked.
+            return stripHtml(first.content.replace(/\n$/, ''));
+        case 'footnote_reference_open':
+            return blocksIn(tokens.slice(1, -1), code);
         default:
             return '';
     }
 }
+// Sibling blocks (list items, blockquote children, footnote bodies) joined one per line;
+// empties dropped so a blank child never widens the gap between its neighbors.
+function blocksIn(tokens, code) {
+    return topLevelBlocks(tokens).map(function(b) {
+        return blockText(b, code);
+    }).filter(function(s) {
+        return s.length > 0;
+    }).join('\n');
+}
+// The task-lists plugin slices 3 of the marker's 4 chars, leaving its trailing space on the text
+// after the checkbox token. Returns a copy with that space dropped, never a mutation: a block is
+// extracted more than once (parse pre-extracts headings, group.ts extracts again).
+function withoutTaskMarker(item) {
+    for(var i = 0; i < item.length; i++){
+        var _token_children;
+        var token = item[i];
+        // The checkbox, if any, sits in the item's first inline only.
+        if (token.type !== 'inline' || !((_token_children = token.children) === null || _token_children === void 0 ? void 0 : _token_children.length)) continue;
+        var second = token.children[1];
+        if (!isTaskCheckbox(token.children[0]) || (second === null || second === void 0 ? void 0 : second.type) !== 'text' || !second.content.startsWith(' ')) return item;
+        var children = token.children.slice();
+        children[1] = _object_spread_props(_object_spread({}, second), {
+            content: second.content.slice(1)
+        });
+        var copy = item.slice();
+        copy[i] = _object_spread_props(_object_spread({}, token), {
+            children: children
+        });
+        return copy;
+    }
+    return item;
+}
+function listItemsText(tokens, code) {
+    var items = [];
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = topLevelBlocks(tokens.slice(1, -1))[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var item = _step.value;
+            items.push(blocksIn(withoutTaskMarker(item).slice(1, -1), code));
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    return items.filter(function(s) {
+        return s.length > 0;
+    }).join('\n');
+}
+// Table rows joined by newline, cells by space (mdast's tableRow/tableCell joiners).
+function tableText(tokens, code) {
+    var rows = [];
+    var cells = [];
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = tokens[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var token = _step.value;
+            var _token_children;
+            if (token.type === 'tr_open') cells = [];
+            else if (token.type === 'inline') cells.push(inlineText((_token_children = token.children) !== null && _token_children !== void 0 ? _token_children : [], code));
+            else if (token.type === 'tr_close') rows.push(cells.join(' '));
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    return rows.join('\n');
+}
+// Inline content (headings, paragraphs, emphasis, links) concatenated with no separator: the
+// source text tokens already carry whatever spacing separates them.
+function inlineText(tokens, code) {
+    var inAlt = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : false;
+    var out = '';
+    for(var i = 0; i < tokens.length; i++){
+        var token = tokens[i];
+        if (token.type === 'link_open') {
+            var skip = 0;
+            var j = i + 1;
+            var depth = 1;
+            while(j < tokens.length && depth > 0){
+                if (tokens[j].type === 'link_open') depth += 1;
+                else if (tokens[j].type === 'link_close') depth -= 1;
+                j += 1;
+            }
+            var text = inlineText(tokens.slice(i + 1, j - 1), code, inAlt);
+            var markup = token.markup;
+            var next = tokens[j];
+            if (inAlt) {
+                // An image's alt is plain text: mdast kept every link's own text there, autolinks included.
+                out += text;
+            } else if (markup === 'linkify' && isGfmUrl(text)) {
+                // GFM drop: the next text token resumes at the old link's boundary, re-emitting any
+                // trimmed trail. <www.x> has no trail to absorb (the brackets bound it), keep the >.
+                // Emails keep their whole remainder: GFM's email machine has no trail, so the span
+                // already ends at the old boundary and the entity rule has decoded the rest.
+                var prev = tokens[i - 1];
+                var bracketed = (prev === null || prev === void 0 ? void 0 : prev.type) === 'text' && prev.content.endsWith('<') && (next === null || next === void 0 ? void 0 : next.type) === 'text' && next.content.startsWith('>');
+                var tail = (next === null || next === void 0 ? void 0 : next.type) === 'text' ? text + next.content : text;
+                var k = bracketed ? text.length : gfmAutolinkEnd(tail);
+                out += tail.slice(k).replace(LEAD_ENTITY, function(m) {
+                    return (0, _parserts.parser)().utils.unescapeAll(m);
+                });
+                // The trail came out of the next text token, so that token is consumed here, not re-read.
+                if ((next === null || next === void 0 ? void 0 : next.type) === 'text') skip = 1;
+            } else if (markup === 'linkify' && isGfmEmail(text)) {
+            // Email drop: nothing after the span to trim, so the span alone is dropped.
+            } else if (markup === 'autolink') {
+            // <...> leaf: dropped whole, its brackets live inside the token so nothing follows to trim.
+            } else {
+                out += text; // user link or non-GFM linkify target: keep the display text
+            }
+            i = j - 1 + skip;
+        } else if (token.type === 'text') {
+            out += token.content;
+        } else if (token.type === 'softbreak') {
+            out += '\n';
+        } else if (token.type === 'hardbreak') {
+            out += ' ';
+        } else if (token.type === 'code_inline') {
+            out += codePlaceholder(token.content, code);
+        } else if (token.type === 'html_inline') {
+            out += stripHtml(token.content);
+        } else if (token.type === 'image') {
+            var _token_children;
+            out += inlineText((_token_children = token.children) !== null && _token_children !== void 0 ? _token_children : [], code, true);
+        }
+    // Emphasis, strikethrough and footnote markers carry no text of their own.
+    }
+    return out;
+}
 var CODE_PLACEHOLDER = /\uE000(\d+)\uE000/g;
-function extractText(node) {
+function extractText(block) {
     return extractTexts([
-        node
+        block
     ]);
 }
-function extractTexts(nodes) {
+function extractTexts(blocks) {
     var code = [];
-    var joined = nodes.map(function(n) {
-        return extractNode(n, code);
+    var joined = blocks.map(function(b) {
+        return blockText(b.node, code);
     }).filter(function(s) {
         return s.length > 0;
     }).join('\n');

package/dist/cjs/chunk/extract.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/extract.ts"],"sourcesContent":["import type { Nodes } from 'mdast';\n\n// mdast has no wikilink, embed, or %%comment%% node -- literal text resolved by regex passes\n// over non-code text (code/inlineCode is placeholder-held and spliced back verbatim, below).\nconst COMMENT_PAIR = /%%[\\s\\S]*?%%/g;\n// An embed always yields its target, whatever follows a pipe (a resize suffix, never an alias);\n// must run before the wikilink regex below, bang included, or that regex matches first.\nconst EMBED = /!\\[\\[([^\\]]+)\\]\\]/g;\nconst WIKILINK = /\\[\\[([^\\]]+)\\]\\]/g;\n// A trailing \" ^id\" is an Obsidian block anchor, valid only at a line's end; a caret elsewhere\n// (mid-line) is ordinary text and is left alone.\nconst BLOCK_ID = / \\^[A-Za-z0-9-]+$/gm;\n// A callout marker is only ever the first thing in a blockquote's text (Obsidian's grammar),\n// so this anchors to blockquote output alone -- never applied to prose in general.\nconst CALLOUT_MARKER = /^\\[!\\w[\\w-]*\\][+-]?[ \\t]?/;\n// An html node's value is raw HTML, block or inline: a full <!-- --> comment is dropped, any\n// remaining tags are stripped, and the text a browser would still render survives.\nconst HTML_COMMENT = /<!--[\\s\\S]*?-->/g;\nconst HTML_TAG = /<[^>]+>/g;\n\nfunction stripHtml(value: string): string {\n  return value.replace(HTML_COMMENT, '').replace(HTML_TAG, '');\n}\n\n// A `#anchor` keeps its text (hash dropped), unlike links.ts's parseWikilinkInner (the resolver\n// authority), which discards it entirely -- replicated here since that return value doesn't fit.\nfunction anchorText(base: string): string {\n  const hashIdx = base.indexOf('#');\n  if (hashIdx === -1) return base.trim();\n  const target = base.slice(0, hashIdx).trim();\n  const anchor = base.slice(hashIdx + 1).trim();\n  return target ? `${target} ${anchor}` : anchor;\n}\n\nfunction wikilinkText(inner: string): string {\n  const pipeIdx = inner.indexOf('|');\n  return pipeIdx === -1 ? anchorText(inner) : inner.slice(pipeIdx + 1).trim();\n}\n\nfunction embedText(inner: string): string {\n  const pipeIdx = inner.indexOf('|');\n  return anchorText(pipeIdx === -1 ? inner : inner.slice(0, pipeIdx));\n}\n\nfunction resolveFlavor(text: string): string {\n  return text\n    .replace(HTML_COMMENT, '')\n    .replace(COMMENT_PAIR, '')\n    .replace(EMBED, (_, inner: string) => embedText(inner))\n    .replace(WIKILINK, (_, inner: string) => wikilinkText(inner))\n    .replace(BLOCK_ID, '');\n}\n\n// Sibling blocks (list items, blockquote content, footnote bodies) joined one per line;\n// empties dropped so a blank child never widens the gap between its neighbors.\nfunction joinBlocks(nodes: Nodes[], code: string[]): string {\n  return nodes\n    .map((n) => extractNode(n, code))\n    .filter((s) => s.length > 0)\n    .join('\\n');\n}\n\n// Inline content (headings, paragraphs, emphasis, links) concatenated with no separator: the\n// source text nodes already carry whatever spacing separates them.\nfunction joinInline(nodes: Nodes[], code: string[]): string {\n  return nodes.map((n) => extractNode(n, code)).join('');\n}\n\n// U+E000 (Private Use Area) never appears in real markdown text, so it is a collision-free\n// placeholder delimiter, spliced back to the literal code value after flavor resolution.\nfunction codePlaceholder(value: string, code: string[]): string {\n  const idx = code.push(value) - 1;\n  return `\\uE000${idx}\\uE000`;\n}\n\n// A GFM autolink's display text is its own target, missing only the scheme\n// mdast-util-gfm-autolink-literal fills into node.url (http:// for www., mailto: for an email).\nconst SCHEME_PREFIX = /^[a-z][a-z0-9+.-]*:\\/{0,2}/i;\nfunction isAutolink(url: string, text: string): boolean {\n  return url === text || url.replace(SCHEME_PREFIX, '') === text;\n}\n\nfunction extractNode(node: Nodes, code: string[]): string {\n  switch (node.type) {\n    case 'root':\n    case 'list':\n    case 'listItem':\n    case 'footnoteDefinition':\n      return joinBlocks(node.children, code);\n    case 'blockquote':\n      return joinBlocks(node.children, code).replace(CALLOUT_MARKER, '');\n    case 'link': {\n      const text = joinInline(node.children, code);\n      return isAutolink(node.url, text) ? '' : text;\n    }\n    case 'heading':\n    case 'paragraph':\n    case 'linkReference':\n    case 'emphasis':\n    case 'strong':\n    case 'delete':\n      return joinInline(node.children, code);\n    case 'table':\n      return node.children.map((n) => extractNode(n, code)).join('\\n');\n    case 'tableRow':\n      return node.children.map((n) => extractNode(n, code)).join(' ');\n    case 'tableCell':\n      return joinInline(node.children, code);\n    case 'image':\n    case 'imageReference':\n      return node.alt ?? '';\n    case 'text':\n      return node.value;\n    case 'inlineCode':\n    case 'code':\n      return codePlaceholder(node.value, code);\n    case 'html':\n      return stripHtml(node.value);\n    case 'break':\n      return ' ';\n    default:\n      return '';\n  }\n}\n\nconst CODE_PLACEHOLDER = /\\uE000(\\d+)\\uE000/g;\n\n// Plain text of one mdast node (or a whole tree): heading/list/table structure is kept as text,\n// markup (emphasis, link targets, task and callout markers) is dropped. Pure, synchronous.\nexport function extractText(node: Nodes): string {\n  return extractTexts([node]);\n}\n\n// Blocks resolved together: a blank line inside %%...%% splits it across blocks, and a per-block\n// strip then never sees the closing %%. Code stays placeholder-held across all of them.\nexport function extractTexts(nodes: Nodes[]): string {\n  const code: string[] = [];\n  const joined = nodes\n    .map((n) => extractNode(n, code))\n    .filter((s) => s.length > 0)\n    .join('\\n');\n  const resolved = resolveFlavor(joined);\n  return code.length === 0 ? resolved : resolved.replace(CODE_PLACEHOLDER, (_, i: string) => code[Number(i)]);\n}\n"],"names":["extractText","extractTexts","COMMENT_PAIR","EMBED","WIKILINK","BLOCK_ID","CALLOUT_MARKER","HTML_COMMENT","HTML_TAG","stripHtml","value","replace","anchorText","base","hashIdx","indexOf","trim","target","slice","anchor","wikilinkText","inner","pipeIdx","embedText","resolveFlavor","text","_","joinBlocks","nodes","code","map","n","extractNode","filter","s","length","join","joinInline","codePlaceholder","idx","push","SCHEME_PREFIX","isAutolink","url","node","type","children","alt","CODE_PLACEHOLDER","joined","resolved","i","Number"],"mappings":";;;;;;;;;;;QAiIgBA;eAAAA;;QAMAC;eAAAA;;;AArIhB,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,eAAe;AACrB,gGAAgG;AAChG,wFAAwF;AACxF,IAAMC,QAAQ;AACd,IAAMC,WAAW;AACjB,+FAA+F;AAC/F,iDAAiD;AACjD,IAAMC,WAAW;AACjB,6FAA6F;AAC7F,mFAAmF;AACnF,IAAMC,iBAAiB;AACvB,6FAA6F;AAC7F,mFAAmF;AACnF,IAAMC,eAAe;AACrB,IAAMC,WAAW;AAEjB,SAASC,UAAUC,KAAa;IAC9B,OAAOA,MAAMC,OAAO,CAACJ,cAAc,IAAII,OAAO,CAACH,UAAU;AAC3D;AAEA,gGAAgG;AAChG,iGAAiG;AACjG,SAASI,WAAWC,IAAY;IAC9B,IAAMC,UAAUD,KAAKE,OAAO,CAAC;IAC7B,IAAID,YAAY,CAAC,GAAG,OAAOD,KAAKG,IAAI;IACpC,IAAMC,SAASJ,KAAKK,KAAK,CAAC,GAAGJ,SAASE,IAAI;IAC1C,IAAMG,SAASN,KAAKK,KAAK,CAACJ,UAAU,GAAGE,IAAI;IAC3C,OAAOC,SAAS,AAAC,GAAYE,OAAVF,QAAO,KAAU,OAAPE,UAAWA;AAC1C;AAEA,SAASC,aAAaC,KAAa;IACjC,IAAMC,UAAUD,MAAMN,OAAO,CAAC;IAC9B,OAAOO,YAAY,CAAC,IAAIV,WAAWS,SAASA,MAAMH,KAAK,CAACI,UAAU,GAAGN,IAAI;AAC3E;AAEA,SAASO,UAAUF,KAAa;IAC9B,IAAMC,UAAUD,MAAMN,OAAO,CAAC;IAC9B,OAAOH,WAAWU,YAAY,CAAC,IAAID,QAAQA,MAAMH,KAAK,CAAC,GAAGI;AAC5D;AAEA,SAASE,cAAcC,IAAY;IACjC,OAAOA,KACJd,OAAO,CAACJ,cAAc,IACtBI,OAAO,CAACT,cAAc,IACtBS,OAAO,CAACR,OAAO,SAACuB,GAAGL;eAAkBE,UAAUF;OAC/CV,OAAO,CAACP,UAAU,SAACsB,GAAGL;eAAkBD,aAAaC;OACrDV,OAAO,CAACN,UAAU;AACvB;AAEA,wFAAwF;AACxF,+EAA+E;AAC/E,SAASsB,WAAWC,KAAc,EAAEC,IAAc;IAChD,OAAOD,MACJE,GAAG,CAAC,SAACC;eAAMC,YAAYD,GAAGF;OAC1BI,MAAM,CAAC,SAACC;eAAMA,EAAEC,MAAM,GAAG;OACzBC,IAAI,CAAC;AACV;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,SAASC,WAAWT,KAAc,EAAEC,IAAc;IAChD,OAAOD,MAAME,GAAG,CAAC,SAACC;eAAMC,YAAYD,GAAGF;OAAOO,IAAI,CAAC;AACrD;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASE,gBAAgB5B,KAAa,EAAEmB,IAAc;IACpD,IAAMU,MAAMV,KAAKW,IAAI,CAAC9B,SAAS;IAC/B,OAAO,AAAC,IAAY,OAAJ6B,KAAI;AACtB;AAEA,2EAA2E;AAC3E,gGAAgG;AAChG,IAAME,gBAAgB;AACtB,SAASC,WAAWC,GAAW,EAAElB,IAAY;IAC3C,OAAOkB,QAAQlB,QAAQkB,IAAIhC,OAAO,CAAC8B,eAAe,QAAQhB;AAC5D;AAEA,SAASO,YAAYY,IAAW,EAAEf,IAAc;IAC9C,OAAQe,KAAKC,IAAI;QACf,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAOlB,WAAWiB,KAAKE,QAAQ,EAAEjB;QACnC,KAAK;YACH,OAAOF,WAAWiB,KAAKE,QAAQ,EAAEjB,MAAMlB,OAAO,CAACL,gBAAgB;QACjE,KAAK;YAAQ;gBACX,IAAMmB,OAAOY,WAAWO,KAAKE,QAAQ,EAAEjB;gBACvC,OAAOa,WAAWE,KAAKD,GAAG,EAAElB,QAAQ,KAAKA;YAC3C;QACA,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAOY,WAAWO,KAAKE,QAAQ,EAAEjB;QACnC,KAAK;YACH,OAAOe,KAAKE,QAAQ,CAAChB,GAAG,CAAC,SAACC;uBAAMC,YAAYD,GAAGF;eAAOO,IAAI,CAAC;QAC7D,KAAK;YACH,OAAOQ,KAAKE,QAAQ,CAAChB,GAAG,CAAC,SAACC;uBAAMC,YAAYD,GAAGF;eAAOO,IAAI,CAAC;QAC7D,KAAK;YACH,OAAOC,WAAWO,KAAKE,QAAQ,EAAEjB;QACnC,KAAK;QACL,KAAK;gBACIe;YAAP,QAAOA,YAAAA,KAAKG,GAAG,cAARH,uBAAAA,YAAY;QACrB,KAAK;YACH,OAAOA,KAAKlC,KAAK;QACnB,KAAK;QACL,KAAK;YACH,OAAO4B,gBAAgBM,KAAKlC,KAAK,EAAEmB;QACrC,KAAK;YACH,OAAOpB,UAAUmC,KAAKlC,KAAK;QAC7B,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,IAAMsC,mBAAmB;AAIlB,SAAShD,YAAY4C,IAAW;IACrC,OAAO3C,aAAa;QAAC2C;KAAK;AAC5B;AAIO,SAAS3C,aAAa2B,KAAc;IACzC,IAAMC,OAAiB,EAAE;IACzB,IAAMoB,SAASrB,MACZE,GAAG,CAAC,SAACC;eAAMC,YAAYD,GAAGF;OAC1BI,MAAM,CAAC,SAACC;eAAMA,EAAEC,MAAM,GAAG;OACzBC,IAAI,CAAC;IACR,IAAMc,WAAW1B,cAAcyB;IAC/B,OAAOpB,KAAKM,MAAM,KAAK,IAAIe,WAAWA,SAASvC,OAAO,CAACqC,kBAAkB,SAACtB,GAAGyB;eAActB,IAAI,CAACuB,OAAOD,GAAG;;AAC5G"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/extract.ts"],"sourcesContent":["import type { Token } from 'markdown-it';\nimport { parser } from './parser.ts';\nimport type { Block } from './types.ts';\n\n// markdown-it, like mdast, has no wikilink, embed, or %%comment%% token -- literal text resolved\n// by regex passes over non-code text (code/inline code is placeholder-held and spliced back verbatim).\nconst COMMENT_PAIR = /%%[\\s\\S]*?%%/g;\n// An embed always yields its target, whatever follows a pipe (a resize suffix, never an alias);\n// must run before the wikilink regex below, bang included, or that regex matches first.\nconst EMBED = /!\\[\\[([^\\]]+)\\]\\]/g;\nconst WIKILINK = /\\[\\[([^\\]]+)\\]\\]/g;\n// A trailing \" ^id\" is an Obsidian block anchor, valid only at a line's end; a caret elsewhere\n// (mid-line) is ordinary text and is left alone.\nconst BLOCK_ID = / \\^[A-Za-z0-9-]+$/gm;\n// A callout marker is only ever the first thing in a blockquote's text (Obsidian's grammar),\n// so this anchors to blockquote output alone -- never applied to prose in general.\nconst CALLOUT_MARKER = /^\\[!\\w[\\w-]*\\][+-]?[ \\t]?/;\n// An html token's content is raw HTML, block or inline: a full <!-- --> comment is dropped, any\n// remaining tags are stripped, and the text a browser would still render survives.\nconst HTML_COMMENT = /<!--[\\s\\S]*?-->/g;\nconst HTML_TAG = /<[^>]+>/g;\n\nfunction stripHtml(value: string): string {\n  return value.replace(HTML_COMMENT, '').replace(HTML_TAG, '');\n}\n\n// A `#anchor` keeps its text (hash dropped), unlike links.ts's parseWikilinkInner (the resolver\n// authority), which discards it entirely -- replicated here since that return value doesn't fit.\nfunction anchorText(base: string): string {\n  const hashIdx = base.indexOf('#');\n  if (hashIdx === -1) return base.trim();\n  const target = base.slice(0, hashIdx).trim();\n  const anchor = base.slice(hashIdx + 1).trim();\n  return target ? `${target} ${anchor}` : anchor;\n}\n\nfunction wikilinkText(inner: string): string {\n  const pipeIdx = inner.indexOf('|');\n  return pipeIdx === -1 ? anchorText(inner) : inner.slice(pipeIdx + 1).trim();\n}\n\nfunction embedText(inner: string): string {\n  const pipeIdx = inner.indexOf('|');\n  return anchorText(pipeIdx === -1 ? inner : inner.slice(0, pipeIdx));\n}\n\nfunction resolveFlavor(text: string): string {\n  return text\n    .replace(HTML_COMMENT, '')\n    .replace(COMMENT_PAIR, '')\n    .replace(EMBED, (_, inner: string) => embedText(inner))\n    .replace(WIKILINK, (_, inner: string) => wikilinkText(inner))\n    .replace(BLOCK_ID, '');\n}\n\n// U+E000 (Private Use Area) never appears in real markdown text, so it is a collision-free\n// placeholder delimiter, spliced back to the literal code value after flavor resolution.\nfunction codePlaceholder(value: string, code: string[]): string {\n  const idx = code.push(value) - 1;\n  return `\\uE000${idx}\\uE000`;\n}\n\n// GFM's autolink literal ends where its path machine stops: a trail of punctuation is a genuine\n// end (link stops before it) only when trailEndsAt accepts it, else the marks are part of the link.\nconst PUNCT = new Set(['!', '\"', '&', \"'\", ')', '*', ',', '.', ';', ':', '<', '?', '_', '~', ']']);\nconst TRAIL_PUNCT = new Set(['!', '\"', \"'\", ')', '*', ',', '.', ';', ':', '?', '_', '~']);\nconst isAlpha = (c: string | undefined): boolean => c !== undefined && ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'));\nfunction trailEndsAt(s: string, j: number): boolean {\n  for (;;) {\n    const c = s[j];\n    if (c === undefined) return true;\n    if (TRAIL_PUNCT.has(c)) {\n      j += 1;\n      continue;\n    }\n    if (c === '&') {\n      j += 1;\n      if (!isAlpha(s[j])) return false;\n      j += 1;\n      while (isAlpha(s[j])) j += 1;\n      if (s[j] !== ';') return false;\n      j += 1;\n      continue;\n    }\n    if (c === ']') {\n      j += 1;\n      const d = s[j];\n      return d === undefined || d === '(' || d === '[' || /\\s/u.test(d ?? ' ');\n    }\n    if (c === '<') return true;\n    if (c !== undefined && /^\\s$/u.test(c)) return true;\n    return false;\n  }\n}\n// Where the GFM autolink ends in S (the linkifier's text plus the following text), old-style:\n// punctuation that fails the trail test, or a ) with closes <= opens, extends the link.\nfunction gfmAutolinkEnd(s: string): number {\n  let open = 0;\n  let close = 0;\n  for (let i = 0; i < s.length; i++) {\n    const c = s[i];\n    if (c === '(') {\n      open += 1;\n      continue;\n    }\n    if (c === ')' && close < open) {\n      close += 1;\n      continue;\n    }\n    if (PUNCT.has(c) && trailEndsAt(s, i)) return i;\n    if (c !== undefined && /^\\s$/u.test(c)) return i;\n  }\n  return s.length;\n}\n// linkify-it links more than GFM's autolink literal (bare domains, ftp://, //host); only the\n// GFM forms (http(s)://, www., email) drop their text.\nfunction isGfmUrl(text: string): boolean {\n  return /^https?:\\/\\//i.test(text) || /^www\\./i.test(text);\n}\n// Only a bare address is GFM's email autolink: a url keeps its @, since a path can hold one.\nfunction isGfmEmail(text: string): boolean {\n  return !isGfmUrl(text) && text.includes('@');\n}\n// The task-lists plugin injects its checkbox as the item's first inline child.\nfunction isTaskCheckbox(token: Token | undefined): boolean {\n  return token?.type === 'html_inline' && /^<input\\b/i.test(token.content);\n}\n// A linkify span can swallow the & of a following entity (span ...c& + text amp; z); the\n// re-emitted lead is then a reference the entity rule never saw, so decode it old-style.\nconst LEAD_ENTITY = /^&[a-z#][a-z0-9]{1,31};/i;\n\n// Split a flat token range into its top-level blocks: nesting 1 opens, nesting 0 is one block.\nfunction topLevelBlocks(tokens: Token[]): Token[][] {\n  const out: Token[][] = [];\n  let i = 0;\n  while (i < tokens.length) {\n    const token = tokens[i];\n    if (token.nesting === 1) {\n      let depth = 1;\n      let j = i + 1;\n      while (j < tokens.length && depth > 0) {\n        depth += tokens[j].nesting === 1 ? 1 : tokens[j].nesting === -1 ? -1 : 0;\n        j += 1;\n      }\n      out.push(tokens.slice(i, j));\n      i = j;\n    } else if (token.nesting === 0) {\n      out.push(tokens.slice(i, i + 1));\n      i += 1;\n    } else {\n      i += 1;\n    }\n  }\n  return out;\n}\n\n// A first line with no source content (a bare task checkbox, whitespace) contributes no line to\n// the value, so its softbreak is dropped; a first line whose content extraction removes (an\n// autolink, an image) still leaves its line ending behind.\nfunction paragraphText(children: Token[], code: string[]): string {\n  let k = 0;\n  const first = children[0];\n  if (isTaskCheckbox(first)) k = 1;\n  while (k < children.length && children[k].type === 'text' && children[k].content.trim() === '') k += 1;\n  if (k > 0 && children[k]?.type === 'softbreak') return inlineText(children.slice(k + 1), code);\n  return inlineText(children, code);\n}\n\nfunction blockText(tokens: Token[], code: string[]): string {\n  const first = tokens[0];\n  if (!first) return '';\n  switch (first.type) {\n    case 'heading_open':\n    case 'paragraph_open': {\n      const inline = tokens.find((t) => t.type === 'inline');\n      return inline ? paragraphText(inline.children ?? [], code) : '';\n    }\n    case 'fence':\n    case 'code_block':\n      // Fence and indented code tokens carry the closing newline their lines were joined with.\n      return codePlaceholder(first.content.replace(/\\n$/, ''), code);\n    case 'table_open':\n      return tableText(tokens, code);\n    case 'ordered_list_open':\n    case 'bullet_list_open':\n      return listItemsText(tokens, code);\n    case 'blockquote_open':\n      return blocksIn(tokens.slice(1, -1), code).replace(CALLOUT_MARKER, '');\n    case 'html_block':\n      // A non-final-line html block carries its last line's terminator, which the mdast value lacked.\n      return stripHtml(first.content.replace(/\\n$/, ''));\n    case 'footnote_reference_open':\n      return blocksIn(tokens.slice(1, -1), code);\n    default:\n      return '';\n  }\n}\n\n// Sibling blocks (list items, blockquote children, footnote bodies) joined one per line;\n// empties dropped so a blank child never widens the gap between its neighbors.\nfunction blocksIn(tokens: Token[], code: string[]): string {\n  return topLevelBlocks(tokens)\n    .map((b) => blockText(b, code))\n    .filter((s) => s.length > 0)\n    .join('\\n');\n}\n\n// The task-lists plugin slices 3 of the marker's 4 chars, leaving its trailing space on the text\n// after the checkbox token. Returns a copy with that space dropped, never a mutation: a block is\n// extracted more than once (parse pre-extracts headings, group.ts extracts again).\nfunction withoutTaskMarker(item: Token[]): Token[] {\n  for (let i = 0; i < item.length; i++) {\n    const token = item[i];\n    // The checkbox, if any, sits in the item's first inline only.\n    if (token.type !== 'inline' || !token.children?.length) continue;\n    const second = token.children[1];\n    if (!isTaskCheckbox(token.children[0]) || second?.type !== 'text' || !second.content.startsWith(' ')) return item;\n    const children = token.children.slice();\n    children[1] = { ...second, content: second.content.slice(1) } as Token;\n    const copy = item.slice();\n    copy[i] = { ...token, children } as Token;\n    return copy;\n  }\n  return item;\n}\n\nfunction listItemsText(tokens: Token[], code: string[]): string {\n  const items: string[] = [];\n  for (const item of topLevelBlocks(tokens.slice(1, -1))) {\n    items.push(blocksIn(withoutTaskMarker(item).slice(1, -1), code));\n  }\n  return items.filter((s) => s.length > 0).join('\\n');\n}\n\n// Table rows joined by newline, cells by space (mdast's tableRow/tableCell joiners).\nfunction tableText(tokens: Token[], code: string[]): string {\n  const rows: string[] = [];\n  let cells: string[] = [];\n  for (const token of tokens) {\n    if (token.type === 'tr_open') cells = [];\n    else if (token.type === 'inline') cells.push(inlineText(token.children ?? [], code));\n    else if (token.type === 'tr_close') rows.push(cells.join(' '));\n  }\n  return rows.join('\\n');\n}\n\n// Inline content (headings, paragraphs, emphasis, links) concatenated with no separator: the\n// source text tokens already carry whatever spacing separates them.\nfunction inlineText(tokens: Token[], code: string[], inAlt = false): string {\n  let out = '';\n  for (let i = 0; i < tokens.length; i++) {\n    const token = tokens[i];\n    if (token.type === 'link_open') {\n      let skip = 0;\n      let j = i + 1;\n      let depth = 1;\n      while (j < tokens.length && depth > 0) {\n        if (tokens[j].type === 'link_open') depth += 1;\n        else if (tokens[j].type === 'link_close') depth -= 1;\n        j += 1;\n      }\n      const text = inlineText(tokens.slice(i + 1, j - 1), code, inAlt);\n      const markup = token.markup;\n      const next = tokens[j];\n      if (inAlt) {\n        // An image's alt is plain text: mdast kept every link's own text there, autolinks included.\n        out += text;\n      } else if (markup === 'linkify' && isGfmUrl(text)) {\n        // GFM drop: the next text token resumes at the old link's boundary, re-emitting any\n        // trimmed trail. <www.x> has no trail to absorb (the brackets bound it), keep the >.\n        // Emails keep their whole remainder: GFM's email machine has no trail, so the span\n        // already ends at the old boundary and the entity rule has decoded the rest.\n        const prev = tokens[i - 1];\n        const bracketed = prev?.type === 'text' && prev.content.endsWith('<') && next?.type === 'text' && next.content.startsWith('>');\n        const tail = next?.type === 'text' ? text + next.content : text;\n        const k = bracketed ? text.length : gfmAutolinkEnd(tail);\n        out += tail.slice(k).replace(LEAD_ENTITY, (m) => parser().utils.unescapeAll(m));\n        // The trail came out of the next text token, so that token is consumed here, not re-read.\n        if (next?.type === 'text') skip = 1;\n      } else if (markup === 'linkify' && isGfmEmail(text)) {\n        // Email drop: nothing after the span to trim, so the span alone is dropped.\n      } else if (markup === 'autolink') {\n        // <...> leaf: dropped whole, its brackets live inside the token so nothing follows to trim.\n      } else {\n        out += text; // user link or non-GFM linkify target: keep the display text\n      }\n      i = j - 1 + skip;\n    } else if (token.type === 'text') {\n      out += token.content;\n    } else if (token.type === 'softbreak') {\n      out += '\\n';\n    } else if (token.type === 'hardbreak') {\n      out += ' ';\n    } else if (token.type === 'code_inline') {\n      out += codePlaceholder(token.content, code);\n    } else if (token.type === 'html_inline') {\n      out += stripHtml(token.content);\n    } else if (token.type === 'image') {\n      out += inlineText(token.children ?? [], code, true);\n    }\n    // Emphasis, strikethrough and footnote markers carry no text of their own.\n  }\n  return out;\n}\n\nconst CODE_PLACEHOLDER = /\\uE000(\\d+)\\uE000/g;\n\n// Plain text of one parsed block: heading/list/table structure is kept as text, markup (emphasis,\n// link targets, task and callout markers) is dropped. Pure, synchronous.\nexport function extractText(block: Block): string {\n  return extractTexts([block]);\n}\n\n// Blocks resolved together: a blank line inside %%...%% splits it across blocks, and a per-block\n// strip then never sees the closing %%. Code stays placeholder-held across all of them.\nexport function extractTexts(blocks: Block[]): string {\n  const code: string[] = [];\n  const joined = blocks\n    .map((b) => blockText(b.node, code))\n    .filter((s) => s.length > 0)\n    .join('\\n');\n  const resolved = resolveFlavor(joined);\n  return code.length === 0 ? resolved : resolved.replace(CODE_PLACEHOLDER, (_, i: string) => code[Number(i)]);\n}\n"],"names":["extractText","extractTexts","COMMENT_PAIR","EMBED","WIKILINK","BLOCK_ID","CALLOUT_MARKER","HTML_COMMENT","HTML_TAG","stripHtml","value","replace","anchorText","base","hashIdx","indexOf","trim","target","slice","anchor","wikilinkText","inner","pipeIdx","embedText","resolveFlavor","text","_","codePlaceholder","code","idx","push","PUNCT","Set","TRAIL_PUNCT","isAlpha","c","undefined","trailEndsAt","s","j","has","d","test","gfmAutolinkEnd","open","close","i","length","isGfmUrl","isGfmEmail","includes","isTaskCheckbox","token","type","content","LEAD_ENTITY","topLevelBlocks","tokens","out","nesting","depth","paragraphText","children","k","first","inlineText","blockText","inline","find","t","tableText","listItemsText","blocksIn","map","b","filter","join","withoutTaskMarker","item","second","startsWith","copy","items","rows","cells","inAlt","skip","markup","next","prev","bracketed","endsWith","tail","m","parser","utils","unescapeAll","CODE_PLACEHOLDER","block","blocks","joined","node","resolved","Number"],"mappings":";;;;;;;;;;;QAqTgBA;eAAAA;;QAMAC;eAAAA;;;wBA1TO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGvB,iGAAiG;AACjG,uGAAuG;AACvG,IAAMC,eAAe;AACrB,gGAAgG;AAChG,wFAAwF;AACxF,IAAMC,QAAQ;AACd,IAAMC,WAAW;AACjB,+FAA+F;AAC/F,iDAAiD;AACjD,IAAMC,WAAW;AACjB,6FAA6F;AAC7F,mFAAmF;AACnF,IAAMC,iBAAiB;AACvB,gGAAgG;AAChG,mFAAmF;AACnF,IAAMC,eAAe;AACrB,IAAMC,WAAW;AAEjB,SAASC,UAAUC,KAAa;IAC9B,OAAOA,MAAMC,OAAO,CAACJ,cAAc,IAAII,OAAO,CAACH,UAAU;AAC3D;AAEA,gGAAgG;AAChG,iGAAiG;AACjG,SAASI,WAAWC,IAAY;IAC9B,IAAMC,UAAUD,KAAKE,OAAO,CAAC;IAC7B,IAAID,YAAY,CAAC,GAAG,OAAOD,KAAKG,IAAI;IACpC,IAAMC,SAASJ,KAAKK,KAAK,CAAC,GAAGJ,SAASE,IAAI;IAC1C,IAAMG,SAASN,KAAKK,KAAK,CAACJ,UAAU,GAAGE,IAAI;IAC3C,OAAOC,SAAS,AAAC,GAAYE,OAAVF,QAAO,KAAU,OAAPE,UAAWA;AAC1C;AAEA,SAASC,aAAaC,KAAa;IACjC,IAAMC,UAAUD,MAAMN,OAAO,CAAC;IAC9B,OAAOO,YAAY,CAAC,IAAIV,WAAWS,SAASA,MAAMH,KAAK,CAACI,UAAU,GAAGN,IAAI;AAC3E;AAEA,SAASO,UAAUF,KAAa;IAC9B,IAAMC,UAAUD,MAAMN,OAAO,CAAC;IAC9B,OAAOH,WAAWU,YAAY,CAAC,IAAID,QAAQA,MAAMH,KAAK,CAAC,GAAGI;AAC5D;AAEA,SAASE,cAAcC,IAAY;IACjC,OAAOA,KACJd,OAAO,CAACJ,cAAc,IACtBI,OAAO,CAACT,cAAc,IACtBS,OAAO,CAACR,OAAO,SAACuB,GAAGL;eAAkBE,UAAUF;OAC/CV,OAAO,CAACP,UAAU,SAACsB,GAAGL;eAAkBD,aAAaC;OACrDV,OAAO,CAACN,UAAU;AACvB;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASsB,gBAAgBjB,KAAa,EAAEkB,IAAc;IACpD,IAAMC,MAAMD,KAAKE,IAAI,CAACpB,SAAS;IAC/B,OAAO,AAAC,IAAY,OAAJmB,KAAI;AACtB;AAEA,gGAAgG;AAChG,oGAAoG;AACpG,IAAME,QAAQ,IAAIC,IAAI;IAAC;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;CAAI;AACjG,IAAMC,cAAc,IAAID,IAAI;IAAC;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;IAAK;CAAI;AACxF,IAAME,UAAU,iBAACC;WAAmCA,MAAMC,aAAc,CAAA,AAACD,KAAK,OAAOA,KAAK,OAASA,KAAK,OAAOA,KAAK,GAAG;;AACvH,SAASE,YAAYC,CAAS,EAAEC,CAAS;IACvC,OAAS;QACP,IAAMJ,IAAIG,CAAC,CAACC,EAAE;QACd,IAAIJ,MAAMC,WAAW,OAAO;QAC5B,IAAIH,YAAYO,GAAG,CAACL,IAAI;YACtBI,KAAK;YACL;QACF;QACA,IAAIJ,MAAM,KAAK;YACbI,KAAK;YACL,IAAI,CAACL,QAAQI,CAAC,CAACC,EAAE,GAAG,OAAO;YAC3BA,KAAK;YACL,MAAOL,QAAQI,CAAC,CAACC,EAAE,EAAGA,KAAK;YAC3B,IAAID,CAAC,CAACC,EAAE,KAAK,KAAK,OAAO;YACzBA,KAAK;YACL;QACF;QACA,IAAIJ,MAAM,KAAK;YACbI,KAAK;YACL,IAAME,IAAIH,CAAC,CAACC,EAAE;YACd,OAAOE,MAAML,aAAaK,MAAM,OAAOA,MAAM,OAAO,mBAAMC,IAAI,CAACD,cAAAA,eAAAA,IAAK;QACtE;QACA,IAAIN,MAAM,KAAK,OAAO;QACtB,IAAIA,MAAMC,aAAa,qBAAQM,IAAI,CAACP,IAAI,OAAO;QAC/C,OAAO;IACT;AACF;AACA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASQ,eAAeL,CAAS;IAC/B,IAAIM,OAAO;IACX,IAAIC,QAAQ;IACZ,IAAK,IAAIC,IAAI,GAAGA,IAAIR,EAAES,MAAM,EAAED,IAAK;QACjC,IAAMX,IAAIG,CAAC,CAACQ,EAAE;QACd,IAAIX,MAAM,KAAK;YACbS,QAAQ;YACR;QACF;QACA,IAAIT,MAAM,OAAOU,QAAQD,MAAM;YAC7BC,SAAS;YACT;QACF;QACA,IAAId,MAAMS,GAAG,CAACL,MAAME,YAAYC,GAAGQ,IAAI,OAAOA;QAC9C,IAAIX,MAAMC,aAAa,qBAAQM,IAAI,CAACP,IAAI,OAAOW;IACjD;IACA,OAAOR,EAAES,MAAM;AACjB;AACA,6FAA6F;AAC7F,uDAAuD;AACvD,SAASC,SAASvB,IAAY;IAC5B,OAAO,gBAAgBiB,IAAI,CAACjB,SAAS,UAAUiB,IAAI,CAACjB;AACtD;AACA,6FAA6F;AAC7F,SAASwB,WAAWxB,IAAY;IAC9B,OAAO,CAACuB,SAASvB,SAASA,KAAKyB,QAAQ,CAAC;AAC1C;AACA,+EAA+E;AAC/E,SAASC,eAAeC,KAAwB;IAC9C,OAAOA,CAAAA,kBAAAA,4BAAAA,MAAOC,IAAI,MAAK,iBAAiB,aAAaX,IAAI,CAACU,MAAME,OAAO;AACzE;AACA,yFAAyF;AACzF,yFAAyF;AACzF,IAAMC,cAAc;AAEpB,+FAA+F;AAC/F,SAASC,eAAeC,MAAe;IACrC,IAAMC,MAAiB,EAAE;IACzB,IAAIZ,IAAI;IACR,MAAOA,IAAIW,OAAOV,MAAM,CAAE;QACxB,IAAMK,QAAQK,MAAM,CAACX,EAAE;QACvB,IAAIM,MAAMO,OAAO,KAAK,GAAG;YACvB,IAAIC,QAAQ;YACZ,IAAIrB,IAAIO,IAAI;YACZ,MAAOP,IAAIkB,OAAOV,MAAM,IAAIa,QAAQ,EAAG;gBACrCA,SAASH,MAAM,CAAClB,EAAE,CAACoB,OAAO,KAAK,IAAI,IAAIF,MAAM,CAAClB,EAAE,CAACoB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI;gBACvEpB,KAAK;YACP;YACAmB,IAAI5B,IAAI,CAAC2B,OAAOvC,KAAK,CAAC4B,GAAGP;YACzBO,IAAIP;QACN,OAAO,IAAIa,MAAMO,OAAO,KAAK,GAAG;YAC9BD,IAAI5B,IAAI,CAAC2B,OAAOvC,KAAK,CAAC4B,GAAGA,IAAI;YAC7BA,KAAK;QACP,OAAO;YACLA,KAAK;QACP;IACF;IACA,OAAOY;AACT;AAEA,gGAAgG;AAChG,4FAA4F;AAC5F,2DAA2D;AAC3D,SAASG,cAAcC,QAAiB,EAAElC,IAAc;QAKzCkC;IAJb,IAAIC,IAAI;IACR,IAAMC,QAAQF,QAAQ,CAAC,EAAE;IACzB,IAAIX,eAAea,QAAQD,IAAI;IAC/B,MAAOA,IAAID,SAASf,MAAM,IAAIe,QAAQ,CAACC,EAAE,CAACV,IAAI,KAAK,UAAUS,QAAQ,CAACC,EAAE,CAACT,OAAO,CAACtC,IAAI,OAAO,GAAI+C,KAAK;IACrG,IAAIA,IAAI,KAAKD,EAAAA,cAAAA,QAAQ,CAACC,EAAE,cAAXD,kCAAAA,YAAaT,IAAI,MAAK,aAAa,OAAOY,WAAWH,SAAS5C,KAAK,CAAC6C,IAAI,IAAInC;IACzF,OAAOqC,WAAWH,UAAUlC;AAC9B;AAEA,SAASsC,UAAUT,MAAe,EAAE7B,IAAc;IAChD,IAAMoC,QAAQP,MAAM,CAAC,EAAE;IACvB,IAAI,CAACO,OAAO,OAAO;IACnB,OAAQA,MAAMX,IAAI;QAChB,KAAK;QACL,KAAK;YAAkB;oBAESc;gBAD9B,IAAMA,SAASV,OAAOW,IAAI,CAAC,SAACC;2BAAMA,EAAEhB,IAAI,KAAK;;gBAC7C,OAAOc,SAASN,eAAcM,mBAAAA,OAAOL,QAAQ,cAAfK,8BAAAA,mBAAmB,EAAE,EAAEvC,QAAQ;YAC/D;QACA,KAAK;QACL,KAAK;YACH,yFAAyF;YACzF,OAAOD,gBAAgBqC,MAAMV,OAAO,CAAC3C,OAAO,CAAC,OAAO,KAAKiB;QAC3D,KAAK;YACH,OAAO0C,UAAUb,QAAQ7B;QAC3B,KAAK;QACL,KAAK;YACH,OAAO2C,cAAcd,QAAQ7B;QAC/B,KAAK;YACH,OAAO4C,SAASf,OAAOvC,KAAK,CAAC,GAAG,CAAC,IAAIU,MAAMjB,OAAO,CAACL,gBAAgB;QACrE,KAAK;YACH,gGAAgG;YAChG,OAAOG,UAAUuD,MAAMV,OAAO,CAAC3C,OAAO,CAAC,OAAO;QAChD,KAAK;YACH,OAAO6D,SAASf,OAAOvC,KAAK,CAAC,GAAG,CAAC,IAAIU;QACvC;YACE,OAAO;IACX;AACF;AAEA,yFAAyF;AACzF,+EAA+E;AAC/E,SAAS4C,SAASf,MAAe,EAAE7B,IAAc;IAC/C,OAAO4B,eAAeC,QACnBgB,GAAG,CAAC,SAACC;eAAMR,UAAUQ,GAAG9C;OACxB+C,MAAM,CAAC,SAACrC;eAAMA,EAAES,MAAM,GAAG;OACzB6B,IAAI,CAAC;AACV;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,mFAAmF;AACnF,SAASC,kBAAkBC,IAAa;IACtC,IAAK,IAAIhC,IAAI,GAAGA,IAAIgC,KAAK/B,MAAM,EAAED,IAAK;YAGJM;QAFhC,IAAMA,QAAQ0B,IAAI,CAAChC,EAAE;QACrB,8DAA8D;QAC9D,IAAIM,MAAMC,IAAI,KAAK,YAAY,GAACD,kBAAAA,MAAMU,QAAQ,cAAdV,sCAAAA,gBAAgBL,MAAM,GAAE;QACxD,IAAMgC,SAAS3B,MAAMU,QAAQ,CAAC,EAAE;QAChC,IAAI,CAACX,eAAeC,MAAMU,QAAQ,CAAC,EAAE,KAAKiB,CAAAA,mBAAAA,6BAAAA,OAAQ1B,IAAI,MAAK,UAAU,CAAC0B,OAAOzB,OAAO,CAAC0B,UAAU,CAAC,MAAM,OAAOF;QAC7G,IAAMhB,WAAWV,MAAMU,QAAQ,CAAC5C,KAAK;QACrC4C,QAAQ,CAAC,EAAE,GAAG,wCAAKiB;YAAQzB,SAASyB,OAAOzB,OAAO,CAACpC,KAAK,CAAC;;QACzD,IAAM+D,OAAOH,KAAK5D,KAAK;QACvB+D,IAAI,CAACnC,EAAE,GAAG,wCAAKM;YAAOU,UAAAA;;QACtB,OAAOmB;IACT;IACA,OAAOH;AACT;AAEA,SAASP,cAAcd,MAAe,EAAE7B,IAAc;IACpD,IAAMsD,QAAkB,EAAE;QACrB,kCAAA,2BAAA;;QAAL,QAAK,YAAc1B,eAAeC,OAAOvC,KAAK,CAAC,GAAG,CAAC,wBAA9C,SAAA,6BAAA,QAAA,yBAAA,iCAAmD;YAAnD,IAAM4D,OAAN;YACHI,MAAMpD,IAAI,CAAC0C,SAASK,kBAAkBC,MAAM5D,KAAK,CAAC,GAAG,CAAC,IAAIU;QAC5D;;QAFK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAGL,OAAOsD,MAAMP,MAAM,CAAC,SAACrC;eAAMA,EAAES,MAAM,GAAG;OAAG6B,IAAI,CAAC;AAChD;AAEA,qFAAqF;AACrF,SAASN,UAAUb,MAAe,EAAE7B,IAAc;IAChD,IAAMuD,OAAiB,EAAE;IACzB,IAAIC,QAAkB,EAAE;QACnB,kCAAA,2BAAA;;QAAL,QAAK,YAAe3B,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;YAAvB,IAAML,QAAN;gBAEqDA;YADxD,IAAIA,MAAMC,IAAI,KAAK,WAAW+B,QAAQ,EAAE;iBACnC,IAAIhC,MAAMC,IAAI,KAAK,UAAU+B,MAAMtD,IAAI,CAACmC,YAAWb,kBAAAA,MAAMU,QAAQ,cAAdV,6BAAAA,kBAAkB,EAAE,EAAExB;iBACzE,IAAIwB,MAAMC,IAAI,KAAK,YAAY8B,KAAKrD,IAAI,CAACsD,MAAMR,IAAI,CAAC;QAC3D;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOO,KAAKP,IAAI,CAAC;AACnB;AAEA,6FAA6F;AAC7F,oEAAoE;AACpE,SAASX,WAAWR,MAAe,EAAE7B,IAAc;QAAEyD,QAAAA,iEAAQ;IAC3D,IAAI3B,MAAM;IACV,IAAK,IAAIZ,IAAI,GAAGA,IAAIW,OAAOV,MAAM,EAAED,IAAK;QACtC,IAAMM,QAAQK,MAAM,CAACX,EAAE;QACvB,IAAIM,MAAMC,IAAI,KAAK,aAAa;YAC9B,IAAIiC,OAAO;YACX,IAAI/C,IAAIO,IAAI;YACZ,IAAIc,QAAQ;YACZ,MAAOrB,IAAIkB,OAAOV,MAAM,IAAIa,QAAQ,EAAG;gBACrC,IAAIH,MAAM,CAAClB,EAAE,CAACc,IAAI,KAAK,aAAaO,SAAS;qBACxC,IAAIH,MAAM,CAAClB,EAAE,CAACc,IAAI,KAAK,cAAcO,SAAS;gBACnDrB,KAAK;YACP;YACA,IAAMd,OAAOwC,WAAWR,OAAOvC,KAAK,CAAC4B,IAAI,GAAGP,IAAI,IAAIX,MAAMyD;YAC1D,IAAME,SAASnC,MAAMmC,MAAM;YAC3B,IAAMC,OAAO/B,MAAM,CAAClB,EAAE;YACtB,IAAI8C,OAAO;gBACT,4FAA4F;gBAC5F3B,OAAOjC;YACT,OAAO,IAAI8D,WAAW,aAAavC,SAASvB,OAAO;gBACjD,oFAAoF;gBACpF,qFAAqF;gBACrF,mFAAmF;gBACnF,6EAA6E;gBAC7E,IAAMgE,OAAOhC,MAAM,CAACX,IAAI,EAAE;gBAC1B,IAAM4C,YAAYD,CAAAA,iBAAAA,2BAAAA,KAAMpC,IAAI,MAAK,UAAUoC,KAAKnC,OAAO,CAACqC,QAAQ,CAAC,QAAQH,CAAAA,iBAAAA,2BAAAA,KAAMnC,IAAI,MAAK,UAAUmC,KAAKlC,OAAO,CAAC0B,UAAU,CAAC;gBAC1H,IAAMY,OAAOJ,CAAAA,iBAAAA,2BAAAA,KAAMnC,IAAI,MAAK,SAAS5B,OAAO+D,KAAKlC,OAAO,GAAG7B;gBAC3D,IAAMsC,IAAI2B,YAAYjE,KAAKsB,MAAM,GAAGJ,eAAeiD;gBACnDlC,OAAOkC,KAAK1E,KAAK,CAAC6C,GAAGpD,OAAO,CAAC4C,aAAa,SAACsC;2BAAMC,IAAAA,gBAAM,IAAGC,KAAK,CAACC,WAAW,CAACH;;gBAC5E,0FAA0F;gBAC1F,IAAIL,CAAAA,iBAAAA,2BAAAA,KAAMnC,IAAI,MAAK,QAAQiC,OAAO;YACpC,OAAO,IAAIC,WAAW,aAAatC,WAAWxB,OAAO;YACnD,4EAA4E;YAC9E,OAAO,IAAI8D,WAAW,YAAY;YAChC,4FAA4F;YAC9F,OAAO;gBACL7B,OAAOjC,MAAM,6DAA6D;YAC5E;YACAqB,IAAIP,IAAI,IAAI+C;QACd,OAAO,IAAIlC,MAAMC,IAAI,KAAK,QAAQ;YAChCK,OAAON,MAAME,OAAO;QACtB,OAAO,IAAIF,MAAMC,IAAI,KAAK,aAAa;YACrCK,OAAO;QACT,OAAO,IAAIN,MAAMC,IAAI,KAAK,aAAa;YACrCK,OAAO;QACT,OAAO,IAAIN,MAAMC,IAAI,KAAK,eAAe;YACvCK,OAAO/B,gBAAgByB,MAAME,OAAO,EAAE1B;QACxC,OAAO,IAAIwB,MAAMC,IAAI,KAAK,eAAe;YACvCK,OAAOjD,UAAU2C,MAAME,OAAO;QAChC,OAAO,IAAIF,MAAMC,IAAI,KAAK,SAAS;gBACfD;YAAlBM,OAAOO,YAAWb,kBAAAA,MAAMU,QAAQ,cAAdV,6BAAAA,kBAAkB,EAAE,EAAExB,MAAM;QAChD;IACA,2EAA2E;IAC7E;IACA,OAAO8B;AACT;AAEA,IAAMuC,mBAAmB;AAIlB,SAASjG,YAAYkG,KAAY;IACtC,OAAOjG,aAAa;QAACiG;KAAM;AAC7B;AAIO,SAASjG,aAAakG,MAAe;IAC1C,IAAMvE,OAAiB,EAAE;IACzB,IAAMwE,SAASD,OACZ1B,GAAG,CAAC,SAACC;eAAMR,UAAUQ,EAAE2B,IAAI,EAAEzE;OAC7B+C,MAAM,CAAC,SAACrC;eAAMA,EAAES,MAAM,GAAG;OACzB6B,IAAI,CAAC;IACR,IAAM0B,WAAW9E,cAAc4E;IAC/B,OAAOxE,KAAKmB,MAAM,KAAK,IAAIuD,WAAWA,SAAS3F,OAAO,CAACsF,kBAAkB,SAACvE,GAAGoB;eAAclB,IAAI,CAAC2E,OAAOzD,GAAG;;AAC5G"}

package/dist/cjs/chunk/group.js

@@ -250,7 +250,7 @@ var ATOMIC_TYPES = new Set([
 // without their header/delimiter rows would demote them to paragraph text.
 function piece(pieceLines, startLine, endLine, blockType, textMode) {
     var text = ATOMIC_TYPES.has(blockType) || textMode === 'raw' ? pieceLines.join('\n') : (0, _parsets.parse)(pieceLines.join('\n')).map(function(b) {
-        return (0, _extractts.extractText)(b.node);
+        return (0, _extractts.extractText)(b);
     }).join('\n');
     return {
         startLine: startLine,
@@ -362,7 +362,7 @@ function groupScope(scopeBlocks, lines, resolved) {
                 }
                 continue;
             }
-            var extracted = (0, _extractts.extractText)(block.node);
+            var extracted = (0, _extractts.extractText)(block);
             var sizeText = resolved.text === 'raw' ? raw : extracted;
             if (block.type === 'heading') {
                 addPart(extracted, sizeText, block.startLine, block.endLine);