sensemaking 0.14.0 → 0.15.0

package/dist/esm/features/tags.js

@@ -1,14 +1,92 @@
+import { fenceTracker } from '../fences.js';
 // tags(path, tag): Obsidian's file.tags grain -- frontmatter list/string tags plus inline
 // #tags from the prose, deduplicated, source not distinguished. Nested tags store full
 // (book/scifi); `tag = 'book' OR tag LIKE 'book/%'` is how a caller matches the parent too.
-const FENCE_RE = /^(```|~~~)/;
-const INLINE_CODE_RE = /`[^`]*`/g;
 // Obsidian treats [[#Heading]] as a same-note link, not a tag.
 const WIKILINK_RE = /\[\[.*?\]\]/g; // to the first ]], so a heading holding a lone ] still masks
 // Obsidian doesn't read tags inside HTML markup.
 const HTML_TAG_RE = /<\/?[a-zA-Z][^>]*>/g; // tag-shaped only: a comparison's `< 5` must not open a span
 // Anchors on start-of-line or a preceding whitespace/(/[ so `a#b` and URL fragments don't count.
 const INLINE_TAG_RE = /(?:^|[\s([])#([\p{L}\p{N}_/-]+)/gu;
+// A markdown link destination `](...)` -- `[text](#anchor)` is a same-page link, not a tag.
+const LINK_DEST_RE = /\]\((?:[^()]|\([^()]*\))*\)/g; // one paren-nesting level, as CommonMark destinations allow: (https://x/a_(b)#frag)
+// CommonMark's HTML-block type-6 list (fixed by the spec, not a drifting enumeration): a line
+// starting with an open or close tag of one of these, at column 0, opens a block that swallows
+// following lines -- including any #tag in them -- until a blank line closes it.
+const HTML_BLOCK_TAGS = new Set([
+    'address',
+    'article',
+    'aside',
+    'base',
+    'basefont',
+    'blockquote',
+    'body',
+    'caption',
+    'center',
+    'col',
+    'colgroup',
+    'dd',
+    'details',
+    'dialog',
+    'dir',
+    'div',
+    'dl',
+    'dt',
+    'fieldset',
+    'figcaption',
+    'figure',
+    'footer',
+    'form',
+    'frame',
+    'frameset',
+    'h1',
+    'h2',
+    'h3',
+    'h4',
+    'h5',
+    'h6',
+    'head',
+    'header',
+    'hr',
+    'html',
+    'iframe',
+    'legend',
+    'li',
+    'link',
+    'main',
+    'menu',
+    'menuitem',
+    'nav',
+    'noframes',
+    'ol',
+    'optgroup',
+    'option',
+    'p',
+    'param',
+    'search',
+    'section',
+    'summary',
+    'table',
+    'tbody',
+    'td',
+    'tfoot',
+    'th',
+    'thead',
+    'title',
+    'tr',
+    'track',
+    'ul'
+]);
+// Type-1 blocks (script/pre/style/textarea): closes on the line holding the matching end tag,
+// not on a blank line, and that line is the last one skipped.
+const HTML_PRE_TAGS = new Set([
+    'script',
+    'pre',
+    'style',
+    'textarea'
+]);
+// An opening or closing tag at column 0, tag name captured for the lookups above.
+const HTML_BLOCK_OPEN_RE = /^<\/?([a-zA-Z][a-zA-Z0-9]*)(?:[ \t]|\/?>|$)/;
 // Strips a leading # (frontmatter entries may carry one) and a trailing /; rejects an
 // all-digit result -- a tag needs at least one non-digit character.
 function normalizeTag(raw) {
@@ -31,19 +109,94 @@ function frontmatterTags(data) {
     }
     return found;
 }
-// #tag tokens outside fenced code blocks, inline code spans, wikilinks, and HTML tags.
+// A code span opens on a run of N backticks and closes at the next run of exactly N -- a
+// shorter or longer run in between is literal text, not a delimiter (CommonMark code spans).
+// Masked with spaces so column positions and tag-boundary whitespace are unaffected.
+function maskCodeSpans(line) {
+    let out = '';
+    let i = 0;
+    while(i < line.length){
+        if (line[i] !== '`') {
+            out += line[i];
+            i++;
+            continue;
+        }
+        let j = i;
+        while(line[j] === '`')j++;
+        const n = j - i;
+        let k = j;
+        let closeStart = -1;
+        let closeEnd = -1;
+        while(k < line.length){
+            if (line[k] !== '`') {
+                k++;
+                continue;
+            }
+            let m = k;
+            while(line[m] === '`')m++;
+            if (m - k === n) {
+                closeStart = k;
+                closeEnd = m;
+                break;
+            }
+            k = m;
+        }
+        if (closeStart >= 0) {
+            out += ' '.repeat(closeEnd - i);
+            i = closeEnd;
+        } else {
+            out += line.slice(i, j);
+            i = j;
+        }
+    }
+    return out;
+}
+// #tag tokens outside fenced code blocks, inline code spans, wikilinks, HTML tags, HTML blocks,
+// and link destinations.
 function inlineTags(body) {
     const found = [];
-    let inFence = false;
+    const fence = fenceTracker();
+    let inHtmlBlock = false;
+    let htmlBlockClose = null; // set while inside a type-1 (script/pre/style/textarea) block
     for (const line of body.split('\n')){
-        if (FENCE_RE.test(line)) {
-            inFence = !inFence;
+        if (inHtmlBlock) {
+            // A fence-like line here is still HTML-block content -- the block wins until it closes.
+            if (htmlBlockClose) {
+                if (htmlBlockClose.test(line)) {
+                    inHtmlBlock = false;
+                    htmlBlockClose = null;
+                }
+            } else if (/^[ \t>]*$/.test(line)) {
+                inHtmlBlock = false;
+            }
             continue;
         }
-        if (inFence) continue;
+        if (fence.feed(line)) continue;
+        if (fence.inFence) continue;
+        // Indented or blockquoted HTML blocks still swallow their content in Obsidian, so the
+        // opener test runs after stripping leading whitespace and > markers.
+        const stripped = line.replace(/^[ \t>]*/, '');
+        if (stripped[0] === '<') {
+            const openMatch = HTML_BLOCK_OPEN_RE.exec(stripped);
+            if (openMatch) {
+                const tagName = openMatch[1].toLowerCase();
+                const isClosingTag = stripped[1] === '/';
+                if (!isClosingTag && HTML_PRE_TAGS.has(tagName)) {
+                    inHtmlBlock = true;
+                    // Any of the four type-1 closers ends the block, not only the tag that opened it.
+                    htmlBlockClose = /<\/(?:script|pre|style|textarea)>/i;
+                    continue;
+                }
+                if (HTML_BLOCK_TAGS.has(tagName)) {
+                    inHtmlBlock = true;
+                    continue;
+                }
+            }
+        }
         if (!line.includes('#')) continue; // most lines; skip the regex work
-        let cleaned = line.includes('`') ? line.replace(INLINE_CODE_RE, (m)=>' '.repeat(m.length)) : line;
+        let cleaned = line.includes('`') ? maskCodeSpans(line) : line;
         if (cleaned.includes('[[')) cleaned = cleaned.replace(WIKILINK_RE, (m)=>' '.repeat(m.length));
+        if (cleaned.includes('](')) cleaned = cleaned.replace(LINK_DEST_RE, (m)=>' '.repeat(m.length));
         if (cleaned.includes('<')) cleaned = cleaned.replace(HTML_TAG_RE, (m)=>' '.repeat(m.length));
         for (const m of cleaned.matchAll(INLINE_TAG_RE)){
             const tag = normalizeTag(m[1]);

package/dist/esm/features/tags.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/tags.ts"],"sourcesContent":["import type { Feature } from './types.ts';\n\n// tags(path, tag): Obsidian's file.tags grain -- frontmatter list/string tags plus inline\n// #tags from the prose, deduplicated, source not distinguished. Nested tags store full\n// (book/scifi); `tag = 'book' OR tag LIKE 'book/%'` is how a caller matches the parent too.\n\nconst FENCE_RE = /^(```|~~~)/;\nconst INLINE_CODE_RE = /`[^`]*`/g;\n// Obsidian treats [[#Heading]] as a same-note link, not a tag.\nconst WIKILINK_RE = /\\[\\[.*?\\]\\]/g; // to the first ]], so a heading holding a lone ] still masks\n// Obsidian doesn't read tags inside HTML markup.\nconst HTML_TAG_RE = /<\\/?[a-zA-Z][^>]*>/g; // tag-shaped only: a comparison's `< 5` must not open a span\n// Anchors on start-of-line or a preceding whitespace/(/[ so `a#b` and URL fragments don't count.\nconst INLINE_TAG_RE = /(?:^|[\\s([])#([\\p{L}\\p{N}_/-]+)/gu;\n\n// Strips a leading # (frontmatter entries may carry one) and a trailing /; rejects an\n// all-digit result -- a tag needs at least one non-digit character.\nfunction normalizeTag(raw: string): string | null {\n  const stripped = raw.replace(/^#/, '').replace(/\\/+$/, '');\n  if (!stripped || /^\\d+$/.test(stripped)) return null;\n  return stripped;\n}\n\n// data.tags: a YAML list (Obsidian also accepts a bare string). Null members and non-string\n// members are skipped rather than throwing -- `tags:\\n  -` parses to [null].\nfunction frontmatterTags(data?: Record<string, unknown>): string[] {\n  const raw = data?.tags;\n  const list = Array.isArray(raw) ? raw : typeof raw === 'string' ? [raw] : [];\n  const found: string[] = [];\n  for (const item of list) {\n    if (typeof item !== 'string') continue;\n    const tag = normalizeTag(item);\n    if (tag) found.push(tag);\n  }\n  return found;\n}\n\n// #tag tokens outside fenced code blocks, inline code spans, wikilinks, and HTML tags.\nfunction inlineTags(body: string): string[] {\n  const found: string[] = [];\n  let inFence = false;\n  for (const line of body.split('\\n')) {\n    if (FENCE_RE.test(line)) {\n      inFence = !inFence;\n      continue;\n    }\n    if (inFence) continue;\n    if (!line.includes('#')) continue; // most lines; skip the regex work\n    let cleaned = line.includes('`') ? line.replace(INLINE_CODE_RE, (m) => ' '.repeat(m.length)) : line;\n    if (cleaned.includes('[[')) cleaned = cleaned.replace(WIKILINK_RE, (m) => ' '.repeat(m.length));\n    if (cleaned.includes('<')) cleaned = cleaned.replace(HTML_TAG_RE, (m) => ' '.repeat(m.length));\n    for (const m of cleaned.matchAll(INLINE_TAG_RE)) {\n      const tag = normalizeTag(m[1]);\n      if (tag) found.push(tag);\n    }\n  }\n  return found;\n}\n\nfunction extract(_raw: string, body: string, _search?: { title: string; summary: string }, data?: Record<string, unknown>): string[] {\n  return [...new Set([...frontmatterTags(data), ...inlineTags(body)])].sort();\n}\n\nexport const tags: Feature = {\n  name: 'tags',\n  schema(db) {\n    db.exec('CREATE TABLE IF NOT EXISTS tags (\"path\" TEXT, tag TEXT, PRIMARY KEY (\"path\", tag))');\n    db.exec('CREATE INDEX IF NOT EXISTS tags_tag ON tags(tag)');\n  },\n  extract,\n  // Per-file rows with nothing else to resolve, so only a vanished file needs a delete here;\n  // store() below handles a reparse's stale rows itself.\n  remove(db, path, delta) {\n    if (!delta.vanished.includes(path)) return;\n    db.prepare('DELETE FROM tags WHERE \"path\" = ?').run(path);\n  },\n  store(db, path, extracted) {\n    const found = extracted as string[];\n    if (found.length === 0) {\n      db.prepare('DELETE FROM tags WHERE \"path\" = ?').run(path);\n    } else {\n      const placeholders = found.map(() => '?').join(', ');\n      db.prepare(`DELETE FROM tags WHERE \"path\" = ? AND tag NOT IN (${placeholders})`).run(path, ...found);\n    }\n    const insert = db.prepare('INSERT OR IGNORE INTO tags (\"path\", tag) VALUES (?, ?)');\n    for (const tag of found) insert.run(path, tag);\n  },\n};\n"],"names":["FENCE_RE","INLINE_CODE_RE","WIKILINK_RE","HTML_TAG_RE","INLINE_TAG_RE","normalizeTag","raw","stripped","replace","test","frontmatterTags","data","tags","list","Array","isArray","found","item","tag","push","inlineTags","body","inFence","line","split","includes","cleaned","m","repeat","length","matchAll","extract","_raw","_search","Set","sort","name","schema","db","exec","remove","path","delta","vanished","prepare","run","store","extracted","placeholders","map","join","insert"],"mappings":"AAEA,0FAA0F;AAC1F,uFAAuF;AACvF,4FAA4F;AAE5F,MAAMA,WAAW;AACjB,MAAMC,iBAAiB;AACvB,+DAA+D;AAC/D,MAAMC,cAAc,gBAAgB,6DAA6D;AACjG,iDAAiD;AACjD,MAAMC,cAAc,uBAAuB,6DAA6D;AACxG,iGAAiG;AACjG,MAAMC,gBAAgB;AAEtB,sFAAsF;AACtF,oEAAoE;AACpE,SAASC,aAAaC,GAAW;IAC/B,MAAMC,WAAWD,IAAIE,OAAO,CAAC,MAAM,IAAIA,OAAO,CAAC,QAAQ;IACvD,IAAI,CAACD,YAAY,QAAQE,IAAI,CAACF,WAAW,OAAO;IAChD,OAAOA;AACT;AAEA,4FAA4F;AAC5F,6EAA6E;AAC7E,SAASG,gBAAgBC,IAA8B;IACrD,MAAML,MAAMK,iBAAAA,2BAAAA,KAAMC,IAAI;IACtB,MAAMC,OAAOC,MAAMC,OAAO,CAACT,OAAOA,MAAM,OAAOA,QAAQ,WAAW;QAACA;KAAI,GAAG,EAAE;IAC5E,MAAMU,QAAkB,EAAE;IAC1B,KAAK,MAAMC,QAAQJ,KAAM;QACvB,IAAI,OAAOI,SAAS,UAAU;QAC9B,MAAMC,MAAMb,aAAaY;QACzB,IAAIC,KAAKF,MAAMG,IAAI,CAACD;IACtB;IACA,OAAOF;AACT;AAEA,uFAAuF;AACvF,SAASI,WAAWC,IAAY;IAC9B,MAAML,QAAkB,EAAE;IAC1B,IAAIM,UAAU;IACd,KAAK,MAAMC,QAAQF,KAAKG,KAAK,CAAC,MAAO;QACnC,IAAIxB,SAASS,IAAI,CAACc,OAAO;YACvBD,UAAU,CAACA;YACX;QACF;QACA,IAAIA,SAAS;QACb,IAAI,CAACC,KAAKE,QAAQ,CAAC,MAAM,UAAU,kCAAkC;QACrE,IAAIC,UAAUH,KAAKE,QAAQ,CAAC,OAAOF,KAAKf,OAAO,CAACP,gBAAgB,CAAC0B,IAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM,KAAKN;QAC/F,IAAIG,QAAQD,QAAQ,CAAC,OAAOC,UAAUA,QAAQlB,OAAO,CAACN,aAAa,CAACyB,IAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;QAC7F,IAAIH,QAAQD,QAAQ,CAAC,MAAMC,UAAUA,QAAQlB,OAAO,CAACL,aAAa,CAACwB,IAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;QAC5F,KAAK,MAAMF,KAAKD,QAAQI,QAAQ,CAAC1B,eAAgB;YAC/C,MAAMc,MAAMb,aAAasB,CAAC,CAAC,EAAE;YAC7B,IAAIT,KAAKF,MAAMG,IAAI,CAACD;QACtB;IACF;IACA,OAAOF;AACT;AAEA,SAASe,QAAQC,IAAY,EAAEX,IAAY,EAAEY,OAA4C,EAAEtB,IAA8B;IACvH,OAAO;WAAI,IAAIuB,IAAI;eAAIxB,gBAAgBC;eAAUS,WAAWC;SAAM;KAAE,CAACc,IAAI;AAC3E;AAEA,OAAO,MAAMvB,OAAgB;IAC3BwB,MAAM;IACNC,QAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC;QACRD,GAAGC,IAAI,CAAC;IACV;IACAR;IACA,2FAA2F;IAC3F,uDAAuD;IACvDS,QAAOF,EAAE,EAAEG,IAAI,EAAEC,KAAK;QACpB,IAAI,CAACA,MAAMC,QAAQ,CAAClB,QAAQ,CAACgB,OAAO;QACpCH,GAAGM,OAAO,CAAC,qCAAqCC,GAAG,CAACJ;IACtD;IACAK,OAAMR,EAAE,EAAEG,IAAI,EAAEM,SAAS;QACvB,MAAM/B,QAAQ+B;QACd,IAAI/B,MAAMa,MAAM,KAAK,GAAG;YACtBS,GAAGM,OAAO,CAAC,qCAAqCC,GAAG,CAACJ;QACtD,OAAO;YACL,MAAMO,eAAehC,MAAMiC,GAAG,CAAC,IAAM,KAAKC,IAAI,CAAC;YAC/CZ,GAAGM,OAAO,CAAC,CAAC,kDAAkD,EAAEI,aAAa,CAAC,CAAC,EAAEH,GAAG,CAACJ,SAASzB;QAChG;QACA,MAAMmC,SAASb,GAAGM,OAAO,CAAC;QAC1B,KAAK,MAAM1B,OAAOF,MAAOmC,OAAON,GAAG,CAACJ,MAAMvB;IAC5C;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/tags.ts"],"sourcesContent":["import { fenceTracker } from '../fences.ts';\nimport type { Feature } from './types.ts';\n\n// tags(path, tag): Obsidian's file.tags grain -- frontmatter list/string tags plus inline\n// #tags from the prose, deduplicated, source not distinguished. Nested tags store full\n// (book/scifi); `tag = 'book' OR tag LIKE 'book/%'` is how a caller matches the parent too.\n\n// Obsidian treats [[#Heading]] as a same-note link, not a tag.\nconst WIKILINK_RE = /\\[\\[.*?\\]\\]/g; // to the first ]], so a heading holding a lone ] still masks\n// Obsidian doesn't read tags inside HTML markup.\nconst HTML_TAG_RE = /<\\/?[a-zA-Z][^>]*>/g; // tag-shaped only: a comparison's `< 5` must not open a span\n// Anchors on start-of-line or a preceding whitespace/(/[ so `a#b` and URL fragments don't count.\nconst INLINE_TAG_RE = /(?:^|[\\s([])#([\\p{L}\\p{N}_/-]+)/gu;\n// A markdown link destination `](...)` -- `[text](#anchor)` is a same-page link, not a tag.\nconst LINK_DEST_RE = /\\]\\((?:[^()]|\\([^()]*\\))*\\)/g; // one paren-nesting level, as CommonMark destinations allow: (https://x/a_(b)#frag)\n\n// CommonMark's HTML-block type-6 list (fixed by the spec, not a drifting enumeration): a line\n// starting with an open or close tag of one of these, at column 0, opens a block that swallows\n// following lines -- including any #tag in them -- until a blank line closes it.\nconst HTML_BLOCK_TAGS = new Set([\n  'address',\n  'article',\n  'aside',\n  'base',\n  'basefont',\n  'blockquote',\n  'body',\n  'caption',\n  'center',\n  'col',\n  'colgroup',\n  'dd',\n  'details',\n  'dialog',\n  'dir',\n  'div',\n  'dl',\n  'dt',\n  'fieldset',\n  'figcaption',\n  'figure',\n  'footer',\n  'form',\n  'frame',\n  'frameset',\n  'h1',\n  'h2',\n  'h3',\n  'h4',\n  'h5',\n  'h6',\n  'head',\n  'header',\n  'hr',\n  'html',\n  'iframe',\n  'legend',\n  'li',\n  'link',\n  'main',\n  'menu',\n  'menuitem',\n  'nav',\n  'noframes',\n  'ol',\n  'optgroup',\n  'option',\n  'p',\n  'param',\n  'search',\n  'section',\n  'summary',\n  'table',\n  'tbody',\n  'td',\n  'tfoot',\n  'th',\n  'thead',\n  'title',\n  'tr',\n  'track',\n  'ul',\n]);\n// Type-1 blocks (script/pre/style/textarea): closes on the line holding the matching end tag,\n// not on a blank line, and that line is the last one skipped.\nconst HTML_PRE_TAGS = new Set(['script', 'pre', 'style', 'textarea']);\n// An opening or closing tag at column 0, tag name captured for the lookups above.\nconst HTML_BLOCK_OPEN_RE = /^<\\/?([a-zA-Z][a-zA-Z0-9]*)(?:[ \\t]|\\/?>|$)/;\n\n// Strips a leading # (frontmatter entries may carry one) and a trailing /; rejects an\n// all-digit result -- a tag needs at least one non-digit character.\nfunction normalizeTag(raw: string): string | null {\n  const stripped = raw.replace(/^#/, '').replace(/\\/+$/, '');\n  if (!stripped || /^\\d+$/.test(stripped)) return null;\n  return stripped;\n}\n\n// data.tags: a YAML list (Obsidian also accepts a bare string). Null members and non-string\n// members are skipped rather than throwing -- `tags:\\n  -` parses to [null].\nfunction frontmatterTags(data?: Record<string, unknown>): string[] {\n  const raw = data?.tags;\n  const list = Array.isArray(raw) ? raw : typeof raw === 'string' ? [raw] : [];\n  const found: string[] = [];\n  for (const item of list) {\n    if (typeof item !== 'string') continue;\n    const tag = normalizeTag(item);\n    if (tag) found.push(tag);\n  }\n  return found;\n}\n\n// A code span opens on a run of N backticks and closes at the next run of exactly N -- a\n// shorter or longer run in between is literal text, not a delimiter (CommonMark code spans).\n// Masked with spaces so column positions and tag-boundary whitespace are unaffected.\nfunction maskCodeSpans(line: string): string {\n  let out = '';\n  let i = 0;\n  while (i < line.length) {\n    if (line[i] !== '`') {\n      out += line[i];\n      i++;\n      continue;\n    }\n    let j = i;\n    while (line[j] === '`') j++;\n    const n = j - i;\n    let k = j;\n    let closeStart = -1;\n    let closeEnd = -1;\n    while (k < line.length) {\n      if (line[k] !== '`') {\n        k++;\n        continue;\n      }\n      let m = k;\n      while (line[m] === '`') m++;\n      if (m - k === n) {\n        closeStart = k;\n        closeEnd = m;\n        break;\n      }\n      k = m;\n    }\n    if (closeStart >= 0) {\n      out += ' '.repeat(closeEnd - i);\n      i = closeEnd;\n    } else {\n      out += line.slice(i, j);\n      i = j;\n    }\n  }\n  return out;\n}\n\n// #tag tokens outside fenced code blocks, inline code spans, wikilinks, HTML tags, HTML blocks,\n// and link destinations.\nfunction inlineTags(body: string): string[] {\n  const found: string[] = [];\n  const fence = fenceTracker();\n  let inHtmlBlock = false;\n  let htmlBlockClose: RegExp | null = null; // set while inside a type-1 (script/pre/style/textarea) block\n  for (const line of body.split('\\n')) {\n    if (inHtmlBlock) {\n      // A fence-like line here is still HTML-block content -- the block wins until it closes.\n      if (htmlBlockClose) {\n        if (htmlBlockClose.test(line)) {\n          inHtmlBlock = false;\n          htmlBlockClose = null;\n        }\n      } else if (/^[ \\t>]*$/.test(line)) {\n        inHtmlBlock = false;\n      }\n      continue;\n    }\n    if (fence.feed(line)) continue;\n    if (fence.inFence) continue;\n    // Indented or blockquoted HTML blocks still swallow their content in Obsidian, so the\n    // opener test runs after stripping leading whitespace and > markers.\n    const stripped = line.replace(/^[ \\t>]*/, '');\n    if (stripped[0] === '<') {\n      const openMatch = HTML_BLOCK_OPEN_RE.exec(stripped);\n      if (openMatch) {\n        const tagName = openMatch[1].toLowerCase();\n        const isClosingTag = stripped[1] === '/';\n        if (!isClosingTag && HTML_PRE_TAGS.has(tagName)) {\n          inHtmlBlock = true;\n          // Any of the four type-1 closers ends the block, not only the tag that opened it.\n          htmlBlockClose = /<\\/(?:script|pre|style|textarea)>/i;\n          continue;\n        }\n        if (HTML_BLOCK_TAGS.has(tagName)) {\n          inHtmlBlock = true;\n          continue;\n        }\n      }\n    }\n    if (!line.includes('#')) continue; // most lines; skip the regex work\n    let cleaned = line.includes('`') ? maskCodeSpans(line) : line;\n    if (cleaned.includes('[[')) cleaned = cleaned.replace(WIKILINK_RE, (m) => ' '.repeat(m.length));\n    if (cleaned.includes('](')) cleaned = cleaned.replace(LINK_DEST_RE, (m) => ' '.repeat(m.length));\n    if (cleaned.includes('<')) cleaned = cleaned.replace(HTML_TAG_RE, (m) => ' '.repeat(m.length));\n    for (const m of cleaned.matchAll(INLINE_TAG_RE)) {\n      const tag = normalizeTag(m[1]);\n      if (tag) found.push(tag);\n    }\n  }\n  return found;\n}\n\nfunction extract(_raw: string, body: string, _search?: { title: string; summary: string }, data?: Record<string, unknown>): string[] {\n  return [...new Set([...frontmatterTags(data), ...inlineTags(body)])].sort();\n}\n\nexport const tags: Feature = {\n  name: 'tags',\n  schema(db) {\n    db.exec('CREATE TABLE IF NOT EXISTS tags (\"path\" TEXT, tag TEXT, PRIMARY KEY (\"path\", tag))');\n    db.exec('CREATE INDEX IF NOT EXISTS tags_tag ON tags(tag)');\n  },\n  extract,\n  // Per-file rows with nothing else to resolve, so only a vanished file needs a delete here;\n  // store() below handles a reparse's stale rows itself.\n  remove(db, path, delta) {\n    if (!delta.vanished.includes(path)) return;\n    db.prepare('DELETE FROM tags WHERE \"path\" = ?').run(path);\n  },\n  store(db, path, extracted) {\n    const found = extracted as string[];\n    if (found.length === 0) {\n      db.prepare('DELETE FROM tags WHERE \"path\" = ?').run(path);\n    } else {\n      const placeholders = found.map(() => '?').join(', ');\n      db.prepare(`DELETE FROM tags WHERE \"path\" = ? AND tag NOT IN (${placeholders})`).run(path, ...found);\n    }\n    const insert = db.prepare('INSERT OR IGNORE INTO tags (\"path\", tag) VALUES (?, ?)');\n    for (const tag of found) insert.run(path, tag);\n  },\n};\n"],"names":["fenceTracker","WIKILINK_RE","HTML_TAG_RE","INLINE_TAG_RE","LINK_DEST_RE","HTML_BLOCK_TAGS","Set","HTML_PRE_TAGS","HTML_BLOCK_OPEN_RE","normalizeTag","raw","stripped","replace","test","frontmatterTags","data","tags","list","Array","isArray","found","item","tag","push","maskCodeSpans","line","out","i","length","j","n","k","closeStart","closeEnd","m","repeat","slice","inlineTags","body","fence","inHtmlBlock","htmlBlockClose","split","feed","inFence","openMatch","exec","tagName","toLowerCase","isClosingTag","has","includes","cleaned","matchAll","extract","_raw","_search","sort","name","schema","db","remove","path","delta","vanished","prepare","run","store","extracted","placeholders","map","join","insert"],"mappings":"AAAA,SAASA,YAAY,QAAQ,eAAe;AAG5C,0FAA0F;AAC1F,uFAAuF;AACvF,4FAA4F;AAE5F,+DAA+D;AAC/D,MAAMC,cAAc,gBAAgB,6DAA6D;AACjG,iDAAiD;AACjD,MAAMC,cAAc,uBAAuB,6DAA6D;AACxG,iGAAiG;AACjG,MAAMC,gBAAgB;AACtB,4FAA4F;AAC5F,MAAMC,eAAe,gCAAgC,oFAAoF;AAEzI,8FAA8F;AAC9F,+FAA+F;AAC/F,iFAAiF;AACjF,MAAMC,kBAAkB,IAAIC,IAAI;IAC9B;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AACD,8FAA8F;AAC9F,8DAA8D;AAC9D,MAAMC,gBAAgB,IAAID,IAAI;IAAC;IAAU;IAAO;IAAS;CAAW;AACpE,kFAAkF;AAClF,MAAME,qBAAqB;AAE3B,sFAAsF;AACtF,oEAAoE;AACpE,SAASC,aAAaC,GAAW;IAC/B,MAAMC,WAAWD,IAAIE,OAAO,CAAC,MAAM,IAAIA,OAAO,CAAC,QAAQ;IACvD,IAAI,CAACD,YAAY,QAAQE,IAAI,CAACF,WAAW,OAAO;IAChD,OAAOA;AACT;AAEA,4FAA4F;AAC5F,6EAA6E;AAC7E,SAASG,gBAAgBC,IAA8B;IACrD,MAAML,MAAMK,iBAAAA,2BAAAA,KAAMC,IAAI;IACtB,MAAMC,OAAOC,MAAMC,OAAO,CAACT,OAAOA,MAAM,OAAOA,QAAQ,WAAW;QAACA;KAAI,GAAG,EAAE;IAC5E,MAAMU,QAAkB,EAAE;IAC1B,KAAK,MAAMC,QAAQJ,KAAM;QACvB,IAAI,OAAOI,SAAS,UAAU;QAC9B,MAAMC,MAAMb,aAAaY;QACzB,IAAIC,KAAKF,MAAMG,IAAI,CAACD;IACtB;IACA,OAAOF;AACT;AAEA,yFAAyF;AACzF,6FAA6F;AAC7F,qFAAqF;AACrF,SAASI,cAAcC,IAAY;IACjC,IAAIC,MAAM;IACV,IAAIC,IAAI;IACR,MAAOA,IAAIF,KAAKG,MAAM,CAAE;QACtB,IAAIH,IAAI,CAACE,EAAE,KAAK,KAAK;YACnBD,OAAOD,IAAI,CAACE,EAAE;YACdA;YACA;QACF;QACA,IAAIE,IAAIF;QACR,MAAOF,IAAI,CAACI,EAAE,KAAK,IAAKA;QACxB,MAAMC,IAAID,IAAIF;QACd,IAAII,IAAIF;QACR,IAAIG,aAAa,CAAC;QAClB,IAAIC,WAAW,CAAC;QAChB,MAAOF,IAAIN,KAAKG,MAAM,CAAE;YACtB,IAAIH,IAAI,CAACM,EAAE,KAAK,KAAK;gBACnBA;gBACA;YACF;YACA,IAAIG,IAAIH;YACR,MAAON,IAAI,CAACS,EAAE,KAAK,IAAKA;YACxB,IAAIA,IAAIH,MAAMD,GAAG;gBACfE,aAAaD;gBACbE,WAAWC;gBACX;YACF;YACAH,IAAIG;QACN;QACA,IAAIF,cAAc,GAAG;YACnBN,OAAO,IAAIS,MAAM,CAACF,WAAWN;YAC7BA,IAAIM;QACN,OAAO;YACLP,OAAOD,KAAKW,KAAK,CAACT,GAAGE;YACrBF,IAAIE;QACN;IACF;IACA,OAAOH;AACT;AAEA,gGAAgG;AAChG,yBAAyB;AACzB,SAASW,WAAWC,IAAY;IAC9B,MAAMlB,QAAkB,EAAE;IAC1B,MAAMmB,QAAQvC;IACd,IAAIwC,cAAc;IAClB,IAAIC,iBAAgC,MAAM,8DAA8D;IACxG,KAAK,MAAMhB,QAAQa,KAAKI,KAAK,CAAC,MAAO;QACnC,IAAIF,aAAa;YACf,wFAAwF;YACxF,IAAIC,gBAAgB;gBAClB,IAAIA,eAAe5B,IAAI,CAACY,OAAO;oBAC7Be,cAAc;oBACdC,iBAAiB;gBACnB;YACF,OAAO,IAAI,YAAY5B,IAAI,CAACY,OAAO;gBACjCe,cAAc;YAChB;YACA;QACF;QACA,IAAID,MAAMI,IAAI,CAAClB,OAAO;QACtB,IAAIc,MAAMK,OAAO,EAAE;QACnB,sFAAsF;QACtF,qEAAqE;QACrE,MAAMjC,WAAWc,KAAKb,OAAO,CAAC,YAAY;QAC1C,IAAID,QAAQ,CAAC,EAAE,KAAK,KAAK;YACvB,MAAMkC,YAAYrC,mBAAmBsC,IAAI,CAACnC;YAC1C,IAAIkC,WAAW;gBACb,MAAME,UAAUF,SAAS,CAAC,EAAE,CAACG,WAAW;gBACxC,MAAMC,eAAetC,QAAQ,CAAC,EAAE,KAAK;gBACrC,IAAI,CAACsC,gBAAgB1C,cAAc2C,GAAG,CAACH,UAAU;oBAC/CP,cAAc;oBACd,kFAAkF;oBAClFC,iBAAiB;oBACjB;gBACF;gBACA,IAAIpC,gBAAgB6C,GAAG,CAACH,UAAU;oBAChCP,cAAc;oBACd;gBACF;YACF;QACF;QACA,IAAI,CAACf,KAAK0B,QAAQ,CAAC,MAAM,UAAU,kCAAkC;QACrE,IAAIC,UAAU3B,KAAK0B,QAAQ,CAAC,OAAO3B,cAAcC,QAAQA;QACzD,IAAI2B,QAAQD,QAAQ,CAAC,OAAOC,UAAUA,QAAQxC,OAAO,CAACX,aAAa,CAACiC,IAAM,IAAIC,MAAM,CAACD,EAAEN,MAAM;QAC7F,IAAIwB,QAAQD,QAAQ,CAAC,OAAOC,UAAUA,QAAQxC,OAAO,CAACR,cAAc,CAAC8B,IAAM,IAAIC,MAAM,CAACD,EAAEN,MAAM;QAC9F,IAAIwB,QAAQD,QAAQ,CAAC,MAAMC,UAAUA,QAAQxC,OAAO,CAACV,aAAa,CAACgC,IAAM,IAAIC,MAAM,CAACD,EAAEN,MAAM;QAC5F,KAAK,MAAMM,KAAKkB,QAAQC,QAAQ,CAAClD,eAAgB;YAC/C,MAAMmB,MAAMb,aAAayB,CAAC,CAAC,EAAE;YAC7B,IAAIZ,KAAKF,MAAMG,IAAI,CAACD;QACtB;IACF;IACA,OAAOF;AACT;AAEA,SAASkC,QAAQC,IAAY,EAAEjB,IAAY,EAAEkB,OAA4C,EAAEzC,IAA8B;IACvH,OAAO;WAAI,IAAIT,IAAI;eAAIQ,gBAAgBC;eAAUsB,WAAWC;SAAM;KAAE,CAACmB,IAAI;AAC3E;AAEA,OAAO,MAAMzC,OAAgB;IAC3B0C,MAAM;IACNC,QAAOC,EAAE;QACPA,GAAGd,IAAI,CAAC;QACRc,GAAGd,IAAI,CAAC;IACV;IACAQ;IACA,2FAA2F;IAC3F,uDAAuD;IACvDO,QAAOD,EAAE,EAAEE,IAAI,EAAEC,KAAK;QACpB,IAAI,CAACA,MAAMC,QAAQ,CAACb,QAAQ,CAACW,OAAO;QACpCF,GAAGK,OAAO,CAAC,qCAAqCC,GAAG,CAACJ;IACtD;IACAK,OAAMP,EAAE,EAAEE,IAAI,EAAEM,SAAS;QACvB,MAAMhD,QAAQgD;QACd,IAAIhD,MAAMQ,MAAM,KAAK,GAAG;YACtBgC,GAAGK,OAAO,CAAC,qCAAqCC,GAAG,CAACJ;QACtD,OAAO;YACL,MAAMO,eAAejD,MAAMkD,GAAG,CAAC,IAAM,KAAKC,IAAI,CAAC;YAC/CX,GAAGK,OAAO,CAAC,CAAC,kDAAkD,EAAEI,aAAa,CAAC,CAAC,EAAEH,GAAG,CAACJ,SAAS1C;QAChG;QACA,MAAMoD,SAASZ,GAAGK,OAAO,CAAC;QAC1B,KAAK,MAAM3C,OAAOF,MAAOoD,OAAON,GAAG,CAACJ,MAAMxC;IAC5C;AACF,EAAE"}

package/dist/esm/fences.d.ts

@@ -0,0 +1,5 @@
+export interface FenceTracker {
+    feed(line: string): boolean;
+    readonly inFence: boolean;
+}
+export declare function fenceTracker(): FenceTracker;

package/dist/esm/fences.js

@@ -0,0 +1,43 @@
+// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column
+// 0 -- no indent allowance, a deliberate divergence from CommonMark's up-to-3-space rule (see
+// test/unit/fences.test.ts DIVERGENCES table). A backtick opener's info string must not itself
+// contain a backtick (spec rule). A closer is a run of the SAME character, length >= the
+// opener's, holding nothing but trailing spaces after the run.
+const BACKTICK_OPEN = /^(`{3,})(.*)$/;
+const TILDE_OPEN = /^(~{3,})(.*)$/;
+export function fenceTracker() {
+    let inFence = false;
+    let fenceChar = '';
+    let fenceLen = 0;
+    return {
+        feed (line) {
+            if (!inFence) {
+                const bt = BACKTICK_OPEN.exec(line);
+                if (bt && !bt[2].includes('`')) {
+                    inFence = true;
+                    fenceChar = '`';
+                    fenceLen = bt[1].length;
+                    return true;
+                }
+                const td = TILDE_OPEN.exec(line);
+                if (td) {
+                    inFence = true;
+                    fenceChar = '~';
+                    fenceLen = td[1].length;
+                    return true;
+                }
+                return false;
+            }
+            const run = fenceChar === '`' ? /^(`+)(.*)$/ : /^(~+)(.*)$/;
+            const m = run.exec(line);
+            if (m && m[1].length >= fenceLen && m[2].trim() === '') {
+                inFence = false;
+                return true;
+            }
+            return false;
+        },
+        get inFence () {
+            return inFence;
+        }
+    };
+}

package/dist/esm/fences.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/fences.ts"],"sourcesContent":["// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column\n// 0 -- no indent allowance, a deliberate divergence from CommonMark's up-to-3-space rule (see\n// test/unit/fences.test.ts DIVERGENCES table). A backtick opener's info string must not itself\n// contain a backtick (spec rule). A closer is a run of the SAME character, length >= the\n// opener's, holding nothing but trailing spaces after the run.\n\nexport interface FenceTracker {\n  feed(line: string): boolean; // true iff this line is a fence delimiter (open or close)\n  readonly inFence: boolean;\n}\n\nconst BACKTICK_OPEN = /^(`{3,})(.*)$/;\nconst TILDE_OPEN = /^(~{3,})(.*)$/;\n\nexport function fenceTracker(): FenceTracker {\n  let inFence = false;\n  let fenceChar = '';\n  let fenceLen = 0;\n  return {\n    feed(line: string): boolean {\n      if (!inFence) {\n        const bt = BACKTICK_OPEN.exec(line);\n        if (bt && !bt[2].includes('`')) {\n          inFence = true;\n          fenceChar = '`';\n          fenceLen = bt[1].length;\n          return true;\n        }\n        const td = TILDE_OPEN.exec(line);\n        if (td) {\n          inFence = true;\n          fenceChar = '~';\n          fenceLen = td[1].length;\n          return true;\n        }\n        return false;\n      }\n      const run = fenceChar === '`' ? /^(`+)(.*)$/ : /^(~+)(.*)$/;\n      const m = run.exec(line);\n      if (m && m[1].length >= fenceLen && m[2].trim() === '') {\n        inFence = false;\n        return true;\n      }\n      return false;\n    },\n    get inFence() {\n      return inFence;\n    },\n  };\n}\n"],"names":["BACKTICK_OPEN","TILDE_OPEN","fenceTracker","inFence","fenceChar","fenceLen","feed","line","bt","exec","includes","length","td","run","m","trim"],"mappings":"AAAA,+FAA+F;AAC/F,8FAA8F;AAC9F,+FAA+F;AAC/F,yFAAyF;AACzF,+DAA+D;AAO/D,MAAMA,gBAAgB;AACtB,MAAMC,aAAa;AAEnB,OAAO,SAASC;IACd,IAAIC,UAAU;IACd,IAAIC,YAAY;IAChB,IAAIC,WAAW;IACf,OAAO;QACLC,MAAKC,IAAY;YACf,IAAI,CAACJ,SAAS;gBACZ,MAAMK,KAAKR,cAAcS,IAAI,CAACF;gBAC9B,IAAIC,MAAM,CAACA,EAAE,CAAC,EAAE,CAACE,QAAQ,CAAC,MAAM;oBAC9BP,UAAU;oBACVC,YAAY;oBACZC,WAAWG,EAAE,CAAC,EAAE,CAACG,MAAM;oBACvB,OAAO;gBACT;gBACA,MAAMC,KAAKX,WAAWQ,IAAI,CAACF;gBAC3B,IAAIK,IAAI;oBACNT,UAAU;oBACVC,YAAY;oBACZC,WAAWO,EAAE,CAAC,EAAE,CAACD,MAAM;oBACvB,OAAO;gBACT;gBACA,OAAO;YACT;YACA,MAAME,MAAMT,cAAc,MAAM,eAAe;YAC/C,MAAMU,IAAID,IAAIJ,IAAI,CAACF;YACnB,IAAIO,KAAKA,CAAC,CAAC,EAAE,CAACH,MAAM,IAAIN,YAAYS,CAAC,CAAC,EAAE,CAACC,IAAI,OAAO,IAAI;gBACtDZ,UAAU;gBACV,OAAO;YACT;YACA,OAAO;QACT;QACA,IAAIA,WAAU;YACZ,OAAOA;QACT;IACF;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",