sensemaking 0.14.0 → 0.15.1

package/dist/esm/fences.js

@@ -0,0 +1,138 @@
+// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column
+// 0 -- no indent allowance, a deliberate divergence from CommonMark's up-to-3-space rule (see
+// test/unit/fences.test.ts DIVERGENCES table). A backtick opener's info string must not itself
+// contain a backtick (spec rule). A closer is a run of the SAME character, length >= the
+// opener's, holding nothing but trailing spaces after the run.
+const BACKTICK_OPEN = /^(`{3,})(.*)$/;
+const TILDE_OPEN = /^(~{3,})(.*)$/;
+export function fenceTracker() {
+    let inFence = false;
+    let fenceChar = '';
+    let fenceLen = 0;
+    return {
+        feed (line) {
+            if (!inFence) {
+                const bt = BACKTICK_OPEN.exec(line);
+                if (bt && !bt[2].includes('`')) {
+                    inFence = true;
+                    fenceChar = '`';
+                    fenceLen = bt[1].length;
+                    return true;
+                }
+                const td = TILDE_OPEN.exec(line);
+                if (td) {
+                    inFence = true;
+                    fenceChar = '~';
+                    fenceLen = td[1].length;
+                    return true;
+                }
+                return false;
+            }
+            const run = fenceChar === '`' ? /^(`+)(.*)$/ : /^(~+)(.*)$/;
+            const m = run.exec(line);
+            if (m && m[1].length >= fenceLen && m[2].trim() === '') {
+                inFence = false;
+                return true;
+            }
+            return false;
+        },
+        get inFence () {
+            return inFence;
+        }
+    };
+}
+export function commentTracker() {
+    let inComment = false;
+    return {
+        mask (line) {
+            let out = '';
+            let i = 0;
+            while(i < line.length){
+                if (inComment) {
+                    const end = line.indexOf('-->', i);
+                    if (end === -1) {
+                        out += ' '.repeat(line.length - i);
+                        i = line.length;
+                    } else {
+                        out += ' '.repeat(end + 3 - i);
+                        i = end + 3;
+                        inComment = false;
+                    }
+                    continue;
+                }
+                const start = line.indexOf('<!--', i);
+                if (start === -1) {
+                    out += line.slice(i);
+                    i = line.length;
+                } else {
+                    out += `${line.slice(i, start)}    `;
+                    i = start + 4;
+                    inComment = true;
+                }
+            }
+            return out;
+        },
+        get inComment () {
+            return inComment;
+        }
+    };
+}
+// Body with fenced-code regions (delimiter and content lines) and <!-- --> comment spans
+// replaced by spaces, newlines preserved so line numbers and offsets survive. Bodies without a
+// backtick/tilde/`<!--` fast-path out untouched.
+export function maskRegions(body) {
+    if (!/[`~]/.test(body) && !body.includes('<!--')) return body;
+    const fence = fenceTracker();
+    const comment = commentTracker();
+    return body.split('\n').map((line)=>{
+        if (!comment.inComment) {
+            const isDelim = fence.feed(line);
+            if (isDelim || fence.inFence) return ' '.repeat(line.length);
+        }
+        const uncommented = comment.mask(line);
+        // Inline code spans hide their content too: `[[x]]` in prose is not a link.
+        return uncommented.includes('`') ? maskCodeSpans(uncommented) : uncommented;
+    }).join('\n');
+}
+// A code span opens on a run of N backticks and closes at the next run of exactly N -- a
+// shorter or longer run in between is literal text, not a delimiter (CommonMark code spans).
+// Masked with spaces so column positions and tag-boundary whitespace are unaffected.
+export function maskCodeSpans(line) {
+    let out = '';
+    let i = 0;
+    while(i < line.length){
+        if (line[i] !== '`') {
+            out += line[i];
+            i++;
+            continue;
+        }
+        let j = i;
+        while(line[j] === '`')j++;
+        const n = j - i;
+        let k = j;
+        let closeStart = -1;
+        let closeEnd = -1;
+        while(k < line.length){
+            if (line[k] !== '`') {
+                k++;
+                continue;
+            }
+            let m = k;
+            while(line[m] === '`')m++;
+            if (m - k === n) {
+                closeStart = k;
+                closeEnd = m;
+                break;
+            }
+            k = m;
+        }
+        if (closeStart >= 0) {
+            out += ' '.repeat(closeEnd - i);
+            i = closeEnd;
+        } else {
+            out += line.slice(i, j);
+            i = j;
+        }
+    }
+    return out;
+}

package/dist/esm/fences.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/fences.ts"],"sourcesContent":["// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column\n// 0 -- no indent allowance, a deliberate divergence from CommonMark's up-to-3-space rule (see\n// test/unit/fences.test.ts DIVERGENCES table). A backtick opener's info string must not itself\n// contain a backtick (spec rule). A closer is a run of the SAME character, length >= the\n// opener's, holding nothing but trailing spaces after the run.\n\nexport interface FenceTracker {\n  feed(line: string): boolean; // true iff this line is a fence delimiter (open or close)\n  readonly inFence: boolean;\n}\n\nconst BACKTICK_OPEN = /^(`{3,})(.*)$/;\nconst TILDE_OPEN = /^(~{3,})(.*)$/;\n\nexport function fenceTracker(): FenceTracker {\n  let inFence = false;\n  let fenceChar = '';\n  let fenceLen = 0;\n  return {\n    feed(line: string): boolean {\n      if (!inFence) {\n        const bt = BACKTICK_OPEN.exec(line);\n        if (bt && !bt[2].includes('`')) {\n          inFence = true;\n          fenceChar = '`';\n          fenceLen = bt[1].length;\n          return true;\n        }\n        const td = TILDE_OPEN.exec(line);\n        if (td) {\n          inFence = true;\n          fenceChar = '~';\n          fenceLen = td[1].length;\n          return true;\n        }\n        return false;\n      }\n      const run = fenceChar === '`' ? /^(`+)(.*)$/ : /^(~+)(.*)$/;\n      const m = run.exec(line);\n      if (m && m[1].length >= fenceLen && m[2].trim() === '') {\n        inFence = false;\n        return true;\n      }\n      return false;\n    },\n    get inFence() {\n      return inFence;\n    },\n  };\n}\n\n// Masks <!-- ... --> spans that may cross line boundaries, one line at a time so callers can\n// interleave it with their own per-line state. An unclosed <!-- masks to end of input, matching\n// CommonMark type-2 HTML comments' practical effect (everything after is swallowed).\nexport interface CommentTracker {\n  mask(line: string): string;\n  readonly inComment: boolean;\n}\n\nexport function commentTracker(): CommentTracker {\n  let inComment = false;\n  return {\n    mask(line: string): string {\n      let out = '';\n      let i = 0;\n      while (i < line.length) {\n        if (inComment) {\n          const end = line.indexOf('-->', i);\n          if (end === -1) {\n            out += ' '.repeat(line.length - i);\n            i = line.length;\n          } else {\n            out += ' '.repeat(end + 3 - i);\n            i = end + 3;\n            inComment = false;\n          }\n          continue;\n        }\n        const start = line.indexOf('<!--', i);\n        if (start === -1) {\n          out += line.slice(i);\n          i = line.length;\n        } else {\n          out += `${line.slice(i, start)}    `;\n          i = start + 4;\n          inComment = true;\n        }\n      }\n      return out;\n    },\n    get inComment() {\n      return inComment;\n    },\n  };\n}\n\n// Body with fenced-code regions (delimiter and content lines) and <!-- --> comment spans\n// replaced by spaces, newlines preserved so line numbers and offsets survive. Bodies without a\n// backtick/tilde/`<!--` fast-path out untouched.\nexport function maskRegions(body: string): string {\n  if (!/[`~]/.test(body) && !body.includes('<!--')) return body;\n  const fence = fenceTracker();\n  const comment = commentTracker();\n  return body\n    .split('\\n')\n    .map((line) => {\n      if (!comment.inComment) {\n        const isDelim = fence.feed(line);\n        if (isDelim || fence.inFence) return ' '.repeat(line.length);\n      }\n      const uncommented = comment.mask(line);\n      // Inline code spans hide their content too: `[[x]]` in prose is not a link.\n      return uncommented.includes('`') ? maskCodeSpans(uncommented) : uncommented;\n    })\n    .join('\\n');\n}\n\n// A code span opens on a run of N backticks and closes at the next run of exactly N -- a\n// shorter or longer run in between is literal text, not a delimiter (CommonMark code spans).\n// Masked with spaces so column positions and tag-boundary whitespace are unaffected.\nexport function maskCodeSpans(line: string): string {\n  let out = '';\n  let i = 0;\n  while (i < line.length) {\n    if (line[i] !== '`') {\n      out += line[i];\n      i++;\n      continue;\n    }\n    let j = i;\n    while (line[j] === '`') j++;\n    const n = j - i;\n    let k = j;\n    let closeStart = -1;\n    let closeEnd = -1;\n    while (k < line.length) {\n      if (line[k] !== '`') {\n        k++;\n        continue;\n      }\n      let m = k;\n      while (line[m] === '`') m++;\n      if (m - k === n) {\n        closeStart = k;\n        closeEnd = m;\n        break;\n      }\n      k = m;\n    }\n    if (closeStart >= 0) {\n      out += ' '.repeat(closeEnd - i);\n      i = closeEnd;\n    } else {\n      out += line.slice(i, j);\n      i = j;\n    }\n  }\n  return out;\n}\n"],"names":["BACKTICK_OPEN","TILDE_OPEN","fenceTracker","inFence","fenceChar","fenceLen","feed","line","bt","exec","includes","length","td","run","m","trim","commentTracker","inComment","mask","out","i","end","indexOf","repeat","start","slice","maskRegions","body","test","fence","comment","split","map","isDelim","uncommented","maskCodeSpans","join","j","n","k","closeStart","closeEnd"],"mappings":"AAAA,+FAA+F;AAC/F,8FAA8F;AAC9F,+FAA+F;AAC/F,yFAAyF;AACzF,+DAA+D;AAO/D,MAAMA,gBAAgB;AACtB,MAAMC,aAAa;AAEnB,OAAO,SAASC;IACd,IAAIC,UAAU;IACd,IAAIC,YAAY;IAChB,IAAIC,WAAW;IACf,OAAO;QACLC,MAAKC,IAAY;YACf,IAAI,CAACJ,SAAS;gBACZ,MAAMK,KAAKR,cAAcS,IAAI,CAACF;gBAC9B,IAAIC,MAAM,CAACA,EAAE,CAAC,EAAE,CAACE,QAAQ,CAAC,MAAM;oBAC9BP,UAAU;oBACVC,YAAY;oBACZC,WAAWG,EAAE,CAAC,EAAE,CAACG,MAAM;oBACvB,OAAO;gBACT;gBACA,MAAMC,KAAKX,WAAWQ,IAAI,CAACF;gBAC3B,IAAIK,IAAI;oBACNT,UAAU;oBACVC,YAAY;oBACZC,WAAWO,EAAE,CAAC,EAAE,CAACD,MAAM;oBACvB,OAAO;gBACT;gBACA,OAAO;YACT;YACA,MAAME,MAAMT,cAAc,MAAM,eAAe;YAC/C,MAAMU,IAAID,IAAIJ,IAAI,CAACF;YACnB,IAAIO,KAAKA,CAAC,CAAC,EAAE,CAACH,MAAM,IAAIN,YAAYS,CAAC,CAAC,EAAE,CAACC,IAAI,OAAO,IAAI;gBACtDZ,UAAU;gBACV,OAAO;YACT;YACA,OAAO;QACT;QACA,IAAIA,WAAU;YACZ,OAAOA;QACT;IACF;AACF;AAUA,OAAO,SAASa;IACd,IAAIC,YAAY;IAChB,OAAO;QACLC,MAAKX,IAAY;YACf,IAAIY,MAAM;YACV,IAAIC,IAAI;YACR,MAAOA,IAAIb,KAAKI,MAAM,CAAE;gBACtB,IAAIM,WAAW;oBACb,MAAMI,MAAMd,KAAKe,OAAO,CAAC,OAAOF;oBAChC,IAAIC,QAAQ,CAAC,GAAG;wBACdF,OAAO,IAAII,MAAM,CAAChB,KAAKI,MAAM,GAAGS;wBAChCA,IAAIb,KAAKI,MAAM;oBACjB,OAAO;wBACLQ,OAAO,IAAII,MAAM,CAACF,MAAM,IAAID;wBAC5BA,IAAIC,MAAM;wBACVJ,YAAY;oBACd;oBACA;gBACF;gBACA,MAAMO,QAAQjB,KAAKe,OAAO,CAAC,QAAQF;gBACnC,IAAII,UAAU,CAAC,GAAG;oBAChBL,OAAOZ,KAAKkB,KAAK,CAACL;oBAClBA,IAAIb,KAAKI,MAAM;gBACjB,OAAO;oBACLQ,OAAO,GAAGZ,KAAKkB,KAAK,CAACL,GAAGI,OAAO,IAAI,CAAC;oBACpCJ,IAAII,QAAQ;oBACZP,YAAY;gBACd;YACF;YACA,OAAOE;QACT;QACA,IAAIF,aAAY;YACd,OAAOA;QACT;IACF;AACF;AAEA,yFAAyF;AACzF,+FAA+F;AAC/F,iDAAiD;AACjD,OAAO,SAASS,YAAYC,IAAY;IACtC,IAAI,CAAC,OAAOC,IAAI,CAACD,SAAS,CAACA,KAAKjB,QAAQ,CAAC,SAAS,OAAOiB;IACzD,MAAME,QAAQ3B;IACd,MAAM4B,UAAUd;IAChB,OAAOW,KACJI,KAAK,CAAC,MACNC,GAAG,CAAC,CAACzB;QACJ,IAAI,CAACuB,QAAQb,SAAS,EAAE;YACtB,MAAMgB,UAAUJ,MAAMvB,IAAI,CAACC;YAC3B,IAAI0B,WAAWJ,MAAM1B,OAAO,EAAE,OAAO,IAAIoB,MAAM,CAAChB,KAAKI,MAAM;QAC7D;QACA,MAAMuB,cAAcJ,QAAQZ,IAAI,CAACX;QACjC,4EAA4E;QAC5E,OAAO2B,YAAYxB,QAAQ,CAAC,OAAOyB,cAAcD,eAAeA;IAClE,GACCE,IAAI,CAAC;AACV;AAEA,yFAAyF;AACzF,6FAA6F;AAC7F,qFAAqF;AACrF,OAAO,SAASD,cAAc5B,IAAY;IACxC,IAAIY,MAAM;IACV,IAAIC,IAAI;IACR,MAAOA,IAAIb,KAAKI,MAAM,CAAE;QACtB,IAAIJ,IAAI,CAACa,EAAE,KAAK,KAAK;YACnBD,OAAOZ,IAAI,CAACa,EAAE;YACdA;YACA;QACF;QACA,IAAIiB,IAAIjB;QACR,MAAOb,IAAI,CAAC8B,EAAE,KAAK,IAAKA;QACxB,MAAMC,IAAID,IAAIjB;QACd,IAAImB,IAAIF;QACR,IAAIG,aAAa,CAAC;QAClB,IAAIC,WAAW,CAAC;QAChB,MAAOF,IAAIhC,KAAKI,MAAM,CAAE;YACtB,IAAIJ,IAAI,CAACgC,EAAE,KAAK,KAAK;gBACnBA;gBACA;YACF;YACA,IAAIzB,IAAIyB;YACR,MAAOhC,IAAI,CAACO,EAAE,KAAK,IAAKA;YACxB,IAAIA,IAAIyB,MAAMD,GAAG;gBACfE,aAAaD;gBACbE,WAAW3B;gBACX;YACF;YACAyB,IAAIzB;QACN;QACA,IAAI0B,cAAc,GAAG;YACnBrB,OAAO,IAAII,MAAM,CAACkB,WAAWrB;YAC7BA,IAAIqB;QACN,OAAO;YACLtB,OAAOZ,KAAKkB,KAAK,CAACL,GAAGiB;YACrBjB,IAAIiB;QACN;IACF;IACA,OAAOlB;AACT"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.14.0",
+  "version": "0.15.1",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",

package/skills/sense/SKILL.md

@@ -5,7 +5,7 @@ description: "Query a markdown tree with the sense CLI: filter notes by frontmat
 
 # sense
 
-SQL over a markdown tree, kept fresh by a filesystem check on every query. Every file becomes rows in `frontmatter` (one column per key, plus `path`/`_mtime`/`_ctime`/`_size`/`_rank`/`_parse_error`; `_ctime` is filesystem birthtime, which a clone or copy resets just like `_mtime`), `content` (FTS5: `title`, `summary`, `text`, `path`, plus machine-written `title_seg`/`summary_seg`/`text_seg` sidecars used for matching Chinese, Japanese, Thai, Khmer, Lao, and Burmese text, not for reading), `links` (`src`, `target`, `dst`, `embed`; `NULL` dst = dead link; `embed` 1 for `![[...]]` embeds, 0 for links; one row per distinct written target and kind with alias and anchor stripped, so `[[Foo]]` and `[[Foo|alias]]` are one row while `[[Foo]]` and `[[notes/Foo]]` are two rows that can share a `dst`, and a target both linked and embedded is a row of each kind), `tags` (`path`, `tag`: frontmatter and inline `#tags` merged and deduplicated; nested tags stored full, so `book/scifi` matches `tag = 'book' OR tag LIKE 'book/%'`), `sections` (heading outline with line ranges and token estimates), and `preset_files` (`path`, `preset`: which presets cover which files). Features add their own storage; `map` and `status` report which are on.
+SQL over a markdown tree, kept fresh by a filesystem check on every query. Every file becomes rows in `frontmatter` (one column per key, plus `path`/`_mtime`/`_ctime`/`_size`/`_rank`/`_parse_error`; `_ctime` is filesystem birthtime, which a clone or copy resets just like `_mtime`), `content` (FTS5: `title`, `summary`, `text`, `path`, plus machine-written `title_seg`/`summary_seg`/`text_seg` sidecars used for matching Chinese, Japanese, Thai, Khmer, Lao, and Burmese text, not for reading), `links` (`src`, `target`, `dst`, `embed`; `NULL` dst = dead link; `embed` 1 for `![[...]]` embeds, 0 for links; one row per distinct written target and kind with alias and anchor stripped, so `[[Foo]]` and `[[Foo|alias]]` are one row while `[[Foo]]` and `[[notes/Foo]]` are two rows that can share a `dst`, and a target both linked and embedded is a row of each kind; extraction matches Obsidian's own graph: comments, code, and link-syntax text yield no rows, `[[#Anchor]]` is a self-edge, a frontmatter value that is exactly `[[X]]` is a link, and a basename collision resolves to the linking note itself, else the shortest path), `tags` (`path`, `tag`: frontmatter and inline `#tags` merged and deduplicated; nested tags stored full, so `book/scifi` matches `tag = 'book' OR tag LIKE 'book/%'`), `sections` (heading outline with line ranges and token estimates), and `preset_files` (`path`, `preset`: which presets cover which files). Features add their own storage; `map` and `status` report which are on.
 
 ## What each tool is for