sensemaking 0.1.0 → 0.2.0

package/dist/esm/db.js

@@ -3,21 +3,14 @@ import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 import { STATE_DIR } from './config.js';
 import { listFiles, parseFile } from './scan.js';
-// Rows -> SQLite only: schema/ALTER, the reconcile diff + transaction,
-// pragmas, the meta table, has() registration. Filesystem reading and
-// frontmatter parsing live in scan.ts; this module never touches
-// node:fs/gray-matter directly and never prints (throws are for callers).
+// Rows -> SQLite: schema, reconcile, has(). Filesystem/frontmatter parsing lives in scan.ts.
 export const DB_FILENAME = 'cache.db';
-export const SCHEMA_VERSION = '1';
-// Quote a SQL identifier, escaping embedded double quotes so an unusual
-// frontmatter key (however unlikely) can't break out of the identifier.
+// Cache shape version, independent of the config's own `version`.
+export const SCHEMA_VERSION = '2';
 function quoteIdent(name) {
     return `"${name.split('"').join('""')}"`;
 }
-// The one custom SQL function in the whole tool: has(field, value).
-//   - JSON-array field (stored as JSON text, e.g. `["a","b"]`) -> membership
-//   - string field -> substring
-//   - NULL -> false
+// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.
 function registerFunctions(db) {
     db.function('has', {
         deterministic: true,
@@ -26,30 +19,27 @@ function registerFunctions(db) {
         if (field === null || field === undefined) return 0;
         const needle = String(value);
         if (typeof field === 'string') {
-            // Try JSON array first (arrays are stored as JSON text).
             if (field.startsWith('[')) {
                 try {
                     const parsed = JSON.parse(field);
                     if (Array.isArray(parsed)) {
                         return parsed.some((item)=>String(item) === needle) ? 1 : 0;
                     }
-                } catch  {
-                // fall through to substring match
-                }
+                } catch  {}
             }
             return field.includes(needle) ? 1 : 0;
         }
-        // Numbers, etc: coerce to string and substring-match.
         return String(field).includes(needle) ? 1 : 0;
     });
 }
 function getColumns(db) {
-    const rows = db.prepare('PRAGMA table_info(docs)').all();
+    const rows = db.prepare('PRAGMA table_info(frontmatter)').all();
     return new Set(rows.map((r)=>r.name));
 }
+// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter` can't dump file text into context.
 function ensureSchema(db) {
-    db.exec(`CREATE TABLE IF NOT EXISTS docs ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_size" INTEGER)`);
-    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
+    db.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_size" INTEGER)`);
+    db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);
     if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);
 }
 export function getMeta(db, key) {
@@ -64,18 +54,13 @@ export function setMeta(db, key, value) {
     db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);
 }
 export function docCount(db) {
-    const row = db.prepare('SELECT COUNT(*) AS n FROM docs').get();
+    const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get();
     return row.n;
 }
-// Reconcile the `docs` table against the filesystem: glob current files via
-// scan.ts, reparse stale (mtime/size changed) + new files in one
-// transaction, DELETE vanished paths, ALTER TABLE ADD COLUMN for newly
-// discovered frontmatter keys. Returns the number of files (re)parsed and
-// any warnings scan.ts collected (e.g. reserved-key collisions).
 export function reconcile(db, cfg, baseDir) {
     const files = listFiles(cfg, baseDir);
     const currentSet = new Set(files.map((f)=>f.relPath));
-    const existingRows = db.prepare(`SELECT "path", "_mtime", "_size" FROM docs`).all();
+    const existingRows = db.prepare(`SELECT "path", "_mtime", "_size" FROM frontmatter`).all();
     const existing = new Map(existingRows.map((r)=>[
             r.path,
             r
@@ -104,21 +89,25 @@ export function reconcile(db, cfg, baseDir) {
         }
         parsedDocs.push(doc);
     }
-    // seenColumns already contains the reserved columns (they're real columns
-    // of `docs`), so spreading it alone avoids duplicate names in the INSERT.
     const allColumns = [
         ...seenColumns
     ];
-    const insertSql = `INSERT OR REPLACE INTO docs (${allColumns.map(quoteIdent).join(', ')}) VALUES (${allColumns.map(()=>'?').join(', ')})`;
+    const insertSql = `INSERT OR REPLACE INTO frontmatter (${allColumns.map(quoteIdent).join(', ')}) VALUES (${allColumns.map(()=>'?').join(', ')})`;
     db.exec('BEGIN');
     try {
-        for (const col of newColumns)db.exec(`ALTER TABLE docs ADD COLUMN ${quoteIdent(col)}`);
+        for (const col of newColumns)db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);
+        // FTS5 has no upsert, so delete-before-insert into `content`.
+        const delBody = db.prepare(`DELETE FROM content WHERE "path" = ?`);
         if (vanished.length > 0) {
-            const del = db.prepare(`DELETE FROM docs WHERE "path" = ?`);
-            for (const path of vanished)del.run(path);
+            const del = db.prepare(`DELETE FROM frontmatter WHERE "path" = ?`);
+            for (const path of vanished){
+                del.run(path);
+                delBody.run(path);
+            }
         }
         if (parsedDocs.length > 0) {
             const insert = db.prepare(insertSql);
+            const insertBody = db.prepare(`INSERT INTO content (title, summary, text, "path") VALUES (?, ?, ?, ?)`);
             for (const doc of parsedDocs){
                 const values = allColumns.map((col)=>{
                     var _doc_data_col;
@@ -128,6 +117,8 @@ export function reconcile(db, cfg, baseDir) {
                     return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
                 });
                 insert.run(...values);
+                delBody.run(doc.relPath);
+                insertBody.run(doc.search.title, doc.search.summary, doc.search.text, doc.relPath);
             }
         }
         db.exec('COMMIT');
@@ -140,11 +131,6 @@ export function reconcile(db, cfg, baseDir) {
         warnings
     };
 }
-// open(resolved config): open (or create) the on-disk SQLite cache at
-// `<baseDir>/.sense/cache.db`, reconcile against the filesystem, and return
-// a live handle. The DB is a warm start, never a source of truth -- the
-// `.md` files remain truth. Takes only an already-resolved config; discovery
-// and version-gating are config.ts's job, not this one's.
 export function open(cfg) {
     const stateDir = join(cfg.baseDir, STATE_DIR);
     mkdirSync(stateDir, {
@@ -155,6 +141,17 @@ export function open(cfg) {
     db.exec('PRAGMA journal_mode = WAL');
     db.exec('PRAGMA busy_timeout = 5000');
     registerFunctions(db);
+    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
+    // Schema version mismatch: reconcile only reparses changed files, so an old cache can't be patched up incrementally -- rebuild instead.
+    const version = getMeta(db, 'schema_version');
+    if (version !== null && version !== SCHEMA_VERSION) {
+        db.close();
+        rmSync(stateDir, {
+            recursive: true,
+            force: true
+        });
+        return open(cfg);
+    }
     ensureSchema(db);
     const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);
     return {
@@ -165,8 +162,7 @@ export function open(cfg) {
         warnings
     };
 }
-// Delete the `.sense/` state dir entirely and reconcile fresh -- the manual
-// reset for lingering columns or a doubted cache.
+// Manual reset for a doubted cache.
 export function rebuild(cfg) {
     rmSync(join(cfg.baseDir, STATE_DIR), {
         recursive: true,

package/dist/esm/db.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["import { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { STATE_DIR } from './config.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile } from './scan.ts';\n\n// Rows -> SQLite only: schema/ALTER, the reconcile diff + transaction,\n// pragmas, the meta table, has() registration. Filesystem reading and\n// frontmatter parsing live in scan.ts; this module never touches\n// node:fs/gray-matter directly and never prints (throws are for callers).\n\nexport const DB_FILENAME = 'cache.db';\nexport const SCHEMA_VERSION = '1';\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  // Number of files (re)parsed by this open's reconcile -- 0 on a fully\n  // warm cache. Tests use this as reconcile instrumentation.\n  parsed: number;\n  warnings: string[];\n}\n\n// Quote a SQL identifier, escaping embedded double quotes so an unusual\n// frontmatter key (however unlikely) can't break out of the identifier.\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// The one custom SQL function in the whole tool: has(field, value).\n//   - JSON-array field (stored as JSON text, e.g. `[\"a\",\"b\"]`) -> membership\n//   - string field -> substring\n//   - NULL -> false\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      // Try JSON array first (arrays are stored as JSON text).\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {\n          // fall through to substring match\n        }\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    // Numbers, etc: coerce to string and substring-match.\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(docs)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nfunction ensureSchema(db: DatabaseSync): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS docs (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER)`);\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM docs').get() as { n: number };\n  return row.n;\n}\n\n// Reconcile the `docs` table against the filesystem: glob current files via\n// scan.ts, reparse stale (mtime/size changed) + new files in one\n// transaction, DELETE vanished paths, ALTER TABLE ADD COLUMN for newly\n// discovered frontmatter keys. Returns the number of files (re)parsed and\n// any warnings scan.ts collected (e.g. reserved-key collisions).\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM docs`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  for (const file of toReparse) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n\n  // seenColumns already contains the reserved columns (they're real columns\n  // of `docs`), so spreading it alone avoids duplicate names in the INSERT.\n  const allColumns = [...seenColumns];\n  const insertSql = `INSERT OR REPLACE INTO docs (${allColumns.map(quoteIdent).join(', ')}) VALUES (${allColumns.map(() => '?').join(', ')})`;\n\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE docs ADD COLUMN ${quoteIdent(col)}`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM docs WHERE \"path\" = ?`);\n      for (const path of vanished) del.run(path);\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      for (const doc of parsedDocs) {\n        const values = allColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          return doc.data[col] ?? null;\n        });\n        insert.run(...values);\n      }\n    }\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// open(resolved config): open (or create) the on-disk SQLite cache at\n// `<baseDir>/.sense/cache.db`, reconcile against the filesystem, and return\n// a live handle. The DB is a warm start, never a source of truth -- the\n// `.md` files remain truth. Takes only an already-resolved config; discovery\n// and version-gating are config.ts's job, not this one's.\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  db.exec('PRAGMA busy_timeout = 5000');\n  registerFunctions(db);\n\n  ensureSchema(db);\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Delete the `.sense/` state dir entirely and reconcile fresh -- the manual\n// reset for lingering columns or a doubted cache.\nexport function rebuild(cfg: ResolvedConfig): OpenResult {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n  return open(cfg);\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","STATE_DIR","listFiles","parseFile","DB_FILENAME","SCHEMA_VERSION","quoteIdent","name","split","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","Set","map","r","ensureSchema","exec","getMeta","setMeta","key","row","get","run","docCount","n","reconcile","cfg","baseDir","files","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","seenColumns","newColumns","parsedDocs","file","doc","fileWarnings","push","Object","keys","data","add","allColumns","insertSql","col","del","insert","values","err","open","stateDir","recursive","dbPath","rebuild","force"],"mappings":"AAAA,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,SAAS,QAAQ,cAAc;AAExC,SAASC,SAAS,EAAEC,SAAS,QAAQ,YAAY;AAEjD,uEAAuE;AACvE,sEAAsE;AACtE,iEAAiE;AACjE,0EAA0E;AAE1E,OAAO,MAAMC,cAAc,WAAW;AACtC,OAAO,MAAMC,iBAAiB,IAAI;AAYlC,wEAAwE;AACxE,wEAAwE;AACxE,SAASC,WAAWC,IAAY;IAC9B,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKT,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,oEAAoE;AACpE,6EAA6E;AAC7E,gCAAgC;AAChC,oBAAoB;AACpB,SAASU,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,MAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,yDAAyD;YACzD,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;oBAC9D;gBACF,EAAE,OAAM;gBACN,kCAAkC;gBACpC;YACF;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,sDAAsD;QACtD,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,MAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,2BAA2BC,GAAG;IACtD,OAAO,IAAIC,IAAIH,KAAKI,GAAG,CAAC,CAACC,IAAMA,EAAE3B,IAAI;AACvC;AAEA,SAAS4B,aAAazB,EAAgB;IACpCA,GAAG0B,IAAI,CAAC,CAAC,yFAAyF,CAAC;IACnG1B,GAAG0B,IAAI,CAAC;IACR,IAAIC,QAAQ3B,IAAI,sBAAsB,MAAM4B,QAAQ5B,IAAI,kBAAkBL;AAC5E;AAEA,OAAO,SAASgC,QAAQ3B,EAAgB,EAAE6B,GAAW;IACnD,MAAMC,MAAM9B,GAAGoB,OAAO,CAAC,wCAAwCW,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAIzB,KAAK,GAAG;AAC3B;AAEA,OAAO,SAASuB,QAAQ5B,EAAgB,EAAE6B,GAAW,EAAExB,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCY,GAAG,CAACH;QACjD;IACF;IACA7B,GAAGoB,OAAO,CAAC,qGAAqGY,GAAG,CAACH,KAAKxB;AAC3H;AAEA,OAAO,SAAS4B,SAASjC,EAAgB;IACvC,MAAM8B,MAAM9B,GAAGoB,OAAO,CAAC,kCAAkCW,GAAG;IAC5D,OAAOD,IAAII,CAAC;AACd;AAEA,4EAA4E;AAC5E,iEAAiE;AACjE,uEAAuE;AACvE,0EAA0E;AAC1E,iEAAiE;AACjE,OAAO,SAASC,UAAUnC,EAAgB,EAAEoC,GAAW,EAAEC,OAAe;IACtE,MAAMC,QAAQ9C,UAAU4C,KAAKC;IAC7B,MAAME,aAAa,IAAIjB,IAAIgB,MAAMf,GAAG,CAAC,CAACiB,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe1C,GAAGoB,OAAO,CAAC,CAAC,0CAA0C,CAAC,EAAEC,GAAG;IAKjF,MAAMsB,WAAW,IAAIC,IAAIF,aAAanB,GAAG,CAAC,CAACC,IAAM;YAACA,EAAEqB,IAAI;YAAErB;SAAE;IAC5D,MAAMsB,WAAWJ,aAAaK,MAAM,CAAC,CAACvB,IAAM,CAACe,WAAWS,GAAG,CAACxB,EAAEqB,IAAI,GAAGtB,GAAG,CAAC,CAACC,IAAMA,EAAEqB,IAAI;IAEtF,MAAMI,YAAYX,MAAMS,MAAM,CAAC,CAACP;QAC9B,MAAMV,MAAMa,SAASZ,GAAG,CAACS,EAAEC,OAAO;QAClC,OAAO,CAACX,OAAOA,IAAIoB,MAAM,KAAKV,EAAEW,OAAO,IAAIrB,IAAIsB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE5C,QAAQ;QAAG6C,UAAU,EAAE;IAAC;IAEtF,MAAMC,cAActC,WAAWlB;IAC/B,MAAMyD,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMH,WAAqB,EAAE;IAE7B,KAAK,MAAMI,QAAQV,UAAW;QAC5B,MAAM,EAAEW,GAAG,EAAEL,UAAUM,YAAY,EAAE,GAAGpE,UAAUkE;QAClDJ,SAASO,IAAI,IAAID;QACjB,KAAK,MAAMhC,OAAOkC,OAAOC,IAAI,CAACJ,IAAIK,IAAI,EAAG;YACvC,IAAI,CAACT,YAAYR,GAAG,CAACnB,MAAM;gBACzB2B,YAAYU,GAAG,CAACrC;gBAChB4B,WAAWK,IAAI,CAACjC;YAClB;QACF;QACA6B,WAAWI,IAAI,CAACF;IAClB;IAEA,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAMO,aAAa;WAAIX;KAAY;IACnC,MAAMY,YAAY,CAAC,6BAA6B,EAAED,WAAW5C,GAAG,CAAC3B,YAAYP,IAAI,CAAC,MAAM,UAAU,EAAE8E,WAAW5C,GAAG,CAAC,IAAM,KAAKlC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE3IW,GAAG0B,IAAI,CAAC;IACR,IAAI;QACF,KAAK,MAAM2C,OAAOZ,WAAYzD,GAAG0B,IAAI,CAAC,CAAC,4BAA4B,EAAE9B,WAAWyE,MAAM;QACtF,IAAIvB,SAASQ,MAAM,GAAG,GAAG;YACvB,MAAMgB,MAAMtE,GAAGoB,OAAO,CAAC,CAAC,iCAAiC,CAAC;YAC1D,KAAK,MAAMyB,QAAQC,SAAUwB,IAAItC,GAAG,CAACa;QACvC;QACA,IAAIa,WAAWJ,MAAM,GAAG,GAAG;YACzB,MAAMiB,SAASvE,GAAGoB,OAAO,CAACgD;YAC1B,KAAK,MAAMR,OAAOF,WAAY;gBAC5B,MAAMc,SAASL,WAAW5C,GAAG,CAAC,CAAC8C;wBAItBT;oBAHP,IAAIS,QAAQ,QAAQ,OAAOT,IAAInB,OAAO;oBACtC,IAAI4B,QAAQ,UAAU,OAAOT,IAAIT,OAAO;oBACxC,IAAIkB,QAAQ,SAAS,OAAOT,IAAIP,IAAI;oBACpC,QAAOO,gBAAAA,IAAIK,IAAI,CAACI,IAAI,cAAbT,2BAAAA,gBAAiB;gBAC1B;gBACAW,OAAOvC,GAAG,IAAIwC;YAChB;QACF;QACAxE,GAAG0B,IAAI,CAAC;IACV,EAAE,OAAO+C,KAAK;QACZzE,GAAG0B,IAAI,CAAC;QACR,MAAM+C;IACR;IAEA,OAAO;QAAE/D,QAAQgD,WAAWJ,MAAM;QAAEC;IAAS;AAC/C;AAEA,sEAAsE;AACtE,4EAA4E;AAC5E,wEAAwE;AACxE,6EAA6E;AAC7E,0DAA0D;AAC1D,OAAO,SAASmB,KAAKtC,GAAmB;IACtC,MAAMuC,WAAWtF,KAAK+C,IAAIC,OAAO,EAAE9C;IACnCJ,UAAUwF,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMC,SAASxF,KAAKsF,UAAUjF;IAE9B,MAAMM,KAAK,IAAIV,aAAauF;IAC5B7E,GAAG0B,IAAI,CAAC;IACR1B,GAAG0B,IAAI,CAAC;IACR3B,kBAAkBC;IAElByB,aAAazB;IACb,MAAM,EAAEU,MAAM,EAAE6C,QAAQ,EAAE,GAAGpB,UAAUnC,IAAIoC,KAAKA,IAAIC,OAAO;IAE3D,OAAO;QAAErC;QAAIoC;QAAKyC;QAAQnE;QAAQ6C;IAAS;AAC7C;AAEA,4EAA4E;AAC5E,kDAAkD;AAClD,OAAO,SAASuB,QAAQ1C,GAAmB;IACzChD,OAAOC,KAAK+C,IAAIC,OAAO,EAAE9C,YAAY;QAAEqF,WAAW;QAAMG,OAAO;IAAK;IACpE,OAAOL,KAAKtC;AACd"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["import { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { STATE_DIR } from './config.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile } from './scan.ts';\n\n// Rows -> SQLite: schema, reconcile, has(). Filesystem/frontmatter parsing lives in scan.ts.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter` can't dump file text into context.\nfunction ensureSchema(db: DatabaseSync): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER)`);\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  for (const file of toReparse) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n\n  const allColumns = [...seenColumns];\n  const insertSql = `INSERT OR REPLACE INTO frontmatter (${allColumns.map(quoteIdent).join(', ')}) VALUES (${allColumns.map(() => '?').join(', ')})`;\n\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`.\n    const delBody = db.prepare(`DELETE FROM content WHERE \"path\" = ?`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        del.run(path);\n        delBody.run(path);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(`INSERT INTO content (title, summary, text, \"path\") VALUES (?, ?, ?, ?)`);\n      for (const doc of parsedDocs) {\n        const values = allColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          return doc.data[col] ?? null;\n        });\n        insert.run(...values);\n        delBody.run(doc.relPath);\n        insertBody.run(doc.search.title, doc.search.summary, doc.search.text, doc.relPath);\n      }\n    }\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  db.exec('PRAGMA busy_timeout = 5000');\n  registerFunctions(db);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Schema version mismatch: reconcile only reparses changed files, so an old cache can't be patched up incrementally -- rebuild instead.\n  const version = getMeta(db, 'schema_version');\n  if (version !== null && version !== SCHEMA_VERSION) {\n    db.close();\n    rmSync(stateDir, { recursive: true, force: true });\n    return open(cfg);\n  }\n\n  ensureSchema(db);\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Manual reset for a doubted cache.\nexport function rebuild(cfg: ResolvedConfig): OpenResult {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n  return open(cfg);\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","STATE_DIR","listFiles","parseFile","DB_FILENAME","SCHEMA_VERSION","quoteIdent","name","split","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","Set","map","r","ensureSchema","exec","getMeta","setMeta","key","row","get","run","docCount","n","reconcile","cfg","baseDir","files","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","seenColumns","newColumns","parsedDocs","file","doc","fileWarnings","push","Object","keys","data","add","allColumns","insertSql","col","delBody","del","insert","insertBody","values","search","title","summary","text","err","open","stateDir","recursive","dbPath","version","close","force","rebuild"],"mappings":"AAAA,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,SAAS,QAAQ,cAAc;AAExC,SAASC,SAAS,EAAEC,SAAS,QAAQ,YAAY;AAEjD,6FAA6F;AAE7F,OAAO,MAAMC,cAAc,WAAW;AACtC,kEAAkE;AAClE,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,SAASC,WAAWC,IAAY;IAC9B,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKT,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,+FAA+F;AAC/F,SAASU,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,MAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;oBAC9D;gBACF,EAAE,OAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,MAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAIC,IAAIH,KAAKI,GAAG,CAAC,CAACC,IAAMA,EAAE3B,IAAI;AACvC;AAEA,8HAA8H;AAC9H,SAAS4B,aAAazB,EAAgB;IACpCA,GAAG0B,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1G1B,GAAG0B,IAAI,CAAC,CAAC,0HAA0H,CAAC;IACpI,IAAIC,QAAQ3B,IAAI,sBAAsB,MAAM4B,QAAQ5B,IAAI,kBAAkBL;AAC5E;AAEA,OAAO,SAASgC,QAAQ3B,EAAgB,EAAE6B,GAAW;IACnD,MAAMC,MAAM9B,GAAGoB,OAAO,CAAC,wCAAwCW,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAIzB,KAAK,GAAG;AAC3B;AAEA,OAAO,SAASuB,QAAQ5B,EAAgB,EAAE6B,GAAW,EAAExB,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCY,GAAG,CAACH;QACjD;IACF;IACA7B,GAAGoB,OAAO,CAAC,qGAAqGY,GAAG,CAACH,KAAKxB;AAC3H;AAEA,OAAO,SAAS4B,SAASjC,EAAgB;IACvC,MAAM8B,MAAM9B,GAAGoB,OAAO,CAAC,yCAAyCW,GAAG;IACnE,OAAOD,IAAII,CAAC;AACd;AAEA,OAAO,SAASC,UAAUnC,EAAgB,EAAEoC,GAAW,EAAEC,OAAe;IACtE,MAAMC,QAAQ9C,UAAU4C,KAAKC;IAC7B,MAAME,aAAa,IAAIjB,IAAIgB,MAAMf,GAAG,CAAC,CAACiB,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe1C,GAAGoB,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMsB,WAAW,IAAIC,IAAIF,aAAanB,GAAG,CAAC,CAACC,IAAM;YAACA,EAAEqB,IAAI;YAAErB;SAAE;IAC5D,MAAMsB,WAAWJ,aAAaK,MAAM,CAAC,CAACvB,IAAM,CAACe,WAAWS,GAAG,CAACxB,EAAEqB,IAAI,GAAGtB,GAAG,CAAC,CAACC,IAAMA,EAAEqB,IAAI;IAEtF,MAAMI,YAAYX,MAAMS,MAAM,CAAC,CAACP;QAC9B,MAAMV,MAAMa,SAASZ,GAAG,CAACS,EAAEC,OAAO;QAClC,OAAO,CAACX,OAAOA,IAAIoB,MAAM,KAAKV,EAAEW,OAAO,IAAIrB,IAAIsB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE5C,QAAQ;QAAG6C,UAAU,EAAE;IAAC;IAEtF,MAAMC,cAActC,WAAWlB;IAC/B,MAAMyD,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMH,WAAqB,EAAE;IAE7B,KAAK,MAAMI,QAAQV,UAAW;QAC5B,MAAM,EAAEW,GAAG,EAAEL,UAAUM,YAAY,EAAE,GAAGpE,UAAUkE;QAClDJ,SAASO,IAAI,IAAID;QACjB,KAAK,MAAMhC,OAAOkC,OAAOC,IAAI,CAACJ,IAAIK,IAAI,EAAG;YACvC,IAAI,CAACT,YAAYR,GAAG,CAACnB,MAAM;gBACzB2B,YAAYU,GAAG,CAACrC;gBAChB4B,WAAWK,IAAI,CAACjC;YAClB;QACF;QACA6B,WAAWI,IAAI,CAACF;IAClB;IAEA,MAAMO,aAAa;WAAIX;KAAY;IACnC,MAAMY,YAAY,CAAC,oCAAoC,EAAED,WAAW5C,GAAG,CAAC3B,YAAYP,IAAI,CAAC,MAAM,UAAU,EAAE8E,WAAW5C,GAAG,CAAC,IAAM,KAAKlC,IAAI,CAAC,MAAM,CAAC,CAAC;IAElJW,GAAG0B,IAAI,CAAC;IACR,IAAI;QACF,KAAK,MAAM2C,OAAOZ,WAAYzD,GAAG0B,IAAI,CAAC,CAAC,mCAAmC,EAAE9B,WAAWyE,MAAM;QAC7F,8DAA8D;QAC9D,MAAMC,UAAUtE,GAAGoB,OAAO,CAAC,CAAC,oCAAoC,CAAC;QACjE,IAAI0B,SAASQ,MAAM,GAAG,GAAG;YACvB,MAAMiB,MAAMvE,GAAGoB,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAMyB,QAAQC,SAAU;gBAC3ByB,IAAIvC,GAAG,CAACa;gBACRyB,QAAQtC,GAAG,CAACa;YACd;QACF;QACA,IAAIa,WAAWJ,MAAM,GAAG,GAAG;YACzB,MAAMkB,SAASxE,GAAGoB,OAAO,CAACgD;YAC1B,MAAMK,aAAazE,GAAGoB,OAAO,CAAC,CAAC,sEAAsE,CAAC;YACtG,KAAK,MAAMwC,OAAOF,WAAY;gBAC5B,MAAMgB,SAASP,WAAW5C,GAAG,CAAC,CAAC8C;wBAItBT;oBAHP,IAAIS,QAAQ,QAAQ,OAAOT,IAAInB,OAAO;oBACtC,IAAI4B,QAAQ,UAAU,OAAOT,IAAIT,OAAO;oBACxC,IAAIkB,QAAQ,SAAS,OAAOT,IAAIP,IAAI;oBACpC,QAAOO,gBAAAA,IAAIK,IAAI,CAACI,IAAI,cAAbT,2BAAAA,gBAAiB;gBAC1B;gBACAY,OAAOxC,GAAG,IAAI0C;gBACdJ,QAAQtC,GAAG,CAAC4B,IAAInB,OAAO;gBACvBgC,WAAWzC,GAAG,CAAC4B,IAAIe,MAAM,CAACC,KAAK,EAAEhB,IAAIe,MAAM,CAACE,OAAO,EAAEjB,IAAIe,MAAM,CAACG,IAAI,EAAElB,IAAInB,OAAO;YACnF;QACF;QACAzC,GAAG0B,IAAI,CAAC;IACV,EAAE,OAAOqD,KAAK;QACZ/E,GAAG0B,IAAI,CAAC;QACR,MAAMqD;IACR;IAEA,OAAO;QAAErE,QAAQgD,WAAWJ,MAAM;QAAEC;IAAS;AAC/C;AAEA,OAAO,SAASyB,KAAK5C,GAAmB;IACtC,MAAM6C,WAAW5F,KAAK+C,IAAIC,OAAO,EAAE9C;IACnCJ,UAAU8F,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMC,SAAS9F,KAAK4F,UAAUvF;IAE9B,MAAMM,KAAK,IAAIV,aAAa6F;IAC5BnF,GAAG0B,IAAI,CAAC;IACR1B,GAAG0B,IAAI,CAAC;IACR3B,kBAAkBC;IAElBA,GAAG0B,IAAI,CAAC;IAER,wIAAwI;IACxI,MAAM0D,UAAUzD,QAAQ3B,IAAI;IAC5B,IAAIoF,YAAY,QAAQA,YAAYzF,gBAAgB;QAClDK,GAAGqF,KAAK;QACRjG,OAAO6F,UAAU;YAAEC,WAAW;YAAMI,OAAO;QAAK;QAChD,OAAON,KAAK5C;IACd;IAEAX,aAAazB;IACb,MAAM,EAAEU,MAAM,EAAE6C,QAAQ,EAAE,GAAGpB,UAAUnC,IAAIoC,KAAKA,IAAIC,OAAO;IAE3D,OAAO;QAAErC;QAAIoC;QAAK+C;QAAQzE;QAAQ6C;IAAS;AAC7C;AAEA,oCAAoC;AACpC,OAAO,SAASgC,QAAQnD,GAAmB;IACzChD,OAAOC,KAAK+C,IAAIC,OAAO,EAAE9C,YAAY;QAAE2F,WAAW;QAAMI,OAAO;IAAK;IACpE,OAAON,KAAK5C;AACd"}

package/dist/esm/errors.js

@@ -1,6 +1,4 @@
-// Typed errors for library modules. Library code (config.ts, scan.ts,
-// db.ts, watch.ts) never calls process.exit and never prints -- it throws.
-// Only cli.ts catches, prints the message, and maps it to an exit code.
+// Library code throws; only cli.ts prints and exits.
 export class SenseError extends Error {
     constructor(code, message){
         super(message);

package/dist/esm/errors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Typed errors for library modules. Library code (config.ts, scan.ts,\n// db.ts, watch.ts) never calls process.exit and never prints -- it throws.\n// Only cli.ts catches, prints the message, and maps it to an exit code.\nexport type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","Error","code","message","name"],"mappings":"AAAA,sEAAsE;AACtE,2EAA2E;AAC3E,wEAAwE;AAGxE,OAAO,MAAMA,mBAAmBC;IAG9B,YAAYC,IAAoB,EAAEC,OAAe,CAAE;QACjD,KAAK,CAACA;QACN,IAAI,CAACC,IAAI,GAAG;QACZ,IAAI,CAACF,IAAI,GAAGA;IACd;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","Error","code","message","name"],"mappings":"AAAA,qDAAqD;AAGrD,OAAO,MAAMA,mBAAmBC;IAG9B,YAAYC,IAAoB,EAAEC,OAAe,CAAE;QACjD,KAAK,CAACA;QACN,IAAI,CAACC,IAAI,GAAG;QACZ,IAAI,CAACF,IAAI,GAAGA;IACd;AACF"}

package/dist/esm/index.js

@@ -1,6 +1,4 @@
-// Public library API. cli.ts is a consumer of this surface, not a source of
-// truth for it -- anything a test or another tool needs from sense should be
-// exported here rather than reached into src/ by relative path.
+// Public library API.
 export { CONFIG_FILENAME, findConfigPath, initConfig, loadConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
 export { DB_FILENAME, docCount, getMeta, open, rebuild, reconcile, setMeta } from './db.js';
 export { SenseError } from './errors.js';

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. cli.ts is a consumer of this surface, not a source of\n// truth for it -- anything a test or another tool needs from sense should be\n// exported here rather than reached into src/ by relative path.\n\nexport type { Config, ResolvedConfig } from './config.ts';\nexport { CONFIG_FILENAME, findConfigPath, initConfig, loadConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.ts';\n\nexport type { OpenResult } from './db.ts';\nexport { DB_FILENAME, docCount, getMeta, open, rebuild, reconcile, setMeta } from './db.ts';\n\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\n\nexport type { Row } from './output.ts';\nexport { printRows } from './output.ts';\n\nexport type { FileStat, ParsedDoc } from './scan.ts';\nexport { listFiles, parseFile } from './scan.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CONFIG_FILENAME","findConfigPath","initConfig","loadConfig","STATE_DIR","SUPPORTED_CONFIG_VERSION","DB_FILENAME","docCount","getMeta","open","rebuild","reconcile","setMeta","SenseError","printRows","listFiles","parseFile","runWatch"],"mappings":"AAAA,4EAA4E;AAC5E,6EAA6E;AAC7E,gEAAgE;AAGhE,SAASA,eAAe,EAAEC,cAAc,EAAEC,UAAU,EAAEC,UAAU,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,cAAc;AAG3H,SAASC,WAAW,EAAEC,QAAQ,EAAEC,OAAO,EAAEC,IAAI,EAAEC,OAAO,EAAEC,SAAS,EAAEC,OAAO,QAAQ,UAAU;AAG5F,SAASC,UAAU,QAAQ,cAAc;AAGzC,SAASC,SAAS,QAAQ,cAAc;AAGxC,SAASC,SAAS,EAAEC,SAAS,QAAQ,YAAY;AAGjD,SAASC,QAAQ,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API.\n\nexport type { Config, ResolvedConfig } from './config.ts';\nexport { CONFIG_FILENAME, findConfigPath, initConfig, loadConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.ts';\n\nexport type { OpenResult } from './db.ts';\nexport { DB_FILENAME, docCount, getMeta, open, rebuild, reconcile, setMeta } from './db.ts';\n\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\n\nexport type { Row } from './output.ts';\nexport { printRows } from './output.ts';\n\nexport type { FileStat, ParsedDoc } from './scan.ts';\nexport { listFiles, parseFile } from './scan.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CONFIG_FILENAME","findConfigPath","initConfig","loadConfig","STATE_DIR","SUPPORTED_CONFIG_VERSION","DB_FILENAME","docCount","getMeta","open","rebuild","reconcile","setMeta","SenseError","printRows","listFiles","parseFile","runWatch"],"mappings":"AAAA,sBAAsB;AAGtB,SAASA,eAAe,EAAEC,cAAc,EAAEC,UAAU,EAAEC,UAAU,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,cAAc;AAG3H,SAASC,WAAW,EAAEC,QAAQ,EAAEC,OAAO,EAAEC,IAAI,EAAEC,OAAO,EAAEC,SAAS,EAAEC,OAAO,QAAQ,UAAU;AAG5F,SAASC,UAAU,QAAQ,cAAc;AAGzC,SAASC,SAAS,QAAQ,cAAc;AAGxC,SAASC,SAAS,EAAEC,SAAS,QAAQ,YAAY;AAGjD,SAASC,QAAQ,QAAQ,aAAa"}

package/dist/esm/output.js

@@ -1,25 +1,27 @@
 // rows -> table (default) | json
+// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.
+const OVERSIZE_BYTES = 50000;
+// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.
+function cell(value) {
+    return String(value !== null && value !== void 0 ? value : '').replace(/\s*\n\s*/g, ' ');
+}
 export function printRows(rows, format) {
-    if (format === 'json') {
-        console.log(JSON.stringify(rows, null, 2));
-        return;
-    }
-    if (rows.length === 0) {
-        console.log('(0 rows)');
-        return;
+    const rendered = render(rows, format);
+    console.log(rendered);
+    if (rendered.length > OVERSIZE_BYTES) {
+        console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);
     }
+}
+function render(rows, format) {
+    if (format === 'json') return JSON.stringify(rows, null, 2);
+    if (rows.length === 0) return '(0 rows)';
     const columns = Object.keys(rows[0]);
-    const widths = columns.map((col)=>Math.max(col.length, ...rows.map((row)=>{
-            var _row_col;
-            return String((_row_col = row[col]) !== null && _row_col !== void 0 ? _row_col : '').length;
-        })));
+    const cells = rows.map((row)=>columns.map((col)=>cell(row[col])));
+    const widths = columns.map((col, i)=>Math.max(col.length, ...cells.map((row)=>row[i].length)));
     const formatRow = (values)=>values.map((value, i)=>value.padEnd(widths[i])).join('  ');
-    console.log(formatRow(columns));
-    console.log(widths.map((w)=>'-'.repeat(w)).join('  '));
-    for (const row of rows){
-        console.log(formatRow(columns.map((col)=>{
-            var _row_col;
-            return String((_row_col = row[col]) !== null && _row_col !== void 0 ? _row_col : '');
-        })));
-    }
+    return [
+        formatRow(columns),
+        widths.map((w)=>'-'.repeat(w)).join('  '),
+        ...cells.map(formatRow)
+    ].join('\n');
 }

package/dist/esm/output.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  if (format === 'json') {\n    console.log(JSON.stringify(rows, null, 2));\n    return;\n  }\n\n  if (rows.length === 0) {\n    console.log('(0 rows)');\n    return;\n  }\n\n  const columns = Object.keys(rows[0]);\n  const widths = columns.map((col) => Math.max(col.length, ...rows.map((row) => String(row[col] ?? '').length)));\n\n  const formatRow = (values: string[]) => values.map((value, i) => value.padEnd(widths[i])).join('  ');\n\n  console.log(formatRow(columns));\n  console.log(widths.map((w) => '-'.repeat(w)).join('  '));\n  for (const row of rows) {\n    console.log(formatRow(columns.map((col) => String(row[col] ?? ''))));\n  }\n}\n"],"names":["printRows","rows","format","console","log","JSON","stringify","length","columns","Object","keys","widths","map","col","Math","max","row","String","formatRow","values","value","i","padEnd","join","w","repeat"],"mappings":"AAAA,iCAAiC;AAIjC,OAAO,SAASA,UAAUC,IAAW,EAAEC,MAAwB;IAC7D,IAAIA,WAAW,QAAQ;QACrBC,QAAQC,GAAG,CAACC,KAAKC,SAAS,CAACL,MAAM,MAAM;QACvC;IACF;IAEA,IAAIA,KAAKM,MAAM,KAAK,GAAG;QACrBJ,QAAQC,GAAG,CAAC;QACZ;IACF;IAEA,MAAMI,UAAUC,OAAOC,IAAI,CAACT,IAAI,CAAC,EAAE;IACnC,MAAMU,SAASH,QAAQI,GAAG,CAAC,CAACC,MAAQC,KAAKC,GAAG,CAACF,IAAIN,MAAM,KAAKN,KAAKW,GAAG,CAAC,CAACI;gBAAeA;mBAAPC,QAAOD,WAAAA,GAAG,CAACH,IAAI,cAARG,sBAAAA,WAAY,IAAIT,MAAM;;IAE3G,MAAMW,YAAY,CAACC,SAAqBA,OAAOP,GAAG,CAAC,CAACQ,OAAOC,IAAMD,MAAME,MAAM,CAACX,MAAM,CAACU,EAAE,GAAGE,IAAI,CAAC;IAE/FpB,QAAQC,GAAG,CAACc,UAAUV;IACtBL,QAAQC,GAAG,CAACO,OAAOC,GAAG,CAAC,CAACY,IAAM,IAAIC,MAAM,CAACD,IAAID,IAAI,CAAC;IAClD,KAAK,MAAMP,OAAOf,KAAM;QACtBE,QAAQC,GAAG,CAACc,UAAUV,QAAQI,GAAG,CAAC,CAACC;gBAAeG;mBAAPC,QAAOD,WAAAA,GAAG,CAACH,IAAI,cAARG,sBAAAA,WAAY;;IAChE;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = render(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\nfunction render(rows: Row[], format: 'table' | 'json'): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const widths = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n\n  const formatRow = (values: string[]) => values.map((value, i) => value.padEnd(widths[i])).join('  ');\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","printRows","rows","format","rendered","render","console","log","length","warn","Math","round","JSON","stringify","columns","Object","keys","cells","map","row","col","widths","i","max","formatRow","values","padEnd","join","w","repeat"],"mappings":"AAAA,iCAAiC;AAIjC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,OAAO,SAASC,UAAUC,IAAW,EAAEC,MAAwB;IAC7D,MAAMC,WAAWC,OAAOH,MAAMC;IAC9BG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGZ,gBAAgB;QACpCU,QAAQG,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,MAAM,WAAW,EAAEN,KAAKM,MAAM,CAAC,8EAA8E,CAAC;IAChL;AACF;AAEA,SAASH,OAAOH,IAAW,EAAEC,MAAwB;IACnD,IAAIA,WAAW,QAAQ,OAAOS,KAAKC,SAAS,CAACX,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMM,UAAUC,OAAOC,IAAI,CAACd,IAAI,CAAC,EAAE;IACnC,MAAMe,QAAQf,KAAKgB,GAAG,CAAC,CAACC,MAAQL,QAAQI,GAAG,CAAC,CAACE,MAAQvB,KAAKsB,GAAG,CAACC,IAAI;IAClE,MAAMC,SAASP,QAAQI,GAAG,CAAC,CAACE,KAAKE,IAAMZ,KAAKa,GAAG,CAACH,IAAIZ,MAAM,KAAKS,MAAMC,GAAG,CAAC,CAACC,MAAQA,GAAG,CAACG,EAAE,CAACd,MAAM;IAE/F,MAAMgB,YAAY,CAACC,SAAqBA,OAAOP,GAAG,CAAC,CAACpB,OAAOwB,IAAMxB,MAAM4B,MAAM,CAACL,MAAM,CAACC,EAAE,GAAGK,IAAI,CAAC;IAE/F,OAAO;QAACH,UAAUV;QAAUO,OAAOH,GAAG,CAAC,CAACU,IAAM,IAAIC,MAAM,CAACD,IAAID,IAAI,CAAC;WAAUV,MAAMC,GAAG,CAACM;KAAW,CAACG,IAAI,CAAC;AACzG"}

package/dist/esm/scan.d.ts

@@ -11,6 +11,11 @@ export interface ParsedDoc {
     mtimeMs: number;
     size: number;
     data: Record<string, string | number | null>;
+    search: {
+        title: string;
+        summary: string;
+        text: string;
+    };
 }
 export declare function parseFile(file: FileStat): {
     doc: ParsedDoc;

package/dist/esm/scan.js

@@ -2,20 +2,27 @@ import { readFileSync, statSync } from 'node:fs';
 import { join } from 'node:path';
 import fastGlob from 'fast-glob';
 import matter from 'gray-matter';
-// Filesystem -> rows. Pure data in, data + warnings out -- no node:sqlite,
-// no printing. db.ts is the only thing that knows what to do with the
-// result (diffing against `docs`, writing SQL).
-// Reserved `docs` columns: the real file path plus the mtime/size pair used
-// to detect staleness. A frontmatter key literally named one of these would
-// collide, so it's dropped (with a warning returned to the caller) rather
-// than clobbering the column.
+import removeMarkdown from 'remove-markdown';
+// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.
+// Reserved: colliding with these would clash with the `frontmatter` table's own columns or the `content` FTS5 table.
 const RESERVED_COLUMNS = new Set([
     'path',
     '_mtime',
-    '_size'
+    '_size',
+    'content'
 ]);
-// Glob the files `cfg.scan.include` matches under `baseDir`, stat each.
-// Sorted for deterministic output.
+function normalizeText(value) {
+    if (value === null || value === undefined) return '';
+    return String(value).replace(/\s+/g, ' ').trim();
+}
+// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).
+// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.
+function stripText(value) {
+    const withoutWikilinks = value.replace(/\[\[([^\]|]+)\|([^\]]+)\]\]/g, '$2').replace(/\[\[([^\]]+)\]\]/g, '$1');
+    const withoutMarkdown = removeMarkdown(withoutWikilinks);
+    const withoutTables = withoutMarkdown.replace(/^\s*\|?[-\s|:]+\|\s*$/gm, '').replace(/\|/g, ' ');
+    return normalizeText(withoutTables);
+}
 export function listFiles(cfg, baseDir) {
     const relPaths = fastGlob.sync(cfg.scan.include, {
         cwd: baseDir
@@ -31,23 +38,16 @@ export function listFiles(cfg, baseDir) {
         };
     });
 }
-// Value mapping: strings/numbers as-is; booleans -> 0/1; dates -> ISO
-// strings (lexicographic = chronological, no date lib); arrays/objects ->
-// JSON text. `null`/`undefined` map to SQL NULL.
 function mapValue(value) {
     if (value === null || value === undefined) return null;
     if (value instanceof Date) return value.toISOString();
     if (typeof value === 'boolean') return value ? 1 : 0;
     if (typeof value === 'string' || typeof value === 'number') return value;
-    // arrays and plain objects
     return JSON.stringify(value);
 }
-// Read + parse one file's frontmatter, mapping values and dropping reserved
-// keys. Returns the parsed doc plus any warnings (e.g. a reserved key
-// collision) for the caller to surface.
 export function parseFile(file) {
     const raw = readFileSync(file.absPath, 'utf8');
-    const { data } = matter(raw);
+    const { data, content } = matter(raw);
     const warnings = [];
     const mapped = {};
     for (const key of Object.keys(data)){
@@ -62,7 +62,14 @@ export function parseFile(file) {
             relPath: file.relPath,
             mtimeMs: file.mtimeMs,
             size: file.size,
-            data: mapped
+            data: mapped,
+            search: {
+                // title/summary are plain YAML strings -- whitespace-collapse only;
+                // the prose gets the full markdown strip.
+                title: normalizeText(data.title),
+                summary: normalizeText(data.summary),
+                text: stripText(content)
+            }
         },
         warnings
     };

package/dist/esm/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport matter from 'gray-matter';\nimport type { Config } from './config.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out -- no node:sqlite,\n// no printing. db.ts is the only thing that knows what to do with the\n// result (diffing against `docs`, writing SQL).\n\n// Reserved `docs` columns: the real file path plus the mtime/size pair used\n// to detect staleness. A frontmatter key literally named one of these would\n// collide, so it's dropped (with a warning returned to the caller) rather\n// than clobbering the column.\nconst RESERVED_COLUMNS = new Set(['path', '_mtime', '_size']);\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n}\n\n// Glob the files `cfg.scan.include` matches under `baseDir`, stat each.\n// Sorted for deterministic output.\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const relPaths = fastGlob.sync(cfg.scan.include, { cwd: baseDir }).sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  // Already value-mapped (booleans -> 0/1, dates -> ISO, arrays/objects ->\n  // JSON text), reserved keys already dropped -- ready to bind into SQL.\n  data: Record<string, string | number | null>;\n}\n\n// Value mapping: strings/numbers as-is; booleans -> 0/1; dates -> ISO\n// strings (lexicographic = chronological, no date lib); arrays/objects ->\n// JSON text. `null`/`undefined` map to SQL NULL.\nfunction mapValue(value: unknown): string | number | null {\n  if (value === null || value === undefined) return null;\n  if (value instanceof Date) return value.toISOString();\n  if (typeof value === 'boolean') return value ? 1 : 0;\n  if (typeof value === 'string' || typeof value === 'number') return value;\n  // arrays and plain objects\n  return JSON.stringify(value);\n}\n\n// Read + parse one file's frontmatter, mapping values and dropping reserved\n// keys. Returns the parsed doc plus any warnings (e.g. a reserved key\n// collision) for the caller to surface.\nexport function parseFile(file: FileStat): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const { data } = matter(raw);\n\n  const warnings: string[] = [];\n  const mapped: Record<string, string | number | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  return {\n    doc: { relPath: file.relPath, mtimeMs: file.mtimeMs, size: file.size, data: mapped },\n    warnings,\n  };\n}\n"],"names":["readFileSync","statSync","join","fastGlob","matter","RESERVED_COLUMNS","Set","listFiles","cfg","baseDir","relPaths","sync","scan","include","cwd","sort","map","relPath","absPath","st","mtimeMs","size","mapValue","value","undefined","Date","toISOString","JSON","stringify","parseFile","file","raw","data","warnings","mapped","key","Object","keys","has","push","doc"],"mappings":"AAAA,SAASA,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AACjD,SAASC,IAAI,QAAQ,YAAY;AACjC,OAAOC,cAAc,YAAY;AACjC,OAAOC,YAAY,cAAc;AAGjC,2EAA2E;AAC3E,sEAAsE;AACtE,gDAAgD;AAEhD,4EAA4E;AAC5E,4EAA4E;AAC5E,0EAA0E;AAC1E,8BAA8B;AAC9B,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;CAAQ;AAS5D,wEAAwE;AACxE,mCAAmC;AACnC,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAWP,SAASQ,IAAI,CAACH,IAAII,IAAI,CAACC,OAAO,EAAE;QAAEC,KAAKL;IAAQ,GAAGM,IAAI;IACvE,OAAOL,SAASM,GAAG,CAAC,CAACC;QACnB,MAAMC,UAAUhB,KAAKO,SAASQ;QAC9B,MAAME,KAAKlB,SAASiB;QACpB,OAAO;YAAED;YAASC;YAASE,SAASD,GAAGC,OAAO;YAAEC,MAAMF,GAAGE,IAAI;QAAC;IAChE;AACF;AAWA,sEAAsE;AACtE,0EAA0E;AAC1E,iDAAiD;AACjD,SAASC,SAASC,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAID,iBAAiBE,MAAM,OAAOF,MAAMG,WAAW;IACnD,IAAI,OAAOH,UAAU,WAAW,OAAOA,QAAQ,IAAI;IACnD,IAAI,OAAOA,UAAU,YAAY,OAAOA,UAAU,UAAU,OAAOA;IACnE,2BAA2B;IAC3B,OAAOI,KAAKC,SAAS,CAACL;AACxB;AAEA,4EAA4E;AAC5E,sEAAsE;AACtE,wCAAwC;AACxC,OAAO,SAASM,UAAUC,IAAc;IACtC,MAAMC,MAAM/B,aAAa8B,KAAKZ,OAAO,EAAE;IACvC,MAAM,EAAEc,IAAI,EAAE,GAAG5B,OAAO2B;IAExB,MAAME,WAAqB,EAAE;IAC7B,MAAMC,SAAiD,CAAC;IAExD,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACL,MAAO;QACnC,IAAI3B,iBAAiBiC,GAAG,CAACH,MAAM;YAC7BF,SAASM,IAAI,CAAC,CAAC,SAAS,EAAET,KAAKb,OAAO,CAAC,8BAA8B,EAAEkB,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAD,MAAM,CAACC,IAAI,GAAGb,SAASU,IAAI,CAACG,IAAI;IAClC;IAEA,OAAO;QACLK,KAAK;YAAEvB,SAASa,KAAKb,OAAO;YAAEG,SAASU,KAAKV,OAAO;YAAEC,MAAMS,KAAKT,IAAI;YAAEW,MAAME;QAAO;QACnFD;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport matter from 'gray-matter';\nimport removeMarkdown from 'remove-markdown';\nimport type { Config } from './config.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or the `content` FTS5 table.\nconst RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', 'content']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n}\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const relPaths = fastGlob.sync(cfg.scan.include, { cwd: baseDir }).sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  data: Record<string, string | number | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n}\n\nfunction mapValue(value: unknown): string | number | null {\n  if (value === null || value === undefined) return null;\n  if (value instanceof Date) return value.toISOString();\n  if (typeof value === 'boolean') return value ? 1 : 0;\n  if (typeof value === 'string' || typeof value === 'number') return value;\n  return JSON.stringify(value);\n}\n\nexport function parseFile(file: FileStat): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const { data, content } = matter(raw);\n\n  const warnings: string[] = [];\n  const mapped: Record<string, string | number | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      data: mapped,\n      search: {\n        // title/summary are plain YAML strings -- whitespace-collapse only;\n        // the prose gets the full markdown strip.\n        title: normalizeText(data.title),\n        summary: normalizeText(data.summary),\n        text: stripText(content),\n      },\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","statSync","join","fastGlob","matter","removeMarkdown","RESERVED_COLUMNS","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","listFiles","cfg","baseDir","relPaths","sync","scan","include","cwd","sort","map","relPath","absPath","st","mtimeMs","size","mapValue","Date","toISOString","JSON","stringify","parseFile","file","raw","data","content","warnings","mapped","key","Object","keys","has","push","doc","search","title","summary","text"],"mappings":"AAAA,SAASA,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AACjD,SAASC,IAAI,QAAQ,YAAY;AACjC,OAAOC,cAAc,YAAY;AACjC,OAAOC,YAAY,cAAc;AACjC,OAAOC,oBAAoB,kBAAkB;AAG7C,6EAA6E;AAE7E,qHAAqH;AACrH,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;CAAU;AAEvE,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBX,eAAeU;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AASA,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAWlB,SAASmB,IAAI,CAACH,IAAII,IAAI,CAACC,OAAO,EAAE;QAAEC,KAAKL;IAAQ,GAAGM,IAAI;IACvE,OAAOL,SAASM,GAAG,CAAC,CAACC;QACnB,MAAMC,UAAU3B,KAAKkB,SAASQ;QAC9B,MAAME,KAAK7B,SAAS4B;QACpB,OAAO;YAAED;YAASC;YAASE,SAASD,GAAGC,OAAO;YAAEC,MAAMF,GAAGE,IAAI;QAAC;IAChE;AACF;AAWA,SAASC,SAASxB,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAID,iBAAiByB,MAAM,OAAOzB,MAAM0B,WAAW;IACnD,IAAI,OAAO1B,UAAU,WAAW,OAAOA,QAAQ,IAAI;IACnD,IAAI,OAAOA,UAAU,YAAY,OAAOA,UAAU,UAAU,OAAOA;IACnE,OAAO2B,KAAKC,SAAS,CAAC5B;AACxB;AAEA,OAAO,SAAS6B,UAAUC,IAAc;IACtC,MAAMC,MAAMxC,aAAauC,KAAKV,OAAO,EAAE;IACvC,MAAM,EAAEY,IAAI,EAAEC,OAAO,EAAE,GAAGtC,OAAOoC;IAEjC,MAAMG,WAAqB,EAAE;IAC7B,MAAMC,SAAiD,CAAC;IAExD,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACN,MAAO;QACnC,IAAInC,iBAAiB0C,GAAG,CAACH,MAAM;YAC7BF,SAASM,IAAI,CAAC,CAAC,SAAS,EAAEV,KAAKX,OAAO,CAAC,8BAA8B,EAAEiB,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAD,MAAM,CAACC,IAAI,GAAGZ,SAASQ,IAAI,CAACI,IAAI;IAClC;IAEA,OAAO;QACLK,KAAK;YACHtB,SAASW,KAAKX,OAAO;YACrBG,SAASQ,KAAKR,OAAO;YACrBC,MAAMO,KAAKP,IAAI;YACfS,MAAMG;YACNO,QAAQ;gBACN,oEAAoE;gBACpE,0CAA0C;gBAC1CC,OAAO5C,cAAciC,KAAKW,KAAK;gBAC/BC,SAAS7C,cAAciC,KAAKY,OAAO;gBACnCC,MAAMxC,UAAU4B;YAClB;QACF;QACAC;IACF;AACF"}

package/dist/esm/watch.js

@@ -2,19 +2,11 @@ import { watch as fsWatch } from 'node:fs';
 import { STATE_DIR } from './config.js';
 import { docCount, getMeta, open, reconcile, setMeta } from './db.js';
 import { SenseError } from './errors.js';
-// Watch is a cache pre-warmer, not a correctness mechanism: every query
-// `open()` reconciles against the filesystem regardless, so a missed or
-// coalesced fs event can never change a query result -- it only moves one
-// file's parse from ahead-of-time to query-time. That's why any event, of
-// any kind, triggers the exact same response: a debounced full reconcile.
-// Per-file event handling would be complexity with no correctness payoff.
+// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.
 const DEBOUNCE_MS = 200;
 const HEARTBEAT_INTERVAL_MS = 5000;
 const STALE_HEARTBEAT_MS = 15000;
-// Runs in the foreground until SIGINT/SIGTERM; never forks or daemonizes --
-// process lifecycle belongs to the OS (launchd/systemd/terminal). Resolves
-// on clean shutdown; throws SenseError("WATCH_ACTIVE", ...) if another
-// watcher's heartbeat is still fresh and `--force` wasn't given.
+// Runs in the foreground until SIGINT/SIGTERM. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.
 export async function runWatch(cfg, opts = {}) {
     var _opts_onEvent;
     const onEvent = (_opts_onEvent = opts.onEvent) !== null && _opts_onEvent !== void 0 ? _opts_onEvent : ()=>{};
@@ -67,9 +59,7 @@ export async function runWatch(cfg, opts = {}) {
             }
         }, DEBOUNCE_MS);
     };
-    // Ignore events from our own state dir: the heartbeat writes cache.db
-    // every few seconds, and without this filter each write would schedule a
-    // (harmless but pointless) reconcile forever — a perpetual self-trigger.
+    // Ignore our own state dir, or the heartbeat write would retrigger itself forever.
     const watcher = fsWatch(baseDir, {
         recursive: true
     }, (_event, filename)=>{

package/dist/esm/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config.ts';\nimport { STATE_DIR } from './config.ts';\nimport { docCount, getMeta, open, reconcile, setMeta } from './db.ts';\nimport { SenseError } from './errors.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: every query\n// `open()` reconciles against the filesystem regardless, so a missed or\n// coalesced fs event can never change a query result -- it only moves one\n// file's parse from ahead-of-time to query-time. That's why any event, of\n// any kind, triggers the exact same response: a debounced full reconcile.\n// Per-file event handling would be complexity with no correctness payoff.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\n// This module never prints -- it emits events via `onEvent` and throws on\n// the one exit-worthy condition (a live watcher already present). cli.ts is\n// the only place that touches console/process.exit.\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM; never forks or daemonizes --\n// process lifecycle belongs to the OS (launchd/systemd/terminal). Resolves\n// on clean shutdown; throws SenseError(\"WATCH_ACTIVE\", ...) if another\n// watcher's heartbeat is still fresh and `--force` wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  const onEvent = opts.onEvent ?? (() => {});\n  const { db, dbPath, warnings: initialWarnings, parsed: initialParsed } = open(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = getMeta(db, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      db.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: docCount(db), warnings: initialWarnings });\n  }\n\n  const touchHeartbeat = () => {\n    setMeta(db, 'watch_heartbeat', new Date().toISOString());\n    setMeta(db, 'watch_pid', String(process.pid));\n  };\n  touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      try {\n        const { parsed, warnings } = reconcile(db, cfg, baseDir);\n        onEvent({ type: 'reconciled', parsed, total: docCount(db), warnings });\n      } catch (err) {\n        onEvent({ type: 'reconcile-error', message: (err as Error).message });\n      }\n    }, DEBOUNCE_MS);\n  };\n\n  // Ignore events from our own state dir: the heartbeat writes cache.db\n  // every few seconds, and without this filter each write would schedule a\n  // (harmless but pointless) reconcile forever — a perpetual self-trigger.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(touchHeartbeat, HEARTBEAT_INTERVAL_MS);\n\n  return new Promise<void>((resolveShutdown) => {\n    const cleanExit = () => {\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      setMeta(db, 'watch_heartbeat', null);\n      setMeta(db, 'watch_pid', null);\n      db.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', cleanExit);\n    process.once('SIGTERM', cleanExit);\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","docCount","getMeta","open","reconcile","setMeta","SenseError","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","db","dbPath","warnings","initialWarnings","parsed","initialParsed","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","type","length","total","touchHeartbeat","toISOString","String","process","pid","debounceTimer","scheduleReconcile","clearTimeout","setTimeout","err","message","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","cleanExit","clearInterval","once"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,cAAc;AACxC,SAASC,QAAQ,EAAEC,OAAO,EAAEC,IAAI,EAAEC,SAAS,EAAEC,OAAO,QAAQ,UAAU;AACtE,SAASC,UAAU,QAAQ,cAAc;AAEzC,wEAAwE;AACxE,wEAAwE;AACxE,0EAA0E;AAC1E,0EAA0E;AAC1E,0EAA0E;AAC1E,0EAA0E;AAC1E,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAY3B,4EAA4E;AAC5E,2EAA2E;AAC3E,uEAAuE;AACvE,iEAAiE;AACjE,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QACzDA;IAAhB,MAAMC,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAM,EAAEE,EAAE,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAGhB,KAAKQ;IAC9E,MAAMS,UAAUT,IAAIS,OAAO;IAE3B,MAAMC,oBAAoBnB,QAAQY,IAAI;IACtC,IAAIO,qBAAqB,CAACT,KAAKU,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMd,oBAAoB;YACxCK,GAAGa,KAAK;YACR,MAAM,IAAIrB,WAAW,gBAAgB,CAAC,0CAA0C,EAAEsB,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEAV,QAAQ;QAAEiB,MAAM;QAAWV;QAASL;IAAO;IAC3C,IAAIE,gBAAgBc,MAAM,GAAG,KAAKZ,gBAAgB,GAAG;QACnDN,QAAQ;YAAEiB,MAAM;YAAcZ,QAAQC;YAAea,OAAO/B,SAASa;YAAKE,UAAUC;QAAgB;IACtG;IAEA,MAAMgB,iBAAiB;QACrB5B,QAAQS,IAAI,mBAAmB,IAAIU,OAAOU,WAAW;QACrD7B,QAAQS,IAAI,aAAaqB,OAAOC,QAAQC,GAAG;IAC7C;IACAJ;IAEA,IAAIK,gBAAuC;IAC3C,MAAMC,oBAAoB;QACxB,IAAID,eAAeE,aAAaF;QAChCA,gBAAgBG,WAAW;YACzBH,gBAAgB;YAChB,IAAI;gBACF,MAAM,EAAEpB,MAAM,EAAEF,QAAQ,EAAE,GAAGZ,UAAUU,IAAIH,KAAKS;gBAChDP,QAAQ;oBAAEiB,MAAM;oBAAcZ;oBAAQc,OAAO/B,SAASa;oBAAKE;gBAAS;YACtE,EAAE,OAAO0B,KAAK;gBACZ7B,QAAQ;oBAAEiB,MAAM;oBAAmBa,SAAS,AAACD,IAAcC,OAAO;gBAAC;YACrE;QACF,GAAGpC;IACL;IAEA,sEAAsE;IACtE,yEAAyE;IACzE,yEAAyE;IACzE,MAAMqC,UAAU7C,QAAQqB,SAAS;QAAEyB,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAAChD,YAAY;QACpEuC;IACF;IACA,MAAMU,iBAAiBC,YAAYjB,gBAAgBzB;IAEnD,OAAO,IAAI2C,QAAc,CAACC;QACxB,MAAMC,YAAY;YAChBC,cAAcL;YACd,IAAIX,eAAeE,aAAaF;YAChCM,QAAQjB,KAAK;YACbtB,QAAQS,IAAI,mBAAmB;YAC/BT,QAAQS,IAAI,aAAa;YACzBA,GAAGa,KAAK;YACRyB;QACF;QACAhB,QAAQmB,IAAI,CAAC,UAAUF;QACvBjB,QAAQmB,IAAI,CAAC,WAAWF;IAC1B;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config.ts';\nimport { STATE_DIR } from './config.ts';\nimport { docCount, getMeta, open, reconcile, setMeta } from './db.ts';\nimport { SenseError } from './errors.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  const onEvent = opts.onEvent ?? (() => {});\n  const { db, dbPath, warnings: initialWarnings, parsed: initialParsed } = open(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = getMeta(db, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      db.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: docCount(db), warnings: initialWarnings });\n  }\n\n  const touchHeartbeat = () => {\n    setMeta(db, 'watch_heartbeat', new Date().toISOString());\n    setMeta(db, 'watch_pid', String(process.pid));\n  };\n  touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      try {\n        const { parsed, warnings } = reconcile(db, cfg, baseDir);\n        onEvent({ type: 'reconciled', parsed, total: docCount(db), warnings });\n      } catch (err) {\n        onEvent({ type: 'reconcile-error', message: (err as Error).message });\n      }\n    }, DEBOUNCE_MS);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(touchHeartbeat, HEARTBEAT_INTERVAL_MS);\n\n  return new Promise<void>((resolveShutdown) => {\n    const cleanExit = () => {\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      setMeta(db, 'watch_heartbeat', null);\n      setMeta(db, 'watch_pid', null);\n      db.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', cleanExit);\n    process.once('SIGTERM', cleanExit);\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","docCount","getMeta","open","reconcile","setMeta","SenseError","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","db","dbPath","warnings","initialWarnings","parsed","initialParsed","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","type","length","total","touchHeartbeat","toISOString","String","process","pid","debounceTimer","scheduleReconcile","clearTimeout","setTimeout","err","message","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","cleanExit","clearInterval","once"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,cAAc;AACxC,SAASC,QAAQ,EAAEC,OAAO,EAAEC,IAAI,EAAEC,SAAS,EAAEC,OAAO,QAAQ,UAAU;AACtE,SAASC,UAAU,QAAQ,cAAc;AAEzC,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAS3B,2IAA2I;AAC3I,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QACzDA;IAAhB,MAAMC,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAM,EAAEE,EAAE,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAGhB,KAAKQ;IAC9E,MAAMS,UAAUT,IAAIS,OAAO;IAE3B,MAAMC,oBAAoBnB,QAAQY,IAAI;IACtC,IAAIO,qBAAqB,CAACT,KAAKU,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMd,oBAAoB;YACxCK,GAAGa,KAAK;YACR,MAAM,IAAIrB,WAAW,gBAAgB,CAAC,0CAA0C,EAAEsB,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEAV,QAAQ;QAAEiB,MAAM;QAAWV;QAASL;IAAO;IAC3C,IAAIE,gBAAgBc,MAAM,GAAG,KAAKZ,gBAAgB,GAAG;QACnDN,QAAQ;YAAEiB,MAAM;YAAcZ,QAAQC;YAAea,OAAO/B,SAASa;YAAKE,UAAUC;QAAgB;IACtG;IAEA,MAAMgB,iBAAiB;QACrB5B,QAAQS,IAAI,mBAAmB,IAAIU,OAAOU,WAAW;QACrD7B,QAAQS,IAAI,aAAaqB,OAAOC,QAAQC,GAAG;IAC7C;IACAJ;IAEA,IAAIK,gBAAuC;IAC3C,MAAMC,oBAAoB;QACxB,IAAID,eAAeE,aAAaF;QAChCA,gBAAgBG,WAAW;YACzBH,gBAAgB;YAChB,IAAI;gBACF,MAAM,EAAEpB,MAAM,EAAEF,QAAQ,EAAE,GAAGZ,UAAUU,IAAIH,KAAKS;gBAChDP,QAAQ;oBAAEiB,MAAM;oBAAcZ;oBAAQc,OAAO/B,SAASa;oBAAKE;gBAAS;YACtE,EAAE,OAAO0B,KAAK;gBACZ7B,QAAQ;oBAAEiB,MAAM;oBAAmBa,SAAS,AAACD,IAAcC,OAAO;gBAAC;YACrE;QACF,GAAGpC;IACL;IAEA,mFAAmF;IACnF,MAAMqC,UAAU7C,QAAQqB,SAAS;QAAEyB,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAAChD,YAAY;QACpEuC;IACF;IACA,MAAMU,iBAAiBC,YAAYjB,gBAAgBzB;IAEnD,OAAO,IAAI2C,QAAc,CAACC;QACxB,MAAMC,YAAY;YAChBC,cAAcL;YACd,IAAIX,eAAeE,aAAaF;YAChCM,QAAQjB,KAAK;YACbtB,QAAQS,IAAI,mBAAmB;YAC/BT,QAAQS,IAAI,aAAa;YACzBA,GAAGa,KAAK;YACRyB;QACF;QACAhB,QAAQmB,IAAI,CAAC,UAAUF;QACvBjB,QAAQmB,IAAI,CAAC,WAAWF;IAC1B;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Query a knowledge base you build with an agent: filter notes by frontmatter, then search inside them",
   "keywords": [
     "markdown",
@@ -48,7 +48,8 @@
   },
   "dependencies": {
     "fast-glob": "^3.3.3",
-    "gray-matter": "^4.0.3"
+    "gray-matter": "^4.0.3",
+    "remove-markdown": "^0.6.4"
   },
   "devDependencies": {
     "@types/mocha": "*",

package/schema.json

@@ -19,7 +19,7 @@
       "type": "object",
       "additionalProperties": false,
       "required": ["include"],
-      "description": "Which files become rows in the `docs` table.",
+      "description": "Which files become rows in the `frontmatter` table.",
       "properties": {
         "include": {
           "type": "array",
@@ -31,7 +31,7 @@
     },
     "queries": {
       "type": "object",
-      "description": "Named SQL queries runnable as `sense <name> [params...]`. Each value is a SQL SELECT against the `docs` table (one row per file, one column per discovered frontmatter key, plus reserved `path`/`_mtime`/`_size`). `?` placeholders bind to CLI positional arguments in order. The custom `has(field, value)` function does array-membership on a JSON-array field, substring match on a string field, and is always false on NULL. The names `init`, `query`, `watch`, `status`, and `rebuild` are reserved subcommands — a query with one of those names is unreachable from the CLI.",
+      "description": "Named SQL queries runnable as `sense <name> [params...]`. Tables: `frontmatter` (one row per file, one column per discovered frontmatter key, plus `path`/`_mtime`/`_size`) and `content`, an FTS5 index (`title`, `summary`, `text`, `path`) for content search. `?` placeholders bind to CLI positional args in order. `has(field, value)`: array membership on a JSON-array field, substring match on a string, false on NULL. Canonical query: `SELECT f.path, content.title, content.summary, snippet(content, -1, '«', '»', '…', 10) AS hit FROM frontmatter f JOIN content ON content.path = f.path WHERE content MATCH ? ORDER BY bm25(content, 10.0, 5.0, 1.0) LIMIT 10`. Reserved frontmatter keys: `path`, `_mtime`, `_size`, `content`. Reserved query names (unreachable as subcommands): `init`, `query`, `watch`, `status`, `rebuild`.",
       "additionalProperties": { "type": "string" }
     }
   }