sensemaking 0.11.4 → 0.12.0

package/dist/esm/db.js

@@ -3,11 +3,12 @@
 import { mkdirSync, rmSync } from 'node:fs';
 import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
-import { featureSignature, STATE_DIR } from './config.js';
+import { contentTokenize, featureSignature, STATE_DIR } from './config.js';
 import { SenseError } from './errors.js';
 import { activeFeatures } from './features/index.js';
 import { progress } from './progress.js';
 import { listFiles, parseFile, RESERVED_COLUMNS } from './scan.js';
+import { segmentField, segmentMatch } from './segment.js';
 // Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last
 // computed value on every touch, not just the reconciles that recompute it.
 const CORE_FRONTMATTER_COLUMNS = new Set([
@@ -21,14 +22,14 @@ const CORE_FRONTMATTER_COLUMNS = new Set([
 export const DB_FILENAME = 'cache.db';
 // Cache shape version, independent of the config's own `version`. Bumping it rebuilds
 // existing trees on first query.
-export const SCHEMA_VERSION = '10';
+export const SCHEMA_VERSION = '12';
 // SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).
 const MAX_FRONTMATTER_COLUMNS = 2000;
 function quoteIdent(name) {
     return `"${name.split('"').join('""')}"`;
 }
 // has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.
-function registerFunctions(db) {
+function registerFunctions(db, segmenting) {
     db.function('has', {
         deterministic: true,
         varargs: false
@@ -48,16 +49,64 @@ function registerFunctions(db) {
         }
         return String(field).includes(needle) ? 1 : 0;
     });
+    // Raw `content MATCH '<unspaced text>'` cannot be rewritten behind the author's back, and
+    // matches nothing, so the same transform is available to hand-written SQL by name. `segmenting`
+    // is the same config predicate commands.ts's search() gates on (contentTokenize(cfg) ===
+    // undefined), passed in by open() rather than read back from persisted state.
+    db.function('segment', {
+        deterministic: true,
+        varargs: false
+    }, (terms)=>{
+        const text = terms === null || terms === undefined ? '' : String(terms);
+        return segmenting ? segmentMatch(text) : text;
+    });
 }
 function getColumns(db) {
     const rows = db.prepare('PRAGMA table_info(frontmatter)').all();
     return new Set(rows.map((r)=>r.name));
 }
+// Stemming is English-only, but the segmentation underneath it is what decides coverage:
+// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)
+// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how
+// such a tree picks trigram instead.
+const DEFAULT_TOKENIZE = 'porter unicode61';
+// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a
+// configured value has to be concatenated. Probing a throwaway table is what makes that safe
+// and is also the whole validation: anything the linked SQLite accepts passes, anything else
+// fails here with SQLite's own message rather than against the real table. It means no table
+// of which version added which tokenizer has to be maintained.
+function resolveTokenize(db, cfg) {
+    const configured = contentTokenize(cfg);
+    if (configured === undefined) return DEFAULT_TOKENIZE;
+    const literal = configured.replace(/'/g, "''");
+    try {
+        db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');
+        db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);
+        db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');
+    } catch (err) {
+        throw new SenseError('CONFIG_INVALID', `content.tokenize "${configured}" is not a tokenizer this SQLite accepts (${err.message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);
+    }
+    return literal;
+}
+// The tokenizer the content table was actually built with, from its own DDL -- the one
+// record that cannot desynchronize from the table. NULL when the table does not exist yet.
+function storedTokenize(db) {
+    const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get();
+    if (!row) return null;
+    const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);
+    return m ? m[1] : null;
+}
 // Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`
 // can't dump file text into context. Features add their own tables after the core ones.
-function ensureSchema(db, cfg) {
+function ensureSchema(db, cfg, tokenize) {
     db.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_size" INTEGER, "_parse_error" TEXT)`);
-    db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);
+    // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL
+    // against the resolved tokenizer before this runs, so a stale table is already gone by now.
+    // The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and
+    // snippet(content, 2, ...) are documented against the first three columns and keep working
+    // (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced
+    // runs for text that needs it, and an empty string for text that does not.
+    db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);
     // Coverage, not ownership: a path can appear under several presets. path leads the PK so the
     // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.
     db.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
@@ -77,6 +126,14 @@ export function setMeta(db, key, value) {
     }
     db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);
 }
+// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same
+// literal instead of two copies drifting apart.
+const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, "path", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE "path" = ?), ?, ?, ?, ?, ?, ?, ?)`;
+// Shared by reconcile() and the tokenize-only rebuild in open(), so content population is
+// defined once. Assumes the frontmatter row for doc.relPath already exists (rowid lookup).
+function insertContentRow(insertBody, doc, segmenting) {
+    insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : '');
+}
 export function docCount(db) {
     const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get();
     return row.n;
@@ -169,7 +226,10 @@ export function reconcile(db, cfg, baseDir) {
         }
         if (parsedDocs.length > 0) {
             const insert = db.prepare(insertSql);
-            const insertBody = db.prepare(`INSERT INTO content (rowid, title, summary, text, "path") VALUES ((SELECT rowid FROM frontmatter WHERE "path" = ?), ?, ?, ?, ?)`);
+            const insertBody = db.prepare(INSERT_CONTENT_SQL);
+            // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over
+            // grapheme runs would be nonsense against trigram, so the sidecars stay empty.
+            const segmenting = contentTokenize(cfg) === undefined;
             for (const doc of parsedDocs){
                 var _feature_store;
                 const values = writableColumns.map((col)=>{
@@ -189,7 +249,7 @@ export function reconcile(db, cfg, baseDir) {
                     delBody.run(doc.relPath);
                     for (const feature of features)(_feature_remove1 = feature.remove) === null || _feature_remove1 === void 0 ? void 0 : _feature_remove1.call(feature, db, doc.relPath, delta);
                 }
-                insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath);
+                insertContentRow(insertBody, doc, segmenting);
                 // A preset edit forces a full rebuild, so an unchanged doc's coverage is already
                 // correct; new docs have nothing to clear, which keeps cold builds linear.
                 if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);
@@ -216,10 +276,9 @@ export function reconcile(db, cfg, baseDir) {
         warnings
     };
 }
-// Names what moved between two feature signatures (see config.featureSignature's format:
-// global features, embed provider, then one segment per preset) for the rebuild notice.
-function signatureDiff(before, after) {
-    // Segment keys: `features`, `embed`, `preset:<name>` (config.featureSignature's format).
+// Segment keys that moved between two feature signatures (see config.featureSignature's
+// format: global features, embed provider, tokenize, then one segment per preset).
+function changedSignatureKeys(before, after) {
     const keyOf = (part)=>part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0];
     const parse = (sig)=>new Map(sig.split('|').map((part)=>[
                 keyOf(part),
@@ -230,11 +289,42 @@ function signatureDiff(before, after) {
     const changed = new Set();
     for (const [key, val] of b)if (a.get(key) !== val) changed.add(key);
     for (const key of a.keys())if (!b.has(key)) changed.add(key);
-    const label = (key)=>key === 'embed' ? 'embed settings' : key.startsWith('preset:') ? `preset "${key.slice(7)}"` : 'features';
+    return changed;
+}
+// Names what moved, for the rebuild notice.
+function signatureDiff(before, after) {
+    const changed = changedSignatureKeys(before, after);
+    const label = (key)=>key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset "${key.slice(7)}"` : 'features';
     return changed.size === 0 ? 'features' : [
         ...changed
     ].map(label).join(', ');
 }
+// The tokenize-only rebuild in open(): content is dropped and repopulated from files already
+// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and
+// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors
+// run here -- doc.search (title/summary/text) is all content population needs. Returns
+// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are
+// untouched, so reconcile() never reparses these files and would otherwise never emit them again.
+function rebuildContentTable(db, cfg, baseDir) {
+    const known = new Set(db.prepare('SELECT "path" FROM frontmatter').all().map((r)=>r.path));
+    const files = listFiles(cfg, baseDir).filter((f)=>known.has(f.relPath));
+    const segmenting = contentTokenize(cfg) === undefined;
+    const insertBody = db.prepare(INSERT_CONTENT_SQL);
+    const warnings = [];
+    db.exec('BEGIN');
+    try {
+        for (const file of files){
+            const { doc, warnings: fileWarnings } = parseFile(file);
+            warnings.push(...fileWarnings);
+            insertContentRow(insertBody, doc, segmenting);
+        }
+        db.exec('COMMIT');
+    } catch (err) {
+        db.exec('ROLLBACK');
+        throw err;
+    }
+    return warnings;
+}
 export function open(cfg) {
     var _getMeta;
     const stateDir = join(cfg.baseDir, STATE_DIR);
@@ -247,27 +337,59 @@ export function open(cfg) {
     // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query
     // that outwaits it still fails loudly.
     db.exec('PRAGMA busy_timeout = 30000');
-    registerFunctions(db);
+    registerFunctions(db, contentTokenize(cfg) === undefined);
     db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
+    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd
+    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.
+    const tokenize = resolveTokenize(db, cfg);
     // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an
     // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).
     const version = getMeta(db, 'schema_version');
     const features = getMeta(db, 'features');
     const wantFeatures = featureSignature(cfg);
+    let tokenizeOnlyRebuild = false;
     if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
         // Indexing derives from presets, so a config edit rebuilding the cache must say so and
         // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.
         if (version !== null && version !== SCHEMA_VERSION) {
             console.error('sense: cache format changed (new sensemaking version); rebuilding the index');
+            db.close();
+            clearCache(cfg);
+            return open(cfg);
+        }
+        const changedKeys = changedSignatureKeys(features !== null && features !== void 0 ? features : '', wantFeatures);
+        // Only the tokenizer moved: frontmatter, links, sections, and embeddings are file-derived
+        // and tokenizer-independent, so they don't need re-deriving. Everything else -- a preset
+        // edit, an embed model change -- still takes the full clear/reopen below.
+        if (changedKeys.size === 1 && changedKeys.has('tokenize')) {
+            console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');
+            db.exec('DROP TABLE content');
+            tokenizeOnlyRebuild = true;
         } else {
             const changed = signatureDiff(features !== null && features !== void 0 ? features : '', wantFeatures);
             console.error(`sense: config change (${changed}) rebuilds the index`);
+            db.close();
+            clearCache(cfg);
+            return open(cfg);
         }
+    }
+    // Meta can lie after a crash between table creation and the signature write; the table's
+    // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild
+    // just dropped content above, so storedTokenize sees no table and this guard is skipped
+    // naturally rather than needing its own case.
+    const stored = storedTokenize(db);
+    if (stored !== null && stored !== tokenize) {
+        console.error('sense: cache was built with a different content tokenizer; rebuilding the index');
         db.close();
         clearCache(cfg);
         return open(cfg);
     }
-    ensureSchema(db, cfg);
+    ensureSchema(db, cfg, tokenize);
+    let rebuildWarnings = [];
+    if (tokenizeOnlyRebuild) {
+        rebuildWarnings = rebuildContentTable(db, cfg, cfg.baseDir);
+        setMeta(db, 'features', wantFeatures);
+    }
     // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.
     // Installed before reconcile() -- that call is the one that races a watcher's transaction.
     const recordedMaxMs = Number((_getMeta = getMeta(db, 'reconcile_max_ms')) !== null && _getMeta !== void 0 ? _getMeta : '0');
@@ -278,7 +400,10 @@ export function open(cfg) {
         cfg,
         dbPath,
         parsed,
-        warnings
+        warnings: [
+            ...rebuildWarnings,
+            ...warnings
+        ]
     };
 }
 // Deletes the cache directory, and only that. The rebuild is not this function's job: the next

package/dist/esm/db.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { featureSignature, STATE_DIR } from './config.ts';\nimport { SenseError } from './errors.ts';\nimport { activeFeatures } from './features/index.ts';\nimport type { ReconcileDelta } from './features/types.ts';\nimport { progress } from './progress.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from './scan.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_size', '_parse_error']);\n\n// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '10';\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  db.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(`INSERT INTO content (rowid, title, summary, text, \"path\") VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?)`);\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Only for docs that have rows: an unconditional delete made cold crawls quadratic.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath);\n        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already\n        // correct; new docs have nothing to clear, which keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Names what moved between two feature signatures (see config.featureSignature's format:\n// global features, embed provider, then one segment per preset) for the rebuild notice.\nfunction signatureDiff(before: string, after: string): string {\n  // Segment keys: `features`, `embed`, `preset:<name>` (config.featureSignature's format).\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n  // that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n    // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n    if (version !== null && version !== SCHEMA_VERSION) {\n      console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n    } else {\n      const changed = signatureDiff(features ?? '', wantFeatures);\n      console.error(`sense: config change (${changed}) rebuilds the index`);\n    }\n    db.close();\n    clearCache(cfg);\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg);\n\n  // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n  // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n  const recordedMaxMs = Number(getMeta(db, 'reconcile_max_ms') ?? '0');\n  db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","featureSignature","STATE_DIR","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","CORE_FRONTMATTER_COLUMNS","Set","DB_FILENAME","SCHEMA_VERSION","MAX_FRONTMATTER_COLUMNS","quoteIdent","name","split","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","map","r","ensureSchema","cfg","exec","feature","schema","getMeta","setMeta","key","row","get","run","docCount","n","reconcile","baseDir","files","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","features","seenColumns","newColumns","parsedDocs","report","parsedCount","file","fileFeatures","enabledForFile","doc","fileWarnings","tick","push","Object","keys","data","add","finish","allColumns","writableColumns","c","insertSql","added","delta","reparsed","d","txStart","Date","now","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","insertBody","values","parseError","search","title","summary","text","presetName","presets","store","extracted","afterReconcile","err","durationMs","prevRaw","prevMax","Number","signatureDiff","before","after","keyOf","part","slice","sig","a","b","changed","val","label","open","stateDir","recursive","dbPath","version","wantFeatures","console","error","close","clearCache","recordedMaxMs","Math","min","max","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,cAAc;AAC1D,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,cAAc,QAAQ,sBAAsB;AAErD,SAASC,QAAQ,QAAQ,gBAAgB;AAEzC,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,YAAY;AAEnE,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;CAAe;AAEpF,gFAAgF;AAChF,kEAAkE;AAElE,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAEnC,8FAA8F;AAC9F,MAAMC,0BAA0B;AAUhC,SAASC,WAAWC,IAAY;IAC9B,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKjB,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,+FAA+F;AAC/F,SAASkB,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,MAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;oBAC9D;gBACF,EAAE,OAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,MAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAI7B,IAAI2B,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAE1B,IAAI;AACvC;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAS2B,aAAaxB,EAAgB,EAAEyB,GAAW;IACjDzB,GAAG0B,IAAI,CAAC,CAAC,qHAAqH,CAAC;IAC/H1B,GAAG0B,IAAI,CAAC,CAAC,0HAA0H,CAAC;IACpI,6FAA6F;IAC7F,qFAAqF;IACrF1B,GAAG0B,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1G1B,GAAG0B,IAAI,CAAC;IACR,KAAK,MAAMC,WAAWzC,eAAeuC,KAAME,QAAQC,MAAM,CAAC5B;IAC1D,IAAI6B,QAAQ7B,IAAI,sBAAsB,MAAM8B,QAAQ9B,IAAI,kBAAkBN;IAC1E,IAAImC,QAAQ7B,IAAI,gBAAgB,MAAM8B,QAAQ9B,IAAI,YAAYjB,iBAAiB0C;AACjF;AAEA,OAAO,SAASI,QAAQ7B,EAAgB,EAAE+B,GAAW;IACnD,MAAMC,MAAMhC,GAAGoB,OAAO,CAAC,wCAAwCa,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAI3B,KAAK,GAAG;AAC3B;AAEA,OAAO,SAASyB,QAAQ9B,EAAgB,EAAE+B,GAAW,EAAE1B,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCc,GAAG,CAACH;QACjD;IACF;IACA/B,GAAGoB,OAAO,CAAC,qGAAqGc,GAAG,CAACH,KAAK1B;AAC3H;AAEA,OAAO,SAAS8B,SAASnC,EAAgB;IACvC,MAAMgC,MAAMhC,GAAGoB,OAAO,CAAC,yCAAyCa,GAAG;IACnE,OAAOD,IAAII,CAAC;AACd;AAEA,OAAO,SAASC,UAAUrC,EAAgB,EAAEyB,GAAW,EAAEa,OAAe;IACtE,MAAMC,QAAQnD,UAAUqC,KAAKa;IAC7B,MAAME,aAAa,IAAIhD,IAAI+C,MAAMjB,GAAG,CAAC,CAACmB,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe3C,GAAGoB,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMuB,WAAW,IAAIC,IAAIF,aAAarB,GAAG,CAAC,CAACC,IAAM;YAACA,EAAEuB,IAAI;YAAEvB;SAAE;IAC5D,MAAMwB,WAAWJ,aAAaK,MAAM,CAAC,CAACzB,IAAM,CAACiB,WAAWS,GAAG,CAAC1B,EAAEuB,IAAI,GAAGxB,GAAG,CAAC,CAACC,IAAMA,EAAEuB,IAAI;IAEtF,MAAMI,YAAYX,MAAMS,MAAM,CAAC,CAACP;QAC9B,MAAMT,MAAMY,SAASX,GAAG,CAACQ,EAAEC,OAAO;QAClC,OAAO,CAACV,OAAOA,IAAImB,MAAM,KAAKV,EAAEW,OAAO,IAAIpB,IAAIqB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE7C,QAAQ;QAAG8C,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWvE,eAAeuC;IAChC,MAAMiC,cAAcxC,WAAWlB;IAC/B,MAAM2D,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,MAAMK,SAAS1E,SAAS,mBAAmB+D,UAAUK,MAAM;IAC3D,IAAIO,cAAc;IAClB,KAAK,MAAMC,QAAQb,UAAW;QAC5B,sFAAsF;QACtF,mEAAmE;QACnE,MAAMc,eAAeP,SAAST,MAAM,CAAC,CAACrB,UAAY,CAACA,QAAQsC,cAAc,IAAItC,QAAQsC,cAAc,CAACxC,KAAKsC;QACzG,MAAM,EAAEG,GAAG,EAAEV,UAAUW,YAAY,EAAE,GAAG9E,UAAU0E,MAAMC;QACxDH,OAAOO,IAAI,CAAC,EAAEN;QACdN,SAASa,IAAI,IAAIF;QACjB,KAAK,MAAMpC,OAAOuC,OAAOC,IAAI,CAACL,IAAIM,IAAI,EAAG;YACvC,IAAI,CAACd,YAAYT,GAAG,CAAClB,MAAM;gBACzB2B,YAAYe,GAAG,CAAC1C;gBAChB4B,WAAWU,IAAI,CAACtC;YAClB;QACF;QACA6B,WAAWS,IAAI,CAACH;IAClB;IACAL,OAAOa,MAAM;IAEb,MAAMC,aAAa;WAAIjB;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIiB,WAAWpB,MAAM,GAAG5D,yBAAyB;QAC/C,MAAM,IAAIV,WACR,gBACA,CAAC,uBAAuB,EAAE0F,WAAWpB,MAAM,CAAC,0EAA0E,EAAE5D,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMiF,kBAAkBD,WAAW3B,MAAM,CAAC,CAAC6B,IAAMtF,yBAAyB0D,GAAG,CAAC4B,MAAM,CAACvF,iBAAiB2D,GAAG,CAAC4B;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBtD,GAAG,CAAC1B,YAAYf,IAAI,CAAC,MAAM,UAAU,EAAE+F,gBAAgBtD,GAAG,CAAC,IAAM,KAAKzC,IAAI,CAAC,MAAM,oCAAoC,EAAE+F,gBAClL5B,MAAM,CAAC,CAAC6B,IAAMA,MAAM,QACpBvD,GAAG,CAAC,CAACuD,IAAM,GAAGjF,WAAWiF,GAAG,YAAY,EAAEjF,WAAWiF,IAAI,EACzDhG,IAAI,CAAC,OAAO;IAEf,MAAMkG,QAAQ7B,UAAUF,MAAM,CAAC,CAACP,IAAM,CAACG,SAASK,GAAG,CAACR,EAAEC,OAAO,GAAGpB,GAAG,CAAC,CAACmB,IAAMA,EAAEC,OAAO;IACpF,MAAMsC,QAAwB;QAAEzC;QAAO0C,UAAUrB,WAAWtC,GAAG,CAAC,CAAC4D,IAAMA,EAAExC,OAAO;QAAGqC;QAAOhC;IAAS;IAEnG,MAAMoC,UAAUC,KAAKC,GAAG;IACxBrF,GAAG0B,IAAI,CAAC;IACR,IAAI;YA8C8BC;QA7ChC,KAAK,MAAM2D,OAAO3B,WAAY3D,GAAG0B,IAAI,CAAC,CAAC,mCAAmC,EAAE9B,WAAW0F,MAAM;QAC7F,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,MAAMC,UAAUvF,GAAGoB,OAAO,CAAC,CAAC,kFAAkF,CAAC;QAC/G,MAAMoE,iBAAiBxF,GAAGoB,OAAO,CAAC,CAAC,yCAAyC,CAAC;QAC7E,MAAMqE,mBAAmBzF,GAAGoB,OAAO,CAAC,CAAC,uDAAuD,CAAC;QAC7F,IAAI2B,SAASQ,MAAM,GAAG,GAAG;YACvB,MAAMmC,MAAM1F,GAAGoB,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAM0B,QAAQC,SAAU;oBAMKpB;gBALhC,kFAAkF;gBAClF,qEAAqE;gBACrE4D,QAAQrD,GAAG,CAACY;gBACZ4C,IAAIxD,GAAG,CAACY;gBACR0C,eAAetD,GAAG,CAACY;gBACnB,KAAK,MAAMnB,WAAW8B,UAAU9B,kBAAAA,QAAQgE,MAAM,cAAdhE,sCAAAA,qBAAAA,SAAiB3B,IAAI8C,MAAMkC;YAC7D;QACF;QACA,IAAIpB,WAAWL,MAAM,GAAG,GAAG;YACzB,MAAMqC,SAAS5F,GAAGoB,OAAO,CAAC0D;YAC1B,MAAMe,aAAa7F,GAAGoB,OAAO,CAAC,CAAC,+HAA+H,CAAC;YAC/J,KAAK,MAAM8C,OAAON,WAAY;oBAqBIjC;gBApBhC,MAAMmE,SAASlB,gBAAgBtD,GAAG,CAAC,CAACgE;wBAM3BpB;oBALP,IAAIoB,QAAQ,QAAQ,OAAOpB,IAAIxB,OAAO;oBACtC,IAAI4C,QAAQ,UAAU,OAAOpB,IAAId,OAAO;oBACxC,IAAIkC,QAAQ,SAAS,OAAOpB,IAAIZ,IAAI;oBACpC,mFAAmF;oBACnF,IAAIgC,QAAQ,gBAAgB,OAAOpB,IAAI6B,UAAU;oBACjD,QAAO7B,gBAAAA,IAAIM,IAAI,CAACc,IAAI,cAAbpB,2BAAAA,gBAAiB;gBAC1B;gBACA,uFAAuF;gBACvF0B,OAAO1D,GAAG,IAAI4D;gBACd,oFAAoF;gBACpF,IAAIlD,SAASK,GAAG,CAACiB,IAAIxB,OAAO,GAAG;wBAEGf;oBADhC4D,QAAQrD,GAAG,CAACgC,IAAIxB,OAAO;oBACvB,KAAK,MAAMf,WAAW8B,UAAU9B,mBAAAA,QAAQgE,MAAM,cAAdhE,uCAAAA,sBAAAA,SAAiB3B,IAAIkE,IAAIxB,OAAO,EAAEsC;gBACpE;gBACAa,WAAW3D,GAAG,CAACgC,IAAIxB,OAAO,EAAEwB,IAAI8B,MAAM,CAACC,KAAK,EAAE/B,IAAI8B,MAAM,CAACE,OAAO,EAAEhC,IAAI8B,MAAM,CAACG,IAAI,EAAEjC,IAAIxB,OAAO;gBAC9F,iFAAiF;gBACjF,2EAA2E;gBAC3E,IAAIE,SAASK,GAAG,CAACiB,IAAIxB,OAAO,GAAG8C,eAAetD,GAAG,CAACgC,IAAIxB,OAAO;gBAC7D,KAAK,MAAM0D,cAAclC,IAAImC,OAAO,CAAEZ,iBAAiBvD,GAAG,CAACgC,IAAIxB,OAAO,EAAE0D;gBACxE,KAAK,MAAMzE,WAAW8B,UAAU9B,iBAAAA,QAAQ2E,KAAK,cAAb3E,qCAAAA,oBAAAA,SAAgB3B,IAAIkE,IAAIxB,OAAO,EAAEwB,IAAIqC,SAAS,CAAC5E,QAAQ9B,IAAI,CAAC,EAAEmF;YAChG;QACF;QACA,KAAK,MAAMrD,WAAW8B,UAAU9B,0BAAAA,QAAQ6E,cAAc,cAAtB7E,8CAAAA,6BAAAA,SAAyB3B,IAAIgF;QAC7DhF,GAAG0B,IAAI,CAAC;IACV,EAAE,OAAO+E,KAAK;QACZzG,GAAG0B,IAAI,CAAC;QACR,MAAM+E;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMC,aAAatB,KAAKC,GAAG,KAAKF;IAChC,MAAMwB,UAAU9E,QAAQ7B,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM4G,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS9E,QAAQ9B,IAAI,oBAAoBQ,OAAOkG;IAEjE,OAAO;QAAEhG,QAAQkD,WAAWL,MAAM;QAAEC;IAAS;AAC/C;AAEA,yFAAyF;AACzF,wFAAwF;AACxF,SAASsD,cAAcC,MAAc,EAAEC,KAAa;IAClD,yFAAyF;IACzF,MAAMC,QAAQ,CAACC,OAAkBA,KAAKzG,UAAU,CAAC,aAAayG,KAAKpH,KAAK,CAAC,KAAKqH,KAAK,CAAC,GAAG,GAAGtI,IAAI,CAAC,OAAOqI,KAAKpH,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAMc,QAAQ,CAACwG,MAAgB,IAAIvE,IAAIuE,IAAItH,KAAK,CAAC,KAAKwB,GAAG,CAAC,CAAC4F,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMG,IAAIzG,MAAMmG;IAChB,MAAMO,IAAI1G,MAAMoG;IAChB,MAAMO,UAAU,IAAI/H;IACpB,KAAK,MAAM,CAACuC,KAAKyF,IAAI,IAAIF,EAAG,IAAID,EAAEpF,GAAG,CAACF,SAASyF,KAAKD,QAAQ9C,GAAG,CAAC1C;IAChE,KAAK,MAAMA,OAAOsF,EAAE9C,IAAI,GAAI,IAAI,CAAC+C,EAAErE,GAAG,CAAClB,MAAMwF,QAAQ9C,GAAG,CAAC1C;IACzD,MAAM0F,QAAQ,CAAC1F,MAAiBA,QAAQ,UAAU,mBAAmBA,IAAItB,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEsB,IAAIoF,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IAC9H,OAAOI,QAAQjE,IAAI,KAAK,IAAI,aAAa;WAAIiE;KAAQ,CAACjG,GAAG,CAACmG,OAAO5I,IAAI,CAAC;AACxE;AAEA,OAAO,SAAS6I,KAAKjG,GAAmB;QAqCTI;IApC7B,MAAM8F,WAAW9I,KAAK4C,IAAIa,OAAO,EAAEtD;IACnCL,UAAUgJ,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMC,SAAShJ,KAAK8I,UAAUlI;IAE9B,MAAMO,KAAK,IAAIlB,aAAa+I;IAC5B7H,GAAG0B,IAAI,CAAC;IACR,yFAAyF;IACzF,uCAAuC;IACvC1B,GAAG0B,IAAI,CAAC;IACR3B,kBAAkBC;IAElBA,GAAG0B,IAAI,CAAC;IAER,uFAAuF;IACvF,qGAAqG;IACrG,MAAMoG,UAAUjG,QAAQ7B,IAAI;IAC5B,MAAMyD,WAAW5B,QAAQ7B,IAAI;IAC7B,MAAM+H,eAAehJ,iBAAiB0C;IACtC,IAAI,AAACqG,YAAY,QAAQA,YAAYpI,kBAAoB+D,aAAa,QAAQA,aAAasE,cAAe;QACxG,uFAAuF;QACvF,wFAAwF;QACxF,IAAID,YAAY,QAAQA,YAAYpI,gBAAgB;YAClDsI,QAAQC,KAAK,CAAC;QAChB,OAAO;YACL,MAAMV,UAAUT,cAAcrD,qBAAAA,sBAAAA,WAAY,IAAIsE;YAC9CC,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEV,QAAQ,oBAAoB,CAAC;QACtE;QACAvH,GAAGkI,KAAK;QACRC,WAAW1G;QACX,OAAOiG,KAAKjG;IACd;IAEAD,aAAaxB,IAAIyB;IAEjB,wFAAwF;IACxF,2FAA2F;IAC3F,MAAM2G,gBAAgBvB,QAAOhF,WAAAA,QAAQ7B,IAAI,iCAAZ6B,sBAAAA,WAAmC;IAChE7B,GAAG0B,IAAI,CAAC,CAAC,sBAAsB,EAAE2G,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIH,gBAAgB,SAAU;IAExF,MAAM,EAAE1H,MAAM,EAAE8C,QAAQ,EAAE,GAAGnB,UAAUrC,IAAIyB,KAAKA,IAAIa,OAAO;IAE3D,OAAO;QAAEtC;QAAIyB;QAAKoG;QAAQnH;QAAQ8C;IAAS;AAC7C;AAEA,+FAA+F;AAC/F,gGAAgG;AAChG,gGAAgG;AAChG,sDAAsD;AACtD,OAAO,SAAS2E,WAAW1G,GAAmB;IAC5C7C,OAAOC,KAAK4C,IAAIa,OAAO,EAAEtD,YAAY;QAAE4I,WAAW;QAAMY,OAAO;IAAK;AACtE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync, type StatementSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from './config.ts';\nimport { SenseError } from './errors.ts';\nimport { activeFeatures } from './features/index.ts';\nimport type { ReconcileDelta } from './features/types.ts';\nimport { progress } from './progress.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from './scan.ts';\nimport { segmentField, segmentMatch } from './segment.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_size', '_parse_error']);\n\n// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '12';\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync, segmenting: boolean): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n\n  // Raw `content MATCH '<unspaced text>'` cannot be rewritten behind the author's back, and\n  // matches nothing, so the same transform is available to hand-written SQL by name. `segmenting`\n  // is the same config predicate commands.ts's search() gates on (contentTokenize(cfg) ===\n  // undefined), passed in by open() rather than read back from persisted state.\n  db.function('segment', { deterministic: true, varargs: false }, (terms: unknown): string => {\n    const text = terms === null || terms === undefined ? '' : String(terms);\n    return segmenting ? segmentMatch(text) : text;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Stemming is English-only, but the segmentation underneath it is what decides coverage:\n// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)\n// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how\n// such a tree picks trigram instead.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a\n// configured value has to be concatenated. Probing a throwaway table is what makes that safe\n// and is also the whole validation: anything the linked SQLite accepts passes, anything else\n// fails here with SQLite's own message rather than against the real table. It means no table\n// of which version added which tokenizer has to be maintained.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config, tokenize: string): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  // The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and\n  // snippet(content, 2, ...) are documented against the first three columns and keep working\n  // (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced\n  // runs for text that needs it, and an empty string for text that does not.\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  db.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so content population is\n// defined once. Assumes the frontmatter row for doc.relPath already exists (rowid lookup).\nfunction insertContentRow(insertBody: StatementSync, doc: ParsedDoc, segmenting: boolean): void {\n  insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : '');\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(INSERT_CONTENT_SQL);\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Only for docs that have rows: an unconditional delete made cold crawls quadratic.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertContentRow(insertBody, doc, segmenting);\n        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already\n        // correct; new docs have nothing to clear, which keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nfunction changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Names what moved, for the rebuild notice.\nfunction signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// The tokenize-only rebuild in open(): content is dropped and repopulated from files already\n// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and\n// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors\n// run here -- doc.search (title/summary/text) is all content population needs. Returns\n// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are\n// untouched, so reconcile() never reparses these files and would otherwise never emit them again.\nfunction rebuildContentTable(db: DatabaseSync, cfg: Config, baseDir: string): string[] {\n  const known = new Set((db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const insertBody = db.prepare(INSERT_CONTENT_SQL);\n  const warnings: string[] = [];\n  db.exec('BEGIN');\n  try {\n    for (const file of files) {\n      const { doc, warnings: fileWarnings } = parseFile(file);\n      warnings.push(...fileWarnings);\n      insertContentRow(insertBody, doc, segmenting);\n    }\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n  return warnings;\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n  // that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db, contentTokenize(cfg) === undefined);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n  // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n  const tokenize = resolveTokenize(db, cfg);\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  let tokenizeOnlyRebuild = false;\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n    // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n    if (version !== null && version !== SCHEMA_VERSION) {\n      console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n      db.close();\n      clearCache(cfg);\n      return open(cfg);\n    }\n    const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n    // Only the tokenizer moved: frontmatter, links, sections, and embeddings are file-derived\n    // and tokenizer-independent, so they don't need re-deriving. Everything else -- a preset\n    // edit, an embed model change -- still takes the full clear/reopen below.\n    if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n      console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n      db.exec('DROP TABLE content');\n      tokenizeOnlyRebuild = true;\n    } else {\n      const changed = signatureDiff(features ?? '', wantFeatures);\n      console.error(`sense: config change (${changed}) rebuilds the index`);\n      db.close();\n      clearCache(cfg);\n      return open(cfg);\n    }\n  }\n\n  // Meta can lie after a crash between table creation and the signature write; the table's\n  // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild\n  // just dropped content above, so storedTokenize sees no table and this guard is skipped\n  // naturally rather than needing its own case.\n  const stored = storedTokenize(db);\n  if (stored !== null && stored !== tokenize) {\n    console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n    db.close();\n    clearCache(cfg);\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg, tokenize);\n\n  let rebuildWarnings: string[] = [];\n  if (tokenizeOnlyRebuild) {\n    rebuildWarnings = rebuildContentTable(db, cfg, cfg.baseDir);\n    setMeta(db, 'features', wantFeatures);\n  }\n\n  // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n  // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n  const recordedMaxMs = Number(getMeta(db, 'reconcile_max_ms') ?? '0');\n  db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","contentTokenize","featureSignature","STATE_DIR","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","segmentField","segmentMatch","CORE_FRONTMATTER_COLUMNS","Set","DB_FILENAME","SCHEMA_VERSION","MAX_FRONTMATTER_COLUMNS","quoteIdent","name","split","registerFunctions","db","segmenting","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","terms","text","getColumns","rows","prepare","all","map","r","DEFAULT_TOKENIZE","resolveTokenize","cfg","configured","literal","replace","exec","err","message","storedTokenize","row","get","m","sql","match","ensureSchema","tokenize","feature","schema","getMeta","setMeta","key","run","INSERT_CONTENT_SQL","insertContentRow","insertBody","doc","relPath","search","title","summary","docCount","n","reconcile","baseDir","files","currentSet","f","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","features","seenColumns","newColumns","parsedDocs","report","parsedCount","file","fileFeatures","enabledForFile","fileWarnings","tick","push","Object","keys","data","add","finish","allColumns","writableColumns","c","insertSql","added","delta","reparsed","d","txStart","Date","now","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","values","parseError","presetName","presets","store","extracted","afterReconcile","durationMs","prevRaw","prevMax","Number","changedSignatureKeys","before","after","keyOf","part","slice","sig","a","b","changed","val","signatureDiff","label","rebuildContentTable","known","open","stateDir","recursive","dbPath","version","wantFeatures","tokenizeOnlyRebuild","console","error","close","clearCache","changedKeys","stored","rebuildWarnings","recordedMaxMs","Math","min","max","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAA4B,cAAc;AAE/D,SAASC,eAAe,EAAEC,gBAAgB,EAAEC,SAAS,QAAQ,cAAc;AAC3E,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,cAAc,QAAQ,sBAAsB;AAErD,SAASC,QAAQ,QAAQ,gBAAgB;AAEzC,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,YAAY;AACnE,SAASC,YAAY,EAAEC,YAAY,QAAQ,eAAe;AAE1D,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;CAAe;AAEpF,gFAAgF;AAChF,kEAAkE;AAElE,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAEnC,8FAA8F;AAC9F,MAAMC,0BAA0B;AAUhC,SAASC,WAAWC,IAAY;IAC9B,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKpB,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,+FAA+F;AAC/F,SAASqB,kBAAkBC,EAAgB,EAAEC,UAAmB;IAC9DD,GAAGE,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,MAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;oBAC9D;gBACF,EAAE,OAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;IAEA,0FAA0F;IAC1F,gGAAgG;IAChG,yFAAyF;IACzF,8EAA8E;IAC9ER,GAAGE,QAAQ,CAAC,WAAW;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACe;QAC/D,MAAMC,OAAOD,UAAU,QAAQA,UAAUZ,YAAY,KAAKE,OAAOU;QACjE,OAAOlB,aAAaX,aAAa8B,QAAQA;IAC3C;AACF;AAEA,SAASC,WAAWrB,EAAgB;IAClC,MAAMsB,OAAOtB,GAAGuB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAIhC,IAAI8B,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAE7B,IAAI;AACvC;AAEA,yFAAyF;AACzF,2FAA2F;AAC3F,4FAA4F;AAC5F,qCAAqC;AACrC,MAAM8B,mBAAmB;AAEzB,wFAAwF;AACxF,6FAA6F;AAC7F,6FAA6F;AAC7F,6FAA6F;AAC7F,+DAA+D;AAC/D,SAASC,gBAAgB5B,EAAgB,EAAE6B,GAAW;IACpD,MAAMC,aAAalD,gBAAgBiD;IACnC,IAAIC,eAAevB,WAAW,OAAOoB;IACrC,MAAMI,UAAUD,WAAWE,OAAO,CAAC,MAAM;IACzC,IAAI;QACFhC,GAAGiC,IAAI,CAAC;QACRjC,GAAGiC,IAAI,CAAC,CAAC,yEAAyE,EAAEF,QAAQ,EAAE,CAAC;QAC/F/B,GAAGiC,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAInD,WAAW,kBAAkB,CAAC,kBAAkB,EAAE+C,WAAW,0CAA0C,EAAE,AAACI,IAAcC,OAAO,CAAC,8FAA8F,CAAC;IAC3O;IACA,OAAOJ;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASK,eAAepC,EAAgB;IACtC,MAAMqC,MAAMrC,GAAGuB,OAAO,CAAC,CAAC,oDAAoD,CAAC,EAAEe,GAAG;IAClF,IAAI,CAACD,KAAK,OAAO;IACjB,MAAME,IAAIF,IAAIG,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAASG,aAAa1C,EAAgB,EAAE6B,GAAW,EAAEc,QAAgB;IACnE3C,GAAGiC,IAAI,CAAC,CAAC,qHAAqH,CAAC;IAC/H,wFAAwF;IACxF,4FAA4F;IAC5F,4FAA4F;IAC5F,2FAA2F;IAC3F,2FAA2F;IAC3F,2EAA2E;IAC3EjC,GAAGiC,IAAI,CAAC,CAAC,0IAA0I,EAAEU,SAAS,EAAE,CAAC;IACjK,6FAA6F;IAC7F,qFAAqF;IACrF3C,GAAGiC,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1GjC,GAAGiC,IAAI,CAAC;IACR,KAAK,MAAMW,WAAW5D,eAAe6C,KAAMe,QAAQC,MAAM,CAAC7C;IAC1D,IAAI8C,QAAQ9C,IAAI,sBAAsB,MAAM+C,QAAQ/C,IAAI,kBAAkBN;IAC1E,IAAIoD,QAAQ9C,IAAI,gBAAgB,MAAM+C,QAAQ/C,IAAI,YAAYnB,iBAAiBgD;AACjF;AAEA,OAAO,SAASiB,QAAQ9C,EAAgB,EAAEgD,GAAW;IACnD,MAAMX,MAAMrC,GAAGuB,OAAO,CAAC,wCAAwCe,GAAG,CAACU;IACnE,OAAOX,MAAMA,IAAI/B,KAAK,GAAG;AAC3B;AAEA,OAAO,SAASyC,QAAQ/C,EAAgB,EAAEgD,GAAW,EAAE1C,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBN,GAAGuB,OAAO,CAAC,kCAAkC0B,GAAG,CAACD;QACjD;IACF;IACAhD,GAAGuB,OAAO,CAAC,qGAAqG0B,GAAG,CAACD,KAAK1C;AAC3H;AAEA,0FAA0F;AAC1F,gDAAgD;AAChD,MAAM4C,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,0FAA0F;AAC1F,2FAA2F;AAC3F,SAASC,iBAAiBC,UAAyB,EAAEC,GAAc,EAAEpD,UAAmB;IACtFmD,WAAWH,GAAG,CAACI,IAAIC,OAAO,EAAED,IAAIE,MAAM,CAACC,KAAK,EAAEH,IAAIE,MAAM,CAACE,OAAO,EAAEJ,IAAIE,MAAM,CAACnC,IAAI,EAAEiC,IAAIC,OAAO,EAAErD,aAAaZ,aAAagE,IAAIE,MAAM,CAACC,KAAK,IAAI,IAAIvD,aAAaZ,aAAagE,IAAIE,MAAM,CAACE,OAAO,IAAI,IAAIxD,aAAaZ,aAAagE,IAAIE,MAAM,CAACnC,IAAI,IAAI;AACrP;AAEA,OAAO,SAASsC,SAAS1D,EAAgB;IACvC,MAAMqC,MAAMrC,GAAGuB,OAAO,CAAC,yCAAyCe,GAAG;IACnE,OAAOD,IAAIsB,CAAC;AACd;AAEA,OAAO,SAASC,UAAU5D,EAAgB,EAAE6B,GAAW,EAAEgC,OAAe;IACtE,MAAMC,QAAQ5E,UAAU2C,KAAKgC;IAC7B,MAAME,aAAa,IAAIvE,IAAIsE,MAAMrC,GAAG,CAAC,CAACuC,IAAMA,EAAEV,OAAO;IAErD,MAAMW,eAAejE,GAAGuB,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAM0C,WAAW,IAAIC,IAAIF,aAAaxC,GAAG,CAAC,CAACC,IAAM;YAACA,EAAE0C,IAAI;YAAE1C;SAAE;IAC5D,MAAM2C,WAAWJ,aAAaK,MAAM,CAAC,CAAC5C,IAAM,CAACqC,WAAWQ,GAAG,CAAC7C,EAAE0C,IAAI,GAAG3C,GAAG,CAAC,CAACC,IAAMA,EAAE0C,IAAI;IAEtF,MAAMI,YAAYV,MAAMQ,MAAM,CAAC,CAACN;QAC9B,MAAM3B,MAAM6B,SAAS5B,GAAG,CAAC0B,EAAEV,OAAO;QAClC,OAAO,CAACjB,OAAOA,IAAIoC,MAAM,KAAKT,EAAEU,OAAO,IAAIrC,IAAIsC,KAAK,KAAKX,EAAEY,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAElE,QAAQ;QAAGmE,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAW/F,eAAe6C;IAChC,MAAMmD,cAAc3D,WAAWrB;IAC/B,MAAMiF,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,MAAMK,SAASlG,SAAS,mBAAmBuF,UAAUK,MAAM;IAC3D,IAAIO,cAAc;IAClB,KAAK,MAAMC,QAAQb,UAAW;QAC5B,sFAAsF;QACtF,mEAAmE;QACnE,MAAMc,eAAeP,SAAST,MAAM,CAAC,CAAC1B,UAAY,CAACA,QAAQ2C,cAAc,IAAI3C,QAAQ2C,cAAc,CAAC1D,KAAKwD;QACzG,MAAM,EAAEhC,GAAG,EAAEyB,UAAUU,YAAY,EAAE,GAAGrG,UAAUkG,MAAMC;QACxDH,OAAOM,IAAI,CAAC,EAAEL;QACdN,SAASY,IAAI,IAAIF;QACjB,KAAK,MAAMxC,OAAO2C,OAAOC,IAAI,CAACvC,IAAIwC,IAAI,EAAG;YACvC,IAAI,CAACb,YAAYT,GAAG,CAACvB,MAAM;gBACzBgC,YAAYc,GAAG,CAAC9C;gBAChBiC,WAAWS,IAAI,CAAC1C;YAClB;QACF;QACAkC,WAAWQ,IAAI,CAACrC;IAClB;IACA8B,OAAOY,MAAM;IAEb,MAAMC,aAAa;WAAIhB;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIgB,WAAWnB,MAAM,GAAGlF,yBAAyB;QAC/C,MAAM,IAAIZ,WACR,gBACA,CAAC,uBAAuB,EAAEiH,WAAWnB,MAAM,CAAC,0EAA0E,EAAElF,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMsG,kBAAkBD,WAAW1B,MAAM,CAAC,CAAC4B,IAAM3G,yBAAyBgF,GAAG,CAAC2B,MAAM,CAAC9G,iBAAiBmF,GAAG,CAAC2B;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBxE,GAAG,CAAC7B,YAAYlB,IAAI,CAAC,MAAM,UAAU,EAAEuH,gBAAgBxE,GAAG,CAAC,IAAM,KAAK/C,IAAI,CAAC,MAAM,oCAAoC,EAAEuH,gBAClL3B,MAAM,CAAC,CAAC4B,IAAMA,MAAM,QACpBzE,GAAG,CAAC,CAACyE,IAAM,GAAGtG,WAAWsG,GAAG,YAAY,EAAEtG,WAAWsG,IAAI,EACzDxH,IAAI,CAAC,OAAO;IAEf,MAAM0H,QAAQ5B,UAAUF,MAAM,CAAC,CAACN,IAAM,CAACE,SAASK,GAAG,CAACP,EAAEV,OAAO,GAAG7B,GAAG,CAAC,CAACuC,IAAMA,EAAEV,OAAO;IACpF,MAAM+C,QAAwB;QAAEvC;QAAOwC,UAAUpB,WAAWzD,GAAG,CAAC,CAAC8E,IAAMA,EAAEjD,OAAO;QAAG8C;QAAO/B;IAAS;IAEnG,MAAMmC,UAAUC,KAAKC,GAAG;IACxB1G,GAAGiC,IAAI,CAAC;IACR,IAAI;YAiD8BW;QAhDhC,KAAK,MAAM+D,OAAO1B,WAAYjF,GAAGiC,IAAI,CAAC,CAAC,mCAAmC,EAAErC,WAAW+G,MAAM;QAC7F,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,MAAMC,UAAU5G,GAAGuB,OAAO,CAAC,CAAC,kFAAkF,CAAC;QAC/G,MAAMsF,iBAAiB7G,GAAGuB,OAAO,CAAC,CAAC,yCAAyC,CAAC;QAC7E,MAAMuF,mBAAmB9G,GAAGuB,OAAO,CAAC,CAAC,uDAAuD,CAAC;QAC7F,IAAI8C,SAASQ,MAAM,GAAG,GAAG;YACvB,MAAMkC,MAAM/G,GAAGuB,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAM6C,QAAQC,SAAU;oBAMKzB;gBALhC,kFAAkF;gBAClF,qEAAqE;gBACrEgE,QAAQ3D,GAAG,CAACmB;gBACZ2C,IAAI9D,GAAG,CAACmB;gBACRyC,eAAe5D,GAAG,CAACmB;gBACnB,KAAK,MAAMxB,WAAWmC,UAAUnC,kBAAAA,QAAQoE,MAAM,cAAdpE,sCAAAA,qBAAAA,SAAiB5C,IAAIoE,MAAMiC;YAC7D;QACF;QACA,IAAInB,WAAWL,MAAM,GAAG,GAAG;YACzB,MAAMoC,SAASjH,GAAGuB,OAAO,CAAC4E;YAC1B,MAAM/C,aAAapD,GAAGuB,OAAO,CAAC2B;YAC9B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAMjD,aAAarB,gBAAgBiD,SAAStB;YAC5C,KAAK,MAAM8C,OAAO6B,WAAY;oBAqBItC;gBApBhC,MAAMsE,SAASjB,gBAAgBxE,GAAG,CAAC,CAACkF;wBAM3BtD;oBALP,IAAIsD,QAAQ,QAAQ,OAAOtD,IAAIC,OAAO;oBACtC,IAAIqD,QAAQ,UAAU,OAAOtD,IAAIqB,OAAO;oBACxC,IAAIiC,QAAQ,SAAS,OAAOtD,IAAIuB,IAAI;oBACpC,mFAAmF;oBACnF,IAAI+B,QAAQ,gBAAgB,OAAOtD,IAAI8D,UAAU;oBACjD,QAAO9D,gBAAAA,IAAIwC,IAAI,CAACc,IAAI,cAAbtD,2BAAAA,gBAAiB;gBAC1B;gBACA,uFAAuF;gBACvF4D,OAAOhE,GAAG,IAAIiE;gBACd,oFAAoF;gBACpF,IAAIhD,SAASK,GAAG,CAAClB,IAAIC,OAAO,GAAG;wBAEGV;oBADhCgE,QAAQ3D,GAAG,CAACI,IAAIC,OAAO;oBACvB,KAAK,MAAMV,WAAWmC,UAAUnC,mBAAAA,QAAQoE,MAAM,cAAdpE,uCAAAA,sBAAAA,SAAiB5C,IAAIqD,IAAIC,OAAO,EAAE+C;gBACpE;gBACAlD,iBAAiBC,YAAYC,KAAKpD;gBAClC,iFAAiF;gBACjF,2EAA2E;gBAC3E,IAAIiE,SAASK,GAAG,CAAClB,IAAIC,OAAO,GAAGuD,eAAe5D,GAAG,CAACI,IAAIC,OAAO;gBAC7D,KAAK,MAAM8D,cAAc/D,IAAIgE,OAAO,CAAEP,iBAAiB7D,GAAG,CAACI,IAAIC,OAAO,EAAE8D;gBACxE,KAAK,MAAMxE,WAAWmC,UAAUnC,iBAAAA,QAAQ0E,KAAK,cAAb1E,qCAAAA,oBAAAA,SAAgB5C,IAAIqD,IAAIC,OAAO,EAAED,IAAIkE,SAAS,CAAC3E,QAAQ/C,IAAI,CAAC,EAAEwG;YAChG;QACF;QACA,KAAK,MAAMzD,WAAWmC,UAAUnC,0BAAAA,QAAQ4E,cAAc,cAAtB5E,8CAAAA,6BAAAA,SAAyB5C,IAAIqG;QAC7DrG,GAAGiC,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZlC,GAAGiC,IAAI,CAAC;QACR,MAAMC;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMuF,aAAahB,KAAKC,GAAG,KAAKF;IAChC,MAAMkB,UAAU5E,QAAQ9C,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM2H,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS5E,QAAQ/C,IAAI,oBAAoBS,OAAOgH;IAEjE,OAAO;QAAE9G,QAAQuE,WAAWL,MAAM;QAAEC;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,SAAS+C,qBAAqBC,MAAc,EAAEC,KAAa;IACzD,MAAMC,QAAQ,CAACC,OAAkBA,KAAKvH,UAAU,CAAC,aAAauH,KAAKnI,KAAK,CAAC,KAAKoI,KAAK,CAAC,GAAG,GAAGxJ,IAAI,CAAC,OAAOuJ,KAAKnI,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAMe,QAAQ,CAACsH,MAAgB,IAAIhE,IAAIgE,IAAIrI,KAAK,CAAC,KAAK2B,GAAG,CAAC,CAACwG,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMG,IAAIvH,MAAMiH;IAChB,MAAMO,IAAIxH,MAAMkH;IAChB,MAAMO,UAAU,IAAI9I;IACpB,KAAK,MAAM,CAACwD,KAAKuF,IAAI,IAAIF,EAAG,IAAID,EAAE9F,GAAG,CAACU,SAASuF,KAAKD,QAAQxC,GAAG,CAAC9C;IAChE,KAAK,MAAMA,OAAOoF,EAAExC,IAAI,GAAI,IAAI,CAACyC,EAAE9D,GAAG,CAACvB,MAAMsF,QAAQxC,GAAG,CAAC9C;IACzD,OAAOsF;AACT;AAEA,4CAA4C;AAC5C,SAASE,cAAcV,MAAc,EAAEC,KAAa;IAClD,MAAMO,UAAUT,qBAAqBC,QAAQC;IAC7C,MAAMU,QAAQ,CAACzF,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAItC,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEsC,IAAIkF,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOI,QAAQ1D,IAAI,KAAK,IAAI,aAAa;WAAI0D;KAAQ,CAAC7G,GAAG,CAACgH,OAAO/J,IAAI,CAAC;AACxE;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,gGAAgG;AAChG,uFAAuF;AACvF,2FAA2F;AAC3F,kGAAkG;AAClG,SAASgK,oBAAoB1I,EAAgB,EAAE6B,GAAW,EAAEgC,OAAe;IACzE,MAAM8E,QAAQ,IAAInJ,IAAI,AAACQ,GAAGuB,OAAO,CAAC,kCAAkCC,GAAG,GAA+BC,GAAG,CAAC,CAACC,IAAMA,EAAE0C,IAAI;IACvH,MAAMN,QAAQ5E,UAAU2C,KAAKgC,SAASS,MAAM,CAAC,CAACN,IAAM2E,MAAMpE,GAAG,CAACP,EAAEV,OAAO;IACvE,MAAMrD,aAAarB,gBAAgBiD,SAAStB;IAC5C,MAAM6C,aAAapD,GAAGuB,OAAO,CAAC2B;IAC9B,MAAM4B,WAAqB,EAAE;IAC7B9E,GAAGiC,IAAI,CAAC;IACR,IAAI;QACF,KAAK,MAAMoD,QAAQvB,MAAO;YACxB,MAAM,EAAET,GAAG,EAAEyB,UAAUU,YAAY,EAAE,GAAGrG,UAAUkG;YAClDP,SAASY,IAAI,IAAIF;YACjBrC,iBAAiBC,YAAYC,KAAKpD;QACpC;QACAD,GAAGiC,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZlC,GAAGiC,IAAI,CAAC;QACR,MAAMC;IACR;IACA,OAAO4C;AACT;AAEA,OAAO,SAAS8D,KAAK/G,GAAmB;QAwETiB;IAvE7B,MAAM+F,WAAWnK,KAAKmD,IAAIgC,OAAO,EAAE/E;IACnCN,UAAUqK,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMC,SAASrK,KAAKmK,UAAUpJ;IAE9B,MAAMO,KAAK,IAAIrB,aAAaoK;IAC5B/I,GAAGiC,IAAI,CAAC;IACR,yFAAyF;IACzF,uCAAuC;IACvCjC,GAAGiC,IAAI,CAAC;IACRlC,kBAAkBC,IAAIpB,gBAAgBiD,SAAStB;IAE/CP,GAAGiC,IAAI,CAAC;IAER,2FAA2F;IAC3F,8FAA8F;IAC9F,MAAMU,WAAWf,gBAAgB5B,IAAI6B;IAErC,uFAAuF;IACvF,qGAAqG;IACrG,MAAMmH,UAAUlG,QAAQ9C,IAAI;IAC5B,MAAM+E,WAAWjC,QAAQ9C,IAAI;IAC7B,MAAMiJ,eAAepK,iBAAiBgD;IACtC,IAAIqH,sBAAsB;IAC1B,IAAI,AAACF,YAAY,QAAQA,YAAYtJ,kBAAoBqF,aAAa,QAAQA,aAAakE,cAAe;QACxG,uFAAuF;QACvF,wFAAwF;QACxF,IAAID,YAAY,QAAQA,YAAYtJ,gBAAgB;YAClDyJ,QAAQC,KAAK,CAAC;YACdpJ,GAAGqJ,KAAK;YACRC,WAAWzH;YACX,OAAO+G,KAAK/G;QACd;QACA,MAAM0H,cAAc1B,qBAAqB9C,qBAAAA,sBAAAA,WAAY,IAAIkE;QACzD,0FAA0F;QAC1F,yFAAyF;QACzF,0EAA0E;QAC1E,IAAIM,YAAY3E,IAAI,KAAK,KAAK2E,YAAYhF,GAAG,CAAC,aAAa;YACzD4E,QAAQC,KAAK,CAAC;YACdpJ,GAAGiC,IAAI,CAAC;YACRiH,sBAAsB;QACxB,OAAO;YACL,MAAMZ,UAAUE,cAAczD,qBAAAA,sBAAAA,WAAY,IAAIkE;YAC9CE,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEd,QAAQ,oBAAoB,CAAC;YACpEtI,GAAGqJ,KAAK;YACRC,WAAWzH;YACX,OAAO+G,KAAK/G;QACd;IACF;IAEA,yFAAyF;IACzF,6FAA6F;IAC7F,wFAAwF;IACxF,8CAA8C;IAC9C,MAAM2H,SAASpH,eAAepC;IAC9B,IAAIwJ,WAAW,QAAQA,WAAW7G,UAAU;QAC1CwG,QAAQC,KAAK,CAAC;QACdpJ,GAAGqJ,KAAK;QACRC,WAAWzH;QACX,OAAO+G,KAAK/G;IACd;IAEAa,aAAa1C,IAAI6B,KAAKc;IAEtB,IAAI8G,kBAA4B,EAAE;IAClC,IAAIP,qBAAqB;QACvBO,kBAAkBf,oBAAoB1I,IAAI6B,KAAKA,IAAIgC,OAAO;QAC1Dd,QAAQ/C,IAAI,YAAYiJ;IAC1B;IAEA,wFAAwF;IACxF,2FAA2F;IAC3F,MAAMS,gBAAgB9B,QAAO9E,WAAAA,QAAQ9C,IAAI,iCAAZ8C,sBAAAA,WAAmC;IAChE9C,GAAGiC,IAAI,CAAC,CAAC,sBAAsB,EAAE0H,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIH,gBAAgB,SAAU;IAExF,MAAM,EAAE/I,MAAM,EAAEmE,QAAQ,EAAE,GAAGlB,UAAU5D,IAAI6B,KAAKA,IAAIgC,OAAO;IAE3D,OAAO;QAAE7D;QAAI6B;QAAKkH;QAAQpI;QAAQmE,UAAU;eAAI2E;eAAoB3E;SAAS;IAAC;AAChF;AAEA,+FAA+F;AAC/F,gGAAgG;AAChG,gGAAgG;AAChG,sDAAsD;AACtD,OAAO,SAASwE,WAAWzH,GAAmB;IAC5CpD,OAAOC,KAAKmD,IAAIgC,OAAO,EAAE/E,YAAY;QAAEgK,WAAW;QAAMgB,OAAO;IAAK;AACtE"}

package/dist/esm/output.d.ts

@@ -1,5 +1,8 @@
 export type Row = Record<string, unknown>;
-export declare function printRows(rows: Row[], format: 'table' | 'json'): void;
+export type Format = 'table' | 'json';
+export type RowFormat = Format | 'csv';
+export declare function printRows(rows: Row[], format: RowFormat, columns?: string[]): void;
+export declare function printRowStream(rows: Iterable<Row>, format: RowFormat, columns: string[]): void;
 export declare function featureOffHint(name: string): string;
 export declare function featuresLine(features: {
     on: string[];

package/dist/esm/output.js

@@ -1,16 +1,116 @@
-// rows -> table (default) | json
+// rows -> table (default) | json | csv
 // Warned about on stderr (keeps --format json stdout machine-readable), never truncated.
 const OVERSIZE_BYTES = 50000;
 // Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.
 function cell(value) {
     return String(value !== null && value !== void 0 ? value : '').replace(/\s*\n\s*/g, ' ');
 }
-export function printRows(rows, format) {
+// RFC 4180, and deliberately not cell(): csv is the format for redirecting a result to a
+// file, so it keeps every character, embedded newlines included. What it cannot carry is
+// NULL vs empty string, or a value's type -- that is what json is for.
+function csvField(value) {
+    const text = value === null || value === undefined ? '' : String(value);
+    return /["\r\n,]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
+}
+function csvLine(values) {
+    return values.map(csvField).join(',');
+}
+function warnOversize(bytes, rowCount) {
+    if (bytes <= OVERSIZE_BYTES) return;
+    console.warn(`warning: result is ${Math.round(bytes / 1000)} KB across ${rowCount} row(s); consider a LIMIT, fewer columns, snippet() instead of whole values, or --format csv redirected to a file`);
+}
+// Header names come from a row wherever there is one: a statement's column list keeps
+// duplicates that the row object has already collapsed, so `SELECT a AS x, b AS x` would print
+// b's value under both headers. `columns` is the fallback that labels a 0-row result, and with
+// no rows and no statement (a bounded command that found nothing) csv writes nothing at all --
+// a bare newline would read to a csv parser as one empty record.
+export function printRows(rows, format, columns) {
+    if (format === 'csv') {
+        // Collapsed the same way a row object collapses them, so a statement naming a column
+        // twice describes itself identically whether it matched anything or not.
+        const names = rows.length > 0 ? Object.keys(rows[0]) : [
+            ...new Set(columns !== null && columns !== void 0 ? columns : [])
+        ];
+        if (names.length === 0) return;
+        const rendered = [
+            csvLine(names),
+            ...rows.map((row)=>csvLine(names.map((name)=>row[name])))
+        ].join('\n');
+        console.log(rendered);
+        // Bytes, not code units -- CJK is 3 bytes per character.
+        warnOversize(Buffer.byteLength(rendered), rows.length);
+        return;
+    }
     const rendered = renderRows(rows, format);
     console.log(rendered);
-    if (rendered.length > OVERSIZE_BYTES) {
-        console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);
+    warnOversize(Buffer.byteLength(rendered), rows.length);
+}
+// Streaming counterpart for the one unbounded caller, `sense sql`: rows arrive from an
+// iterator, so a large result is never held here as rows and again as a rendered string.
+// Table buffers by necessity (a column is as wide as its widest cell, which the last row can
+// change); json and csv write row by row. The writes are synchronous, so when stdout is a pipe
+// Node buffers what the reader has not taken yet -- the saving is this module's copies, not
+// the operating system's.
+export function printRowStream(rows, format, columns) {
+    const iterator = rows[Symbol.iterator]();
+    // One step before any write. FTS5 parses a MATCH expression at step time rather than at
+    // prepare time, so a bad search string throws here, with stdout still clean for the error.
+    const first = iterator.next();
+    const rest = {
+        [Symbol.iterator]: ()=>iterator
+    };
+    if (format === 'table') {
+        printRows(first.done ? [] : [
+            first.value,
+            ...rest
+        ], format, columns);
+        return;
+    }
+    let bytes = 0;
+    let rowCount = 0;
+    const write = (text)=>{
+        // Bytes, not code units -- CJK is 3 bytes per character.
+        bytes += Buffer.byteLength(text);
+        process.stdout.write(text);
+    };
+    if (format === 'json') {
+        // Byte-identical to JSON.stringify(rows, null, 2) + newline: each row re-indented one
+        // level, comma-joined, inside the array brackets written here.
+        // JSON.stringify throws on BigInt, so a row with an int64 past 2^53 (setReadBigInts) is
+        // converted via the replacer: a value that fits a safe integer stays a number, a larger one
+        // becomes its decimal string, since a JSON number cannot carry it losslessly.
+        const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);
+        const bigintSafe = (_key, value)=>typeof value === 'bigint' ? value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value.toString() : value;
+        const indent = (row)=>JSON.stringify(row, bigintSafe, 2).split('\n').map((line)=>`  ${line}`).join('\n');
+        if (first.done) {
+            write('[]\n');
+        } else {
+            write('[\n');
+            write(indent(first.value));
+            rowCount = 1;
+            for (const row of rest){
+                write(',\n');
+                write(indent(row));
+                rowCount++;
+            }
+            write('\n]\n');
+        }
+    } else if (first.done) {
+        const names = [
+            ...new Set(columns)
+        ];
+        if (names.length > 0) write(`${csvLine(names)}\n`);
+    } else {
+        const names = Object.keys(first.value);
+        write(`${csvLine(names)}\n`);
+        write(`${csvLine(names.map((name)=>first.value[name]))}\n`);
+        rowCount = 1;
+        for (const row of rest){
+            write(`${csvLine(names.map((name)=>row[name]))}\n`);
+            rowCount++;
+        }
     }
+    warnOversize(bytes, rowCount);
 }
 // Tables are for reading, and a row wider than the terminal wraps unreadably: widest columns
 // give up space first, down to a floor. --format json is never truncated.