sensemaking 0.9.4 → 0.10.0

package/dist/esm/db.js

@@ -1,8 +1,5 @@
-// The package's Node floor (>=22.16) is set here and nowhere else: node:sqlite arrived in
-// 22.5, but FTS5 -- which search.ts's whole lexical half is built on -- and
-// StatementSync.columns() both landed in 22.16. 22.15 fails with "no such module: fts5".
-// Nothing outside this file, features/, and commands.ts needs anything past Node 12, so
-// raise the floor only for a sqlite capability, and lower it for nothing.
+// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with
+// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.
 import { mkdirSync, rmSync } from 'node:fs';
 import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
@@ -11,11 +8,8 @@ import { SenseError } from './errors.js';
 import { activeFeatures } from './features/index.js';
 import { progress } from './progress.js';
 import { listFiles, parseFile, RESERVED_COLUMNS } from './scan.js';
-// path/_mtime/_size are core: every reparse legitimately rewrites them. Every other
-// RESERVED_COLUMNS entry that shows up as a real frontmatter column (currently only
-// rank's `_rank`) is feature-owned -- scan.ts already refuses to let frontmatter set it,
-// so it must never appear in the upsert below, or a reparse would blow its last computed
-// value away with NULL on every touch, not just the reconciles that recompute it.
+// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last
+// computed value on every touch, not just the reconciles that recompute it.
 const CORE_FRONTMATTER_COLUMNS = new Set([
     'path',
     '_mtime',
@@ -24,10 +18,8 @@ const CORE_FRONTMATTER_COLUMNS = new Set([
 // Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;
 // everything beyond frontmatter + content lives in src/features/.
 export const DB_FILENAME = 'cache.db';
-// Cache shape version, independent of the config's own `version`.
-// 7: presets replace layers -- frontmatter drops the `layer` column, a new
-// preset_files(preset, path) table tracks per-preset coverage for status/map (was: 6,
-// frontmatter gains the `layer` column).
+// Cache shape version, independent of the config's own `version`. Bumping it rebuilds
+// existing trees on first query.
 export const SCHEMA_VERSION = '8';
 // SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).
 const MAX_FRONTMATTER_COLUMNS = 2000;
@@ -65,12 +57,8 @@ function getColumns(db) {
 function ensureSchema(db, cfg) {
     db.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_size" INTEGER)`);
     db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);
-    // Coverage, not ownership: a path can appear under several presets. Rebuilt per-file
-    // alongside frontmatter/content at reconcile so status/map can report matched/embedded
-    // counts per preset without recomputing globs at read time.
-    // path leads the PK so the per-doc delete in reconcile is an index hit -- keyed the other
-    // way it scans the whole table per doc, which made cold builds quadratic (measured 3x cost
-    // per note-count doubling at 13k/26k). Coverage-by-preset reads get their own index.
+    // Coverage, not ownership: a path can appear under several presets. path leads the PK so the
+    // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.
     db.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
     db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
     for (const feature of activeFeatures(cfg))feature.schema(db);
@@ -120,7 +108,7 @@ export function reconcile(db, cfg, baseDir) {
     let parsedCount = 0;
     for (const file of toReparse){
         // A doc only gets extract/store from features that apply to it (currently: embed, via
-        // FileStat.embed -- true iff a covering preset has semantic on).
+        // FileStat.embed -- true iff the config names an embedding model).
         const fileFeatures = features.filter((feature)=>!feature.enabledForFile || feature.enabledForFile(cfg, file));
         const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);
         report.tick(++parsedCount);
@@ -192,20 +180,15 @@ export function reconcile(db, cfg, baseDir) {
                 });
                 // Frontmatter upsert first: content's rowid lookup below depends on this row existing.
                 insert.run(...values);
-                // Delete-before-insert only for docs that have rows: an FTS5 DELETE by rowid on a
-                // cold build (empty table, nothing to delete) is wasted work, and doing it
-                // unconditionally previously made the crawl quadratic when it scanned by column --
-                // measured 4x time per note-count doubling at 13k/26k notes.
+                // Only for docs that have rows: an unconditional delete made cold crawls quadratic.
                 if (existing.has(doc.relPath)) {
                     var _feature_remove1;
                     delBody.run(doc.relPath);
                     for (const feature of features)(_feature_remove1 = feature.remove) === null || _feature_remove1 === void 0 ? void 0 : _feature_remove1.call(feature, db, doc.relPath, delta);
                 }
                 insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath);
-                // Coverage is glob-derived, not content-derived, but only reparsed/added docs are
-                // touched here: a preset edit changes featureSignature and forces a full rebuild
-                // (see open()), so an unchanged doc's coverage is already correct on disk. New docs
-                // have no rows to clear -- skipping the delete keeps cold builds linear.
+                // A preset edit forces a full rebuild, so an unchanged doc's coverage is already
+                // correct; new docs have nothing to clear, which keeps cold builds linear.
                 if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);
                 for (const presetName of doc.presets)insertPresetFile.run(doc.relPath, presetName);
                 for (const feature of features)(_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, db, doc.relPath, doc.extracted[feature.name], delta);
@@ -258,10 +241,8 @@ export function open(cfg) {
     const dbPath = join(stateDir, DB_FILENAME);
     const db = new DatabaseSync(dbPath);
     db.exec('PRAGMA journal_mode = WAL');
-    // Covers a concurrent watcher's bulk reconcile: the write transaction for 500 changed
-    // files measures ~5s at 26k notes, so 5s expired exactly at the boundary and queries
-    // racing the watcher got SQLITE_BUSY. 30s bounds the wait at ~3x the largest measured
-    // reconcile; a query that outwaits it still fails loudly.
+    // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query
+    // that outwaits it still fails loudly.
     db.exec('PRAGMA busy_timeout = 30000');
     registerFunctions(db);
     db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
@@ -287,11 +268,8 @@ export function open(cfg) {
         return open(cfg);
     }
     ensureSchema(db, cfg);
-    // Derived from reconcile's own recorded max (F): 3x the largest reconcile this cache has
-    // ever held its write transaction for, floored at the 30s default and capped at 10min so
-    // one pathological build can't pin every later open to an unbounded wait. Installed
-    // before reconcile() below -- this open's own reconcile is exactly the operation that
-    // races a concurrent watcher's transaction and needs the derived wait.
+    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.
+    // Installed before reconcile() -- that call is the one that races a watcher's transaction.
     const recordedMaxMs = Number((_getMeta = getMeta(db, 'reconcile_max_ms')) !== null && _getMeta !== void 0 ? _getMeta : '0');
     db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600000)}`);
     const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);

package/dist/esm/db.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["// The package's Node floor (>=22.16) is set here and nowhere else: node:sqlite arrived in\n// 22.5, but FTS5 -- which search.ts's whole lexical half is built on -- and\n// StatementSync.columns() both landed in 22.16. 22.15 fails with \"no such module: fts5\".\n// Nothing outside this file, features/, and commands.ts needs anything past Node 12, so\n// raise the floor only for a sqlite capability, and lower it for nothing.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { featureSignature, STATE_DIR } from './config.ts';\nimport { SenseError } from './errors.ts';\nimport { activeFeatures } from './features/index.ts';\nimport type { ReconcileDelta } from './features/types.ts';\nimport { progress } from './progress.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from './scan.ts';\n\n// path/_mtime/_size are core: every reparse legitimately rewrites them. Every other\n// RESERVED_COLUMNS entry that shows up as a real frontmatter column (currently only\n// rank's `_rank`) is feature-owned -- scan.ts already refuses to let frontmatter set it,\n// so it must never appear in the upsert below, or a reparse would blow its last computed\n// value away with NULL on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_size']);\n\n// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`.\n// 7: presets replace layers -- frontmatter drops the `layer` column, a new\n// preset_files(preset, path) table tracks per-preset coverage for status/map (was: 6,\n// frontmatter gains the `layer` column).\nexport const SCHEMA_VERSION = '8';\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER)`);\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);\n  // Coverage, not ownership: a path can appear under several presets. Rebuilt per-file\n  // alongside frontmatter/content at reconcile so status/map can report matched/embedded\n  // counts per preset without recomputing globs at read time.\n  // path leads the PK so the per-doc delete in reconcile is an index hit -- keyed the other\n  // way it scans the whole table per doc, which made cold builds quadratic (measured 3x cost\n  // per note-count doubling at 13k/26k). Coverage-by-preset reads get their own index.\n  db.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff a covering preset has semantic on).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(`INSERT INTO content (rowid, title, summary, text, \"path\") VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?)`);\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Delete-before-insert only for docs that have rows: an FTS5 DELETE by rowid on a\n        // cold build (empty table, nothing to delete) is wasted work, and doing it\n        // unconditionally previously made the crawl quadratic when it scanned by column --\n        // measured 4x time per note-count doubling at 13k/26k notes.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath);\n        // Coverage is glob-derived, not content-derived, but only reparsed/added docs are\n        // touched here: a preset edit changes featureSignature and forces a full rebuild\n        // (see open()), so an unchanged doc's coverage is already correct on disk. New docs\n        // have no rows to clear -- skipping the delete keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Names what moved between two feature signatures (see config.featureSignature's format:\n// global features, embed provider, then one segment per preset) for the rebuild notice.\nfunction signatureDiff(before: string, after: string): string {\n  // Segment keys: `features`, `embed`, `preset:<name>` (config.featureSignature's format).\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile: the write transaction for 500 changed\n  // files measures ~5s at 26k notes, so 5s expired exactly at the boundary and queries\n  // racing the watcher got SQLITE_BUSY. 30s bounds the wait at ~3x the largest measured\n  // reconcile; a query that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n    // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n    if (version !== null && version !== SCHEMA_VERSION) {\n      console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n    } else {\n      const changed = signatureDiff(features ?? '', wantFeatures);\n      console.error(`sense: config change (${changed}) rebuilds the index`);\n    }\n    db.close();\n    rmSync(stateDir, { recursive: true, force: true });\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg);\n\n  // Derived from reconcile's own recorded max (F): 3x the largest reconcile this cache has\n  // ever held its write transaction for, floored at the 30s default and capped at 10min so\n  // one pathological build can't pin every later open to an unbounded wait. Installed\n  // before reconcile() below -- this open's own reconcile is exactly the operation that\n  // races a concurrent watcher's transaction and needs the derived wait.\n  const recordedMaxMs = Number(getMeta(db, 'reconcile_max_ms') ?? '0');\n  db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Manual reset for a doubted cache.\nexport function rebuild(cfg: ResolvedConfig): OpenResult {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n  return open(cfg);\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","featureSignature","STATE_DIR","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","CORE_FRONTMATTER_COLUMNS","Set","DB_FILENAME","SCHEMA_VERSION","MAX_FRONTMATTER_COLUMNS","quoteIdent","name","split","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","map","r","ensureSchema","cfg","exec","feature","schema","getMeta","setMeta","key","row","get","run","docCount","n","reconcile","baseDir","files","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","features","seenColumns","newColumns","parsedDocs","report","parsedCount","file","fileFeatures","enabledForFile","doc","fileWarnings","tick","push","Object","keys","data","add","finish","allColumns","writableColumns","c","insertSql","added","delta","reparsed","d","txStart","Date","now","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","insertBody","values","search","title","summary","text","presetName","presets","store","extracted","afterReconcile","err","durationMs","prevRaw","prevMax","Number","signatureDiff","before","after","keyOf","part","slice","sig","a","b","changed","val","label","open","stateDir","recursive","dbPath","version","wantFeatures","console","error","close","force","recordedMaxMs","Math","min","max","rebuild"],"mappings":"AAAA,0FAA0F;AAC1F,4EAA4E;AAC5E,yFAAyF;AACzF,wFAAwF;AACxF,0EAA0E;AAC1E,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,cAAc;AAC1D,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,cAAc,QAAQ,sBAAsB;AAErD,SAASC,QAAQ,QAAQ,gBAAgB;AAEzC,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,YAAY;AAEnE,oFAAoF;AACpF,oFAAoF;AACpF,yFAAyF;AACzF,yFAAyF;AACzF,kFAAkF;AAClF,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;CAAQ;AAEpE,gFAAgF;AAChF,kEAAkE;AAElE,OAAO,MAAMC,cAAc,WAAW;AACtC,kEAAkE;AAClE,2EAA2E;AAC3E,sFAAsF;AACtF,yCAAyC;AACzC,OAAO,MAAMC,iBAAiB,IAAI;AAElC,8FAA8F;AAC9F,MAAMC,0BAA0B;AAUhC,SAASC,WAAWC,IAAY;IAC9B,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKjB,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,+FAA+F;AAC/F,SAASkB,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,MAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;oBAC9D;gBACF,EAAE,OAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,MAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAI7B,IAAI2B,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAE1B,IAAI;AACvC;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAS2B,aAAaxB,EAAgB,EAAEyB,GAAW;IACjDzB,GAAG0B,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1G1B,GAAG0B,IAAI,CAAC,CAAC,0HAA0H,CAAC;IACpI,qFAAqF;IACrF,uFAAuF;IACvF,4DAA4D;IAC5D,0FAA0F;IAC1F,2FAA2F;IAC3F,qFAAqF;IACrF1B,GAAG0B,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1G1B,GAAG0B,IAAI,CAAC;IACR,KAAK,MAAMC,WAAWzC,eAAeuC,KAAME,QAAQC,MAAM,CAAC5B;IAC1D,IAAI6B,QAAQ7B,IAAI,sBAAsB,MAAM8B,QAAQ9B,IAAI,kBAAkBN;IAC1E,IAAImC,QAAQ7B,IAAI,gBAAgB,MAAM8B,QAAQ9B,IAAI,YAAYjB,iBAAiB0C;AACjF;AAEA,OAAO,SAASI,QAAQ7B,EAAgB,EAAE+B,GAAW;IACnD,MAAMC,MAAMhC,GAAGoB,OAAO,CAAC,wCAAwCa,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAI3B,KAAK,GAAG;AAC3B;AAEA,OAAO,SAASyB,QAAQ9B,EAAgB,EAAE+B,GAAW,EAAE1B,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCc,GAAG,CAACH;QACjD;IACF;IACA/B,GAAGoB,OAAO,CAAC,qGAAqGc,GAAG,CAACH,KAAK1B;AAC3H;AAEA,OAAO,SAAS8B,SAASnC,EAAgB;IACvC,MAAMgC,MAAMhC,GAAGoB,OAAO,CAAC,yCAAyCa,GAAG;IACnE,OAAOD,IAAII,CAAC;AACd;AAEA,OAAO,SAASC,UAAUrC,EAAgB,EAAEyB,GAAW,EAAEa,OAAe;IACtE,MAAMC,QAAQnD,UAAUqC,KAAKa;IAC7B,MAAME,aAAa,IAAIhD,IAAI+C,MAAMjB,GAAG,CAAC,CAACmB,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe3C,GAAGoB,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMuB,WAAW,IAAIC,IAAIF,aAAarB,GAAG,CAAC,CAACC,IAAM;YAACA,EAAEuB,IAAI;YAAEvB;SAAE;IAC5D,MAAMwB,WAAWJ,aAAaK,MAAM,CAAC,CAACzB,IAAM,CAACiB,WAAWS,GAAG,CAAC1B,EAAEuB,IAAI,GAAGxB,GAAG,CAAC,CAACC,IAAMA,EAAEuB,IAAI;IAEtF,MAAMI,YAAYX,MAAMS,MAAM,CAAC,CAACP;QAC9B,MAAMT,MAAMY,SAASX,GAAG,CAACQ,EAAEC,OAAO;QAClC,OAAO,CAACV,OAAOA,IAAImB,MAAM,KAAKV,EAAEW,OAAO,IAAIpB,IAAIqB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE7C,QAAQ;QAAG8C,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWvE,eAAeuC;IAChC,MAAMiC,cAAcxC,WAAWlB;IAC/B,MAAM2D,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,MAAMK,SAAS1E,SAAS,mBAAmB+D,UAAUK,MAAM;IAC3D,IAAIO,cAAc;IAClB,KAAK,MAAMC,QAAQb,UAAW;QAC5B,sFAAsF;QACtF,iEAAiE;QACjE,MAAMc,eAAeP,SAAST,MAAM,CAAC,CAACrB,UAAY,CAACA,QAAQsC,cAAc,IAAItC,QAAQsC,cAAc,CAACxC,KAAKsC;QACzG,MAAM,EAAEG,GAAG,EAAEV,UAAUW,YAAY,EAAE,GAAG9E,UAAU0E,MAAMC;QACxDH,OAAOO,IAAI,CAAC,EAAEN;QACdN,SAASa,IAAI,IAAIF;QACjB,KAAK,MAAMpC,OAAOuC,OAAOC,IAAI,CAACL,IAAIM,IAAI,EAAG;YACvC,IAAI,CAACd,YAAYT,GAAG,CAAClB,MAAM;gBACzB2B,YAAYe,GAAG,CAAC1C;gBAChB4B,WAAWU,IAAI,CAACtC;YAClB;QACF;QACA6B,WAAWS,IAAI,CAACH;IAClB;IACAL,OAAOa,MAAM;IAEb,MAAMC,aAAa;WAAIjB;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIiB,WAAWpB,MAAM,GAAG5D,yBAAyB;QAC/C,MAAM,IAAIV,WACR,gBACA,CAAC,uBAAuB,EAAE0F,WAAWpB,MAAM,CAAC,0EAA0E,EAAE5D,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMiF,kBAAkBD,WAAW3B,MAAM,CAAC,CAAC6B,IAAMtF,yBAAyB0D,GAAG,CAAC4B,MAAM,CAACvF,iBAAiB2D,GAAG,CAAC4B;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBtD,GAAG,CAAC1B,YAAYf,IAAI,CAAC,MAAM,UAAU,EAAE+F,gBAAgBtD,GAAG,CAAC,IAAM,KAAKzC,IAAI,CAAC,MAAM,oCAAoC,EAAE+F,gBAClL5B,MAAM,CAAC,CAAC6B,IAAMA,MAAM,QACpBvD,GAAG,CAAC,CAACuD,IAAM,GAAGjF,WAAWiF,GAAG,YAAY,EAAEjF,WAAWiF,IAAI,EACzDhG,IAAI,CAAC,OAAO;IAEf,MAAMkG,QAAQ7B,UAAUF,MAAM,CAAC,CAACP,IAAM,CAACG,SAASK,GAAG,CAACR,EAAEC,OAAO,GAAGpB,GAAG,CAAC,CAACmB,IAAMA,EAAEC,OAAO;IACpF,MAAMsC,QAAwB;QAAEzC;QAAO0C,UAAUrB,WAAWtC,GAAG,CAAC,CAAC4D,IAAMA,EAAExC,OAAO;QAAGqC;QAAOhC;IAAS;IAEnG,MAAMoC,UAAUC,KAAKC,GAAG;IACxBrF,GAAG0B,IAAI,CAAC;IACR,IAAI;YAiD8BC;QAhDhC,KAAK,MAAM2D,OAAO3B,WAAY3D,GAAG0B,IAAI,CAAC,CAAC,mCAAmC,EAAE9B,WAAW0F,MAAM;QAC7F,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,MAAMC,UAAUvF,GAAGoB,OAAO,CAAC,CAAC,kFAAkF,CAAC;QAC/G,MAAMoE,iBAAiBxF,GAAGoB,OAAO,CAAC,CAAC,yCAAyC,CAAC;QAC7E,MAAMqE,mBAAmBzF,GAAGoB,OAAO,CAAC,CAAC,uDAAuD,CAAC;QAC7F,IAAI2B,SAASQ,MAAM,GAAG,GAAG;YACvB,MAAMmC,MAAM1F,GAAGoB,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAM0B,QAAQC,SAAU;oBAMKpB;gBALhC,kFAAkF;gBAClF,qEAAqE;gBACrE4D,QAAQrD,GAAG,CAACY;gBACZ4C,IAAIxD,GAAG,CAACY;gBACR0C,eAAetD,GAAG,CAACY;gBACnB,KAAK,MAAMnB,WAAW8B,UAAU9B,kBAAAA,QAAQgE,MAAM,cAAdhE,sCAAAA,qBAAAA,SAAiB3B,IAAI8C,MAAMkC;YAC7D;QACF;QACA,IAAIpB,WAAWL,MAAM,GAAG,GAAG;YACzB,MAAMqC,SAAS5F,GAAGoB,OAAO,CAAC0D;YAC1B,MAAMe,aAAa7F,GAAGoB,OAAO,CAAC,CAAC,+HAA+H,CAAC;YAC/J,KAAK,MAAM8C,OAAON,WAAY;oBAwBIjC;gBAvBhC,MAAMmE,SAASlB,gBAAgBtD,GAAG,CAAC,CAACgE;wBAI3BpB;oBAHP,IAAIoB,QAAQ,QAAQ,OAAOpB,IAAIxB,OAAO;oBACtC,IAAI4C,QAAQ,UAAU,OAAOpB,IAAId,OAAO;oBACxC,IAAIkC,QAAQ,SAAS,OAAOpB,IAAIZ,IAAI;oBACpC,QAAOY,gBAAAA,IAAIM,IAAI,CAACc,IAAI,cAAbpB,2BAAAA,gBAAiB;gBAC1B;gBACA,uFAAuF;gBACvF0B,OAAO1D,GAAG,IAAI4D;gBACd,kFAAkF;gBAClF,2EAA2E;gBAC3E,mFAAmF;gBACnF,6DAA6D;gBAC7D,IAAIlD,SAASK,GAAG,CAACiB,IAAIxB,OAAO,GAAG;wBAEGf;oBADhC4D,QAAQrD,GAAG,CAACgC,IAAIxB,OAAO;oBACvB,KAAK,MAAMf,WAAW8B,UAAU9B,mBAAAA,QAAQgE,MAAM,cAAdhE,uCAAAA,sBAAAA,SAAiB3B,IAAIkE,IAAIxB,OAAO,EAAEsC;gBACpE;gBACAa,WAAW3D,GAAG,CAACgC,IAAIxB,OAAO,EAAEwB,IAAI6B,MAAM,CAACC,KAAK,EAAE9B,IAAI6B,MAAM,CAACE,OAAO,EAAE/B,IAAI6B,MAAM,CAACG,IAAI,EAAEhC,IAAIxB,OAAO;gBAC9F,kFAAkF;gBAClF,iFAAiF;gBACjF,oFAAoF;gBACpF,yEAAyE;gBACzE,IAAIE,SAASK,GAAG,CAACiB,IAAIxB,OAAO,GAAG8C,eAAetD,GAAG,CAACgC,IAAIxB,OAAO;gBAC7D,KAAK,MAAMyD,cAAcjC,IAAIkC,OAAO,CAAEX,iBAAiBvD,GAAG,CAACgC,IAAIxB,OAAO,EAAEyD;gBACxE,KAAK,MAAMxE,WAAW8B,UAAU9B,iBAAAA,QAAQ0E,KAAK,cAAb1E,qCAAAA,oBAAAA,SAAgB3B,IAAIkE,IAAIxB,OAAO,EAAEwB,IAAIoC,SAAS,CAAC3E,QAAQ9B,IAAI,CAAC,EAAEmF;YAChG;QACF;QACA,KAAK,MAAMrD,WAAW8B,UAAU9B,0BAAAA,QAAQ4E,cAAc,cAAtB5E,8CAAAA,6BAAAA,SAAyB3B,IAAIgF;QAC7DhF,GAAG0B,IAAI,CAAC;IACV,EAAE,OAAO8E,KAAK;QACZxG,GAAG0B,IAAI,CAAC;QACR,MAAM8E;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMC,aAAarB,KAAKC,GAAG,KAAKF;IAChC,MAAMuB,UAAU7E,QAAQ7B,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM2G,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS7E,QAAQ9B,IAAI,oBAAoBQ,OAAOiG;IAEjE,OAAO;QAAE/F,QAAQkD,WAAWL,MAAM;QAAEC;IAAS;AAC/C;AAEA,yFAAyF;AACzF,wFAAwF;AACxF,SAASqD,cAAcC,MAAc,EAAEC,KAAa;IAClD,yFAAyF;IACzF,MAAMC,QAAQ,CAACC,OAAkBA,KAAKxG,UAAU,CAAC,aAAawG,KAAKnH,KAAK,CAAC,KAAKoH,KAAK,CAAC,GAAG,GAAGrI,IAAI,CAAC,OAAOoI,KAAKnH,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAMc,QAAQ,CAACuG,MAAgB,IAAItE,IAAIsE,IAAIrH,KAAK,CAAC,KAAKwB,GAAG,CAAC,CAAC2F,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMG,IAAIxG,MAAMkG;IAChB,MAAMO,IAAIzG,MAAMmG;IAChB,MAAMO,UAAU,IAAI9H;IACpB,KAAK,MAAM,CAACuC,KAAKwF,IAAI,IAAIF,EAAG,IAAID,EAAEnF,GAAG,CAACF,SAASwF,KAAKD,QAAQ7C,GAAG,CAAC1C;IAChE,KAAK,MAAMA,OAAOqF,EAAE7C,IAAI,GAAI,IAAI,CAAC8C,EAAEpE,GAAG,CAAClB,MAAMuF,QAAQ7C,GAAG,CAAC1C;IACzD,MAAMyF,QAAQ,CAACzF,MAAiBA,QAAQ,UAAU,mBAAmBA,IAAItB,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEsB,IAAImF,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IAC9H,OAAOI,QAAQhE,IAAI,KAAK,IAAI,aAAa;WAAIgE;KAAQ,CAAChG,GAAG,CAACkG,OAAO3I,IAAI,CAAC;AACxE;AAEA,OAAO,SAAS4I,KAAKhG,GAAmB;QA0CTI;IAzC7B,MAAM6F,WAAW7I,KAAK4C,IAAIa,OAAO,EAAEtD;IACnCL,UAAU+I,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMC,SAAS/I,KAAK6I,UAAUjI;IAE9B,MAAMO,KAAK,IAAIlB,aAAa8I;IAC5B5H,GAAG0B,IAAI,CAAC;IACR,sFAAsF;IACtF,qFAAqF;IACrF,sFAAsF;IACtF,0DAA0D;IAC1D1B,GAAG0B,IAAI,CAAC;IACR3B,kBAAkBC;IAElBA,GAAG0B,IAAI,CAAC;IAER,uFAAuF;IACvF,qGAAqG;IACrG,MAAMmG,UAAUhG,QAAQ7B,IAAI;IAC5B,MAAMyD,WAAW5B,QAAQ7B,IAAI;IAC7B,MAAM8H,eAAe/I,iBAAiB0C;IACtC,IAAI,AAACoG,YAAY,QAAQA,YAAYnI,kBAAoB+D,aAAa,QAAQA,aAAaqE,cAAe;QACxG,uFAAuF;QACvF,wFAAwF;QACxF,IAAID,YAAY,QAAQA,YAAYnI,gBAAgB;YAClDqI,QAAQC,KAAK,CAAC;QAChB,OAAO;YACL,MAAMV,UAAUT,cAAcpD,qBAAAA,sBAAAA,WAAY,IAAIqE;YAC9CC,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEV,QAAQ,oBAAoB,CAAC;QACtE;QACAtH,GAAGiI,KAAK;QACRrJ,OAAO8I,UAAU;YAAEC,WAAW;YAAMO,OAAO;QAAK;QAChD,OAAOT,KAAKhG;IACd;IAEAD,aAAaxB,IAAIyB;IAEjB,yFAAyF;IACzF,yFAAyF;IACzF,oFAAoF;IACpF,sFAAsF;IACtF,uEAAuE;IACvE,MAAM0G,gBAAgBvB,QAAO/E,WAAAA,QAAQ7B,IAAI,iCAAZ6B,sBAAAA,WAAmC;IAChE7B,GAAG0B,IAAI,CAAC,CAAC,sBAAsB,EAAE0G,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIH,gBAAgB,SAAU;IAExF,MAAM,EAAEzH,MAAM,EAAE8C,QAAQ,EAAE,GAAGnB,UAAUrC,IAAIyB,KAAKA,IAAIa,OAAO;IAE3D,OAAO;QAAEtC;QAAIyB;QAAKmG;QAAQlH;QAAQ8C;IAAS;AAC7C;AAEA,oCAAoC;AACpC,OAAO,SAAS+E,QAAQ9G,GAAmB;IACzC7C,OAAOC,KAAK4C,IAAIa,OAAO,EAAEtD,YAAY;QAAE2I,WAAW;QAAMO,OAAO;IAAK;IACpE,OAAOT,KAAKhG;AACd"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { featureSignature, STATE_DIR } from './config.ts';\nimport { SenseError } from './errors.ts';\nimport { activeFeatures } from './features/index.ts';\nimport type { ReconcileDelta } from './features/types.ts';\nimport { progress } from './progress.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from './scan.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_size']);\n\n// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '8';\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER)`);\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  db.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(`INSERT INTO content (rowid, title, summary, text, \"path\") VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?)`);\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Only for docs that have rows: an unconditional delete made cold crawls quadratic.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath);\n        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already\n        // correct; new docs have nothing to clear, which keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Names what moved between two feature signatures (see config.featureSignature's format:\n// global features, embed provider, then one segment per preset) for the rebuild notice.\nfunction signatureDiff(before: string, after: string): string {\n  // Segment keys: `features`, `embed`, `preset:<name>` (config.featureSignature's format).\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n  // that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n    // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n    if (version !== null && version !== SCHEMA_VERSION) {\n      console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n    } else {\n      const changed = signatureDiff(features ?? '', wantFeatures);\n      console.error(`sense: config change (${changed}) rebuilds the index`);\n    }\n    db.close();\n    rmSync(stateDir, { recursive: true, force: true });\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg);\n\n  // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n  // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n  const recordedMaxMs = Number(getMeta(db, 'reconcile_max_ms') ?? '0');\n  db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Manual reset for a doubted cache.\nexport function rebuild(cfg: ResolvedConfig): OpenResult {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n  return open(cfg);\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","featureSignature","STATE_DIR","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","CORE_FRONTMATTER_COLUMNS","Set","DB_FILENAME","SCHEMA_VERSION","MAX_FRONTMATTER_COLUMNS","quoteIdent","name","split","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","map","r","ensureSchema","cfg","exec","feature","schema","getMeta","setMeta","key","row","get","run","docCount","n","reconcile","baseDir","files","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","features","seenColumns","newColumns","parsedDocs","report","parsedCount","file","fileFeatures","enabledForFile","doc","fileWarnings","tick","push","Object","keys","data","add","finish","allColumns","writableColumns","c","insertSql","added","delta","reparsed","d","txStart","Date","now","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","insertBody","values","search","title","summary","text","presetName","presets","store","extracted","afterReconcile","err","durationMs","prevRaw","prevMax","Number","signatureDiff","before","after","keyOf","part","slice","sig","a","b","changed","val","label","open","stateDir","recursive","dbPath","version","wantFeatures","console","error","close","force","recordedMaxMs","Math","min","max","rebuild"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,cAAc;AAC1D,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,cAAc,QAAQ,sBAAsB;AAErD,SAASC,QAAQ,QAAQ,gBAAgB;AAEzC,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,YAAY;AAEnE,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;CAAQ;AAEpE,gFAAgF;AAChF,kEAAkE;AAElE,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,IAAI;AAElC,8FAA8F;AAC9F,MAAMC,0BAA0B;AAUhC,SAASC,WAAWC,IAAY;IAC9B,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKjB,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,+FAA+F;AAC/F,SAASkB,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,MAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;oBAC9D;gBACF,EAAE,OAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,MAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAI7B,IAAI2B,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAE1B,IAAI;AACvC;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAS2B,aAAaxB,EAAgB,EAAEyB,GAAW;IACjDzB,GAAG0B,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1G1B,GAAG0B,IAAI,CAAC,CAAC,0HAA0H,CAAC;IACpI,6FAA6F;IAC7F,qFAAqF;IACrF1B,GAAG0B,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1G1B,GAAG0B,IAAI,CAAC;IACR,KAAK,MAAMC,WAAWzC,eAAeuC,KAAME,QAAQC,MAAM,CAAC5B;IAC1D,IAAI6B,QAAQ7B,IAAI,sBAAsB,MAAM8B,QAAQ9B,IAAI,kBAAkBN;IAC1E,IAAImC,QAAQ7B,IAAI,gBAAgB,MAAM8B,QAAQ9B,IAAI,YAAYjB,iBAAiB0C;AACjF;AAEA,OAAO,SAASI,QAAQ7B,EAAgB,EAAE+B,GAAW;IACnD,MAAMC,MAAMhC,GAAGoB,OAAO,CAAC,wCAAwCa,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAI3B,KAAK,GAAG;AAC3B;AAEA,OAAO,SAASyB,QAAQ9B,EAAgB,EAAE+B,GAAW,EAAE1B,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCc,GAAG,CAACH;QACjD;IACF;IACA/B,GAAGoB,OAAO,CAAC,qGAAqGc,GAAG,CAACH,KAAK1B;AAC3H;AAEA,OAAO,SAAS8B,SAASnC,EAAgB;IACvC,MAAMgC,MAAMhC,GAAGoB,OAAO,CAAC,yCAAyCa,GAAG;IACnE,OAAOD,IAAII,CAAC;AACd;AAEA,OAAO,SAASC,UAAUrC,EAAgB,EAAEyB,GAAW,EAAEa,OAAe;IACtE,MAAMC,QAAQnD,UAAUqC,KAAKa;IAC7B,MAAME,aAAa,IAAIhD,IAAI+C,MAAMjB,GAAG,CAAC,CAACmB,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe3C,GAAGoB,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMuB,WAAW,IAAIC,IAAIF,aAAarB,GAAG,CAAC,CAACC,IAAM;YAACA,EAAEuB,IAAI;YAAEvB;SAAE;IAC5D,MAAMwB,WAAWJ,aAAaK,MAAM,CAAC,CAACzB,IAAM,CAACiB,WAAWS,GAAG,CAAC1B,EAAEuB,IAAI,GAAGxB,GAAG,CAAC,CAACC,IAAMA,EAAEuB,IAAI;IAEtF,MAAMI,YAAYX,MAAMS,MAAM,CAAC,CAACP;QAC9B,MAAMT,MAAMY,SAASX,GAAG,CAACQ,EAAEC,OAAO;QAClC,OAAO,CAACV,OAAOA,IAAImB,MAAM,KAAKV,EAAEW,OAAO,IAAIpB,IAAIqB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE7C,QAAQ;QAAG8C,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWvE,eAAeuC;IAChC,MAAMiC,cAAcxC,WAAWlB;IAC/B,MAAM2D,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,MAAMK,SAAS1E,SAAS,mBAAmB+D,UAAUK,MAAM;IAC3D,IAAIO,cAAc;IAClB,KAAK,MAAMC,QAAQb,UAAW;QAC5B,sFAAsF;QACtF,mEAAmE;QACnE,MAAMc,eAAeP,SAAST,MAAM,CAAC,CAACrB,UAAY,CAACA,QAAQsC,cAAc,IAAItC,QAAQsC,cAAc,CAACxC,KAAKsC;QACzG,MAAM,EAAEG,GAAG,EAAEV,UAAUW,YAAY,EAAE,GAAG9E,UAAU0E,MAAMC;QACxDH,OAAOO,IAAI,CAAC,EAAEN;QACdN,SAASa,IAAI,IAAIF;QACjB,KAAK,MAAMpC,OAAOuC,OAAOC,IAAI,CAACL,IAAIM,IAAI,EAAG;YACvC,IAAI,CAACd,YAAYT,GAAG,CAAClB,MAAM;gBACzB2B,YAAYe,GAAG,CAAC1C;gBAChB4B,WAAWU,IAAI,CAACtC;YAClB;QACF;QACA6B,WAAWS,IAAI,CAACH;IAClB;IACAL,OAAOa,MAAM;IAEb,MAAMC,aAAa;WAAIjB;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIiB,WAAWpB,MAAM,GAAG5D,yBAAyB;QAC/C,MAAM,IAAIV,WACR,gBACA,CAAC,uBAAuB,EAAE0F,WAAWpB,MAAM,CAAC,0EAA0E,EAAE5D,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMiF,kBAAkBD,WAAW3B,MAAM,CAAC,CAAC6B,IAAMtF,yBAAyB0D,GAAG,CAAC4B,MAAM,CAACvF,iBAAiB2D,GAAG,CAAC4B;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBtD,GAAG,CAAC1B,YAAYf,IAAI,CAAC,MAAM,UAAU,EAAE+F,gBAAgBtD,GAAG,CAAC,IAAM,KAAKzC,IAAI,CAAC,MAAM,oCAAoC,EAAE+F,gBAClL5B,MAAM,CAAC,CAAC6B,IAAMA,MAAM,QACpBvD,GAAG,CAAC,CAACuD,IAAM,GAAGjF,WAAWiF,GAAG,YAAY,EAAEjF,WAAWiF,IAAI,EACzDhG,IAAI,CAAC,OAAO;IAEf,MAAMkG,QAAQ7B,UAAUF,MAAM,CAAC,CAACP,IAAM,CAACG,SAASK,GAAG,CAACR,EAAEC,OAAO,GAAGpB,GAAG,CAAC,CAACmB,IAAMA,EAAEC,OAAO;IACpF,MAAMsC,QAAwB;QAAEzC;QAAO0C,UAAUrB,WAAWtC,GAAG,CAAC,CAAC4D,IAAMA,EAAExC,OAAO;QAAGqC;QAAOhC;IAAS;IAEnG,MAAMoC,UAAUC,KAAKC,GAAG;IACxBrF,GAAG0B,IAAI,CAAC;IACR,IAAI;YA4C8BC;QA3ChC,KAAK,MAAM2D,OAAO3B,WAAY3D,GAAG0B,IAAI,CAAC,CAAC,mCAAmC,EAAE9B,WAAW0F,MAAM;QAC7F,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,MAAMC,UAAUvF,GAAGoB,OAAO,CAAC,CAAC,kFAAkF,CAAC;QAC/G,MAAMoE,iBAAiBxF,GAAGoB,OAAO,CAAC,CAAC,yCAAyC,CAAC;QAC7E,MAAMqE,mBAAmBzF,GAAGoB,OAAO,CAAC,CAAC,uDAAuD,CAAC;QAC7F,IAAI2B,SAASQ,MAAM,GAAG,GAAG;YACvB,MAAMmC,MAAM1F,GAAGoB,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAM0B,QAAQC,SAAU;oBAMKpB;gBALhC,kFAAkF;gBAClF,qEAAqE;gBACrE4D,QAAQrD,GAAG,CAACY;gBACZ4C,IAAIxD,GAAG,CAACY;gBACR0C,eAAetD,GAAG,CAACY;gBACnB,KAAK,MAAMnB,WAAW8B,UAAU9B,kBAAAA,QAAQgE,MAAM,cAAdhE,sCAAAA,qBAAAA,SAAiB3B,IAAI8C,MAAMkC;YAC7D;QACF;QACA,IAAIpB,WAAWL,MAAM,GAAG,GAAG;YACzB,MAAMqC,SAAS5F,GAAGoB,OAAO,CAAC0D;YAC1B,MAAMe,aAAa7F,GAAGoB,OAAO,CAAC,CAAC,+HAA+H,CAAC;YAC/J,KAAK,MAAM8C,OAAON,WAAY;oBAmBIjC;gBAlBhC,MAAMmE,SAASlB,gBAAgBtD,GAAG,CAAC,CAACgE;wBAI3BpB;oBAHP,IAAIoB,QAAQ,QAAQ,OAAOpB,IAAIxB,OAAO;oBACtC,IAAI4C,QAAQ,UAAU,OAAOpB,IAAId,OAAO;oBACxC,IAAIkC,QAAQ,SAAS,OAAOpB,IAAIZ,IAAI;oBACpC,QAAOY,gBAAAA,IAAIM,IAAI,CAACc,IAAI,cAAbpB,2BAAAA,gBAAiB;gBAC1B;gBACA,uFAAuF;gBACvF0B,OAAO1D,GAAG,IAAI4D;gBACd,oFAAoF;gBACpF,IAAIlD,SAASK,GAAG,CAACiB,IAAIxB,OAAO,GAAG;wBAEGf;oBADhC4D,QAAQrD,GAAG,CAACgC,IAAIxB,OAAO;oBACvB,KAAK,MAAMf,WAAW8B,UAAU9B,mBAAAA,QAAQgE,MAAM,cAAdhE,uCAAAA,sBAAAA,SAAiB3B,IAAIkE,IAAIxB,OAAO,EAAEsC;gBACpE;gBACAa,WAAW3D,GAAG,CAACgC,IAAIxB,OAAO,EAAEwB,IAAI6B,MAAM,CAACC,KAAK,EAAE9B,IAAI6B,MAAM,CAACE,OAAO,EAAE/B,IAAI6B,MAAM,CAACG,IAAI,EAAEhC,IAAIxB,OAAO;gBAC9F,iFAAiF;gBACjF,2EAA2E;gBAC3E,IAAIE,SAASK,GAAG,CAACiB,IAAIxB,OAAO,GAAG8C,eAAetD,GAAG,CAACgC,IAAIxB,OAAO;gBAC7D,KAAK,MAAMyD,cAAcjC,IAAIkC,OAAO,CAAEX,iBAAiBvD,GAAG,CAACgC,IAAIxB,OAAO,EAAEyD;gBACxE,KAAK,MAAMxE,WAAW8B,UAAU9B,iBAAAA,QAAQ0E,KAAK,cAAb1E,qCAAAA,oBAAAA,SAAgB3B,IAAIkE,IAAIxB,OAAO,EAAEwB,IAAIoC,SAAS,CAAC3E,QAAQ9B,IAAI,CAAC,EAAEmF;YAChG;QACF;QACA,KAAK,MAAMrD,WAAW8B,UAAU9B,0BAAAA,QAAQ4E,cAAc,cAAtB5E,8CAAAA,6BAAAA,SAAyB3B,IAAIgF;QAC7DhF,GAAG0B,IAAI,CAAC;IACV,EAAE,OAAO8E,KAAK;QACZxG,GAAG0B,IAAI,CAAC;QACR,MAAM8E;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMC,aAAarB,KAAKC,GAAG,KAAKF;IAChC,MAAMuB,UAAU7E,QAAQ7B,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM2G,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS7E,QAAQ9B,IAAI,oBAAoBQ,OAAOiG;IAEjE,OAAO;QAAE/F,QAAQkD,WAAWL,MAAM;QAAEC;IAAS;AAC/C;AAEA,yFAAyF;AACzF,wFAAwF;AACxF,SAASqD,cAAcC,MAAc,EAAEC,KAAa;IAClD,yFAAyF;IACzF,MAAMC,QAAQ,CAACC,OAAkBA,KAAKxG,UAAU,CAAC,aAAawG,KAAKnH,KAAK,CAAC,KAAKoH,KAAK,CAAC,GAAG,GAAGrI,IAAI,CAAC,OAAOoI,KAAKnH,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAMc,QAAQ,CAACuG,MAAgB,IAAItE,IAAIsE,IAAIrH,KAAK,CAAC,KAAKwB,GAAG,CAAC,CAAC2F,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMG,IAAIxG,MAAMkG;IAChB,MAAMO,IAAIzG,MAAMmG;IAChB,MAAMO,UAAU,IAAI9H;IACpB,KAAK,MAAM,CAACuC,KAAKwF,IAAI,IAAIF,EAAG,IAAID,EAAEnF,GAAG,CAACF,SAASwF,KAAKD,QAAQ7C,GAAG,CAAC1C;IAChE,KAAK,MAAMA,OAAOqF,EAAE7C,IAAI,GAAI,IAAI,CAAC8C,EAAEpE,GAAG,CAAClB,MAAMuF,QAAQ7C,GAAG,CAAC1C;IACzD,MAAMyF,QAAQ,CAACzF,MAAiBA,QAAQ,UAAU,mBAAmBA,IAAItB,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEsB,IAAImF,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IAC9H,OAAOI,QAAQhE,IAAI,KAAK,IAAI,aAAa;WAAIgE;KAAQ,CAAChG,GAAG,CAACkG,OAAO3I,IAAI,CAAC;AACxE;AAEA,OAAO,SAAS4I,KAAKhG,GAAmB;QAqCTI;IApC7B,MAAM6F,WAAW7I,KAAK4C,IAAIa,OAAO,EAAEtD;IACnCL,UAAU+I,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMC,SAAS/I,KAAK6I,UAAUjI;IAE9B,MAAMO,KAAK,IAAIlB,aAAa8I;IAC5B5H,GAAG0B,IAAI,CAAC;IACR,yFAAyF;IACzF,uCAAuC;IACvC1B,GAAG0B,IAAI,CAAC;IACR3B,kBAAkBC;IAElBA,GAAG0B,IAAI,CAAC;IAER,uFAAuF;IACvF,qGAAqG;IACrG,MAAMmG,UAAUhG,QAAQ7B,IAAI;IAC5B,MAAMyD,WAAW5B,QAAQ7B,IAAI;IAC7B,MAAM8H,eAAe/I,iBAAiB0C;IACtC,IAAI,AAACoG,YAAY,QAAQA,YAAYnI,kBAAoB+D,aAAa,QAAQA,aAAaqE,cAAe;QACxG,uFAAuF;QACvF,wFAAwF;QACxF,IAAID,YAAY,QAAQA,YAAYnI,gBAAgB;YAClDqI,QAAQC,KAAK,CAAC;QAChB,OAAO;YACL,MAAMV,UAAUT,cAAcpD,qBAAAA,sBAAAA,WAAY,IAAIqE;YAC9CC,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEV,QAAQ,oBAAoB,CAAC;QACtE;QACAtH,GAAGiI,KAAK;QACRrJ,OAAO8I,UAAU;YAAEC,WAAW;YAAMO,OAAO;QAAK;QAChD,OAAOT,KAAKhG;IACd;IAEAD,aAAaxB,IAAIyB;IAEjB,wFAAwF;IACxF,2FAA2F;IAC3F,MAAM0G,gBAAgBvB,QAAO/E,WAAAA,QAAQ7B,IAAI,iCAAZ6B,sBAAAA,WAAmC;IAChE7B,GAAG0B,IAAI,CAAC,CAAC,sBAAsB,EAAE0G,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIH,gBAAgB,SAAU;IAExF,MAAM,EAAEzH,MAAM,EAAE8C,QAAQ,EAAE,GAAGnB,UAAUrC,IAAIyB,KAAKA,IAAIa,OAAO;IAE3D,OAAO;QAAEtC;QAAIyB;QAAKmG;QAAQlH;QAAQ8C;IAAS;AAC7C;AAEA,oCAAoC;AACpC,OAAO,SAAS+E,QAAQ9G,GAAmB;IACzC7C,OAAOC,KAAK4C,IAAIa,OAAO,EAAEtD,YAAY;QAAE2I,WAAW;QAAMO,OAAO;IAAK;IACpE,OAAOT,KAAKhG;AACd"}

package/dist/esm/errors.d.ts

@@ -1,4 +1,4 @@
-export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN';
+export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN';
 export declare class SenseError extends Error {
     code: SenseErrorCode;
     constructor(code: SenseErrorCode, message: string);

package/dist/esm/errors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","Error","code","message","name"],"mappings":"AAAA,qDAAqD;AAGrD,OAAO,MAAMA,mBAAmBC;IAG9B,YAAYC,IAAoB,EAAEC,OAAe,CAAE;QACjD,KAAK,CAACA;QACN,IAAI,CAACC,IAAI,GAAG;QACZ,IAAI,CAACF,IAAI,GAAGA;IACd;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","Error","code","message","name"],"mappings":"AAAA,qDAAqD;AAGrD,OAAO,MAAMA,mBAAmBC;IAG9B,YAAYC,IAAoB,EAAEC,OAAe,CAAE;QACjD,KAAK,CAACA;QACN,IAAI,CAACC,IAAI,GAAG;QACZ,IAAI,CAACF,IAAI,GAAGA;IACd;AACF"}

package/dist/esm/features/embed.d.ts

@@ -7,9 +7,23 @@ export interface EmbedProvider {
     embed(texts: string[]): Promise<Float32Array[]>;
 }
 export declare const embed: Feature;
+export declare const MODEL_FILENAMES: string;
+export declare function isDownloadable(model: string): boolean;
+export declare function modelDir(model: string): string;
+export declare function hasModelFiles(model: string): boolean;
+export declare function modelPresent(cfg: Config): boolean;
+export declare function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string>;
 export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void>;
 export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{
     path: string;
     lines: string;
     similarity: number;
 }>>;
+export declare function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: {
+    exclude: Set<string>;
+    allowed?: Set<string>;
+    k: number;
+}): Array<{
+    path: string;
+    similarity: number;
+}>;

package/dist/esm/features/embed.js

@@ -5,18 +5,16 @@ import { embedConfig } from '../config.js';
 import { SenseError } from '../errors.js';
 import { progress } from '../progress.js';
 import { parseFile } from '../scan.js';
-// embeddings(path, chunk, start_line, end_line, scale, vector): heading-based chunks,
-// int8 vectors with a per-vector dequantization scale, vector NULL = not yet embedded.
-// Reconcile stores dirty rows inside its transaction; embedding tops up on the next
-// semantic query (embedPending), so staleness costs recall, never correctness.
+// Heading-based chunks, int8 vectors with a per-vector scale, NULL vector = not yet embedded.
+// Reconcile writes dirty rows; embedding tops up on the next search, so staleness costs recall.
 // Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is
 // quality-free vs f32-512 when fused. Queries stay f32 at the same dims.
 const STORE_DIMS = 256;
+// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.
+const TARGET_CHUNK_CAP = 16;
 const BATCH = 64;
-// Deterministic chunker used at reconcile (line ranges stored) and at embed time (text
-// re-derived from the file -- chunk text is never stored). Heading-delimited with the
-// preamble kept; whole file when no headings. The title/summary prefix mirrors the
-// bm25 column weighting.
+// Deterministic, so embed time can re-derive text from stored line ranges. Heading-delimited,
+// preamble kept, whole file when no headings; the title/summary prefix mirrors bm25 weighting.
 function chunksOf(raw, search) {
     const lines = raw.split('\n');
     const starts = [];
@@ -61,8 +59,8 @@ export const embed = {
     remove (db, path) {
         db.prepare('DELETE FROM embeddings WHERE "path" = ?').run(path);
     },
-    // A doc covered only by semantic-false presets never had extract() run for it (db.ts's
-    // per-file filter skips it), so extracted is undefined here -- store nothing, i.e. no rows.
+    // A tree with no embedding model never had extract() run for the doc (db.ts's per-file
+    // filter skips it), so extracted is undefined here -- store nothing, i.e. no rows.
     store (db, path, extracted) {
         if (!extracted) return;
         const insert = db.prepare('INSERT INTO embeddings ("path", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');
@@ -75,6 +73,45 @@ export const embed = {
 // --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---
 // Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,
 // L2-normalize.
+const MODEL_FILES = [
+    'model.safetensors',
+    'tokenizer.json'
+];
+// The pair, for messages that name what a local model directory must contain.
+export const MODEL_FILENAMES = MODEL_FILES.join(' and ');
+// A Hugging Face repo id: one slash, HF's charset. Anything else is a path, used as one rather
+// than flattened into a cache key Windows would reject.
+const HF_MODEL_ID = /^[A-Za-z0-9][\w.-]*\/[A-Za-z0-9][\w.-]*$/;
+// Downloadable iff it names a Hugging Face repo; a local path is the caller's to populate.
+export function isDownloadable(model) {
+    return HF_MODEL_ID.test(model);
+}
+// Machine-wide, not per-tree: `.sense/` is the index a rebuild throws away, while a 124 MB
+// model is shared by every tree on the machine. Honors XDG_CACHE_HOME where it is set;
+// elsewhere ~/.cache, which is also where Hugging Face's own libraries keep theirs.
+function cacheRoot() {
+    const xdg = process.env.XDG_CACHE_HOME;
+    return join(xdg && xdg.length > 0 ? xdg : join(homedir(), '.cache'), 'sensemaking', 'models');
+}
+// A model is either a local directory the caller pointed at, or the cache `sense download`
+// fills. Nothing here fetches: an absent model degrades search to its other signals rather
+// than pulling 124 MB out of a command that reads like a query.
+export function modelDir(model) {
+    if (!HF_MODEL_ID.test(model)) return model;
+    return join(cacheRoot(), model.replace(/\//g, '--'));
+}
+// Both files, so an interrupted download reads as absent and the next one resumes it.
+export function hasModelFiles(model) {
+    const dir = modelDir(model);
+    return MODEL_FILES.every((file)=>existsSync(join(dir, file)));
+}
+// api providers have nothing to download, so they are never "missing" here; an unreachable
+// endpoint surfaces as an EMBED_MODEL error at call time instead.
+export function modelPresent(cfg) {
+    const e = embedConfig(cfg);
+    if (!e) return false;
+    return e.type === 'api' || hasModelFiles(e.model);
+}
 function fetchToFile(url, dest) {
     return fetch(url).then(async (res)=>{
         if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);
@@ -82,23 +119,33 @@ function fetchToFile(url, dest) {
         renameSync(`${dest}.part`, dest);
     });
 }
+// The only code path that touches the network for weights. `sense download` calls it; no
+// query ever does. Idempotent: a file already on disk is left alone.
+export async function downloadModel(model, onFile) {
+    if (!isDownloadable(model)) {
+        throw new SenseError('EMBED_MODEL', `embed.model "${model}" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like "minishlab/potion-retrieval-32M"`);
+    }
+    const dir = modelDir(model);
+    mkdirSync(dir, {
+        recursive: true
+    });
+    for (const file of MODEL_FILES){
+        if (existsSync(join(dir, file))) continue;
+        onFile === null || onFile === void 0 ? void 0 : onFile(file, dir);
+        await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));
+    }
+    return dir;
+}
 async function staticProvider(model) {
     var _ref;
     var _tokenizerJson_model, _tokenizerJson_model_vocab, _tokenizerJson_model1;
-    let dir = model;
-    if (!existsSync(join(dir, 'model.safetensors'))) {
-        dir = join(homedir(), '.cache', 'sensemaking', 'models', model.replace(/\//g, '--'));
-        mkdirSync(dir, {
-            recursive: true
-        });
-        for (const file of [
-            'model.safetensors',
-            'tokenizer.json'
-        ]){
-            if (!existsSync(join(dir, file))) {
-                console.error(`fetching ${model}/${file} into ${dir} (once; delete to refetch)`);
-                await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));
-            }
+    const dir = modelDir(model);
+    for (const file of MODEL_FILES){
+        if (!existsSync(join(dir, file))) {
+            // A local path the caller controls gets told what is missing where; only a repo id can
+            // be fixed by downloading.
+            const fix = isDownloadable(model) ? 'run `sense download`' : `expected ${MODEL_FILENAMES} in that directory`;
+            throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); ${fix}`);
         }
     }
     const raw = readFileSync(join(dir, 'model.safetensors'));
@@ -178,7 +225,7 @@ const providers = new Map();
 function getProvider(cfg) {
     var _e_url;
     const e = embedConfig(cfg);
-    if (!e) throw new SenseError('EMBED_DISABLED', 'semantic search needs at least one preset with vectors on (semantic !== false); every declared preset in sense.config.json has "semantic": false');
+    if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an "embed" block naming one to sense.config.json, then run `sense download`');
     const sig = `${e.type}:${e.model}:${(_e_url = e.url) !== null && _e_url !== void 0 ? _e_url : ''}`;
     let p = providers.get(sig);
     if (!p) {
@@ -224,7 +271,7 @@ export async function embedPending(db, cfg, baseDir) {
         let chunks;
         try {
             // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already
-            // has embeddings rows means it was covered by a semantic-on preset at reconcile time.
+            // has embeddings rows means the tree had an embedding model at reconcile time.
             chunks = parseFile({
                 relPath: path,
                 absPath: join(baseDir, path),
@@ -245,7 +292,7 @@ export async function embedPending(db, cfg, baseDir) {
         });
     }
     const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE "path" = ? AND chunk = ?');
-    // The lazy build is the one long silence a first --semantic query hits (measured 23s at
+    // The lazy build is the one long silence a first search hits (measured 23s at
     // 26k notes); progress makes it distinguishable from a hang.
     const report = progress('embedding chunks', jobs.length);
     for(let i = 0; i < jobs.length; i += BATCH){
@@ -268,14 +315,9 @@ export async function embedPending(db, cfg, baseDir) {
     }
     report.finish();
 }
-// Top candidates by cosine for a semantic find: best chunk per file, its line range
-// riding along. FTS5 operators in the terms are lexical syntax, not meaning -- stripped
-// before embedding.
-// Returns the cosine similarity alongside each candidate: stored and query vectors are both
-// L2-normalised before quantisation, so the dot product is cosine in [-1, 1]. `find` surfaces
-// it because the fused rank score cannot express match quality -- nearest-neighbour search
-// always returns a nearest neighbour, so without a magnitude an agent cannot tell a real hit
-// from the best of a bad lot.
+// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as
+// lexical syntax. Similarity comes back because the fused score cannot express match quality,
+// and nearest-neighbour search always returns a neighbour however far away.
 export async function semanticCandidates(db, cfg, terms, fetch1) {
     var _terms_match;
     const baseDir = cfg.baseDir;
@@ -313,3 +355,36 @@ export async function semanticCandidates(db, cfg, terms, fetch1) {
             similarity: Math.round(b.score * 1000) / 1000
         }));
 }
+// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear
+// scan of stored vectors. Reads only what is stored, so it stays sync.
+export function similarNotes(db, _cfg, path, opts) {
+    // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus
+    // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.
+    const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk').all(path);
+    if (targetRows.length === 0) return [];
+    const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));
+    const target = targetRows.filter((_, i)=>i % step === 0).map((row)=>({
+            v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength),
+            scale: row.scale
+        }));
+    const rows = db.prepare('SELECT "path", scale, vector FROM embeddings WHERE vector IS NOT NULL').all();
+    const best = new Map();
+    for (const row of rows){
+        if (row.path === path || opts.exclude.has(row.path) || opts.allowed && !opts.allowed.has(row.path)) continue;
+        const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
+        for (const t of target){
+            let dot = 0;
+            const len = Math.min(t.v.length, other.length);
+            for(let d = 0; d < len; d++)dot += t.v[d] * other[d];
+            const score = dot * t.scale * row.scale;
+            const existing = best.get(row.path);
+            if (existing === undefined || score > existing) best.set(row.path, score);
+        }
+    }
+    return [
+        ...best.entries()
+    ].sort((a, b)=>b[1] - a[1]).slice(0, opts.k).map(([p, score])=>({
+            path: p,
+            similarity: Math.round(score * 1000) / 1000
+        }));
+}