sensemaking 0.19.1 → 0.19.2

package/dist/esm/store/sqlite/reconcile.js

@@ -1,29 +1,18 @@
 import { contentTokenize } from '../../config/index.js';
 import { SenseError } from '../../errors.js';
-import { activeFeatures } from '../../features/index.js';
-import { progress } from '../../output/progress.js';
-import { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.js';
-import { reparseFiles } from '../../scan/reparse.js';
 import { segmentField } from '../../text/segment.js';
-import { getColumns, getMeta, quoteIdent, setMeta } from '../shared.js';
-import { BEGIN_WRITE, withTransaction } from '../transaction.js';
-// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last
-// computed value on every touch, not just the reconciles that recompute it.
-const CORE_FRONTMATTER_COLUMNS = new Set([
-    'path',
-    '_mtime',
-    '_ctime',
-    '_size',
-    '_parse_error'
-]);
+import { recordReconcileDuration } from '../shared.js';
+import { BEGIN_WRITE } from '../transaction.js';
+// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in
+// store/reconcile.ts. content is FTS5-virtual with "_seg" sidecars for lexical.ts.
 // SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).
 const MAX_FRONTMATTER_COLUMNS = 2000;
-// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same
-// literal instead of two copies drifting apart.
-const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, "path", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE "path" = ?), ?, ?, ?, ?, ?, ?, ?)`;
+// Shared by reconcileContent() and the tokenize-only rebuild in open.ts, so both prepare the
+// same literal instead of two copies drifting apart.
+export const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, "path", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE "path" = ?), ?, ?, ?, ?, ?, ?, ?)`;
 // A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.
 // Assumes the frontmatter row for doc.relPath already exists (rowid subquery).
-function contentRow(doc, segmenting) {
+export function contentRow(doc, segmenting) {
     return [
         doc.relPath,
         doc.search.title,
@@ -35,188 +24,29 @@ function contentRow(doc, segmenting) {
         segmenting ? segmentField(doc.search.text) : ''
     ];
 }
-export async function reconcile(conn, cfg, baseDir) {
-    const files = listFiles(cfg, baseDir);
-    const currentSet = new Set(files.map((f)=>f.relPath));
-    const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
-    const existingRows = await existingStmt.all();
-    const existing = new Map(existingRows.map((r)=>[
-            r.path,
-            r
+async function reconcileContent(conn, touched, docs, _delta, cfg) {
+    // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via
+    // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.
+    if (touched.length > 0) await conn.runBatch('DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE "path" = ?)', touched.map((p)=>[
+            p
         ]));
-    const vanished = existingRows.filter((r)=>!currentSet.has(r.path)).map((r)=>r.path);
-    const toReparse = files.filter((f)=>{
-        const row = existing.get(f.relPath);
-        return !row || row._mtime !== f.mtimeMs || row._size !== f.size;
-    });
-    if (vanished.length === 0 && toReparse.length === 0) return {
-        parsed: 0,
-        warnings: []
-    };
-    const features = activeFeatures(cfg);
-    const seenColumns = await getColumns(conn);
-    // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
-    // reconciles stay silent (progress() has a threshold).
-    const report = progress('reparsing files', toReparse.length);
-    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
-    report.finish();
-    for (const col of newColumns)seenColumns.add(col);
-    const allColumns = [
-        ...seenColumns
-    ];
-    // Fence before ALTERing: SQLite's own failure past this point is a raw
-    // "too many columns on sqlite_altertab_frontmatter" with no indication of the boundary or the levers.
-    if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {
-        throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);
+    if (docs.length > 0) {
+        // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over
+        // grapheme runs would be nonsense against trigram, so the sidecars stay empty.
+        const segmenting = contentTokenize(cfg) === undefined;
+        await conn.runBatch(INSERT_CONTENT_SQL, docs.map((doc)=>contentRow(doc, segmenting)));
     }
-    // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a
-    // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).
-    const writableColumns = allColumns.filter((c)=>CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));
-    // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --
-    // content rows are coupled to that rowid below.
-    const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(()=>'?').join(', ')}) ON CONFLICT("path") DO UPDATE SET ${writableColumns.filter((c)=>c !== 'path').map((c)=>`${quoteIdent(c)} = excluded.${quoteIdent(c)}`).join(', ')}`;
-    const added = toReparse.filter((f)=>!existing.has(f.relPath)).map((f)=>f.relPath);
-    const delta = {
-        files,
-        reparsed: parsedDocs.map((d)=>d.relPath),
-        added,
-        vanished
-    };
-    const addedSet = new Set(added);
-    const reparsedExisting = parsedDocs.map((d)=>d.relPath).filter((p)=>!addedSet.has(p));
-    const txStart = Date.now();
-    await withTransaction(conn, async ()=>{
-        var _feature_remove, _feature_afterReconcile;
-        // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a
-        // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.
-        const present = await getColumns(conn);
-        for (const col of newColumns)if (!present.has(col)) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);
-        // content's rowid lookup depends on its coupled frontmatter row, so the content
-        // deletes/inserts run while every path still has one -- the vanished frontmatter delete comes last.
-        if (vanished.length > 0) {
-            await conn.runBatch('DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE "path" = ?)', vanished.map((p)=>[
-                    p
-                ]));
-        }
-        if (reparsedExisting.length > 0) {
-            // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via
-            // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.
-            await conn.runBatch('DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE "path" = ?)', reparsedExisting.map((p)=>[
-                    p
-                ]));
-        }
-        if (parsedDocs.length > 0) {
-            const rows = parsedDocs.map((doc)=>writableColumns.map((col)=>{
-                    var _doc_data_col;
-                    if (col === 'path') return doc.relPath;
-                    if (col === '_mtime') return doc.mtimeMs;
-                    if (col === '_ctime') return doc.ctimeMs;
-                    if (col === '_size') return doc.size;
-                    // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.
-                    if (col === '_parse_error') return doc.parseError;
-                    return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
-                }));
-            await conn.runBatch(insertSql, rows);
-            // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over
-            // grapheme runs would be nonsense against trigram, so the sidecars stay empty.
-            const segmenting = contentTokenize(cfg) === undefined;
-            await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map((doc)=>contentRow(doc, segmenting)));
-        }
-        if (vanished.length > 0) await conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[
-                p
-            ]));
-        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;
-        // new docs have nothing to clear, which keeps cold builds linear.
-        const presetTouched = [
-            ...vanished,
-            ...reparsedExisting
-        ];
-        if (presetTouched.length > 0) await conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', presetTouched.map((p)=>[
-                p
-            ]));
-        const presetRows = [];
-        for (const doc of parsedDocs)for (const presetName of doc.presets)presetRows.push([
-            doc.relPath,
-            presetName
-        ]);
-        if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?)', presetRows);
-        const removedPaths = [
-            ...vanished,
-            ...reparsedExisting
-        ];
-        if (removedPaths.length > 0) for (const feature of features)await ((_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, removedPaths, delta));
-        for (const feature of features){
-            var _feature_store;
-            const docsForFeature = parsedDocs.map((doc)=>({
-                    path: doc.relPath,
-                    extracted: doc.extracted[feature.name]
-                }));
-            await ((_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta));
-        }
-        for (const feature of features)await ((_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta));
-    }, BEGIN_WRITE);
-    // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):
-    // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.
-    const durationMs = Date.now() - txStart;
-    const prevRaw = await getMeta(conn, 'reconcile_max_ms');
-    // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)
-    // still gets recorded instead of losing to the "nothing recorded yet" default.
-    const prevMax = prevRaw === null ? -1 : Number(prevRaw);
-    if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));
-    return {
-        parsed: parsedDocs.length,
-        warnings
-    };
-}
-// Segment keys that moved between two feature signatures (see config.featureSignature's
-// format: global features, embed provider, tokenize, then one segment per preset).
-export function changedSignatureKeys(before, after) {
-    const keyOf = (part)=>part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0];
-    const parse = (sig)=>new Map(sig.split('|').map((part)=>[
-                keyOf(part),
-                part
-            ]));
-    const a = parse(before);
-    const b = parse(after);
-    const changed = new Set();
-    for (const [key, val] of b)if (a.get(key) !== val) changed.add(key);
-    for (const key of a.keys())if (!b.has(key)) changed.add(key);
-    return changed;
-}
-// Whether the embed segment only gained its resolved weight identity: same provider and
-// model, no identity recorded before, one now -- adopted into meta without a rebuild.
-export function embedIdentityAdopted(before, after) {
-    const embedPart = (sig)=>sig.split('|').find((p)=>p.startsWith('embed:'));
-    const b = embedPart(before);
-    const a = embedPart(after);
-    if (b === undefined || a === undefined) return false;
-    const at = a.indexOf('@');
-    return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;
-}
-// Names what moved, for the rebuild notice.
-export function signatureDiff(before, after) {
-    const changed = changedSignatureKeys(before, after);
-    const label = (key)=>key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset "${key.slice(7)}"` : 'features';
-    return changed.size === 0 ? 'features' : [
-        ...changed
-    ].map(label).join(', ');
-}
-// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)
-// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.
-export async function rebuildContentTable(conn, cfg, baseDir) {
-    const stmt = await conn.prepare('SELECT "path" FROM frontmatter');
-    const known = new Set((await stmt.all()).map((r)=>r.path));
-    const files = listFiles(cfg, baseDir).filter((f)=>known.has(f.relPath));
-    const segmenting = contentTokenize(cfg) === undefined;
-    const warnings = [];
-    const rows = [];
-    for (const file of files){
-        const { doc, warnings: fileWarnings } = parseFile(file);
-        warnings.push(...fileWarnings);
-        rows.push(contentRow(doc, segmenting));
-    }
-    await withTransaction(conn, async ()=>{
-        if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);
-    }, BEGIN_WRITE);
-    return warnings;
 }
+export const sqliteDialect = {
+    beginMode: ()=>BEGIN_WRITE,
+    checkColumnLimit (count) {
+        // Fence before ALTERing: SQLite's own failure past this point is a raw
+        // "too many columns on sqlite_altertab_frontmatter" with no indication of the boundary or the levers.
+        if (count > MAX_FRONTMATTER_COLUMNS) {
+            throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);
+        }
+    },
+    columnTypeSuffix: ()=>'',
+    reconcileContent,
+    recordDuration: recordReconcileDuration
+};

package/dist/esm/store/sqlite/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nfunction contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      for (const col of newColumns) if (!present.has(col)) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n      // content's rowid lookup depends on its coupled frontmatter row, so the content\n      // deletes/inserts run while every path still has one -- the vanished frontmatter delete comes last.\n      if (vanished.length > 0) {\n        await conn.runBatch(\n          'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n          vanished.map((p) => [p])\n        );\n      }\n      if (reparsedExisting.length > 0) {\n        // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via\n        // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.\n        await conn.runBatch(\n          'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n          reparsedExisting.map((p) => [p])\n        );\n      }\n\n      if (parsedDocs.length > 0) {\n        const rows = parsedDocs.map((doc) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          })\n        );\n        await conn.runBatch(insertSql, rows);\n\n        // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n        // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n        const segmenting = contentTokenize(cfg) === undefined;\n        await conn.runBatch(\n          INSERT_CONTENT_SQL,\n          parsedDocs.map((doc) => contentRow(doc, segmenting))\n        );\n      }\n\n      if (vanished.length > 0)\n        await conn.runBatch(\n          'DELETE FROM frontmatter WHERE \"path\" = ?',\n          vanished.map((p) => [p])\n        );\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      const presetTouched = [...vanished, ...reparsedExisting];\n      if (presetTouched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM preset_files WHERE \"path\" = ?',\n          presetTouched.map((p) => [p])\n        );\n      const presetRows: unknown[][] = [];\n      for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n      if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n      const removedPaths = [...vanished, ...reparsedExisting];\n      if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await feature.store?.(conn, docsForFeature, delta);\n      }\n      for (const feature of features) await feature.afterReconcile?.(conn, delta);\n    },\n    BEGIN_WRITE\n  );\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(\n    conn,\n    async () => {\n      if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n    },\n    BEGIN_WRITE\n  );\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","reparseFiles","segmentField","getColumns","getMeta","quoteIdent","setMeta","BEGIN_WRITE","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","txStart","Date","now","feature","present","exec","runBatch","rows","ctimeMs","parseError","data","undefined","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","key","val","keys","embedIdentityAdopted","embedPart","find","at","indexOf","signatureDiff","label","rebuildContentTable","stmt","known","file","fileWarnings"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAC7E,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,eAAe;AACxE,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAGjE,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,uFAAuF;AACvF,+EAA+E;AAC/E,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IACrD,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAab,aAAaY,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAab,aAAaY,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAab,aAAaY,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQ3B,UAAUyB,KAAKC;IAC7B,MAAME,aAAa,IAAIhB,IAAIe,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEZ,OAAO;QAClC,OAAO,CAACyB,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWtD,eAAe2B;IAChC,MAAM4B,cAAc,MAAMhD,WAAWmB;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8B,SAASvD,SAAS,mBAAmB2C,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMtD,aAAauC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIS,WAAWb,MAAM,GAAGpC,yBAAyB;QAC/C,MAAM,IAAIhB,WACR,gBACA,CAAC,uBAAuB,EAAEiE,WAAWb,MAAM,CAAC,0EAA0E,EAAEpC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMkD,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMrD,yBAAyB8B,GAAG,CAACuB,MAAM,CAAC9D,iBAAiBuC,GAAG,CAACuB;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACtB,YAAY2D,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGzD,WAAWyD,GAAG,YAAY,EAAEzD,WAAWyD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAMkD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO;QAAGiD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI3D,IAAIuD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO,EAAEsB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMlE,gBACJc,MACA;YAiEqEqD,iBAK7BA;QArEtC,iGAAiG;QACjG,sFAAsF;QACtF,MAAMC,UAAU,MAAMzE,WAAWmB;QACjC,KAAK,MAAMoC,OAAOH,WAAY,IAAI,CAACqB,QAAQrC,GAAG,CAACmB,MAAM,MAAMpC,KAAKuD,IAAI,CAAC,CAAC,mCAAmC,EAAExE,WAAWqD,MAAM;QAE5H,gFAAgF;QAChF,oGAAoG;QACpG,IAAIrB,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMzB,KAAKwD,QAAQ,CACjB,sFACAzC,SAASV,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAE3B;QACA,IAAID,iBAAiBvB,MAAM,GAAG,GAAG;YAC/B,6FAA6F;YAC7F,uGAAuG;YACvG,MAAMzB,KAAKwD,QAAQ,CACjB,sFACAR,iBAAiB3C,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAEnC;QAEA,IAAIjB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMgC,OAAOzB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAOZ5C;oBANP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIE,OAAO;oBACtC,IAAI0C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAIkE,OAAO;oBACxC,IAAItB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAImE,UAAU;oBACjD,QAAOnE,gBAAAA,IAAIoE,IAAI,CAACxB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKwD,QAAQ,CAACf,WAAWgB;YAE/B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAMhE,aAAarB,gBAAgB6B,SAAS4D;YAC5C,MAAM7D,KAAKwD,QAAQ,CACjBlE,oBACA0C,WAAW3B,GAAG,CAAC,CAACb,MAAQD,WAAWC,KAAKC;QAE5C;QAEA,IAAIsB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKwD,QAAQ,CACjB,4CACAzC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMa,gBAAgB;eAAI/C;eAAaiC;SAAiB;QACxD,IAAIc,cAAcrC,MAAM,GAAG,GACzB,MAAMzB,KAAKwD,QAAQ,CACjB,6CACAM,cAAczD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMc,aAA0B,EAAE;QAClC,KAAK,MAAMvE,OAAOwC,WAAY,KAAK,MAAMgC,cAAcxE,IAAIyE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAAC1E,IAAIE,OAAO;YAAEsE;SAAW;QAC7G,IAAID,WAAWtC,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAC,2DAA2DO;QAE1G,MAAMI,eAAe;eAAIpD;eAAaiC;SAAiB;QACvD,IAAImB,aAAa1C,MAAM,GAAG,GAAG,KAAK,MAAM4B,WAAWzB,SAAU,QAAMyB,kBAAAA,QAAQe,MAAM,cAAdf,sCAAAA,qBAAAA,SAAiBrD,MAAMmE,cAAcvB;QACxG,KAAK,MAAMS,WAAWzB,SAAU;gBAExByB;YADN,MAAMgB,iBAAiCrC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIE,OAAO;oBAAE4E,WAAW9E,IAAI8E,SAAS,CAACjB,QAAQkB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMlB,iBAAAA,QAAQmB,KAAK,cAAbnB,qCAAAA,oBAAAA,SAAgBrD,MAAMqE,gBAAgBzB;QAC9C;QACA,KAAK,MAAMS,WAAWzB,SAAU,QAAMyB,0BAAAA,QAAQoB,cAAc,cAAtBpB,8CAAAA,6BAAAA,SAAyBrD,MAAM4C;IACvE,GACA3D;IAGF,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMyF,aAAavB,KAAKC,GAAG,KAAKF;IAChC,MAAMyB,UAAU,MAAM7F,QAAQkB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM4E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS,MAAM5F,QAAQgB,MAAM,oBAAoB8E,OAAOJ;IAEzE,OAAO;QAAEhD,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASoD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAG5C,IAAI,CAAC,OAAOyC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAI5E,IAAI4E,IAAIH,KAAK,CAAC,KAAKhF,GAAG,CAAC,CAAC8E,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAIvG;IACpB,KAAK,MAAM,CAACwG,KAAKC,IAAI,IAAIH,EAAG,IAAID,EAAErE,GAAG,CAACwE,SAASC,KAAKF,QAAQtD,GAAG,CAACuD;IAChE,KAAK,MAAMA,OAAOH,EAAEK,IAAI,GAAI,IAAI,CAACJ,EAAEzE,GAAG,CAAC2E,MAAMD,QAAQtD,GAAG,CAACuD;IACzD,OAAOD;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASI,qBAAqBf,MAAc,EAAEC,KAAa;IAChE,MAAMe,YAAY,CAACR,MAAgBA,IAAIH,KAAK,CAAC,KAAKY,IAAI,CAAC,CAAChD,IAAMA,EAAEmC,UAAU,CAAC;IAC3E,MAAMM,IAAIM,UAAUhB;IACpB,MAAMS,IAAIO,UAAUf;IACpB,IAAIS,MAAM7B,aAAa4B,MAAM5B,WAAW,OAAO;IAC/C,MAAMqC,KAAKT,EAAEU,OAAO,CAAC;IACrB,OAAOT,EAAES,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKT,EAAEH,KAAK,CAAC,GAAGY,QAAQR;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASU,cAAcpB,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMoB,QAAQ,CAACT,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIR,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEQ,IAAIN,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQnE,IAAI,KAAK,IAAI,aAAa;WAAImE;KAAQ,CAACtF,GAAG,CAACgG,OAAO3D,IAAI,CAAC;AACxE;AAEA,oGAAoG;AACpG,gIAAgI;AAChI,OAAO,eAAe4D,oBAAoBtG,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,MAAMqG,OAAO,MAAMvG,KAAKQ,OAAO,CAAC;IAChC,MAAMgG,QAAQ,IAAIpH,IAAI,AAAE,CAAA,MAAMmH,KAAK7F,GAAG,EAAC,EAA+BL,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IACvF,MAAMX,QAAQ3B,UAAUyB,KAAKC,SAASc,MAAM,CAAC,CAACV,IAAMkG,MAAMvF,GAAG,CAACX,EAAEZ,OAAO;IACvE,MAAMD,aAAarB,gBAAgB6B,SAAS4D;IAC5C,MAAMlC,WAAqB,EAAE;IAC7B,MAAM8B,OAAoB,EAAE;IAC5B,KAAK,MAAMgD,QAAQtG,MAAO;QACxB,MAAM,EAAEX,GAAG,EAAEmC,UAAU+E,YAAY,EAAE,GAAGjI,UAAUgI;QAClD9E,SAASuC,IAAI,IAAIwC;QACjBjD,KAAKS,IAAI,CAAC3E,WAAWC,KAAKC;IAC5B;IACA,MAAMP,gBACJc,MACA;QACE,IAAIyD,KAAKhC,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAClE,oBAAoBmE;IAC/D,GACAxE;IAEF,OAAO0C;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is FTS5-virtual with \"_seg\" sidecars for lexical.ts.\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcileContent() and the tokenize-only rebuild in open.ts, so both prepare the\n// same literal instead of two copies drifting apart.\nexport const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nexport function contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta, cfg: Config): Promise<void> {\n  // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via\n  // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n      touched.map((p) => [p])\n    );\n\n  if (docs.length > 0) {\n    // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n    // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n    const segmenting = contentTokenize(cfg) === undefined;\n    await conn.runBatch(\n      INSERT_CONTENT_SQL,\n      docs.map((doc) => contentRow(doc, segmenting))\n    );\n  }\n}\n\nexport const sqliteDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    // Fence before ALTERing: SQLite's own failure past this point is a raw\n    // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  columnTypeSuffix: () => '',\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["contentTokenize","SenseError","segmentField","recordReconcileDuration","BEGIN_WRITE","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","reconcileContent","conn","touched","docs","_delta","cfg","length","runBatch","map","p","undefined","sqliteDialect","beginMode","checkColumnLimit","count","columnTypeSuffix","recordDuration"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,uBAAuB,QAAQ,eAAe;AACvD,SAASC,WAAW,QAAQ,oBAAoB;AAGhD,qFAAqF;AACrF,mFAAmF;AAEnF,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,6FAA6F;AAC7F,qDAAqD;AACrD,OAAO,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC,CAAC;AAE/M,uFAAuF;AACvF,+EAA+E;AAC/E,OAAO,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IAC5D,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaP,aAAaM,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaP,aAAaM,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaP,aAAaM,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB,EAAEC,GAAW;IACzH,6FAA6F;IAC7F,uGAAuG;IACvG,IAAIH,QAAQI,MAAM,GAAG,GACnB,MAAML,KAAKM,QAAQ,CACjB,sFACAL,QAAQM,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAG1B,IAAIN,KAAKG,MAAM,GAAG,GAAG;QACnB,wFAAwF;QACxF,+EAA+E;QAC/E,MAAMZ,aAAaT,gBAAgBoB,SAASK;QAC5C,MAAMT,KAAKM,QAAQ,CACjBhB,oBACAY,KAAKK,GAAG,CAAC,CAACf,MAAQD,WAAWC,KAAKC;IAEtC;AACF;AAEA,OAAO,MAAMiB,gBAAkC;IAC7CC,WAAW,IAAMvB;IACjBwB,kBAAiBC,KAAK;QACpB,uEAAuE;QACvE,sGAAsG;QACtG,IAAIA,QAAQxB,yBAAyB;YACnC,MAAM,IAAIJ,WACR,gBACA,CAAC,uBAAuB,EAAE4B,MAAM,0EAA0E,EAAExB,wBAAwB,wKAAwK,CAAC;QAEjT;IACF;IACAyB,kBAAkB,IAAM;IACxBf;IACAgB,gBAAgB5B;AAClB,EAAE"}

package/dist/esm/store/sqlite/signature.d.ts

@@ -0,0 +1,6 @@
+import type { Config } from '../../config/index.js';
+import type { Connection } from '../types.js';
+export declare function changedSignatureKeys(before: string, after: string): Set<string>;
+export declare function embedIdentityAdopted(before: string, after: string): boolean;
+export declare function signatureDiff(before: string, after: string): string;
+export declare function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]>;

package/dist/esm/store/sqlite/signature.js

@@ -0,0 +1,58 @@
+import { contentTokenize } from '../../config/index.js';
+import { listFiles, parseFile } from '../../scan/index.js';
+import { BEGIN_WRITE, withTransaction } from '../transaction.js';
+import { contentRow, INSERT_CONTENT_SQL } from './reconcile.js';
+// open()-lifecycle helpers for a config-signature change (see config.featureSignature's format:
+// global features, embed provider, tokenize, then one segment per preset). Never called from
+// reconcile() itself.
+// Segment keys that moved between two feature signatures.
+export function changedSignatureKeys(before, after) {
+    const keyOf = (part)=>part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0];
+    const parse = (sig)=>new Map(sig.split('|').map((part)=>[
+                keyOf(part),
+                part
+            ]));
+    const a = parse(before);
+    const b = parse(after);
+    const changed = new Set();
+    for (const [key, val] of b)if (a.get(key) !== val) changed.add(key);
+    for (const key of a.keys())if (!b.has(key)) changed.add(key);
+    return changed;
+}
+// Whether the embed segment only gained its resolved weight identity: same provider and
+// model, no identity recorded before, one now -- adopted into meta without a rebuild.
+export function embedIdentityAdopted(before, after) {
+    const embedPart = (sig)=>sig.split('|').find((p)=>p.startsWith('embed:'));
+    const b = embedPart(before);
+    const a = embedPart(after);
+    if (b === undefined || a === undefined) return false;
+    const at = a.indexOf('@');
+    return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;
+}
+// Names what moved, for the rebuild notice.
+export function signatureDiff(before, after) {
+    const changed = changedSignatureKeys(before, after);
+    const label = (key)=>key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset "${key.slice(7)}"` : 'features';
+    return changed.size === 0 ? 'features' : [
+        ...changed
+    ].map(label).join(', ');
+}
+// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)
+// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.
+export async function rebuildContentTable(conn, cfg, baseDir) {
+    const stmt = await conn.prepare('SELECT "path" FROM frontmatter');
+    const known = new Set((await stmt.all()).map((r)=>r.path));
+    const files = listFiles(cfg, baseDir).filter((f)=>known.has(f.relPath));
+    const segmenting = contentTokenize(cfg) === undefined;
+    const warnings = [];
+    const rows = [];
+    for (const file of files){
+        const { doc, warnings: fileWarnings } = parseFile(file);
+        warnings.push(...fileWarnings);
+        rows.push(contentRow(doc, segmenting));
+    }
+    await withTransaction(conn, async ()=>{
+        if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);
+    }, BEGIN_WRITE);
+    return warnings;
+}

package/dist/esm/store/sqlite/signature.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/signature.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { listFiles, parseFile } from '../../scan/index.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\nimport { contentRow, INSERT_CONTENT_SQL } from './reconcile.ts';\n\n// open()-lifecycle helpers for a config-signature change (see config.featureSignature's format:\n// global features, embed provider, tokenize, then one segment per preset). Never called from\n// reconcile() itself.\n\n// Segment keys that moved between two feature signatures.\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(\n    conn,\n    async () => {\n      if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n    },\n    BEGIN_WRITE\n  );\n  return warnings;\n}\n"],"names":["contentTokenize","listFiles","parseFile","BEGIN_WRITE","withTransaction","contentRow","INSERT_CONTENT_SQL","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","join","parse","sig","Map","map","a","b","changed","Set","key","val","get","add","keys","has","embedIdentityAdopted","embedPart","find","p","undefined","at","indexOf","signatureDiff","label","size","rebuildContentTable","conn","cfg","baseDir","stmt","prepare","known","all","r","path","files","filter","f","relPath","segmenting","warnings","rows","file","doc","fileWarnings","push","length","runBatch"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,SAAS,EAAEC,SAAS,QAAQ,sBAAsB;AAC3D,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,UAAU,EAAEC,kBAAkB,QAAQ,iBAAiB;AAEhE,gGAAgG;AAChG,6FAA6F;AAC7F,sBAAsB;AAEtB,0DAA0D;AAC1D,OAAO,SAASC,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGC,IAAI,CAAC,OAAOJ,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAMG,QAAQ,CAACC,MAAgB,IAAIC,IAAID,IAAIJ,KAAK,CAAC,KAAKM,GAAG,CAAC,CAACR,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMS,IAAIJ,MAAMR;IAChB,MAAMa,IAAIL,MAAMP;IAChB,MAAMa,UAAU,IAAIC;IACpB,KAAK,MAAM,CAACC,KAAKC,IAAI,IAAIJ,EAAG,IAAID,EAAEM,GAAG,CAACF,SAASC,KAAKH,QAAQK,GAAG,CAACH;IAChE,KAAK,MAAMA,OAAOJ,EAAEQ,IAAI,GAAI,IAAI,CAACP,EAAEQ,GAAG,CAACL,MAAMF,QAAQK,GAAG,CAACH;IACzD,OAAOF;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASQ,qBAAqBtB,MAAc,EAAEC,KAAa;IAChE,MAAMsB,YAAY,CAACd,MAAgBA,IAAIJ,KAAK,CAAC,KAAKmB,IAAI,CAAC,CAACC,IAAMA,EAAErB,UAAU,CAAC;IAC3E,MAAMS,IAAIU,UAAUvB;IACpB,MAAMY,IAAIW,UAAUtB;IACpB,IAAIY,MAAMa,aAAad,MAAMc,WAAW,OAAO;IAC/C,MAAMC,KAAKf,EAAEgB,OAAO,CAAC;IACrB,OAAOf,EAAEe,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKf,EAAEN,KAAK,CAAC,GAAGqB,QAAQd;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASgB,cAAc7B,MAAc,EAAEC,KAAa;IACzD,MAAMa,UAAUf,qBAAqBC,QAAQC;IAC7C,MAAM6B,QAAQ,CAACd,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIZ,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEY,IAAIV,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOQ,QAAQiB,IAAI,KAAK,IAAI,aAAa;WAAIjB;KAAQ,CAACH,GAAG,CAACmB,OAAOvB,IAAI,CAAC;AACxE;AAEA,oGAAoG;AACpG,gIAAgI;AAChI,OAAO,eAAeyB,oBAAoBC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,MAAMC,OAAO,MAAMH,KAAKI,OAAO,CAAC;IAChC,MAAMC,QAAQ,IAAIvB,IAAI,AAAE,CAAA,MAAMqB,KAAKG,GAAG,EAAC,EAA+B5B,GAAG,CAAC,CAAC6B,IAAMA,EAAEC,IAAI;IACvF,MAAMC,QAAQjD,UAAUyC,KAAKC,SAASQ,MAAM,CAAC,CAACC,IAAMN,MAAMjB,GAAG,CAACuB,EAAEC,OAAO;IACvE,MAAMC,aAAatD,gBAAgB0C,SAASR;IAC5C,MAAMqB,WAAqB,EAAE;IAC7B,MAAMC,OAAoB,EAAE;IAC5B,KAAK,MAAMC,QAAQP,MAAO;QACxB,MAAM,EAAEQ,GAAG,EAAEH,UAAUI,YAAY,EAAE,GAAGzD,UAAUuD;QAClDF,SAASK,IAAI,IAAID;QACjBH,KAAKI,IAAI,CAACvD,WAAWqD,KAAKJ;IAC5B;IACA,MAAMlD,gBACJqC,MACA;QACE,IAAIe,KAAKK,MAAM,GAAG,GAAG,MAAMpB,KAAKqB,QAAQ,CAACxD,oBAAoBkD;IAC/D,GACArD;IAEF,OAAOoD;AACT"}

package/dist/esm/store/sqlite/store.js

@@ -1,17 +1,19 @@
 import { contentTokenize } from '../../config/index.js';
+import { reconcile } from '../reconcile.js';
 import { getColumns } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { hasVectorRow, pendingRows } from '../vectors.js';
 import { fieldStats } from './fieldStats.js';
 import { queryLexical } from './lexical.js';
-import { reconcile } from './reconcile.js';
+import { sqliteDialect } from './reconcile.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
 export const CAPABILITIES = new Set([
     'phrases',
     'snippets',
     'watch-concurrency',
     'lexical',
-    'vectors'
+    'vectors',
+    'sql-functions'
 ]);
 // Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one
 // Connection instance (conn) with open()'s own reconcile call so transaction depth is tracked against the same object everywhere.
@@ -32,7 +34,7 @@ export function createStore(db, conn, cfg, baseDir) {
             return withTransaction(conn, fn);
         },
         async reconcile () {
-            return reconcile(conn, cfg, baseDir);
+            return reconcile(conn, cfg, baseDir, sqliteDialect);
         },
         docs: {
             async columns () {

package/dist/esm/store/sqlite/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with open()'s own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["contentTokenize","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","reconcile","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","undefined","vectors","pending","writeVectors","rows","candidates","qv","storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":"AAEA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;CAAU,EAAE;AAEjI,0FAA0F;AAC1F,kIAAkI;AAClI,OAAO,SAASC,YAAYC,EAAgB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC1F,OAAO;QACLC,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAMN,KAAKK,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAON,KAAKO,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMT,KAAKQ,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBa,MAAMW;QAC/B;QACA,MAAMnB;YACJ,OAAOA,UAAUQ,MAAMC,KAAKC;QAC9B;QACAU,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM3B,WAAWc;iBAAO;YACtC;YACA,MAAMV,YAAWuB,OAAO,EAAEC,UAAU;gBAClC,OAAOxB,WAAWU,MAAMa,SAASC;YACnC;QACF;QACAC,SAAS;YACP,MAAMC,OAAMC,KAAK,EAAEC,IAAI;gBACrB,OAAO3B,aAAaS,MAAMiB,OAAOC,MAAMjC,gBAAgBgB,SAASkB;YAClE;QACF;QACAC,SAAS;YACP,MAAMC;gBACJ,OAAOhC,YAAYW;YACrB;YACA,MAAMsB,cAAaC,IAAsB;gBACvC,MAAM5B,iBAAiBK,MAAMuB;YAC/B;YACA,MAAMC,YAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;gBAC5C,OAAOnC,eAAeO,MAAMyB,IAAIC,WAAWC,OAAOC;YACpD;YACA,MAAMC,SAAQC,IAAI,EAAEZ,IAAI;gBACtB,OAAOxB,YAAYM,MAAM8B,MAAMZ;YACjC;YACA,MAAMa,WAAUD,IAAI;gBAClB,OAAO1C,aAAaY,MAAM8B;YAC5B;QACF;QACA,MAAME;YACJ,qFAAqF;YACrF,qFAAqF;YACrF,MAAMC,MAAMlC,GAAGQ,OAAO,CAAC,uBAAuB2B,GAAG;YACjD,OAAO;gBAAEC,cAAc,GAAGF,IAAIG,OAAO,CAAC,kFAAkF,CAAC;YAAC;QAC5H;QACAC,KAAK;YACH,MAAM9B,SAAQD,GAAW;gBACvB,MAAMgC,OAAOvC,GAAGQ,OAAO,CAACD;gBACxBgC,KAAKC,cAAc,CAAC,OAAO,qEAAqE;gBAChG,OAAO;oBACL1B,SAAS,IAAMyB,KAAKzB,OAAO;oBAC3B,2EAA2E;oBAC3E2B,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAKC;oBAC1B;gBACF;YACF;QACF;QACA,MAAMC;YACJ3C,GAAG2C,KAAK;QACV;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with open()'s own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir, sqliteDialect);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["contentTokenize","reconcile","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","sqliteDialect","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","undefined","vectors","pending","writeVectors","rows","candidates","qv","storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":"AAEA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB,EAAE;AAElJ,0FAA0F;AAC1F,kIAAkI;AAClI,OAAO,SAASC,YAAYC,EAAgB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC1F,OAAO;QACLC,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAMN,KAAKK,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAON,KAAKO,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMT,KAAKQ,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBa,MAAMW;QAC/B;QACA,MAAM1B;YACJ,OAAOA,UAAUe,MAAMC,KAAKC,SAASV;QACvC;QACAoB,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM3B,WAAWc;iBAAO;YACtC;YACA,MAAMV,YAAWuB,OAAO,EAAEC,UAAU;gBAClC,OAAOxB,WAAWU,MAAMa,SAASC;YACnC;QACF;QACAC,SAAS;YACP,MAAMC,OAAMC,KAAK,EAAEC,IAAI;gBACrB,OAAO3B,aAAaS,MAAMiB,OAAOC,MAAMlC,gBAAgBiB,SAASkB;YAClE;QACF;QACAC,SAAS;YACP,MAAMC;gBACJ,OAAOhC,YAAYW;YACrB;YACA,MAAMsB,cAAaC,IAAsB;gBACvC,MAAM5B,iBAAiBK,MAAMuB;YAC/B;YACA,MAAMC,YAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;gBAC5C,OAAOnC,eAAeO,MAAMyB,IAAIC,WAAWC,OAAOC;YACpD;YACA,MAAMC,SAAQC,IAAI,EAAEZ,IAAI;gBACtB,OAAOxB,YAAYM,MAAM8B,MAAMZ;YACjC;YACA,MAAMa,WAAUD,IAAI;gBAClB,OAAO1C,aAAaY,MAAM8B;YAC5B;QACF;QACA,MAAME;YACJ,qFAAqF;YACrF,qFAAqF;YACrF,MAAMC,MAAMlC,GAAGQ,OAAO,CAAC,uBAAuB2B,GAAG;YACjD,OAAO;gBAAEC,cAAc,GAAGF,IAAIG,OAAO,CAAC,kFAAkF,CAAC;YAAC;QAC5H;QACAC,KAAK;YACH,MAAM9B,SAAQD,GAAW;gBACvB,MAAMgC,OAAOvC,GAAGQ,OAAO,CAACD;gBACxBgC,KAAKC,cAAc,CAAC,OAAO,qEAAqE;gBAChG,OAAO;oBACL1B,SAAS,IAAMyB,KAAKzB,OAAO;oBAC3B,2EAA2E;oBAC3E2B,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAKC;oBAC1B;gBACF;YACF;QACF;QACA,MAAMC;YACJ3C,GAAG2C,KAAK;QACV;IACF;AACF"}

package/dist/esm/store/turso/open.js

@@ -6,10 +6,11 @@ import { STORE_DIMS } from '../../embed/types.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
 import { clearCache } from '../cache.js';
+import { reconcile } from '../reconcile.js';
 import { getMeta, setMeta } from '../shared.js';
 import { createConnection } from './connection.js';
 import { TURSO_PACKAGE, tursoApi } from './native.js';
-import { reconcile } from './reconcile.js';
+import { tursoDialect } from './reconcile.js';
 import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.turso.db';
 // Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves
@@ -98,7 +99,7 @@ async function connect(cfg) {
         // Installed before reconcile() -- that call is the one that races a watcher's transaction.
         const recordedMaxMs = Number((_ref = await getMeta(conn, 'reconcile_max_ms')) !== null && _ref !== void 0 ? _ref : '0');
         await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600000)}`);
-        const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);
+        const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, tursoDialect);
         return {
             db,
             conn,

package/dist/esm/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:\n// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n  db: Database;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below must release this handle, or the leaked WAL makes the db file undeletable on\n  // Windows. closed guards the catch against double-closing.\n  let closed = false;\n  try {\n    const conn = createConnection(db);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched\n      // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      closed = true;\n      await db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) await db.close();\n    throw err;\n  }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","STORE_DIMS","SenseError","activeFeatures","FEATURES","clearCache","getMeta","setMeta","createConnection","TURSO_PACKAGE","tursoApi","reconcile","createStore","DB_FILENAME","SCHEMA_VERSION","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ensureSchema","conn","cfg","exec","ddl","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","turso","err","message","db","timeout","experimental","closed","version","features","wantFeatures","reason","console","error","close","recordedMaxMs","Number","Math","min","max","parsed","warnings","openTurso","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,+FAA+F;AAC/F,2FAA2F;AAC3F,gGAAgG;AAChG,0FAA0F;AAC1F,OAAO,MAAMC,kBAAkB;IAC7B,CAAC,qIAAqI,CAAC;IACvI,CAAC,gMAAgM,CAAC;CACnM,CAAU;AACX,OAAO,MAAMC,oBAAoB;IAAC;IAAe;CAAoB,CAAU;AAE/E,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAON,gBAAiB,MAAMG,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWnB,eAAegB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEnB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMqB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBJ;IAC5F,IAAI,AAAC,MAAMR,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYpB,iBAAiBqB,KAAKf;AACxG;AAWA,eAAeqB,QAAQN,GAAmB;IACxC,MAAMO,WAAW7B,KAAKsB,IAAIQ,OAAO,EAAE5B;IACnCH,UAAU8B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAShC,KAAK6B,UAAUb;IAE9B,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIiB;IACJ,IAAI;QACFA,QAAQ,MAAMpB;IAChB,EAAE,OAAOqB,KAAK;QACZ,IAAIA,eAAe7B,YAAY,MAAM6B;QACrC,MAAM,IAAI7B,WAAW,4BAA4B,CAAC,wBAAwB,EAAEO,cAAc,UAAU,EAAE,AAACsB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAML,OAAO,CAACI,QAAQ;YAAEK,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAI7B,WAAW,4BAA4B,CAAC,6BAA6B,EAAE2B,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACtH;IACA,6FAA6F;IAC7F,2DAA2D;IAC3D,IAAII,SAAS;IACb,IAAI;YAuB4B;QAtB9B,MAAMlB,OAAOV,iBAAiByB;QAE9B,MAAMf,KAAKE,IAAI,CAAC;QAEhB,MAAMiB,UAAU,MAAM/B,QAAQY,MAAM;QACpC,MAAMoB,WAAW,MAAMhC,QAAQY,MAAM;QACrC,MAAMqB,eAAezC,iBAAiBqB,KAAKf;QAC3C,IAAI,AAACiC,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,cAAe;YACxG,yFAAyF;YACzF,iGAAiG;YACjG,MAAMC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;YACnH2B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtDJ,SAAS;YACT,MAAMH,GAAGU,KAAK;YACdtC,WAAWc;YACX,OAAOM,QAAQN;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMyB,gBAAgBC,QAAQ,OAAA,MAAMvC,QAAQY,MAAM,iCAApB,kBAAA,OAA4C;QAC1E,MAAMA,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAE0B,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAEhG,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMvC,UAAUO,MAAMC,KAAKA,IAAIQ,OAAO;QAEnE,OAAO;YAAEM;YAAIf;YAAMC;YAAKU;YAAQoB;YAAQC;QAAS;IACnD,EAAE,OAAOnB,KAAK;QACZ,IAAI,CAACK,QAAQ,MAAMH,GAAGU,KAAK;QAC3B,MAAMZ;IACR;AACF;AAEA,gGAAgG;AAChG,qCAAqC;AACrC,OAAO,eAAeoB,UAAUhC,GAAmB;IACjD,MAAM,EAAEc,EAAE,EAAEf,IAAI,EAAEC,KAAKiC,WAAW,EAAEvB,MAAM,EAAEoB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzB,QAAQN;IAC/E,MAAMkC,QAAQzC,YAAYqB,IAAIf,MAAMkC,aAAaA,YAAYzB,OAAO;IACpE,0FAA0F;IAC1F3B,eAAekB,MAAMmC;IACrB,OAAO;QAAEA,OAAOA;QAAOlC,KAAKiC;QAAavB;QAAQoB;QAAQC;IAAS;AACpE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:\n// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n  db: Database;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below must release this handle, or the leaked WAL makes the db file undeletable on\n  // Windows. closed guards the catch against double-closing.\n  let closed = false;\n  try {\n    const conn = createConnection(db);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched\n      // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      closed = true;\n      await db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, tursoDialect);\n\n    return { db, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) await db.close();\n    throw err;\n  }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","STORE_DIMS","SenseError","activeFeatures","FEATURES","clearCache","reconcile","getMeta","setMeta","createConnection","TURSO_PACKAGE","tursoApi","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ensureSchema","conn","cfg","exec","ddl","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","turso","err","message","db","timeout","experimental","closed","version","features","wantFeatures","reason","console","error","close","recordedMaxMs","Number","Math","min","max","parsed","warnings","openTurso","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,YAAY,QAAQ,iBAAiB;AAC9C,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,+FAA+F;AAC/F,2FAA2F;AAC3F,gGAAgG;AAChG,0FAA0F;AAC1F,OAAO,MAAMC,kBAAkB;IAC7B,CAAC,qIAAqI,CAAC;IACvI,CAAC,gMAAgM,CAAC;CACnM,CAAU;AACX,OAAO,MAAMC,oBAAoB;IAAC;IAAe;CAAoB,CAAU;AAE/E,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAON,gBAAiB,MAAMG,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWpB,eAAeiB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEpB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMsB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBJ;IAC5F,IAAI,AAAC,MAAMR,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYrB,iBAAiBsB,KAAKhB;AACxG;AAWA,eAAesB,QAAQN,GAAmB;IACxC,MAAMO,WAAW9B,KAAKuB,IAAIQ,OAAO,EAAE7B;IACnCH,UAAU+B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAASjC,KAAK8B,UAAUb;IAE9B,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIiB;IACJ,IAAI;QACFA,QAAQ,MAAMpB;IAChB,EAAE,OAAOqB,KAAK;QACZ,IAAIA,eAAe9B,YAAY,MAAM8B;QACrC,MAAM,IAAI9B,WAAW,4BAA4B,CAAC,wBAAwB,EAAEQ,cAAc,UAAU,EAAE,AAACsB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAML,OAAO,CAACI,QAAQ;YAAEK,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAI9B,WAAW,4BAA4B,CAAC,6BAA6B,EAAE4B,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACtH;IACA,6FAA6F;IAC7F,2DAA2D;IAC3D,IAAII,SAAS;IACb,IAAI;YAuB4B;QAtB9B,MAAMlB,OAAOV,iBAAiByB;QAE9B,MAAMf,KAAKE,IAAI,CAAC;QAEhB,MAAMiB,UAAU,MAAM/B,QAAQY,MAAM;QACpC,MAAMoB,WAAW,MAAMhC,QAAQY,MAAM;QACrC,MAAMqB,eAAe1C,iBAAiBsB,KAAKhB;QAC3C,IAAI,AAACkC,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,cAAe;YACxG,yFAAyF;YACzF,iGAAiG;YACjG,MAAMC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;YACnH2B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtDJ,SAAS;YACT,MAAMH,GAAGU,KAAK;YACdvC,WAAWe;YACX,OAAOM,QAAQN;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMyB,gBAAgBC,QAAQ,OAAA,MAAMvC,QAAQY,MAAM,iCAApB,kBAAA,OAA4C;QAC1E,MAAMA,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAE0B,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAEhG,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAM7C,UAAUa,MAAMC,KAAKA,IAAIQ,OAAO,EAAEhB;QAErE,OAAO;YAAEsB;YAAIf;YAAMC;YAAKU;YAAQoB;YAAQC;QAAS;IACnD,EAAE,OAAOnB,KAAK;QACZ,IAAI,CAACK,QAAQ,MAAMH,GAAGU,KAAK;QAC3B,MAAMZ;IACR;AACF;AAEA,gGAAgG;AAChG,qCAAqC;AACrC,OAAO,eAAeoB,UAAUhC,GAAmB;IACjD,MAAM,EAAEc,EAAE,EAAEf,IAAI,EAAEC,KAAKiC,WAAW,EAAEvB,MAAM,EAAEoB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzB,QAAQN;IAC/E,MAAMkC,QAAQzC,YAAYqB,IAAIf,MAAMkC,aAAaA,YAAYzB,OAAO;IACpE,0FAA0F;IAC1F5B,eAAemB,MAAMmC;IACrB,OAAO;QAAEA,OAAOA;QAAOlC,KAAKiC;QAAavB;QAAQoB;QAAQC;IAAS;AACpE"}

package/dist/esm/store/turso/reconcile.d.ts

@@ -1,6 +1,3 @@
-import type { Config } from '../../config/index.js';
-import type { Connection } from '../types.js';
-export declare function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{
-    parsed: number;
-    warnings: string[];
-}>;
+import type { ReconcileDialect } from '../types.js';
+export declare const FTS_REBUILD_THRESHOLD = 250;
+export declare const tursoDialect: ReconcileDialect;