sensemaking 0.19.0 → 0.19.2

package/dist/esm/store/sqlite/reconcile.d.ts

@@ -1,10 +1,5 @@
-import type { Config } from '../../config/index.js';
-import type { Connection } from '../types.js';
-export declare function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{
-    parsed: number;
-    warnings: string[];
-}>;
-export declare function changedSignatureKeys(before: string, after: string): Set<string>;
-export declare function embedIdentityAdopted(before: string, after: string): boolean;
-export declare function signatureDiff(before: string, after: string): string;
-export declare function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]>;
+import type { ParsedDoc } from '../../scan/index.js';
+import type { ReconcileDialect } from '../types.js';
+export declare const INSERT_CONTENT_SQL = "INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)";
+export declare function contentRow(doc: ParsedDoc, segmenting: boolean): unknown[];
+export declare const sqliteDialect: ReconcileDialect;

package/dist/esm/store/sqlite/reconcile.js

@@ -1,29 +1,18 @@
 import { contentTokenize } from '../../config/index.js';
 import { SenseError } from '../../errors.js';
-import { activeFeatures } from '../../features/index.js';
-import { progress } from '../../output/progress.js';
-import { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.js';
-import { reparseFiles } from '../../scan/reparse.js';
 import { segmentField } from '../../text/segment.js';
-import { getColumns, getMeta, quoteIdent, setMeta } from '../shared.js';
-import { withTransaction } from '../transaction.js';
-// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last
-// computed value on every touch, not just the reconciles that recompute it.
-const CORE_FRONTMATTER_COLUMNS = new Set([
-    'path',
-    '_mtime',
-    '_ctime',
-    '_size',
-    '_parse_error'
-]);
+import { recordReconcileDuration } from '../shared.js';
+import { BEGIN_WRITE } from '../transaction.js';
+// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in
+// store/reconcile.ts. content is FTS5-virtual with "_seg" sidecars for lexical.ts.
 // SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).
 const MAX_FRONTMATTER_COLUMNS = 2000;
-// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same
-// literal instead of two copies drifting apart.
-const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, "path", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE "path" = ?), ?, ?, ?, ?, ?, ?, ?)`;
+// Shared by reconcileContent() and the tokenize-only rebuild in open.ts, so both prepare the
+// same literal instead of two copies drifting apart.
+export const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, "path", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE "path" = ?), ?, ?, ?, ?, ?, ?, ?)`;
 // A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.
 // Assumes the frontmatter row for doc.relPath already exists (rowid subquery).
-function contentRow(doc, segmenting) {
+export function contentRow(doc, segmenting) {
     return [
         doc.relPath,
         doc.search.title,
@@ -35,185 +24,29 @@ function contentRow(doc, segmenting) {
         segmenting ? segmentField(doc.search.text) : ''
     ];
 }
-export async function reconcile(conn, cfg, baseDir) {
-    const files = listFiles(cfg, baseDir);
-    const currentSet = new Set(files.map((f)=>f.relPath));
-    const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
-    const existingRows = await existingStmt.all();
-    const existing = new Map(existingRows.map((r)=>[
-            r.path,
-            r
+async function reconcileContent(conn, touched, docs, _delta, cfg) {
+    // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via
+    // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.
+    if (touched.length > 0) await conn.runBatch('DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE "path" = ?)', touched.map((p)=>[
+            p
         ]));
-    const vanished = existingRows.filter((r)=>!currentSet.has(r.path)).map((r)=>r.path);
-    const toReparse = files.filter((f)=>{
-        const row = existing.get(f.relPath);
-        return !row || row._mtime !== f.mtimeMs || row._size !== f.size;
-    });
-    if (vanished.length === 0 && toReparse.length === 0) return {
-        parsed: 0,
-        warnings: []
-    };
-    const features = activeFeatures(cfg);
-    const seenColumns = await getColumns(conn);
-    // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
-    // reconciles stay silent (progress() has a threshold).
-    const report = progress('reparsing files', toReparse.length);
-    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
-    report.finish();
-    for (const col of newColumns)seenColumns.add(col);
-    const allColumns = [
-        ...seenColumns
-    ];
-    // Fence before ALTERing: SQLite's own failure past this point is a raw
-    // "too many columns on sqlite_altertab_frontmatter" with no indication of the boundary or the levers.
-    if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {
-        throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);
+    if (docs.length > 0) {
+        // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over
+        // grapheme runs would be nonsense against trigram, so the sidecars stay empty.
+        const segmenting = contentTokenize(cfg) === undefined;
+        await conn.runBatch(INSERT_CONTENT_SQL, docs.map((doc)=>contentRow(doc, segmenting)));
     }
-    // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a
-    // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).
-    const writableColumns = allColumns.filter((c)=>CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));
-    // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --
-    // content rows are coupled to that rowid below.
-    const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(()=>'?').join(', ')}) ON CONFLICT("path") DO UPDATE SET ${writableColumns.filter((c)=>c !== 'path').map((c)=>`${quoteIdent(c)} = excluded.${quoteIdent(c)}`).join(', ')}`;
-    const added = toReparse.filter((f)=>!existing.has(f.relPath)).map((f)=>f.relPath);
-    const delta = {
-        files,
-        reparsed: parsedDocs.map((d)=>d.relPath),
-        added,
-        vanished
-    };
-    const addedSet = new Set(added);
-    const reparsedExisting = parsedDocs.map((d)=>d.relPath).filter((p)=>!addedSet.has(p));
-    const txStart = Date.now();
-    await withTransaction(conn, async ()=>{
-        var _feature_remove, _feature_afterReconcile;
-        for (const col of newColumns)await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);
-        // content's rowid lookup depends on its coupled frontmatter row, so the content
-        // deletes/inserts run while every path still has one -- the vanished frontmatter delete comes last.
-        if (vanished.length > 0) {
-            await conn.runBatch('DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE "path" = ?)', vanished.map((p)=>[
-                    p
-                ]));
-        }
-        if (reparsedExisting.length > 0) {
-            // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via
-            // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.
-            await conn.runBatch('DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE "path" = ?)', reparsedExisting.map((p)=>[
-                    p
-                ]));
-        }
-        if (parsedDocs.length > 0) {
-            const rows = parsedDocs.map((doc)=>writableColumns.map((col)=>{
-                    var _doc_data_col;
-                    if (col === 'path') return doc.relPath;
-                    if (col === '_mtime') return doc.mtimeMs;
-                    if (col === '_ctime') return doc.ctimeMs;
-                    if (col === '_size') return doc.size;
-                    // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.
-                    if (col === '_parse_error') return doc.parseError;
-                    return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
-                }));
-            await conn.runBatch(insertSql, rows);
-            // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over
-            // grapheme runs would be nonsense against trigram, so the sidecars stay empty.
-            const segmenting = contentTokenize(cfg) === undefined;
-            await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map((doc)=>contentRow(doc, segmenting)));
-        }
-        if (vanished.length > 0) await conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[
-                p
-            ]));
-        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;
-        // new docs have nothing to clear, which keeps cold builds linear.
-        const presetTouched = [
-            ...vanished,
-            ...reparsedExisting
-        ];
-        if (presetTouched.length > 0) await conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', presetTouched.map((p)=>[
-                p
-            ]));
-        const presetRows = [];
-        for (const doc of parsedDocs)for (const presetName of doc.presets)presetRows.push([
-            doc.relPath,
-            presetName
-        ]);
-        if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?)', presetRows);
-        const removedPaths = [
-            ...vanished,
-            ...reparsedExisting
-        ];
-        if (removedPaths.length > 0) for (const feature of features)await ((_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, removedPaths, delta));
-        for (const feature of features){
-            var _feature_store;
-            const docsForFeature = parsedDocs.map((doc)=>({
-                    path: doc.relPath,
-                    extracted: doc.extracted[feature.name]
-                }));
-            await ((_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta));
-        }
-        for (const feature of features)await ((_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta));
-    });
-    // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):
-    // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.
-    const durationMs = Date.now() - txStart;
-    const prevRaw = await getMeta(conn, 'reconcile_max_ms');
-    // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)
-    // still gets recorded instead of losing to the "nothing recorded yet" default.
-    const prevMax = prevRaw === null ? -1 : Number(prevRaw);
-    if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));
-    return {
-        parsed: parsedDocs.length,
-        warnings
-    };
-}
-// Segment keys that moved between two feature signatures (see config.featureSignature's
-// format: global features, embed provider, tokenize, then one segment per preset).
-export function changedSignatureKeys(before, after) {
-    const keyOf = (part)=>part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0];
-    const parse = (sig)=>new Map(sig.split('|').map((part)=>[
-                keyOf(part),
-                part
-            ]));
-    const a = parse(before);
-    const b = parse(after);
-    const changed = new Set();
-    for (const [key, val] of b)if (a.get(key) !== val) changed.add(key);
-    for (const key of a.keys())if (!b.has(key)) changed.add(key);
-    return changed;
-}
-// Whether the embed segment only gained its resolved weight identity: same provider and
-// model, no identity recorded before, one now -- adopted into meta without a rebuild.
-export function embedIdentityAdopted(before, after) {
-    const embedPart = (sig)=>sig.split('|').find((p)=>p.startsWith('embed:'));
-    const b = embedPart(before);
-    const a = embedPart(after);
-    if (b === undefined || a === undefined) return false;
-    const at = a.indexOf('@');
-    return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;
-}
-// Names what moved, for the rebuild notice.
-export function signatureDiff(before, after) {
-    const changed = changedSignatureKeys(before, after);
-    const label = (key)=>key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset "${key.slice(7)}"` : 'features';
-    return changed.size === 0 ? 'features' : [
-        ...changed
-    ].map(label).join(', ');
-}
-// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)
-// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.
-export async function rebuildContentTable(conn, cfg, baseDir) {
-    const stmt = await conn.prepare('SELECT "path" FROM frontmatter');
-    const known = new Set((await stmt.all()).map((r)=>r.path));
-    const files = listFiles(cfg, baseDir).filter((f)=>known.has(f.relPath));
-    const segmenting = contentTokenize(cfg) === undefined;
-    const warnings = [];
-    const rows = [];
-    for (const file of files){
-        const { doc, warnings: fileWarnings } = parseFile(file);
-        warnings.push(...fileWarnings);
-        rows.push(contentRow(doc, segmenting));
-    }
-    await withTransaction(conn, async ()=>{
-        if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);
-    });
-    return warnings;
 }
+export const sqliteDialect = {
+    beginMode: ()=>BEGIN_WRITE,
+    checkColumnLimit (count) {
+        // Fence before ALTERing: SQLite's own failure past this point is a raw
+        // "too many columns on sqlite_altertab_frontmatter" with no indication of the boundary or the levers.
+        if (count > MAX_FRONTMATTER_COLUMNS) {
+            throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);
+        }
+    },
+    columnTypeSuffix: ()=>'',
+    reconcileContent,
+    recordDuration: recordReconcileDuration
+};

package/dist/esm/store/sqlite/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nfunction contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    // content's rowid lookup depends on its coupled frontmatter row, so the content\n    // deletes/inserts run while every path still has one -- the vanished frontmatter delete comes last.\n    if (vanished.length > 0) {\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        vanished.map((p) => [p])\n      );\n    }\n    if (reparsedExisting.length > 0) {\n      // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via\n      // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        reparsedExisting.map((p) => [p])\n      );\n    }\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      await conn.runBatch(\n        INSERT_CONTENT_SQL,\n        parsedDocs.map((doc) => contentRow(doc, segmenting))\n      );\n    }\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(conn, async () => {\n    if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n  });\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","reparseFiles","segmentField","getColumns","getMeta","quoteIdent","setMeta","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","txStart","Date","now","feature","exec","runBatch","rows","ctimeMs","parseError","data","undefined","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","key","val","keys","embedIdentityAdopted","embedPart","find","at","indexOf","signatureDiff","label","rebuildContentTable","stmt","known","file","fileWarnings"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAC7E,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,eAAe;AACxE,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,uFAAuF;AACvF,+EAA+E;AAC/E,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IACrD,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaZ,aAAaW,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaZ,aAAaW,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaZ,aAAaW,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQ1B,UAAUwB,KAAKC;IAC7B,MAAME,aAAa,IAAIhB,IAAIe,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEZ,OAAO;QAClC,OAAO,CAACyB,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWrD,eAAe0B;IAChC,MAAM4B,cAAc,MAAM/C,WAAWkB;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8B,SAAStD,SAAS,mBAAmB0C,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMrD,aAAasC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIS,WAAWb,MAAM,GAAGpC,yBAAyB;QAC/C,MAAM,IAAIf,WACR,gBACA,CAAC,uBAAuB,EAAEgE,WAAWb,MAAM,CAAC,0EAA0E,EAAEpC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMkD,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMrD,yBAAyB8B,GAAG,CAACuB,MAAM,CAAC7D,iBAAiBsC,GAAG,CAACuB;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACrB,YAAY0D,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGxD,WAAWwD,GAAG,YAAY,EAAExD,WAAWwD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAMkD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO;QAAGiD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI3D,IAAIuD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO,EAAEsB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMlE,gBAAgBc,MAAM;YA8DyCqD,iBAK7BA;QAlEtC,KAAK,MAAMjB,OAAOH,WAAY,MAAMjC,KAAKsD,IAAI,CAAC,CAAC,mCAAmC,EAAEtE,WAAWoD,MAAM;QAErG,gFAAgF;QAChF,oGAAoG;QACpG,IAAIrB,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMzB,KAAKuD,QAAQ,CACjB,sFACAxC,SAASV,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAE3B;QACA,IAAID,iBAAiBvB,MAAM,GAAG,GAAG;YAC/B,6FAA6F;YAC7F,uGAAuG;YACvG,MAAMzB,KAAKuD,QAAQ,CACjB,sFACAP,iBAAiB3C,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAEnC;QAEA,IAAIjB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM+B,OAAOxB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAOZ5C;oBANP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIE,OAAO;oBACtC,IAAI0C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAIiE,OAAO;oBACxC,IAAIrB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAIkE,UAAU;oBACjD,QAAOlE,gBAAAA,IAAImE,IAAI,CAACvB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKuD,QAAQ,CAACd,WAAWe;YAE/B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAM/D,aAAapB,gBAAgB4B,SAAS2D;YAC5C,MAAM5D,KAAKuD,QAAQ,CACjBjE,oBACA0C,WAAW3B,GAAG,CAAC,CAACb,MAAQD,WAAWC,KAAKC;QAE5C;QAEA,IAAIsB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKuD,QAAQ,CACjB,4CACAxC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMY,gBAAgB;eAAI9C;eAAaiC;SAAiB;QACxD,IAAIa,cAAcpC,MAAM,GAAG,GACzB,MAAMzB,KAAKuD,QAAQ,CACjB,6CACAM,cAAcxD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMa,aAA0B,EAAE;QAClC,KAAK,MAAMtE,OAAOwC,WAAY,KAAK,MAAM+B,cAAcvE,IAAIwE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACzE,IAAIE,OAAO;YAAEqE;SAAW;QAC7G,IAAID,WAAWrC,MAAM,GAAG,GAAG,MAAMzB,KAAKuD,QAAQ,CAAC,2DAA2DO;QAE1G,MAAMI,eAAe;eAAInD;eAAaiC;SAAiB;QACvD,IAAIkB,aAAazC,MAAM,GAAG,GAAG,KAAK,MAAM4B,WAAWzB,SAAU,QAAMyB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBrD,MAAMkE,cAActB;QACxG,KAAK,MAAMS,WAAWzB,SAAU;gBAExByB;YADN,MAAMe,iBAAiCpC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIE,OAAO;oBAAE2E,WAAW7E,IAAI6E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBrD,MAAMoE,gBAAgBxB;QAC9C;QACA,KAAK,MAAMS,WAAWzB,SAAU,QAAMyB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBrD,MAAM4C;IACvE;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAM6B,aAAatB,KAAKC,GAAG,KAAKF;IAChC,MAAMwB,UAAU,MAAM3F,QAAQiB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM2E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS,MAAM1F,QAAQe,MAAM,oBAAoB6E,OAAOJ;IAEzE,OAAO;QAAE/C,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASmD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAG3C,IAAI,CAAC,OAAOwC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAI3E,IAAI2E,IAAIH,KAAK,CAAC,KAAK/E,GAAG,CAAC,CAAC6E,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAItG;IACpB,KAAK,MAAM,CAACuG,KAAKC,IAAI,IAAIH,EAAG,IAAID,EAAEpE,GAAG,CAACuE,SAASC,KAAKF,QAAQrD,GAAG,CAACsD;IAChE,KAAK,MAAMA,OAAOH,EAAEK,IAAI,GAAI,IAAI,CAACJ,EAAExE,GAAG,CAAC0E,MAAMD,QAAQrD,GAAG,CAACsD;IACzD,OAAOD;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASI,qBAAqBf,MAAc,EAAEC,KAAa;IAChE,MAAMe,YAAY,CAACR,MAAgBA,IAAIH,KAAK,CAAC,KAAKY,IAAI,CAAC,CAAC/C,IAAMA,EAAEkC,UAAU,CAAC;IAC3E,MAAMM,IAAIM,UAAUhB;IACpB,MAAMS,IAAIO,UAAUf;IACpB,IAAIS,MAAM7B,aAAa4B,MAAM5B,WAAW,OAAO;IAC/C,MAAMqC,KAAKT,EAAEU,OAAO,CAAC;IACrB,OAAOT,EAAES,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKT,EAAEH,KAAK,CAAC,GAAGY,QAAQR;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASU,cAAcpB,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMoB,QAAQ,CAACT,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIR,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEQ,IAAIN,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQlE,IAAI,KAAK,IAAI,aAAa;WAAIkE;KAAQ,CAACrF,GAAG,CAAC+F,OAAO1D,IAAI,CAAC;AACxE;AAEA,oGAAoG;AACpG,gIAAgI;AAChI,OAAO,eAAe2D,oBAAoBrG,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,MAAMoG,OAAO,MAAMtG,KAAKQ,OAAO,CAAC;IAChC,MAAM+F,QAAQ,IAAInH,IAAI,AAAE,CAAA,MAAMkH,KAAK5F,GAAG,EAAC,EAA+BL,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IACvF,MAAMX,QAAQ1B,UAAUwB,KAAKC,SAASc,MAAM,CAAC,CAACV,IAAMiG,MAAMtF,GAAG,CAACX,EAAEZ,OAAO;IACvE,MAAMD,aAAapB,gBAAgB4B,SAAS2D;IAC5C,MAAMjC,WAAqB,EAAE;IAC7B,MAAM6B,OAAoB,EAAE;IAC5B,KAAK,MAAMgD,QAAQrG,MAAO;QACxB,MAAM,EAAEX,GAAG,EAAEmC,UAAU8E,YAAY,EAAE,GAAG/H,UAAU8H;QAClD7E,SAASsC,IAAI,IAAIwC;QACjBjD,KAAKS,IAAI,CAAC1E,WAAWC,KAAKC;IAC5B;IACA,MAAMP,gBAAgBc,MAAM;QAC1B,IAAIwD,KAAK/B,MAAM,GAAG,GAAG,MAAMzB,KAAKuD,QAAQ,CAACjE,oBAAoBkE;IAC/D;IACA,OAAO7B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is FTS5-virtual with \"_seg\" sidecars for lexical.ts.\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcileContent() and the tokenize-only rebuild in open.ts, so both prepare the\n// same literal instead of two copies drifting apart.\nexport const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nexport function contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta, cfg: Config): Promise<void> {\n  // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via\n  // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n      touched.map((p) => [p])\n    );\n\n  if (docs.length > 0) {\n    // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n    // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n    const segmenting = contentTokenize(cfg) === undefined;\n    await conn.runBatch(\n      INSERT_CONTENT_SQL,\n      docs.map((doc) => contentRow(doc, segmenting))\n    );\n  }\n}\n\nexport const sqliteDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    // Fence before ALTERing: SQLite's own failure past this point is a raw\n    // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  columnTypeSuffix: () => '',\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["contentTokenize","SenseError","segmentField","recordReconcileDuration","BEGIN_WRITE","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","reconcileContent","conn","touched","docs","_delta","cfg","length","runBatch","map","p","undefined","sqliteDialect","beginMode","checkColumnLimit","count","columnTypeSuffix","recordDuration"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,uBAAuB,QAAQ,eAAe;AACvD,SAASC,WAAW,QAAQ,oBAAoB;AAGhD,qFAAqF;AACrF,mFAAmF;AAEnF,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,6FAA6F;AAC7F,qDAAqD;AACrD,OAAO,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC,CAAC;AAE/M,uFAAuF;AACvF,+EAA+E;AAC/E,OAAO,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IAC5D,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaP,aAAaM,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaP,aAAaM,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaP,aAAaM,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB,EAAEC,GAAW;IACzH,6FAA6F;IAC7F,uGAAuG;IACvG,IAAIH,QAAQI,MAAM,GAAG,GACnB,MAAML,KAAKM,QAAQ,CACjB,sFACAL,QAAQM,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAG1B,IAAIN,KAAKG,MAAM,GAAG,GAAG;QACnB,wFAAwF;QACxF,+EAA+E;QAC/E,MAAMZ,aAAaT,gBAAgBoB,SAASK;QAC5C,MAAMT,KAAKM,QAAQ,CACjBhB,oBACAY,KAAKK,GAAG,CAAC,CAACf,MAAQD,WAAWC,KAAKC;IAEtC;AACF;AAEA,OAAO,MAAMiB,gBAAkC;IAC7CC,WAAW,IAAMvB;IACjBwB,kBAAiBC,KAAK;QACpB,uEAAuE;QACvE,sGAAsG;QACtG,IAAIA,QAAQxB,yBAAyB;YACnC,MAAM,IAAIJ,WACR,gBACA,CAAC,uBAAuB,EAAE4B,MAAM,0EAA0E,EAAExB,wBAAwB,wKAAwK,CAAC;QAEjT;IACF;IACAyB,kBAAkB,IAAM;IACxBf;IACAgB,gBAAgB5B;AAClB,EAAE"}

package/dist/esm/store/sqlite/signature.d.ts

@@ -0,0 +1,6 @@
+import type { Config } from '../../config/index.js';
+import type { Connection } from '../types.js';
+export declare function changedSignatureKeys(before: string, after: string): Set<string>;
+export declare function embedIdentityAdopted(before: string, after: string): boolean;
+export declare function signatureDiff(before: string, after: string): string;
+export declare function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]>;

package/dist/esm/store/sqlite/signature.js

@@ -0,0 +1,58 @@
+import { contentTokenize } from '../../config/index.js';
+import { listFiles, parseFile } from '../../scan/index.js';
+import { BEGIN_WRITE, withTransaction } from '../transaction.js';
+import { contentRow, INSERT_CONTENT_SQL } from './reconcile.js';
+// open()-lifecycle helpers for a config-signature change (see config.featureSignature's format:
+// global features, embed provider, tokenize, then one segment per preset). Never called from
+// reconcile() itself.
+// Segment keys that moved between two feature signatures.
+export function changedSignatureKeys(before, after) {
+    const keyOf = (part)=>part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0];
+    const parse = (sig)=>new Map(sig.split('|').map((part)=>[
+                keyOf(part),
+                part
+            ]));
+    const a = parse(before);
+    const b = parse(after);
+    const changed = new Set();
+    for (const [key, val] of b)if (a.get(key) !== val) changed.add(key);
+    for (const key of a.keys())if (!b.has(key)) changed.add(key);
+    return changed;
+}
+// Whether the embed segment only gained its resolved weight identity: same provider and
+// model, no identity recorded before, one now -- adopted into meta without a rebuild.
+export function embedIdentityAdopted(before, after) {
+    const embedPart = (sig)=>sig.split('|').find((p)=>p.startsWith('embed:'));
+    const b = embedPart(before);
+    const a = embedPart(after);
+    if (b === undefined || a === undefined) return false;
+    const at = a.indexOf('@');
+    return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;
+}
+// Names what moved, for the rebuild notice.
+export function signatureDiff(before, after) {
+    const changed = changedSignatureKeys(before, after);
+    const label = (key)=>key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset "${key.slice(7)}"` : 'features';
+    return changed.size === 0 ? 'features' : [
+        ...changed
+    ].map(label).join(', ');
+}
+// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)
+// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.
+export async function rebuildContentTable(conn, cfg, baseDir) {
+    const stmt = await conn.prepare('SELECT "path" FROM frontmatter');
+    const known = new Set((await stmt.all()).map((r)=>r.path));
+    const files = listFiles(cfg, baseDir).filter((f)=>known.has(f.relPath));
+    const segmenting = contentTokenize(cfg) === undefined;
+    const warnings = [];
+    const rows = [];
+    for (const file of files){
+        const { doc, warnings: fileWarnings } = parseFile(file);
+        warnings.push(...fileWarnings);
+        rows.push(contentRow(doc, segmenting));
+    }
+    await withTransaction(conn, async ()=>{
+        if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);
+    }, BEGIN_WRITE);
+    return warnings;
+}

package/dist/esm/store/sqlite/signature.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/signature.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { listFiles, parseFile } from '../../scan/index.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\nimport { contentRow, INSERT_CONTENT_SQL } from './reconcile.ts';\n\n// open()-lifecycle helpers for a config-signature change (see config.featureSignature's format:\n// global features, embed provider, tokenize, then one segment per preset). Never called from\n// reconcile() itself.\n\n// Segment keys that moved between two feature signatures.\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(\n    conn,\n    async () => {\n      if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n    },\n    BEGIN_WRITE\n  );\n  return warnings;\n}\n"],"names":["contentTokenize","listFiles","parseFile","BEGIN_WRITE","withTransaction","contentRow","INSERT_CONTENT_SQL","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","join","parse","sig","Map","map","a","b","changed","Set","key","val","get","add","keys","has","embedIdentityAdopted","embedPart","find","p","undefined","at","indexOf","signatureDiff","label","size","rebuildContentTable","conn","cfg","baseDir","stmt","prepare","known","all","r","path","files","filter","f","relPath","segmenting","warnings","rows","file","doc","fileWarnings","push","length","runBatch"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,SAAS,EAAEC,SAAS,QAAQ,sBAAsB;AAC3D,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,UAAU,EAAEC,kBAAkB,QAAQ,iBAAiB;AAEhE,gGAAgG;AAChG,6FAA6F;AAC7F,sBAAsB;AAEtB,0DAA0D;AAC1D,OAAO,SAASC,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGC,IAAI,CAAC,OAAOJ,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAMG,QAAQ,CAACC,MAAgB,IAAIC,IAAID,IAAIJ,KAAK,CAAC,KAAKM,GAAG,CAAC,CAACR,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMS,IAAIJ,MAAMR;IAChB,MAAMa,IAAIL,MAAMP;IAChB,MAAMa,UAAU,IAAIC;IACpB,KAAK,MAAM,CAACC,KAAKC,IAAI,IAAIJ,EAAG,IAAID,EAAEM,GAAG,CAACF,SAASC,KAAKH,QAAQK,GAAG,CAACH;IAChE,KAAK,MAAMA,OAAOJ,EAAEQ,IAAI,GAAI,IAAI,CAACP,EAAEQ,GAAG,CAACL,MAAMF,QAAQK,GAAG,CAACH;IACzD,OAAOF;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASQ,qBAAqBtB,MAAc,EAAEC,KAAa;IAChE,MAAMsB,YAAY,CAACd,MAAgBA,IAAIJ,KAAK,CAAC,KAAKmB,IAAI,CAAC,CAACC,IAAMA,EAAErB,UAAU,CAAC;IAC3E,MAAMS,IAAIU,UAAUvB;IACpB,MAAMY,IAAIW,UAAUtB;IACpB,IAAIY,MAAMa,aAAad,MAAMc,WAAW,OAAO;IAC/C,MAAMC,KAAKf,EAAEgB,OAAO,CAAC;IACrB,OAAOf,EAAEe,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKf,EAAEN,KAAK,CAAC,GAAGqB,QAAQd;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASgB,cAAc7B,MAAc,EAAEC,KAAa;IACzD,MAAMa,UAAUf,qBAAqBC,QAAQC;IAC7C,MAAM6B,QAAQ,CAACd,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIZ,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEY,IAAIV,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOQ,QAAQiB,IAAI,KAAK,IAAI,aAAa;WAAIjB;KAAQ,CAACH,GAAG,CAACmB,OAAOvB,IAAI,CAAC;AACxE;AAEA,oGAAoG;AACpG,gIAAgI;AAChI,OAAO,eAAeyB,oBAAoBC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,MAAMC,OAAO,MAAMH,KAAKI,OAAO,CAAC;IAChC,MAAMC,QAAQ,IAAIvB,IAAI,AAAE,CAAA,MAAMqB,KAAKG,GAAG,EAAC,EAA+B5B,GAAG,CAAC,CAAC6B,IAAMA,EAAEC,IAAI;IACvF,MAAMC,QAAQjD,UAAUyC,KAAKC,SAASQ,MAAM,CAAC,CAACC,IAAMN,MAAMjB,GAAG,CAACuB,EAAEC,OAAO;IACvE,MAAMC,aAAatD,gBAAgB0C,SAASR;IAC5C,MAAMqB,WAAqB,EAAE;IAC7B,MAAMC,OAAoB,EAAE;IAC5B,KAAK,MAAMC,QAAQP,MAAO;QACxB,MAAM,EAAEQ,GAAG,EAAEH,UAAUI,YAAY,EAAE,GAAGzD,UAAUuD;QAClDF,SAASK,IAAI,IAAID;QACjBH,KAAKI,IAAI,CAACvD,WAAWqD,KAAKJ;IAC5B;IACA,MAAMlD,gBACJqC,MACA;QACE,IAAIe,KAAKK,MAAM,GAAG,GAAG,MAAMpB,KAAKqB,QAAQ,CAACxD,oBAAoBkD;IAC/D,GACArD;IAEF,OAAOoD;AACT"}

package/dist/esm/store/sqlite/store.js

@@ -1,17 +1,19 @@
 import { contentTokenize } from '../../config/index.js';
+import { reconcile } from '../reconcile.js';
 import { getColumns } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { hasVectorRow, pendingRows } from '../vectors.js';
 import { fieldStats } from './fieldStats.js';
 import { queryLexical } from './lexical.js';
-import { reconcile } from './reconcile.js';
+import { sqliteDialect } from './reconcile.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
 export const CAPABILITIES = new Set([
     'phrases',
     'snippets',
     'watch-concurrency',
     'lexical',
-    'vectors'
+    'vectors',
+    'sql-functions'
 ]);
 // Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one
 // Connection instance (conn) with open()'s own reconcile call so transaction depth is tracked against the same object everywhere.
@@ -32,7 +34,7 @@ export function createStore(db, conn, cfg, baseDir) {
             return withTransaction(conn, fn);
         },
         async reconcile () {
-            return reconcile(conn, cfg, baseDir);
+            return reconcile(conn, cfg, baseDir, sqliteDialect);
         },
         docs: {
             async columns () {

package/dist/esm/store/sqlite/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with open()'s own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["contentTokenize","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","reconcile","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","undefined","vectors","pending","writeVectors","rows","candidates","qv","storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":"AAEA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;CAAU,EAAE;AAEjI,0FAA0F;AAC1F,kIAAkI;AAClI,OAAO,SAASC,YAAYC,EAAgB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC1F,OAAO;QACLC,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAMN,KAAKK,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAON,KAAKO,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMT,KAAKQ,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBa,MAAMW;QAC/B;QACA,MAAMnB;YACJ,OAAOA,UAAUQ,MAAMC,KAAKC;QAC9B;QACAU,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM3B,WAAWc;iBAAO;YACtC;YACA,MAAMV,YAAWuB,OAAO,EAAEC,UAAU;gBAClC,OAAOxB,WAAWU,MAAMa,SAASC;YACnC;QACF;QACAC,SAAS;YACP,MAAMC,OAAMC,KAAK,EAAEC,IAAI;gBACrB,OAAO3B,aAAaS,MAAMiB,OAAOC,MAAMjC,gBAAgBgB,SAASkB;YAClE;QACF;QACAC,SAAS;YACP,MAAMC;gBACJ,OAAOhC,YAAYW;YACrB;YACA,MAAMsB,cAAaC,IAAsB;gBACvC,MAAM5B,iBAAiBK,MAAMuB;YAC/B;YACA,MAAMC,YAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;gBAC5C,OAAOnC,eAAeO,MAAMyB,IAAIC,WAAWC,OAAOC;YACpD;YACA,MAAMC,SAAQC,IAAI,EAAEZ,IAAI;gBACtB,OAAOxB,YAAYM,MAAM8B,MAAMZ;YACjC;YACA,MAAMa,WAAUD,IAAI;gBAClB,OAAO1C,aAAaY,MAAM8B;YAC5B;QACF;QACA,MAAME;YACJ,qFAAqF;YACrF,qFAAqF;YACrF,MAAMC,MAAMlC,GAAGQ,OAAO,CAAC,uBAAuB2B,GAAG;YACjD,OAAO;gBAAEC,cAAc,GAAGF,IAAIG,OAAO,CAAC,kFAAkF,CAAC;YAAC;QAC5H;QACAC,KAAK;YACH,MAAM9B,SAAQD,GAAW;gBACvB,MAAMgC,OAAOvC,GAAGQ,OAAO,CAACD;gBACxBgC,KAAKC,cAAc,CAAC,OAAO,qEAAqE;gBAChG,OAAO;oBACL1B,SAAS,IAAMyB,KAAKzB,OAAO;oBAC3B,2EAA2E;oBAC3E2B,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAKC;oBAC1B;gBACF;YACF;QACF;QACA,MAAMC;YACJ3C,GAAG2C,KAAK;QACV;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with open()'s own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir, sqliteDialect);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["contentTokenize","reconcile","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","sqliteDialect","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","undefined","vectors","pending","writeVectors","rows","candidates","qv","storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":"AAEA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB,EAAE;AAElJ,0FAA0F;AAC1F,kIAAkI;AAClI,OAAO,SAASC,YAAYC,EAAgB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC1F,OAAO;QACLC,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAMN,KAAKK,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAON,KAAKO,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMT,KAAKQ,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBa,MAAMW;QAC/B;QACA,MAAM1B;YACJ,OAAOA,UAAUe,MAAMC,KAAKC,SAASV;QACvC;QACAoB,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM3B,WAAWc;iBAAO;YACtC;YACA,MAAMV,YAAWuB,OAAO,EAAEC,UAAU;gBAClC,OAAOxB,WAAWU,MAAMa,SAASC;YACnC;QACF;QACAC,SAAS;YACP,MAAMC,OAAMC,KAAK,EAAEC,IAAI;gBACrB,OAAO3B,aAAaS,MAAMiB,OAAOC,MAAMlC,gBAAgBiB,SAASkB;YAClE;QACF;QACAC,SAAS;YACP,MAAMC;gBACJ,OAAOhC,YAAYW;YACrB;YACA,MAAMsB,cAAaC,IAAsB;gBACvC,MAAM5B,iBAAiBK,MAAMuB;YAC/B;YACA,MAAMC,YAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;gBAC5C,OAAOnC,eAAeO,MAAMyB,IAAIC,WAAWC,OAAOC;YACpD;YACA,MAAMC,SAAQC,IAAI,EAAEZ,IAAI;gBACtB,OAAOxB,YAAYM,MAAM8B,MAAMZ;YACjC;YACA,MAAMa,WAAUD,IAAI;gBAClB,OAAO1C,aAAaY,MAAM8B;YAC5B;QACF;QACA,MAAME;YACJ,qFAAqF;YACrF,qFAAqF;YACrF,MAAMC,MAAMlC,GAAGQ,OAAO,CAAC,uBAAuB2B,GAAG;YACjD,OAAO;gBAAEC,cAAc,GAAGF,IAAIG,OAAO,CAAC,kFAAkF,CAAC;YAAC;QAC5H;QACAC,KAAK;YACH,MAAM9B,SAAQD,GAAW;gBACvB,MAAMgC,OAAOvC,GAAGQ,OAAO,CAACD;gBACxBgC,KAAKC,cAAc,CAAC,OAAO,qEAAqE;gBAChG,OAAO;oBACL1B,SAAS,IAAMyB,KAAKzB,OAAO;oBAC3B,2EAA2E;oBAC3E2B,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAKC;oBAC1B;gBACF;YACF;QACF;QACA,MAAMC;YACJ3C,GAAG2C,KAAK;QACV;IACF;AACF"}

package/dist/esm/store/transaction.d.ts

@@ -1,5 +1,6 @@
 interface ExecConnection {
     exec(sql: string): Promise<void>;
 }
-export declare function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>): Promise<T>;
+export declare const BEGIN_WRITE = "BEGIN IMMEDIATE";
+export declare function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>, begin?: string): Promise<T>;
 export {};

package/dist/esm/store/transaction.js

@@ -14,9 +14,9 @@ function stateFor(conn) {
     }
     return state;
 }
-async function enter(conn, state) {
+async function enter(conn, state, begin) {
     if (state.depth === 0) {
-        await conn.exec('BEGIN');
+        await conn.exec(begin);
         state.rollbackOnly = false;
     }
     state.depth++;
@@ -37,11 +37,16 @@ async function leaveErr(conn, state) {
     state.depth--;
     if (state.depth === 0) await conn.exec('ROLLBACK');
 }
-// Depth-0 opens BEGIN; a nested call joins it and issues no SQL of its own. The outermost call
-// owns COMMIT/ROLLBACK; every store and internal write loop goes through this one helper, so join semantics never differ by call site.
-export async function withTransaction(conn, fn) {
+// A writer takes its lock at BEGIN rather than on first write: SQLite will not retry a deferred
+// transaction that read first and then upgrades, so busy_timeout never applies and the second
+// concurrent command fails outright instead of queueing. Reads keep plain BEGIN, or every snapshot
+// read (map, peek) would serialize against writers. DuckDB has no IMMEDIATE and passes nothing.
+export const BEGIN_WRITE = 'BEGIN IMMEDIATE';
+// Depth-0 opens the transaction; a nested call joins it and issues no SQL of its own. The outermost
+// call owns COMMIT/ROLLBACK, so join semantics never differ by call site.
+export async function withTransaction(conn, fn, begin = 'BEGIN') {
     const state = stateFor(conn);
-    await enter(conn, state);
+    await enter(conn, state, begin);
     let result;
     try {
         result = await fn();

package/dist/esm/store/transaction.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/transaction.ts"],"sourcesContent":["// Store-agnostic joining transaction helper: node:sqlite and DuckDB both support only\n// BEGIN/COMMIT/ROLLBACK, no SAVEPOINT -- nesting joins the outer transaction, sharing one semantics through exec(sql).\ninterface ExecConnection {\n  exec(sql: string): Promise<void>;\n}\n\ninterface TxState {\n  depth: number;\n  rollbackOnly: boolean;\n}\n\n// Keyed by connection, not a module singleton: two connections can coexist in one process\n// (e.g. watch.ts alongside a CLI query) and must not share depth.\nconst states = new WeakMap<ExecConnection, TxState>();\n\nfunction stateFor(conn: ExecConnection): TxState {\n  let state = states.get(conn);\n  if (!state) {\n    state = { depth: 0, rollbackOnly: false };\n    states.set(conn, state);\n  }\n  return state;\n}\n\nasync function enter(conn: ExecConnection, state: TxState): Promise<void> {\n  if (state.depth === 0) {\n    await conn.exec('BEGIN');\n    state.rollbackOnly = false;\n  }\n  state.depth++;\n}\n\n// Reaching depth 0 still rollback-only means an inner scope failed and an enclosing catch\n// swallowed it; commit here would persist a partial write, so this rolls back and throws instead.\nasync function leaveOk(conn: ExecConnection, state: TxState): Promise<void> {\n  state.depth--;\n  if (state.depth > 0) return;\n  if (state.rollbackOnly) {\n    await conn.exec('ROLLBACK');\n    throw new Error('transaction rolled back: a nested scope failed');\n  }\n  await conn.exec('COMMIT');\n}\n\nasync function leaveErr(conn: ExecConnection, state: TxState): Promise<void> {\n  state.rollbackOnly = true;\n  state.depth--;\n  if (state.depth === 0) await conn.exec('ROLLBACK');\n}\n\n// Depth-0 opens BEGIN; a nested call joins it and issues no SQL of its own. The outermost call\n// owns COMMIT/ROLLBACK; every store and internal write loop goes through this one helper, so join semantics never differ by call site.\nexport async function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>): Promise<T> {\n  const state = stateFor(conn);\n  await enter(conn, state);\n  let result: T;\n  try {\n    result = await fn();\n  } catch (err) {\n    await leaveErr(conn, state);\n    throw err;\n  }\n  await leaveOk(conn, state);\n  return result;\n}\n"],"names":["states","WeakMap","stateFor","conn","state","get","depth","rollbackOnly","set","enter","exec","leaveOk","Error","leaveErr","withTransaction","fn","result","err"],"mappings":"AAAA,sFAAsF;AACtF,uHAAuH;AAUvH,0FAA0F;AAC1F,kEAAkE;AAClE,MAAMA,SAAS,IAAIC;AAEnB,SAASC,SAASC,IAAoB;IACpC,IAAIC,QAAQJ,OAAOK,GAAG,CAACF;IACvB,IAAI,CAACC,OAAO;QACVA,QAAQ;YAAEE,OAAO;YAAGC,cAAc;QAAM;QACxCP,OAAOQ,GAAG,CAACL,MAAMC;IACnB;IACA,OAAOA;AACT;AAEA,eAAeK,MAAMN,IAAoB,EAAEC,KAAc;IACvD,IAAIA,MAAME,KAAK,KAAK,GAAG;QACrB,MAAMH,KAAKO,IAAI,CAAC;QAChBN,MAAMG,YAAY,GAAG;IACvB;IACAH,MAAME,KAAK;AACb;AAEA,0FAA0F;AAC1F,kGAAkG;AAClG,eAAeK,QAAQR,IAAoB,EAAEC,KAAc;IACzDA,MAAME,KAAK;IACX,IAAIF,MAAME,KAAK,GAAG,GAAG;IACrB,IAAIF,MAAMG,YAAY,EAAE;QACtB,MAAMJ,KAAKO,IAAI,CAAC;QAChB,MAAM,IAAIE,MAAM;IAClB;IACA,MAAMT,KAAKO,IAAI,CAAC;AAClB;AAEA,eAAeG,SAASV,IAAoB,EAAEC,KAAc;IAC1DA,MAAMG,YAAY,GAAG;IACrBH,MAAME,KAAK;IACX,IAAIF,MAAME,KAAK,KAAK,GAAG,MAAMH,KAAKO,IAAI,CAAC;AACzC;AAEA,+FAA+F;AAC/F,uIAAuI;AACvI,OAAO,eAAeI,gBAAmBX,IAAoB,EAAEY,EAAoB;IACjF,MAAMX,QAAQF,SAASC;IACvB,MAAMM,MAAMN,MAAMC;IAClB,IAAIY;IACJ,IAAI;QACFA,SAAS,MAAMD;IACjB,EAAE,OAAOE,KAAK;QACZ,MAAMJ,SAASV,MAAMC;QACrB,MAAMa;IACR;IACA,MAAMN,QAAQR,MAAMC;IACpB,OAAOY;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/transaction.ts"],"sourcesContent":["// Store-agnostic joining transaction helper: node:sqlite and DuckDB both support only\n// BEGIN/COMMIT/ROLLBACK, no SAVEPOINT -- nesting joins the outer transaction, sharing one semantics through exec(sql).\ninterface ExecConnection {\n  exec(sql: string): Promise<void>;\n}\n\ninterface TxState {\n  depth: number;\n  rollbackOnly: boolean;\n}\n\n// Keyed by connection, not a module singleton: two connections can coexist in one process\n// (e.g. watch.ts alongside a CLI query) and must not share depth.\nconst states = new WeakMap<ExecConnection, TxState>();\n\nfunction stateFor(conn: ExecConnection): TxState {\n  let state = states.get(conn);\n  if (!state) {\n    state = { depth: 0, rollbackOnly: false };\n    states.set(conn, state);\n  }\n  return state;\n}\n\nasync function enter(conn: ExecConnection, state: TxState, begin: string): Promise<void> {\n  if (state.depth === 0) {\n    await conn.exec(begin);\n    state.rollbackOnly = false;\n  }\n  state.depth++;\n}\n\n// Reaching depth 0 still rollback-only means an inner scope failed and an enclosing catch\n// swallowed it; commit here would persist a partial write, so this rolls back and throws instead.\nasync function leaveOk(conn: ExecConnection, state: TxState): Promise<void> {\n  state.depth--;\n  if (state.depth > 0) return;\n  if (state.rollbackOnly) {\n    await conn.exec('ROLLBACK');\n    throw new Error('transaction rolled back: a nested scope failed');\n  }\n  await conn.exec('COMMIT');\n}\n\nasync function leaveErr(conn: ExecConnection, state: TxState): Promise<void> {\n  state.rollbackOnly = true;\n  state.depth--;\n  if (state.depth === 0) await conn.exec('ROLLBACK');\n}\n\n// A writer takes its lock at BEGIN rather than on first write: SQLite will not retry a deferred\n// transaction that read first and then upgrades, so busy_timeout never applies and the second\n// concurrent command fails outright instead of queueing. Reads keep plain BEGIN, or every snapshot\n// read (map, peek) would serialize against writers. DuckDB has no IMMEDIATE and passes nothing.\nexport const BEGIN_WRITE = 'BEGIN IMMEDIATE';\n\n// Depth-0 opens the transaction; a nested call joins it and issues no SQL of its own. The outermost\n// call owns COMMIT/ROLLBACK, so join semantics never differ by call site.\nexport async function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>, begin = 'BEGIN'): Promise<T> {\n  const state = stateFor(conn);\n  await enter(conn, state, begin);\n  let result: T;\n  try {\n    result = await fn();\n  } catch (err) {\n    await leaveErr(conn, state);\n    throw err;\n  }\n  await leaveOk(conn, state);\n  return result;\n}\n"],"names":["states","WeakMap","stateFor","conn","state","get","depth","rollbackOnly","set","enter","begin","exec","leaveOk","Error","leaveErr","BEGIN_WRITE","withTransaction","fn","result","err"],"mappings":"AAAA,sFAAsF;AACtF,uHAAuH;AAUvH,0FAA0F;AAC1F,kEAAkE;AAClE,MAAMA,SAAS,IAAIC;AAEnB,SAASC,SAASC,IAAoB;IACpC,IAAIC,QAAQJ,OAAOK,GAAG,CAACF;IACvB,IAAI,CAACC,OAAO;QACVA,QAAQ;YAAEE,OAAO;YAAGC,cAAc;QAAM;QACxCP,OAAOQ,GAAG,CAACL,MAAMC;IACnB;IACA,OAAOA;AACT;AAEA,eAAeK,MAAMN,IAAoB,EAAEC,KAAc,EAAEM,KAAa;IACtE,IAAIN,MAAME,KAAK,KAAK,GAAG;QACrB,MAAMH,KAAKQ,IAAI,CAACD;QAChBN,MAAMG,YAAY,GAAG;IACvB;IACAH,MAAME,KAAK;AACb;AAEA,0FAA0F;AAC1F,kGAAkG;AAClG,eAAeM,QAAQT,IAAoB,EAAEC,KAAc;IACzDA,MAAME,KAAK;IACX,IAAIF,MAAME,KAAK,GAAG,GAAG;IACrB,IAAIF,MAAMG,YAAY,EAAE;QACtB,MAAMJ,KAAKQ,IAAI,CAAC;QAChB,MAAM,IAAIE,MAAM;IAClB;IACA,MAAMV,KAAKQ,IAAI,CAAC;AAClB;AAEA,eAAeG,SAASX,IAAoB,EAAEC,KAAc;IAC1DA,MAAMG,YAAY,GAAG;IACrBH,MAAME,KAAK;IACX,IAAIF,MAAME,KAAK,KAAK,GAAG,MAAMH,KAAKQ,IAAI,CAAC;AACzC;AAEA,gGAAgG;AAChG,8FAA8F;AAC9F,mGAAmG;AACnG,gGAAgG;AAChG,OAAO,MAAMI,cAAc,kBAAkB;AAE7C,oGAAoG;AACpG,0EAA0E;AAC1E,OAAO,eAAeC,gBAAmBb,IAAoB,EAAEc,EAAoB,EAAEP,QAAQ,OAAO;IAClG,MAAMN,QAAQF,SAASC;IACvB,MAAMM,MAAMN,MAAMC,OAAOM;IACzB,IAAIQ;IACJ,IAAI;QACFA,SAAS,MAAMD;IACjB,EAAE,OAAOE,KAAK;QACZ,MAAML,SAASX,MAAMC;QACrB,MAAMe;IACR;IACA,MAAMP,QAAQT,MAAMC;IACpB,OAAOc;AACT"}

package/dist/esm/store/turso/connection.js

@@ -1,4 +1,4 @@
-import { withTransaction } from '../transaction.js';
+import { BEGIN_WRITE, withTransaction } from '../transaction.js';
 // Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this
 // wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.
 let TursoStatementWrapper = class TursoStatementWrapper {
@@ -41,7 +41,7 @@ export function createConnection(db) {
                         sql,
                         args
                     })));
-            });
+            }, BEGIN_WRITE);
         }
     };
     return conn;

package/dist/esm/store/turso/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this\n// wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // db.batch() is the engine's own bulk idiom: one crossing, N rows, each its own statement,\n    // no bind-variable ceiling to chunk around. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        await db.batch(paramRows.map((args) => ({ sql, args })));\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","batch","map","args"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAOpD,+FAA+F;AAC/F,gGAAgG;AAChG,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASQ,iBAAiBC,EAAY;IAC3C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpB,MAAMH,GAAGE,IAAI,CAACC;QAChB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAIf,sBAAsB,MAAMY,GAAGI,OAAO,CAACD;QACpD;QACA,2FAA2F;QAC3F,uLAAuL;QACvL,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMpB,gBAAgBc,MAAM;gBAC1B,MAAMD,GAAGQ,KAAK,CAACF,UAAUG,GAAG,CAAC,CAACC,OAAU,CAAA;wBAAEP;wBAAKO;oBAAK,CAAA;YACtD;QACF;IACF;IACA,OAAOT;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this\n// wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // db.batch() is the engine's own bulk idiom: one crossing, N rows, each its own statement,\n    // no bind-variable ceiling to chunk around. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          await db.batch(paramRows.map((args) => ({ sql, args })));\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["BEGIN_WRITE","withTransaction","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","batch","map","args"],"mappings":"AACA,SAASA,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAOjE,+FAA+F;AAC/F,gGAAgG;AAChG,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASQ,iBAAiBC,EAAY;IAC3C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpB,MAAMH,GAAGE,IAAI,CAACC;QAChB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAIf,sBAAsB,MAAMY,GAAGI,OAAO,CAACD;QACpD;QACA,2FAA2F;QAC3F,uLAAuL;QACvL,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMpB,gBACJc,MACA;gBACE,MAAMD,GAAGQ,KAAK,CAACF,UAAUG,GAAG,CAAC,CAACC,OAAU,CAAA;wBAAEP;wBAAKO;oBAAK,CAAA;YACtD,GACAxB;QAEJ;IACF;IACA,OAAOe;AACT"}

package/dist/esm/store/turso/open.d.ts

@@ -9,4 +9,6 @@ export interface OpenResult {
     parsed: number;
     warnings: string[];
 }
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;