sensemaking 0.19.2 → 0.20.0

package/dist/esm/store/sqlite/open.js

@@ -1,25 +1,21 @@
 // The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with
 // both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.
-import { mkdirSync } from 'node:fs';
-import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
-import { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.js';
-import { rekeyChunkText } from '../../embed/handoff.js';
+import { contentTokenize, featureSignature } from '../../config/index.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
-import { clearCache } from '../cache.js';
-import { reconcile } from '../reconcile.js';
+import { listFiles, parseFile } from '../../scan/index.js';
+import { openWithDialect } from '../open.js';
 import { getMeta, setMeta } from '../shared.js';
 import { BEGIN_WRITE, withTransaction } from '../transaction.js';
 import { createConnection } from './connection.js';
-import { sqliteDialect } from './reconcile.js';
-import { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, signatureDiff } from './signature.js';
+import { contentRow, INSERT_CONTENT_SQL, sqliteDialect } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
 import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.db';
 // Cache shape version, independent of the config's own `version`. Bumping it rebuilds
 // existing trees on first query.
-export const SCHEMA_VERSION = '18';
+export const SCHEMA_VERSION = '19';
 // unicode61 splits on spaces, so a language written without them indexes a whole run as one
 // token and word search finds nothing; `content.tokenize` is how such a tree picks trigram.
 const DEFAULT_TOKENIZE = 'porter unicode61';
@@ -53,7 +49,7 @@ async function createContentTable(conn, tokenize) {
 }
 // Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`
 // can't dump file text into context. Features add their own tables after the core ones.
-async function ensureSchema(conn, cfg, tokenize) {
+async function ensureSchemaTables(conn, cfg, tokenize) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_ctime" REAL, "_size" INTEGER, "_parse_error" TEXT)`);
     // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL
     // against the resolved tokenizer before this runs, so a stale table is already gone by now.
@@ -80,18 +76,9 @@ function setJournalWal(db) {
         }
     }
 }
-async function connect(cfg) {
-    const stateDir = join(cfg.baseDir, STATE_DIR);
-    mkdirSync(stateDir, {
-        recursive: true
-    });
-    const dbPath = join(stateDir, DB_FILENAME);
+async function connect(dbPath, cfg) {
     const db = new DatabaseSync(dbPath);
-    // A throw below must release this handle, or on Windows the leaked WAL db is undeletable and
-    // scratch cleanup fails with EPERM/EBUSY. `closed` keeps the catch from double-closing.
-    let closed = false;
     try {
-        var _ref;
         // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive
         // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.
         db.exec('PRAGMA busy_timeout = 30000');
@@ -100,107 +87,88 @@ async function connect(cfg) {
         setJournalWal(db);
         registerFunctions(db, contentTokenize(cfg) === undefined);
         const conn = createConnection(db);
-        db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
         // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd
         // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.
         const tokenize = resolveTokenize(db, cfg);
-        // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an
-        // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).
-        const version = await getMeta(conn, 'schema_version');
-        const features = await getMeta(conn, 'features');
-        const wantFeatures = featureSignature(cfg, FEATURES);
-        let tokenizeOnlyRebuild = false;
-        if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
-            // Indexing derives from presets, so a config edit rebuilding the cache must say so and
-            // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.
-            if (version !== null && version !== SCHEMA_VERSION) {
-                console.error('sense: cache format changed (new sensemaking version); rebuilding the index');
-                closed = true;
-                db.close();
-                clearCache(cfg);
-                return connect(cfg);
-            }
-            const changedKeys = changedSignatureKeys(features !== null && features !== void 0 ? features : '', wantFeatures);
-            // Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived
-            // and tokenizer-independent. Any other signature change takes the full clear/reopen below.
-            if (changedKeys.size === 1 && changedKeys.has('tokenize')) {
-                console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');
-                // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather
-                // than to no table. IF EXISTS makes a retry after such a crash a no-op, not a raw error.
-                await withTransaction(conn, async ()=>{
-                    await conn.exec('DROP TABLE IF EXISTS content');
-                    await createContentTable(conn, tokenize);
-                }, BEGIN_WRITE);
-                tokenizeOnlyRebuild = true;
-            } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features !== null && features !== void 0 ? features : '', wantFeatures)) {
-                // First sight of a resolved weight identity: the model itself hasn't changed, so
-                // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.
-                console.error("sense: recorded the embedding model's resolved identity; vectors are unaffected");
-                await setMeta(conn, 'features', wantFeatures);
-            } else {
-                const changed = signatureDiff(features !== null && features !== void 0 ? features : '', wantFeatures);
-                console.error(`sense: config change (${changed}) rebuilds the index`);
-                closed = true;
-                db.close();
-                clearCache(cfg);
-                return connect(cfg);
-            }
-        }
-        // Meta can lie after a crash between table creation and the signature write; the table's own
-        // DDL cannot, so a mismatch here rebuilds whatever meta says.
-        const stored = storedTokenize(db);
-        if (stored !== null && stored !== tokenize) {
-            console.error('sense: cache was built with a different content tokenizer; rebuilding the index');
-            closed = true;
-            db.close();
-            clearCache(cfg);
-            return connect(cfg);
-        }
-        // One writer at a time: the feature hooks check a column then add it, so two cold opens racing
-        // here both see it missing and the second ALTER fails with a duplicate column.
-        await withTransaction(conn, ()=>ensureSchema(conn, cfg, tokenize), BEGIN_WRITE);
-        let rebuildWarnings = [];
-        if (tokenizeOnlyRebuild) {
-            rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);
-            await setMeta(conn, 'features', wantFeatures);
-        }
-        // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.
-        // Installed before reconcile() -- that call is the one that races a watcher's transaction.
-        const recordedMaxMs = Number((_ref = await getMeta(conn, 'reconcile_max_ms')) !== null && _ref !== void 0 ? _ref : '0');
-        db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600000)}`);
-        const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, sqliteDialect);
         return {
-            db,
-            conn,
-            cfg,
-            dbPath,
-            parsed,
-            warnings: [
-                ...rebuildWarnings,
-                ...warnings
-            ]
+            handle: {
+                db,
+                tokenize
+            },
+            conn
         };
     } catch (err) {
-        if (!closed) db.close();
+        // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and
+        // scratch cleanup fails with EPERM/EBUSY.
+        db.close();
         throw err;
     }
 }
-// The sqlite store's open: connects synchronously (see connect() above), then wraps the
-// resulting connection in the async Store interface.
-export async function openSqlite(cfg) {
-    const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);
-    const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);
-    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
-    rekeyChunkText(conn, store);
-    return {
-        store: store,
-        cfg: resolvedCfg,
-        dbPath,
-        parsed,
-        warnings
-    };
+async function close(handle) {
+    handle.db.close();
+}
+// Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived
+// and tokenizer-independent. Any other signature change takes the full clear/reopen in open.ts.
+async function partialRebuild(handle, conn, _cfg, changedKeys) {
+    if (!(changedKeys.size === 1 && changedKeys.has('tokenize'))) return false;
+    console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');
+    // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather
+    // than to no table. IF EXISTS makes a retry after such a crash a no-op, not a raw error.
+    await withTransaction(conn, async ()=>{
+        await conn.exec('DROP TABLE IF EXISTS content');
+        await createContentTable(conn, handle.tokenize);
+    }, BEGIN_WRITE);
+    return true;
 }
-export async function docCount(store) {
-    const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');
-    return (await stmt.get()).n;
+// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)
+// and no feature extractors run. Warnings come from the reparse, since mtimes stay untouched and reconcile would never reparse them.
+async function postSchemaRebuild(_handle, conn, cfg, baseDir, wantFeatures) {
+    const stmt = await conn.prepare('SELECT "path" FROM frontmatter');
+    const known = new Set((await stmt.all()).map((r)=>r.path));
+    const files = listFiles(cfg, baseDir).filter((f)=>known.has(f.relPath));
+    const segmenting = contentTokenize(cfg) === undefined;
+    const warnings = [];
+    const rows = [];
+    for (const file of files){
+        const { doc, warnings: fileWarnings } = parseFile(file);
+        warnings.push(...fileWarnings);
+        rows.push(contentRow(doc, segmenting));
+    }
+    await withTransaction(conn, async ()=>{
+        if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);
+    }, BEGIN_WRITE);
+    await setMeta(conn, 'features', wantFeatures);
+    return warnings;
+}
+// Meta can lie after a crash between table creation and the signature write; the table's own
+// DDL cannot, so a mismatch here rebuilds whatever meta says.
+function extraRebuildReason(handle) {
+    const stored = storedTokenize(handle.db);
+    if (stored !== null && stored !== handle.tokenize) return 'cache was built with a different content tokenizer';
+    return null;
+}
+async function ensureSchema(handle, conn, cfg) {
+    // One writer at a time: the feature hooks check a column then add it, so two cold opens racing
+    // here both see it missing and the second ALTER fails with a duplicate column.
+    await withTransaction(conn, ()=>ensureSchemaTables(conn, cfg, handle.tokenize), BEGIN_WRITE);
+}
+async function setDerivedBusyTimeout(handle, _conn, ms) {
+    handle.db.exec(`PRAGMA busy_timeout = ${ms}`);
+}
+// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.
+export const sqliteOpenDialect = {
+    filename: DB_FILENAME,
+    schemaVersion: SCHEMA_VERSION,
+    reconcileDialect: sqliteDialect,
+    connect,
+    close,
+    partialRebuild,
+    postSchemaRebuild,
+    extraRebuildReason,
+    ensureSchema,
+    setDerivedBusyTimeout,
+    createStore: (handle, conn, cfg)=>createStore(handle.db, conn, cfg)
+};
+export async function openSqlite(cfg) {
+    return openWithDialect(cfg, sqliteOpenDialect);
 }

package/dist/esm/store/sqlite/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, signatureDiff } from './signature.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '18';\n\ninterface ConnectResult {\n  db: DatabaseSync;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; `content.tokenize` is how such a tree picks trigram.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a DDL string literal where nothing can bind, so the configured\n// value is concatenated; probing a throwaway table first is both the safety and the validation.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchema(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  await createContentTable(conn, tokenize);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  // A throw below must release this handle, or on Windows the leaked WAL db is undeletable and\n  // scratch cleanup fails with EPERM/EBUSY. `closed` keeps the catch from double-closing.\n  let closed = false;\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    setJournalWal(db);\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n\n    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    let tokenizeOnlyRebuild = false;\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== SCHEMA_VERSION) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived\n      // and tokenizer-independent. Any other signature change takes the full clear/reopen below.\n      if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n        console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n        // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather\n        // than to no table. IF EXISTS makes a retry after such a crash a no-op, not a raw error.\n        await withTransaction(\n          conn,\n          async () => {\n            await conn.exec('DROP TABLE IF EXISTS content');\n            await createContentTable(conn, tokenize);\n          },\n          BEGIN_WRITE\n        );\n        tokenizeOnlyRebuild = true;\n      } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n    }\n\n    // Meta can lie after a crash between table creation and the signature write; the table's own\n    // DDL cannot, so a mismatch here rebuilds whatever meta says.\n    const stored = storedTokenize(db);\n    if (stored !== null && stored !== tokenize) {\n      console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n      closed = true;\n      db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n    // here both see it missing and the second ALTER fails with a duplicate column.\n    await withTransaction(conn, () => ensureSchema(conn, cfg, tokenize), BEGIN_WRITE);\n\n    let rebuildWarnings: string[] = [];\n    if (tokenizeOnlyRebuild) {\n      rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);\n      await setMeta(conn, 'features', wantFeatures);\n    }\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, sqliteDialect);\n\n    return { db, conn, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n  } catch (err) {\n    if (!closed) db.close();\n    throw err;\n  }\n}\n\n// The sqlite store's open: connects synchronously (see connect() above), then wraps the\n// resulting connection in the async Store interface.\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["mkdirSync","join","DatabaseSync","contentTokenize","featureSignature","STATE_DIR","rekeyChunkText","SenseError","activeFeatures","FEATURES","clearCache","reconcile","getMeta","setMeta","BEGIN_WRITE","withTransaction","createConnection","sqliteDialect","changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","signatureDiff","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","undefined","literal","replace","exec","err","message","storedTokenize","row","prepare","get","m","sql","match","createContentTable","conn","tokenize","ensureSchema","feature","schema","setJournalWal","deadline","Date","now","test","Atomics","wait","Int32Array","SharedArrayBuffer","connect","stateDir","baseDir","recursive","dbPath","closed","version","features","wantFeatures","tokenizeOnlyRebuild","console","error","close","changedKeys","size","has","changed","stored","rebuildWarnings","recordedMaxMs","Number","Math","min","max","parsed","warnings","openSqlite","resolvedCfg","store","docCount","stmt","n"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,eAAe,EAAEC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACrF,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,mBAAmB,EAAEC,aAAa,QAAQ,iBAAiB;AAChH,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAmBnC,4FAA4F;AAC5F,4FAA4F;AAC5F,MAAMC,mBAAmB;AAEzB,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,MAAMC,aAAa3B,gBAAgB0B;IACnC,IAAIC,eAAeC,WAAW,OAAOL;IACrC,MAAMM,UAAUF,WAAWG,OAAO,CAAC,MAAM;IACzC,IAAI;QACFL,GAAGM,IAAI,CAAC;QACRN,GAAGM,IAAI,CAAC,CAAC,yEAAyE,EAAEF,QAAQ,EAAE,CAAC;QAC/FJ,GAAGM,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAI5B,WAAW,kBAAkB,CAAC,kBAAkB,EAAEuB,WAAW,0CAA0C,EAAE,AAACK,IAAcC,OAAO,CAAC,8FAA8F,CAAC;IAC3O;IACA,OAAOJ;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASK,eAAeT,EAAgB;IACtC,MAAMU,MAAMV,GAAGW,OAAO,CAAC,CAAC,oDAAoD,CAAC,EAAEC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,MAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,8FAA8F;AAC9F,iGAAiG;AACjG,eAAeG,mBAAmBC,IAAgB,EAAEC,QAAgB;IAClE,MAAMD,KAAKX,IAAI,CAAC,CAAC,0IAA0I,EAAEY,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeC,aAAaF,IAAgB,EAAEhB,GAAW,EAAEiB,QAAgB;IACzE,MAAMD,KAAKX,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,wFAAwF;IACxF,4FAA4F;IAC5F,MAAMU,mBAAmBC,MAAMC;IAC/B,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMD,KAAKX,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMW,KAAKX,IAAI,CAAC;IAChB,KAAK,MAAMc,WAAWxC,eAAeqB,KAAM,MAAMmB,QAAQC,MAAM,CAACJ;IAChE,IAAI,AAAC,MAAMjC,QAAQiC,MAAM,sBAAuB,MAAM,MAAMhC,QAAQgC,MAAM,kBAAkBpB;IAC5F,IAAI,AAAC,MAAMb,QAAQiC,MAAM,gBAAiB,MAAM,MAAMhC,QAAQgC,MAAM,YAAYzC,iBAAiByB,KAAKpB;AACxG;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASyC,cAActB,EAAgB;IACrC,MAAMuB,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFzB,GAAGM,IAAI,CAAC;YACR;QACF,EAAE,OAAOC,KAAK;YACZ,IAAIiB,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BG,IAAI,CAAC,AAACnB,IAAcC,OAAO,GAAG,MAAMD;YAC9FoB,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,eAAeC,QAAQ9B,GAAmB;IACxC,MAAM+B,WAAW3D,KAAK4B,IAAIgC,OAAO,EAAExD;IACnCL,UAAU4D,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS9D,KAAK2D,UAAUpC;IAE9B,MAAMI,KAAK,IAAI1B,aAAa6D;IAC5B,6FAA6F;IAC7F,wFAAwF;IACxF,IAAIC,SAAS;IACb,IAAI;YAsF4B;QArF9B,6FAA6F;QAC7F,yFAAyF;QACzFpC,GAAGM,IAAI,CAAC;QACR,yFAAyF;QACzF,wEAAwE;QACxEgB,cAActB;QACdN,kBAAkBM,IAAIzB,gBAAgB0B,SAASE;QAC/C,MAAMc,OAAO7B,iBAAiBY;QAE9BA,GAAGM,IAAI,CAAC;QAER,2FAA2F;QAC3F,8FAA8F;QAC9F,MAAMY,WAAWnB,gBAAgBC,IAAIC;QAErC,uFAAuF;QACvF,qGAAqG;QACrG,MAAMoC,UAAU,MAAMrD,QAAQiC,MAAM;QACpC,MAAMqB,WAAW,MAAMtD,QAAQiC,MAAM;QACrC,MAAMsB,eAAe/D,iBAAiByB,KAAKpB;QAC3C,IAAI2D,sBAAsB;QAC1B,IAAI,AAACH,YAAY,QAAQA,YAAYxC,kBAAoByC,aAAa,QAAQA,aAAaC,cAAe;YACxG,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAYxC,gBAAgB;gBAClD4C,QAAQC,KAAK,CAAC;gBACdN,SAAS;gBACTpC,GAAG2C,KAAK;gBACR7D,WAAWmB;gBACX,OAAO8B,QAAQ9B;YACjB;YACA,MAAM2C,cAActD,qBAAqBgD,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,6FAA6F;YAC7F,2FAA2F;YAC3F,IAAIK,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,aAAa;gBACzDL,QAAQC,KAAK,CAAC;gBACd,2FAA2F;gBAC3F,yFAAyF;gBACzF,MAAMvD,gBACJ8B,MACA;oBACE,MAAMA,KAAKX,IAAI,CAAC;oBAChB,MAAMU,mBAAmBC,MAAMC;gBACjC,GACAhC;gBAEFsD,sBAAsB;YACxB,OAAO,IAAII,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,YAAYvD,qBAAqB+C,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBACnH,iFAAiF;gBACjF,wFAAwF;gBACxFE,QAAQC,KAAK,CAAC;gBACd,MAAMzD,QAAQgC,MAAM,YAAYsB;YAClC,OAAO;gBACL,MAAMQ,UAAUtD,cAAc6C,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CE,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEK,QAAQ,oBAAoB,CAAC;gBACpEX,SAAS;gBACTpC,GAAG2C,KAAK;gBACR7D,WAAWmB;gBACX,OAAO8B,QAAQ9B;YACjB;QACF;QAEA,6FAA6F;QAC7F,8DAA8D;QAC9D,MAAM+C,SAASvC,eAAeT;QAC9B,IAAIgD,WAAW,QAAQA,WAAW9B,UAAU;YAC1CuB,QAAQC,KAAK,CAAC;YACdN,SAAS;YACTpC,GAAG2C,KAAK;YACR7D,WAAWmB;YACX,OAAO8B,QAAQ9B;QACjB;QAEA,+FAA+F;QAC/F,+EAA+E;QAC/E,MAAMd,gBAAgB8B,MAAM,IAAME,aAAaF,MAAMhB,KAAKiB,WAAWhC;QAErE,IAAI+D,kBAA4B,EAAE;QAClC,IAAIT,qBAAqB;YACvBS,kBAAkB,MAAMzD,oBAAoByB,MAAMhB,KAAKA,IAAIgC,OAAO;YAClE,MAAMhD,QAAQgC,MAAM,YAAYsB;QAClC;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMW,gBAAgBC,QAAQ,OAAA,MAAMnE,QAAQiC,MAAM,iCAApB,kBAAA,OAA4C;QAC1EjB,GAAGM,IAAI,CAAC,CAAC,sBAAsB,EAAE8C,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAExF,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzE,UAAUkC,MAAMhB,KAAKA,IAAIgC,OAAO,EAAE5C;QAErE,OAAO;YAAEW;YAAIiB;YAAMhB;YAAKkC;YAAQoB;YAAQC,UAAU;mBAAIP;mBAAoBO;aAAS;QAAC;IACtF,EAAE,OAAOjD,KAAK;QACZ,IAAI,CAAC6B,QAAQpC,GAAG2C,KAAK;QACrB,MAAMpC;IACR;AACF;AAEA,wFAAwF;AACxF,qDAAqD;AACrD,OAAO,eAAekD,WAAWxD,GAAmB;IAClD,MAAM,EAAED,EAAE,EAAEiB,IAAI,EAAEhB,KAAKyD,WAAW,EAAEvB,MAAM,EAAEoB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzB,QAAQ9B;IAC/E,MAAM0D,QAAQhE,YAAYK,IAAIiB,MAAMyC,aAAaA,YAAYzB,OAAO;IACpE,0FAA0F;IAC1FvD,eAAeuC,MAAM0C;IACrB,OAAO;QAAEA,OAAOA;QAAO1D,KAAKyD;QAAavB;QAAQoB;QAAQC;IAAS;AACpE;AAEA,OAAO,eAAeI,SAASD,KAAY;IACzC,MAAME,OAAO,MAAMF,MAAMhD,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMkD,KAAKjD,GAAG,EAAC,EAAqBkD,CAAC;AAChD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { listFiles, parseFile } from '../../scan/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { contentRow, INSERT_CONTENT_SQL, sqliteDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '19';\n\nexport type { OpenResult };\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; `content.tokenize` is how such a tree picks trigram.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// The connection plus this store's own extra state (types.ts's OpenDialect<Handle>): the\n// resolved tokenizer, threaded through connect, the tokenize-only rebuild, and ensureSchema.\ninterface SqliteHandle {\n  db: DatabaseSync;\n  tokenize: string;\n}\n\n// FTS5 takes its tokenizer as a DDL string literal where nothing can bind, so the configured\n// value is concatenated; probing a throwaway table first is both the safety and the validation.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchemaTables(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  await createContentTable(conn, tokenize);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: SqliteHandle; conn: Connection }> {\n  const db = new DatabaseSync(dbPath);\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    setJournalWal(db);\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n    return { handle: { db, tokenize }, conn };\n  } catch (err) {\n    // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and\n    // scratch cleanup fails with EPERM/EBUSY.\n    db.close();\n    throw err;\n  }\n}\n\nasync function close(handle: SqliteHandle): Promise<void> {\n  handle.db.close();\n}\n\n// Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived\n// and tokenizer-independent. Any other signature change takes the full clear/reopen in open.ts.\nasync function partialRebuild(handle: SqliteHandle, conn: Connection, _cfg: Config, changedKeys: Set<string>): Promise<boolean> {\n  if (!(changedKeys.size === 1 && changedKeys.has('tokenize'))) return false;\n  console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n  // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather\n  // than to no table. IF EXISTS makes a retry after such a crash a no-op, not a raw error.\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec('DROP TABLE IF EXISTS content');\n      await createContentTable(conn, handle.tokenize);\n    },\n    BEGIN_WRITE\n  );\n  return true;\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Warnings come from the reparse, since mtimes stay untouched and reconcile would never reparse them.\nasync function postSchemaRebuild(_handle: SqliteHandle, conn: Connection, cfg: Config, baseDir: string, wantFeatures: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(\n    conn,\n    async () => {\n      if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n    },\n    BEGIN_WRITE\n  );\n  await setMeta(conn, 'features', wantFeatures);\n  return warnings;\n}\n\n// Meta can lie after a crash between table creation and the signature write; the table's own\n// DDL cannot, so a mismatch here rebuilds whatever meta says.\nfunction extraRebuildReason(handle: SqliteHandle): string | null {\n  const stored = storedTokenize(handle.db);\n  if (stored !== null && stored !== handle.tokenize) return 'cache was built with a different content tokenizer';\n  return null;\n}\n\nasync function ensureSchema(handle: SqliteHandle, conn: Connection, cfg: Config): Promise<void> {\n  // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n  // here both see it missing and the second ALTER fails with a duplicate column.\n  await withTransaction(conn, () => ensureSchemaTables(conn, cfg, handle.tokenize), BEGIN_WRITE);\n}\n\nasync function setDerivedBusyTimeout(handle: SqliteHandle, _conn: Connection, ms: number): Promise<void> {\n  handle.db.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const sqliteOpenDialect: OpenDialect<SqliteHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: sqliteDialect,\n  connect,\n  close,\n  partialRebuild,\n  postSchemaRebuild,\n  extraRebuildReason,\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn, cfg) => createStore(handle.db, conn, cfg),\n};\n\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, sqliteOpenDialect);\n}\n"],"names":["DatabaseSync","contentTokenize","featureSignature","SenseError","activeFeatures","FEATURES","listFiles","parseFile","openWithDialect","getMeta","setMeta","BEGIN_WRITE","withTransaction","createConnection","contentRow","INSERT_CONTENT_SQL","sqliteDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","undefined","literal","replace","exec","err","message","storedTokenize","row","prepare","get","m","sql","match","createContentTable","conn","tokenize","ensureSchemaTables","feature","schema","setJournalWal","deadline","Date","now","test","Atomics","wait","Int32Array","SharedArrayBuffer","connect","dbPath","handle","close","partialRebuild","_cfg","changedKeys","size","has","console","error","postSchemaRebuild","_handle","baseDir","wantFeatures","stmt","known","Set","all","map","r","path","files","filter","f","relPath","segmenting","warnings","rows","file","doc","fileWarnings","push","length","runBatch","extraRebuildReason","stored","ensureSchema","setDerivedBusyTimeout","_conn","ms","sqliteOpenDialect","filename","schemaVersion","reconcileDialect","openSqlite"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,YAAY,QAAQ,cAAc;AAE3C,SAASC,eAAe,EAAEC,gBAAgB,QAAQ,wBAAwB;AAC1E,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,SAAS,EAAEC,SAAS,QAAQ,sBAAsB;AAE3D,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,UAAU,EAAEC,kBAAkB,EAAEC,aAAa,QAAQ,iBAAiB;AAC/E,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAInC,4FAA4F;AAC5F,4FAA4F;AAC5F,MAAMC,mBAAmB;AASzB,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,MAAMC,aAAaxB,gBAAgBuB;IACnC,IAAIC,eAAeC,WAAW,OAAOL;IACrC,MAAMM,UAAUF,WAAWG,OAAO,CAAC,MAAM;IACzC,IAAI;QACFL,GAAGM,IAAI,CAAC;QACRN,GAAGM,IAAI,CAAC,CAAC,yEAAyE,EAAEF,QAAQ,EAAE,CAAC;QAC/FJ,GAAGM,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAI3B,WAAW,kBAAkB,CAAC,kBAAkB,EAAEsB,WAAW,0CAA0C,EAAE,AAACK,IAAcC,OAAO,CAAC,8FAA8F,CAAC;IAC3O;IACA,OAAOJ;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASK,eAAeT,EAAgB;IACtC,MAAMU,MAAMV,GAAGW,OAAO,CAAC,CAAC,oDAAoD,CAAC,EAAEC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,MAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,8FAA8F;AAC9F,iGAAiG;AACjG,eAAeG,mBAAmBC,IAAgB,EAAEC,QAAgB;IAClE,MAAMD,KAAKX,IAAI,CAAC,CAAC,0IAA0I,EAAEY,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeC,mBAAmBF,IAAgB,EAAEhB,GAAW,EAAEiB,QAAgB;IAC/E,MAAMD,KAAKX,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,wFAAwF;IACxF,4FAA4F;IAC5F,MAAMU,mBAAmBC,MAAMC;IAC/B,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMD,KAAKX,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMW,KAAKX,IAAI,CAAC;IAChB,KAAK,MAAMc,WAAWvC,eAAeoB,KAAM,MAAMmB,QAAQC,MAAM,CAACJ;IAChE,IAAI,AAAC,MAAM/B,QAAQ+B,MAAM,sBAAuB,MAAM,MAAM9B,QAAQ8B,MAAM,kBAAkBpB;IAC5F,IAAI,AAAC,MAAMX,QAAQ+B,MAAM,gBAAiB,MAAM,MAAM9B,QAAQ8B,MAAM,YAAYtC,iBAAiBsB,KAAKnB;AACxG;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASwC,cAActB,EAAgB;IACrC,MAAMuB,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFzB,GAAGM,IAAI,CAAC;YACR;QACF,EAAE,OAAOC,KAAK;YACZ,IAAIiB,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BG,IAAI,CAAC,AAACnB,IAAcC,OAAO,GAAG,MAAMD;YAC9FoB,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,eAAeC,QAAQC,MAAc,EAAE/B,GAAmB;IACxD,MAAMD,KAAK,IAAIvB,aAAauD;IAC5B,IAAI;QACF,6FAA6F;QAC7F,yFAAyF;QACzFhC,GAAGM,IAAI,CAAC;QACR,yFAAyF;QACzF,wEAAwE;QACxEgB,cAActB;QACdN,kBAAkBM,IAAItB,gBAAgBuB,SAASE;QAC/C,MAAMc,OAAO3B,iBAAiBU;QAC9B,2FAA2F;QAC3F,8FAA8F;QAC9F,MAAMkB,WAAWnB,gBAAgBC,IAAIC;QACrC,OAAO;YAAEgC,QAAQ;gBAAEjC;gBAAIkB;YAAS;YAAGD;QAAK;IAC1C,EAAE,OAAOV,KAAK;QACZ,4FAA4F;QAC5F,0CAA0C;QAC1CP,GAAGkC,KAAK;QACR,MAAM3B;IACR;AACF;AAEA,eAAe2B,MAAMD,MAAoB;IACvCA,OAAOjC,EAAE,CAACkC,KAAK;AACjB;AAEA,6FAA6F;AAC7F,gGAAgG;AAChG,eAAeC,eAAeF,MAAoB,EAAEhB,IAAgB,EAAEmB,IAAY,EAAEC,WAAwB;IAC1G,IAAI,CAAEA,CAAAA,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,WAAU,GAAI,OAAO;IACrEC,QAAQC,KAAK,CAAC;IACd,2FAA2F;IAC3F,yFAAyF;IACzF,MAAMpD,gBACJ4B,MACA;QACE,MAAMA,KAAKX,IAAI,CAAC;QAChB,MAAMU,mBAAmBC,MAAMgB,OAAOf,QAAQ;IAChD,GACA9B;IAEF,OAAO;AACT;AAEA,oGAAoG;AACpG,qIAAqI;AACrI,eAAesD,kBAAkBC,OAAqB,EAAE1B,IAAgB,EAAEhB,GAAW,EAAE2C,OAAe,EAAEC,YAAoB;IAC1H,MAAMC,OAAO,MAAM7B,KAAKN,OAAO,CAAC;IAChC,MAAMoC,QAAQ,IAAIC,IAAI,AAAE,CAAA,MAAMF,KAAKG,GAAG,EAAC,EAA+BC,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACvF,MAAMC,QAAQtE,UAAUkB,KAAK2C,SAASU,MAAM,CAAC,CAACC,IAAMR,MAAMR,GAAG,CAACgB,EAAEC,OAAO;IACvE,MAAMC,aAAa/E,gBAAgBuB,SAASE;IAC5C,MAAMuD,WAAqB,EAAE;IAC7B,MAAMC,OAAoB,EAAE;IAC5B,KAAK,MAAMC,QAAQP,MAAO;QACxB,MAAM,EAAEQ,GAAG,EAAEH,UAAUI,YAAY,EAAE,GAAG9E,UAAU4E;QAClDF,SAASK,IAAI,IAAID;QACjBH,KAAKI,IAAI,CAACxE,WAAWsE,KAAKJ;IAC5B;IACA,MAAMpE,gBACJ4B,MACA;QACE,IAAI0C,KAAKK,MAAM,GAAG,GAAG,MAAM/C,KAAKgD,QAAQ,CAACzE,oBAAoBmE;IAC/D,GACAvE;IAEF,MAAMD,QAAQ8B,MAAM,YAAY4B;IAChC,OAAOa;AACT;AAEA,6FAA6F;AAC7F,8DAA8D;AAC9D,SAASQ,mBAAmBjC,MAAoB;IAC9C,MAAMkC,SAAS1D,eAAewB,OAAOjC,EAAE;IACvC,IAAImE,WAAW,QAAQA,WAAWlC,OAAOf,QAAQ,EAAE,OAAO;IAC1D,OAAO;AACT;AAEA,eAAekD,aAAanC,MAAoB,EAAEhB,IAAgB,EAAEhB,GAAW;IAC7E,+FAA+F;IAC/F,+EAA+E;IAC/E,MAAMZ,gBAAgB4B,MAAM,IAAME,mBAAmBF,MAAMhB,KAAKgC,OAAOf,QAAQ,GAAG9B;AACpF;AAEA,eAAeiF,sBAAsBpC,MAAoB,EAAEqC,KAAiB,EAAEC,EAAU;IACtFtC,OAAOjC,EAAE,CAACM,IAAI,CAAC,CAAC,sBAAsB,EAAEiE,IAAI;AAC9C;AAEA,+FAA+F;AAC/F,OAAO,MAAMC,oBAA+C;IAC1DC,UAAU7E;IACV8E,eAAe7E;IACf8E,kBAAkBlF;IAClBsC;IACAG;IACAC;IACAO;IACAwB;IACAE;IACAC;IACA1E,aAAa,CAACsC,QAAQhB,MAAMhB,MAAQN,YAAYsC,OAAOjC,EAAE,EAAEiB,MAAMhB;AACnE,EAAE;AAEF,OAAO,eAAe2E,WAAW3E,GAAmB;IAClD,OAAOhB,gBAAgBgB,KAAKuE;AAC9B"}

package/dist/esm/store/sqlite/store.d.ts

@@ -2,4 +2,4 @@ import type { DatabaseSync } from 'node:sqlite';
 import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store;
+export declare function createStore(db: DatabaseSync, conn: Connection, cfg: Config): Store;

package/dist/esm/store/sqlite/store.js

@@ -1,11 +1,9 @@
 import { contentTokenize } from '../../config/index.js';
-import { reconcile } from '../reconcile.js';
 import { getColumns } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { hasVectorRow, pendingRows } from '../vectors.js';
 import { fieldStats } from './fieldStats.js';
 import { queryLexical } from './lexical.js';
-import { sqliteDialect } from './reconcile.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
 export const CAPABILITIES = new Set([
     'phrases',
@@ -16,8 +14,8 @@ export const CAPABILITIES = new Set([
     'sql-functions'
 ]);
 // Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one
-// Connection instance (conn) with open()'s own reconcile call so transaction depth is tracked against the same object everywhere.
-export function createStore(db, conn, cfg, baseDir) {
+// Connection instance (conn) with the builder's own reconcile call so transaction depth is tracked against the same object everywhere.
+export function createStore(db, conn, cfg) {
     return {
         name: 'sqlite',
         capabilities: CAPABILITIES,
@@ -33,9 +31,6 @@ export function createStore(db, conn, cfg, baseDir) {
         async transaction (fn) {
             return withTransaction(conn, fn);
         },
-        async reconcile () {
-            return reconcile(conn, cfg, baseDir, sqliteDialect);
-        },
         docs: {
             async columns () {
                 return [

package/dist/esm/store/sqlite/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with open()'s own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir, sqliteDialect);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["contentTokenize","reconcile","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","sqliteDialect","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","undefined","vectors","pending","writeVectors","rows","candidates","qv","storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":"AAEA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB,EAAE;AAElJ,0FAA0F;AAC1F,kIAAkI;AAClI,OAAO,SAASC,YAAYC,EAAgB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC1F,OAAO;QACLC,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAMN,KAAKK,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAON,KAAKO,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMT,KAAKQ,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBa,MAAMW;QAC/B;QACA,MAAM1B;YACJ,OAAOA,UAAUe,MAAMC,KAAKC,SAASV;QACvC;QACAoB,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM3B,WAAWc;iBAAO;YACtC;YACA,MAAMV,YAAWuB,OAAO,EAAEC,UAAU;gBAClC,OAAOxB,WAAWU,MAAMa,SAASC;YACnC;QACF;QACAC,SAAS;YACP,MAAMC,OAAMC,KAAK,EAAEC,IAAI;gBACrB,OAAO3B,aAAaS,MAAMiB,OAAOC,MAAMlC,gBAAgBiB,SAASkB;YAClE;QACF;QACAC,SAAS;YACP,MAAMC;gBACJ,OAAOhC,YAAYW;YACrB;YACA,MAAMsB,cAAaC,IAAsB;gBACvC,MAAM5B,iBAAiBK,MAAMuB;YAC/B;YACA,MAAMC,YAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;gBAC5C,OAAOnC,eAAeO,MAAMyB,IAAIC,WAAWC,OAAOC;YACpD;YACA,MAAMC,SAAQC,IAAI,EAAEZ,IAAI;gBACtB,OAAOxB,YAAYM,MAAM8B,MAAMZ;YACjC;YACA,MAAMa,WAAUD,IAAI;gBAClB,OAAO1C,aAAaY,MAAM8B;YAC5B;QACF;QACA,MAAME;YACJ,qFAAqF;YACrF,qFAAqF;YACrF,MAAMC,MAAMlC,GAAGQ,OAAO,CAAC,uBAAuB2B,GAAG;YACjD,OAAO;gBAAEC,cAAc,GAAGF,IAAIG,OAAO,CAAC,kFAAkF,CAAC;YAAC;QAC5H;QACAC,KAAK;YACH,MAAM9B,SAAQD,GAAW;gBACvB,MAAMgC,OAAOvC,GAAGQ,OAAO,CAACD;gBACxBgC,KAAKC,cAAc,CAAC,OAAO,qEAAqE;gBAChG,OAAO;oBACL1B,SAAS,IAAMyB,KAAKzB,OAAO;oBAC3B,2EAA2E;oBAC3E2B,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAKC;oBAC1B;gBACF;YACF;QACF;QACA,MAAMC;YACJ3C,GAAG2C,KAAK;QACV;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with the builder's own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["contentTokenize","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","cfg","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","undefined","vectors","pending","writeVectors","rows","candidates","qv","storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":"AAEA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB,EAAE;AAElJ,0FAA0F;AAC1F,uIAAuI;AACvI,OAAO,SAASC,YAAYC,EAAgB,EAAEC,IAAgB,EAAEC,GAAW;IACzE,OAAO;QACLC,MAAM;QACNC,cAAcP;QACd,MAAMQ,MAAKC,GAAW;YACpB,MAAML,KAAKI,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOL,KAAKM,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMR,KAAKO,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOtB,gBAAgBY,MAAMU;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMzB,WAAWa;iBAAO;YACtC;YACA,MAAMT,YAAWqB,OAAO,EAAEC,UAAU;gBAClC,OAAOtB,WAAWS,MAAMY,SAASC;YACnC;QACF;QACAC,SAAS;YACP,MAAMC,OAAMC,KAAK,EAAEC,IAAI;gBACrB,OAAOzB,aAAaQ,MAAMgB,OAAOC,MAAM/B,gBAAgBe,SAASiB;YAClE;QACF;QACAC,SAAS;YACP,MAAMC;gBACJ,OAAO9B,YAAYU;YACrB;YACA,MAAMqB,cAAaC,IAAsB;gBACvC,MAAM3B,iBAAiBK,MAAMsB;YAC/B;YACA,MAAMC,YAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;gBAC5C,OAAOlC,eAAeO,MAAMwB,IAAIC,WAAWC,OAAOC;YACpD;YACA,MAAMC,SAAQC,IAAI,EAAEZ,IAAI;gBACtB,OAAOvB,YAAYM,MAAM6B,MAAMZ;YACjC;YACA,MAAMa,WAAUD,IAAI;gBAClB,OAAOxC,aAAaW,MAAM6B;YAC5B;QACF;QACA,MAAME;YACJ,qFAAqF;YACrF,qFAAqF;YACrF,MAAMC,MAAMjC,GAAGO,OAAO,CAAC,uBAAuB2B,GAAG;YACjD,OAAO;gBAAEC,cAAc,GAAGF,IAAIG,OAAO,CAAC,kFAAkF,CAAC;YAAC;QAC5H;QACAC,KAAK;YACH,MAAM9B,SAAQD,GAAW;gBACvB,MAAMgC,OAAOtC,GAAGO,OAAO,CAACD;gBACxBgC,KAAKC,cAAc,CAAC,OAAO,qEAAqE;gBAChG,OAAO;oBACL1B,SAAS,IAAMyB,KAAKzB,OAAO;oBAC3B,2EAA2E;oBAC3E2B,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAKC;oBAC1B;gBACF;YACF;QACF;QACA,MAAMC;YACJ1C,GAAG0C,KAAK;QACV;IACF;AACF"}

package/dist/esm/store/turso/open.d.ts

@@ -1,14 +1,9 @@
+import type { Database } from '@tursodatabase/database';
 import type { ResolvedConfig } from '../../config/index.js';
-import type { Store } from '../types.js';
+import type { OpenResult } from '../open.js';
+import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "3";
-export interface OpenResult {
-    store: Store;
-    cfg: ResolvedConfig;
-    dbPath: string;
-    parsed: number;
-    warnings: string[];
-}
-export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
-export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
+export declare const SCHEMA_VERSION = "4";
+export type { OpenResult };
+export declare const tursoOpenDialect: OpenDialect<Database>;
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/esm/store/turso/open.js

@@ -1,34 +1,20 @@
-import { mkdirSync } from 'node:fs';
-import { join } from 'node:path';
-import { featureSignature, STATE_DIR } from '../../config/index.js';
-import { rekeyChunkText } from '../../embed/handoff.js';
+import { featureSignature } from '../../config/index.js';
 import { STORE_DIMS } from '../../embed/types.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
-import { clearCache } from '../cache.js';
-import { reconcile } from '../reconcile.js';
+import { openWithDialect } from '../open.js';
 import { getMeta, setMeta } from '../shared.js';
 import { createConnection } from './connection.js';
 import { TURSO_PACKAGE, tursoApi } from './native.js';
-import { tursoDialect } from './reconcile.js';
+import { CONTENT_FTS_DDL, tursoDialect } from './reconcile.js';
 import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.turso.db';
 // Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves
 // separately. Covers the FTS indexes, the "_ngram" sidecar columns, and embeddings.vector's width.
-export const SCHEMA_VERSION = '3';
-// The ngram index is scoped to disjoint "_ngram" sidecar columns: a second index over the same
-// columns makes a bare substring match a whole word, defeating the prefix-query rejection.
-// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:
-// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.
-export const CONTENT_FTS_DDL = [
-    `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,
-    `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`
-];
-export const CONTENT_FTS_NAMES = [
-    'content_fts',
-    'content_fts_ngram'
-];
-async function ensureSchema(conn, cfg) {
+export const SCHEMA_VERSION = '4';
+// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no
+// extra native state to thread, unlike duckdb's separate instance/connection pair.
+async function ensureSchema(_handle, conn, cfg) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_ctime" REAL, "_size" INTEGER, "_parse_error" TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);
     for (const ddl of CONTENT_FTS_DDL)await conn.exec(ddl);
@@ -46,12 +32,13 @@ async function ensureSchema(conn, cfg) {
     if (await getMeta(conn, 'schema_version') === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);
     if (await getMeta(conn, 'features') === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));
 }
-async function connect(cfg) {
-    const stateDir = join(cfg.baseDir, STATE_DIR);
-    mkdirSync(stateDir, {
-        recursive: true
-    });
-    const dbPath = join(stateDir, DB_FILENAME);
+async function close(handle) {
+    await handle.close();
+}
+async function setDerivedBusyTimeout(_handle, conn, ms) {
+    await conn.exec(`PRAGMA busy_timeout = ${ms}`);
+}
+async function connect(dbPath, _cfg) {
     // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this
     // optional dependency until a turso tree is actually opened. Installed on first use if missing.
     let turso;
@@ -74,57 +61,22 @@ async function connect(cfg) {
     } catch (err) {
         throw new SenseError('STORE_DEPENDENCY_MISSING', `store "turso" failed to open ${dbPath}: ${err.message}`);
     }
-    // A throw below must release this handle, or the leaked WAL makes the db file undeletable on
-    // Windows. closed guards the catch against double-closing.
-    let closed = false;
-    try {
-        var _ref;
-        const conn = createConnection(db);
-        await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
-        const version = await getMeta(conn, 'schema_version');
-        const features = await getMeta(conn, 'features');
-        const wantFeatures = featureSignature(cfg, FEATURES);
-        if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
-            // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched
-            // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.
-            const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
-            console.error(`sense: ${reason}; rebuilding the index`);
-            closed = true;
-            await db.close();
-            clearCache(cfg);
-            return connect(cfg);
-        }
-        await ensureSchema(conn, cfg);
-        // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.
-        // Installed before reconcile() -- that call is the one that races a watcher's transaction.
-        const recordedMaxMs = Number((_ref = await getMeta(conn, 'reconcile_max_ms')) !== null && _ref !== void 0 ? _ref : '0');
-        await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600000)}`);
-        const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, tursoDialect);
-        return {
-            db,
-            conn,
-            cfg,
-            dbPath,
-            parsed,
-            warnings
-        };
-    } catch (err) {
-        if (!closed) await db.close();
-        throw err;
-    }
-}
-// The turso store's open: connects (fully async, see connect() above), then wraps the resulting
-// connection in the Store interface.
-export async function openTurso(cfg) {
-    const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);
-    const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);
-    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
-    rekeyChunkText(conn, store);
     return {
-        store: store,
-        cfg: resolvedCfg,
-        dbPath,
-        parsed,
-        warnings
+        handle: db,
+        conn: createConnection(db)
     };
 }
+// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.
+export const tursoOpenDialect = {
+    filename: DB_FILENAME,
+    schemaVersion: SCHEMA_VERSION,
+    reconcileDialect: tursoDialect,
+    connect,
+    close,
+    ensureSchema,
+    setDerivedBusyTimeout,
+    createStore: (handle, conn)=>createStore(handle, conn)
+};
+export async function openTurso(cfg) {
+    return openWithDialect(cfg, tursoOpenDialect);
+}

package/dist/esm/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:\n// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n  db: Database;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below must release this handle, or the leaked WAL makes the db file undeletable on\n  // Windows. closed guards the catch against double-closing.\n  let closed = false;\n  try {\n    const conn = createConnection(db);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched\n      // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      closed = true;\n      await db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, tursoDialect);\n\n    return { db, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) await db.close();\n    throw err;\n  }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","STORE_DIMS","SenseError","activeFeatures","FEATURES","clearCache","reconcile","getMeta","setMeta","createConnection","TURSO_PACKAGE","tursoApi","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ensureSchema","conn","cfg","exec","ddl","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","turso","err","message","db","timeout","experimental","closed","version","features","wantFeatures","reason","console","error","close","recordedMaxMs","Number","Math","min","max","parsed","warnings","openTurso","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,YAAY,QAAQ,iBAAiB;AAC9C,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,+FAA+F;AAC/F,2FAA2F;AAC3F,gGAAgG;AAChG,0FAA0F;AAC1F,OAAO,MAAMC,kBAAkB;IAC7B,CAAC,qIAAqI,CAAC;IACvI,CAAC,gMAAgM,CAAC;CACnM,CAAU;AACX,OAAO,MAAMC,oBAAoB;IAAC;IAAe;CAAoB,CAAU;AAE/E,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAON,gBAAiB,MAAMG,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWpB,eAAeiB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEpB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMsB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBJ;IAC5F,IAAI,AAAC,MAAMR,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYrB,iBAAiBsB,KAAKhB;AACxG;AAWA,eAAesB,QAAQN,GAAmB;IACxC,MAAMO,WAAW9B,KAAKuB,IAAIQ,OAAO,EAAE7B;IACnCH,UAAU+B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAASjC,KAAK8B,UAAUb;IAE9B,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIiB;IACJ,IAAI;QACFA,QAAQ,MAAMpB;IAChB,EAAE,OAAOqB,KAAK;QACZ,IAAIA,eAAe9B,YAAY,MAAM8B;QACrC,MAAM,IAAI9B,WAAW,4BAA4B,CAAC,wBAAwB,EAAEQ,cAAc,UAAU,EAAE,AAACsB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAML,OAAO,CAACI,QAAQ;YAAEK,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAI9B,WAAW,4BAA4B,CAAC,6BAA6B,EAAE4B,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACtH;IACA,6FAA6F;IAC7F,2DAA2D;IAC3D,IAAII,SAAS;IACb,IAAI;YAuB4B;QAtB9B,MAAMlB,OAAOV,iBAAiByB;QAE9B,MAAMf,KAAKE,IAAI,CAAC;QAEhB,MAAMiB,UAAU,MAAM/B,QAAQY,MAAM;QACpC,MAAMoB,WAAW,MAAMhC,QAAQY,MAAM;QACrC,MAAMqB,eAAe1C,iBAAiBsB,KAAKhB;QAC3C,IAAI,AAACkC,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,cAAe;YACxG,yFAAyF;YACzF,iGAAiG;YACjG,MAAMC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;YACnH2B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtDJ,SAAS;YACT,MAAMH,GAAGU,KAAK;YACdvC,WAAWe;YACX,OAAOM,QAAQN;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMyB,gBAAgBC,QAAQ,OAAA,MAAMvC,QAAQY,MAAM,iCAApB,kBAAA,OAA4C;QAC1E,MAAMA,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAE0B,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAEhG,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAM7C,UAAUa,MAAMC,KAAKA,IAAIQ,OAAO,EAAEhB;QAErE,OAAO;YAAEsB;YAAIf;YAAMC;YAAKU;YAAQoB;YAAQC;QAAS;IACnD,EAAE,OAAOnB,KAAK;QACZ,IAAI,CAACK,QAAQ,MAAMH,GAAGU,KAAK;QAC3B,MAAMZ;IACR;AACF;AAEA,gGAAgG;AAChG,qCAAqC;AACrC,OAAO,eAAeoB,UAAUhC,GAAmB;IACjD,MAAM,EAAEc,EAAE,EAAEf,IAAI,EAAEC,KAAKiC,WAAW,EAAEvB,MAAM,EAAEoB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzB,QAAQN;IAC/E,MAAMkC,QAAQzC,YAAYqB,IAAIf,MAAMkC,aAAaA,YAAYzB,OAAO;IACpE,0FAA0F;IAC1F5B,eAAemB,MAAMmC;IACrB,OAAO;QAAEA,OAAOA;QAAOlC,KAAKiC;QAAavB;QAAQoB;QAAQC;IAAS;AACpE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","TURSO_PACKAGE","tursoApi","CONTENT_FTS_DDL","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","ddl","feature","name","schema","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","message","db","timeout","experimental","tursoOpenDialect","filename","schemaVersion","reconcileDialect","openTurso"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,eAAe,EAAEC,YAAY,QAAQ,iBAAiB;AAC/D,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAIlC,4FAA4F;AAC5F,mFAAmF;AACnF,eAAeC,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;IAC1E,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAOV,gBAAiB,MAAMO,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWnB,eAAegB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEnB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMqB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKf;AACxG;AAEA,eAAeqB,MAAMC,MAAgB;IACnC,MAAMA,OAAOD,KAAK;AACpB;AAEA,eAAeE,sBAAsBV,OAAiB,EAAEC,IAAgB,EAAEU,EAAU;IAClF,MAAMV,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAEQ,IAAI;AAC/C;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIC;IACJ,IAAI;QACFA,QAAQ,MAAMtB;IAChB,EAAE,OAAOuB,KAAK;QACZ,IAAIA,eAAe/B,YAAY,MAAM+B;QACrC,MAAM,IAAI/B,WAAW,4BAA4B,CAAC,wBAAwB,EAAEO,cAAc,UAAU,EAAE,AAACwB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAMH,OAAO,CAACC,QAAQ;YAAEM,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAI/B,WAAW,4BAA4B,CAAC,6BAA6B,EAAE4B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACtH;IACA,OAAO;QAAER,QAAQS;QAAIjB,MAAMV,iBAAiB2B;IAAI;AAClD;AAEA,+FAA+F;AAC/F,OAAO,MAAMG,mBAA0C;IACrDC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB7B;IAClBiB;IACAJ;IACAT;IACAW;IACAd,aAAa,CAACa,QAAQR,OAASL,YAAYa,QAAQR;AACrD,EAAE;AAEF,OAAO,eAAewB,UAAUvB,GAAmB;IACjD,OAAOd,gBAAgBc,KAAKmB;AAC9B"}