sensemaking 0.23.1 → 0.24.0

package/dist/esm/store/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../errors.ts';\n\n// A store's optional native package, and the strings its error messages and install notice name.\nexport interface NativeDescriptor {\n  store: string;\n  pkg: string;\n  sizeHint: string;\n}\n\n// install-module-linked is small, but a target native package can be large (@duckdb/node-api's\n// ~110MB), so it is required lazily inside the gated branch below, not for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory rather than a fixed dot-count: it sits at a\n// different depth under src/ than under built dist/{cjs,esm}/store output.\nexport function packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install a native dependency into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs: since the install is async, concurrent callers await the same\n// in-flight promise, keyed by specifier and target. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's resolution, including a \"not found\" miss, for the process\n// lifetime; importing the entry by an absolute file: URL instead of the bare name sidesteps that.\nfunction resolveEntryUrl(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  return pathToFileURL(join(pkgDir, entry)).href;\n}\n\n// Import-then-install-then-retry, over an injectable importName so the failure path can be\n// exercised in tests without a real native binding. Thrown errors always name descriptor.pkg.\nexport async function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName: string = descriptor.pkg): Promise<T> {\n  try {\n    return (await import(importName)) as T;\n  } catch {\n    console.error(`sense: store \"${descriptor.store}\" needs ${descriptor.pkg}; installing it now (one-time download, ${descriptor.sizeHint})...`);\n    try {\n      await installOnce(importName, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" needs ${descriptor.pkg}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${descriptor.pkg}\\` yourself and try again`\n      );\n    }\n    try {\n      return (await import(resolveEntryUrl(importName, nodeModulesPath))) as T;\n    } catch (loadErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" installed ${descriptor.pkg} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${descriptor.pkg}\\` to see the underlying error`\n      );\n    }\n  }\n}\n"],"names":["existsSync","readFileSync","Module","dirname","join","fileURLToPath","pathToFileURL","SenseError","_require","require","createRequire","url","_dirname","__filename","packageNodeModules","dir","candidate","pkg","JSON","parse","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntryUrl","importName","pkgDir","split","entry","exports","main","href","loadOrInstall","descriptor","console","error","store","sizeHint","installErr","message","loadErr","process","platform","arch"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAC1C,SAASC,aAAa,EAAEC,aAAa,QAAQ,WAAW;AACxD,SAASC,UAAU,QAAQ,eAAe;AAS1C,+FAA+F;AAC/F,sGAAsG;AACtG,MAAMC,WAAW,OAAOC,YAAY,cAAcP,OAAOQ,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAC1F,MAAMG,WAAWT,QAAQ,OAAOU,eAAe,cAAcA,aAAaR,cAAc,YAAYM,GAAG;AAEvG,sFAAsF;AACtF,2EAA2E;AAC3E,OAAO,SAASG;IACd,IAAIC,MAAMH;IACV,OAAS;QACP,MAAMI,YAAYZ,KAAKW,KAAK;QAC5B,IAAIf,WAAWgB,YAAY;YACzB,MAAMC,MAAMC,KAAKC,KAAK,CAAClB,aAAae,WAAW;YAC/C,IAAIC,IAAIG,IAAI,KAAK,eAAe,OAAOhB,KAAKW,KAAK;QACnD;QACA,MAAMM,SAASlB,QAAQY;QACvB,IAAIM,WAAWN,KAAK,MAAM,IAAIO,MAAM;QACpCP,MAAMM;IACR;AACF;AAEA,MAAME,aAAa,IAAIC;AAEvB,8FAA8F;AAC9F,6GAA6G;AAC7G,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,MAAMC,MAAM,GAAGF,UAAU,EAAE,EAAEC,iBAAiB;IAC9C,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,MAAME,sBAAsB,AAACvB,SAAS,yBAA0GwB,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,KAAO,GACP,CAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,2FAA2F;AAC3F,kGAAkG;AAClG,SAASQ,gBAAgBC,UAAkB,EAAEX,eAAuB;QAGHV;IAF/D,MAAMsB,SAASnC,KAAKuB,oBAAoBW,WAAWE,KAAK,CAAC;IACzD,MAAMvB,MAAMC,KAAKC,KAAK,CAAClB,aAAaG,KAAKmC,QAAQ,iBAAiB;IAClE,MAAME,QAAQ,OAAOxB,IAAIyB,OAAO,KAAK,WAAWzB,IAAIyB,OAAO,IAAIzB,YAAAA,IAAI0B,IAAI,cAAR1B,uBAAAA,YAAY;IAC3E,OAAOX,cAAcF,KAAKmC,QAAQE,QAAQG,IAAI;AAChD;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,OAAO,eAAeC,cAAiBC,UAA4B,EAAEnB,eAAuB,EAAEW,aAAqBQ,WAAW7B,GAAG;IAC/H,IAAI;QACF,OAAQ,MAAM,MAAM,CAACqB;IACvB,EAAE,OAAM;QACNS,QAAQC,KAAK,CAAC,CAAC,cAAc,EAAEF,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW7B,GAAG,CAAC,wCAAwC,EAAE6B,WAAWI,QAAQ,CAAC,IAAI,CAAC;QAC5I,IAAI;YACF,MAAMzB,YAAYa,YAAYX;QAChC,EAAE,OAAOwB,YAAY;YACnB,MAAM,IAAI5C,WACR,4BACA,CAAC,OAAO,EAAEuC,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW7B,GAAG,CAAC,0CAA0C,EAAE,AAACkC,WAAqBC,OAAO,CAAC,sKAAsK,EAAEN,WAAW7B,GAAG,CAAC,yBAAyB,CAAC;QAEnV;QACA,IAAI;YACF,OAAQ,MAAM,MAAM,CAACoB,gBAAgBC,YAAYX;QACnD,EAAE,OAAO0B,SAAS;YAChB,MAAM,IAAI9C,WACR,4BACA,CAAC,OAAO,EAAEuC,WAAWG,KAAK,CAAC,YAAY,EAAEH,WAAW7B,GAAG,CAAC,6BAA6B,EAAE,AAACoC,QAAkBD,OAAO,CAAC,8CAA8C,EAAEE,QAAQC,QAAQ,CAAC,CAAC,EAAED,QAAQE,IAAI,CAAC,uBAAuB,EAAEV,WAAW7B,GAAG,CAAC,8BAA8B,CAAC;QAE9Q;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../errors.ts';\n\n// A store's optional native package, and the strings its error messages and install notice name.\nexport interface NativeDescriptor {\n  store: string;\n  pkg: string;\n  sizeHint: string;\n}\n\n// install-module-linked is small, but a target native package can be large (@duckdb/node-api's\n// ~110MB), so it is required lazily inside the gated branch below, not for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory rather than a fixed dot-count: it sits at a\n// different depth under src/ than under built dist/{cjs,esm}/store output.\nexport function packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install a native dependency into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs: since the install is async, concurrent callers await the same\n// in-flight promise, keyed by specifier and target. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's resolution, including a \"not found\" miss, for the process\n// lifetime; use the installed entry directly, as a URL for ESM and a path for CJS.\nfunction resolveEntrySpecifier(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  const entryPath = join(pkgDir, entry);\n  return typeof require === 'undefined' ? pathToFileURL(entryPath).href : entryPath;\n}\n\n// Import-then-install-then-retry, over an injectable importName so the failure path can be\n// exercised in tests without a real native binding. Thrown errors always name descriptor.pkg.\nexport async function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName: string = descriptor.pkg): Promise<T> {\n  try {\n    return (await import(importName)) as T;\n  } catch {\n    console.error(`sense: store \"${descriptor.store}\" needs ${descriptor.pkg}; installing it now (one-time download, ${descriptor.sizeHint})...`);\n    try {\n      await installOnce(importName, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" needs ${descriptor.pkg}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${descriptor.pkg}\\` yourself and try again`\n      );\n    }\n    try {\n      return (await import(resolveEntrySpecifier(importName, nodeModulesPath))) as T;\n    } catch (loadErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" installed ${descriptor.pkg} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${descriptor.pkg}\\` to see the underlying error`\n      );\n    }\n  }\n}\n"],"names":["existsSync","readFileSync","Module","dirname","join","fileURLToPath","pathToFileURL","SenseError","_require","require","createRequire","url","_dirname","__filename","packageNodeModules","dir","candidate","pkg","JSON","parse","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntrySpecifier","importName","pkgDir","split","entry","exports","main","entryPath","href","loadOrInstall","descriptor","console","error","store","sizeHint","installErr","message","loadErr","process","platform","arch"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAC1C,SAASC,aAAa,EAAEC,aAAa,QAAQ,WAAW;AACxD,SAASC,UAAU,QAAQ,eAAe;AAS1C,+FAA+F;AAC/F,sGAAsG;AACtG,MAAMC,WAAW,OAAOC,YAAY,cAAcP,OAAOQ,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAC1F,MAAMG,WAAWT,QAAQ,OAAOU,eAAe,cAAcA,aAAaR,cAAc,YAAYM,GAAG;AAEvG,sFAAsF;AACtF,2EAA2E;AAC3E,OAAO,SAASG;IACd,IAAIC,MAAMH;IACV,OAAS;QACP,MAAMI,YAAYZ,KAAKW,KAAK;QAC5B,IAAIf,WAAWgB,YAAY;YACzB,MAAMC,MAAMC,KAAKC,KAAK,CAAClB,aAAae,WAAW;YAC/C,IAAIC,IAAIG,IAAI,KAAK,eAAe,OAAOhB,KAAKW,KAAK;QACnD;QACA,MAAMM,SAASlB,QAAQY;QACvB,IAAIM,WAAWN,KAAK,MAAM,IAAIO,MAAM;QACpCP,MAAMM;IACR;AACF;AAEA,MAAME,aAAa,IAAIC;AAEvB,8FAA8F;AAC9F,6GAA6G;AAC7G,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,MAAMC,MAAM,GAAGF,UAAU,EAAE,EAAEC,iBAAiB;IAC9C,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,MAAME,sBAAsB,AAACvB,SAAS,yBAA0GwB,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,KAAO,GACP,CAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,2FAA2F;AAC3F,mFAAmF;AACnF,SAASQ,sBAAsBC,UAAkB,EAAEX,eAAuB;QAGTV;IAF/D,MAAMsB,SAASnC,KAAKuB,oBAAoBW,WAAWE,KAAK,CAAC;IACzD,MAAMvB,MAAMC,KAAKC,KAAK,CAAClB,aAAaG,KAAKmC,QAAQ,iBAAiB;IAClE,MAAME,QAAQ,OAAOxB,IAAIyB,OAAO,KAAK,WAAWzB,IAAIyB,OAAO,IAAIzB,YAAAA,IAAI0B,IAAI,cAAR1B,uBAAAA,YAAY;IAC3E,MAAM2B,YAAYxC,KAAKmC,QAAQE;IAC/B,OAAO,OAAOhC,YAAY,cAAcH,cAAcsC,WAAWC,IAAI,GAAGD;AAC1E;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,OAAO,eAAeE,cAAiBC,UAA4B,EAAEpB,eAAuB,EAAEW,aAAqBS,WAAW9B,GAAG;IAC/H,IAAI;QACF,OAAQ,MAAM,MAAM,CAACqB;IACvB,EAAE,OAAM;QACNU,QAAQC,KAAK,CAAC,CAAC,cAAc,EAAEF,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW9B,GAAG,CAAC,wCAAwC,EAAE8B,WAAWI,QAAQ,CAAC,IAAI,CAAC;QAC5I,IAAI;YACF,MAAM1B,YAAYa,YAAYX;QAChC,EAAE,OAAOyB,YAAY;YACnB,MAAM,IAAI7C,WACR,4BACA,CAAC,OAAO,EAAEwC,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW9B,GAAG,CAAC,0CAA0C,EAAE,AAACmC,WAAqBC,OAAO,CAAC,sKAAsK,EAAEN,WAAW9B,GAAG,CAAC,yBAAyB,CAAC;QAEnV;QACA,IAAI;YACF,OAAQ,MAAM,MAAM,CAACoB,sBAAsBC,YAAYX;QACzD,EAAE,OAAO2B,SAAS;YAChB,MAAM,IAAI/C,WACR,4BACA,CAAC,OAAO,EAAEwC,WAAWG,KAAK,CAAC,YAAY,EAAEH,WAAW9B,GAAG,CAAC,6BAA6B,EAAE,AAACqC,QAAkBD,OAAO,CAAC,8CAA8C,EAAEE,QAAQC,QAAQ,CAAC,CAAC,EAAED,QAAQE,IAAI,CAAC,uBAAuB,EAAEV,WAAW9B,GAAG,CAAC,8BAA8B,CAAC;QAE9Q;IACF;AACF"}

package/dist/esm/store/open.js

@@ -1,3 +1,4 @@
+import { channel } from 'node:diagnostics_channel';
 import { mkdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { featureSignature, STATE_DIR } from '../config/index.js';
@@ -15,6 +16,7 @@ import { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './sig
 // duckdb/turso hold the cache file for their connection's whole life, so a second command waits.
 // Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.
 const LOCK_POLL_MS = 50;
+const lockWait = channel('sensemaking.store.lock-wait');
 async function connectUnlocked(dbPath, cfg, dialect) {
     const budgetMs = lockWaitBudgetMs(cfg.baseDir);
     const deadline = Date.now() + budgetMs;
@@ -24,6 +26,11 @@ async function connectUnlocked(dbPath, cfg, dialect) {
         } catch (err) {
             var _dialect_isLocked;
             if (!((_dialect_isLocked = dialect.isLocked) === null || _dialect_isLocked === void 0 ? void 0 : _dialect_isLocked.call(dialect, err))) throw err;
+            // Observe a real rejected native connection, never merely the start of an async open.
+            if (lockWait.hasSubscribers) lockWait.publish({
+                store: cfg.store,
+                dbPath
+            });
             if (Date.now() >= deadline) {
                 throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${Math.round(budgetMs / 1000)}s; wait for that command to finish, or set "store" to "sqlite" in sense.config.json, which serves concurrent commands`);
             }

package/dist/esm/store/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { SenseError } from '../errors.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport { classifyEmbedChange } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.ts';\nimport { lockWaitBudgetMs } from './lock-wait.ts';\nimport { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Stages } from './stages.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Per-stage wall time for this open's build pass (stages.ts), not for any narrow embed or\n  // feature invalidation that ran beside it -- on a cold build there is none, so it is the whole cost.\n  stages: Stages;\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  stages: Stages;\n}\n\n// duckdb/turso hold the cache file for their connection's whole life, so a second command waits.\n// Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.\nconst LOCK_POLL_MS = 50;\n\nasync function connectUnlocked<Handle>(dbPath: string, cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<{ handle: Handle; conn: Connection }> {\n  const budgetMs = lockWaitBudgetMs(cfg.baseDir);\n  const deadline = Date.now() + budgetMs;\n  for (;;) {\n    try {\n      return await dialect.connect(dbPath, cfg);\n    } catch (err) {\n      if (!dialect.isLocked?.(err as Error)) throw err;\n      if (Date.now() >= deadline) {\n        throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${Math.round(budgetMs / 1000)}s; wait for that command to finish, or set \"store\" to \"sqlite\" in sense.config.json, which serves concurrent commands`);\n      }\n      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));\n    }\n  }\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<ConnectResult<Handle>> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await connectUnlocked(dbPath, cfg, dialect);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): invalidate()/invalidateFeatures() below share its pool\n  // with the build() call that follows them.\n  const builder = createBuilder(conn, cfg, cfg.baseDir, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    // Set only by the preset-membership branch below; reconcile treats these paths as touched\n    // despite an unchanged stamp, for this one build() call.\n    let forcedPaths: Set<string> | undefined;\n    // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.\n    let embedInvalidate: EmbedChangeKind | undefined;\n    // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.\n    let featureToggles: FeatureToggle[] | undefined;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed\n      // preset's stored segment can't be parsed back into old include/exclude/vectors -- either\n      // way the branch below is skipped and the full rebuild stays the fallback.\n      const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.baseDir, features ?? '', changedKeys) : null;\n      // Null unless exactly one segment changed and it decomposes into a recognised embed-only\n      // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.\n      const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features ?? '', wantFeatures) : null;\n      // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed\n      // feature's stored segment can't be parsed back into old/new on-off -- either way the branch\n      // below is skipped and the full rebuild stays the fallback.\n      const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features ?? '', wantFeatures, changedKeys) : null;\n      if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (embedKind !== null) {\n        // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither\n        // case needs any file reparsed for any feature but embed itself.\n        embedInvalidate = embedKind;\n        console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (presetForced !== null) {\n        // Membership only: which files a preset covers moved, not a file's own content, a\n        // global feature, or the embed model. reconcile() already knows how to\n        // add, update, and remove a file correctly (every cross-feature cascade included); it\n        // just needs telling which unchanged files to treat as touched (forcedPaths, above).\n        forcedPaths = presetForced;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the files it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (toggles !== null) {\n        // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's\n        // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a\n        // feature turning back on, fully re-derives across every indexed file rather than\n        // trusting rows written before or during that window.\n        featureToggles = toggles;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the feature it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same\n    // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or\n    // feature-toggle-only changed-key set).\n    const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;\n    const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;\n\n    const { parsed, warnings, stages } = await builder.build(forcedPaths);\n    // Every open reconciles once and is done with it -- a watcher opens fresh per event instead of holding this connection (src/watch.ts).\n    await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed: parsed + embedParsed + featureParsed, warnings, stages };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, stages } = await connectWithDialect(cfg, dialect);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, stages };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","SenseError","FEATURES","createBuilder","clearCache","classifyEmbedChange","classifyFeatureToggles","isFeatureOnlyChange","lockWaitBudgetMs","forcedPresetPaths","isPresetOnlyChange","getMeta","setMeta","changedSignatureKeys","embedIdentityAdopted","signatureDiff","LOCK_POLL_MS","connectUnlocked","dbPath","cfg","dialect","budgetMs","baseDir","deadline","Date","now","connect","err","isLocked","store","Math","round","Promise","resolve","setTimeout","connectWithDialect","stateDir","recursive","filename","handle","conn","closed","builder","reconcileDialect","exec","version","features","wantFeatures","forcedPaths","embedInvalidate","featureToggles","schemaVersion","console","error","close","changedKeys","presetForced","embedKind","size","has","toggles","changed","ensureSchema","setDerivedBusyTimeout","recordedMaxMs","Number","min","max","embedParsed","invalidate","parsed","featureParsed","invalidateFeatures","warnings","stages","build","openWithDialect","resolvedCfg","createStore","docCount","stmt","prepare","get","n"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,qBAAqB;AACjE,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,UAAU,QAAQ,aAAa;AAExC,SAASC,mBAAmB,QAAQ,mBAAmB;AAEvD,SAASC,sBAAsB,EAAEC,mBAAmB,QAAQ,qBAAqB;AACjF,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAC1E,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,aAAa,QAAQ,iBAAiB;AA8B3F,iGAAiG;AACjG,oFAAoF;AACpF,MAAMC,eAAe;AAErB,eAAeC,gBAAwBC,MAAc,EAAEC,GAAmB,EAAEC,OAA4B;IACtG,MAAMC,WAAWb,iBAAiBW,IAAIG,OAAO;IAC7C,MAAMC,WAAWC,KAAKC,GAAG,KAAKJ;IAC9B,OAAS;QACP,IAAI;YACF,OAAO,MAAMD,QAAQM,OAAO,CAACR,QAAQC;QACvC,EAAE,OAAOQ,KAAK;gBACPP;YAAL,IAAI,GAACA,oBAAAA,QAAQQ,QAAQ,cAAhBR,wCAAAA,uBAAAA,SAAmBO,OAAe,MAAMA;YAC7C,IAAIH,KAAKC,GAAG,MAAMF,UAAU;gBAC1B,MAAM,IAAItB,WAAW,cAAc,CAAC,2CAA2C,EAAEkB,IAAIU,KAAK,CAAC,QAAQ,EAAEX,OAAO,gCAAgC,EAAEY,KAAKC,KAAK,CAACV,WAAW,MAAM,qHAAqH,CAAC;YAClS;YACA,MAAM,IAAIW,QAAQ,CAACC,UAAYC,WAAWD,SAASjB;QACrD;IACF;AACF;AAEA,eAAemB,mBAA2BhB,GAAmB,EAAEC,OAA4B;IACzF,MAAMgB,WAAWvC,KAAKsB,IAAIG,OAAO,EAAEvB;IACnCH,UAAUwC,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMnB,SAASrB,KAAKuC,UAAUhB,QAAQkB,QAAQ;IAE9C,MAAM,EAAEC,MAAM,EAAEC,IAAI,EAAE,GAAG,MAAMvB,gBAAgBC,QAAQC,KAAKC;IAC5D,gGAAgG;IAChG,kGAAkG;IAClG,IAAIqB,SAAS;IACb,+FAA+F;IAC/F,2CAA2C;IAC3C,MAAMC,UAAUvC,cAAcqC,MAAMrB,KAAKA,IAAIG,OAAO,EAAEF,QAAQuB,gBAAgB;IAC9E,IAAI;QACF,MAAMH,KAAKI,IAAI,CAAC;QAEhB,uFAAuF;QACvF,qGAAqG;QACrG,MAAMC,UAAU,MAAMlC,QAAQ6B,MAAM;QACpC,MAAMM,WAAW,MAAMnC,QAAQ6B,MAAM;QACrC,MAAMO,eAAejD,iBAAiBqB,KAAKjB;QAC3C,0FAA0F;QAC1F,yDAAyD;QACzD,IAAI8C;QACJ,kGAAkG;QAClG,IAAIC;QACJ,4GAA4G;QAC5G,IAAIC;QACJ,IAAI,AAACL,YAAY,QAAQA,YAAYzB,QAAQ+B,aAAa,IAAML,aAAa,QAAQA,aAAaC,cAAe;YAC/G,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAYzB,QAAQ+B,aAAa,EAAE;gBACzDC,QAAQC,KAAK,CAAC;gBACd,kFAAkF;gBAClFZ,SAAS;gBACT,MAAMrB,QAAQkC,KAAK,CAACf;gBACpBnC,WAAWe;gBACX,OAAOgB,mBAAmBhB,KAAKC;YACjC;YACA,MAAMmC,cAAc1C,qBAAqBiC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,oFAAoF;YACpF,0FAA0F;YAC1F,2EAA2E;YAC3E,MAAMS,eAAe9C,mBAAmB6C,eAAe9C,kBAAkBU,KAAKA,IAAIG,OAAO,EAAEwB,qBAAAA,sBAAAA,WAAY,IAAIS,eAAe;YAC1H,yFAAyF;YACzF,oFAAoF;YACpF,MAAME,YAAYF,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,WAAWtD,oBAAoByC,qBAAAA,sBAAAA,WAAY,IAAIC,gBAAgB;YAC3H,4FAA4F;YAC5F,6FAA6F;YAC7F,4DAA4D;YAC5D,MAAMa,UAAUrD,oBAAoBgD,eAAejD,uBAAuBwC,qBAAAA,sBAAAA,WAAY,IAAIC,cAAcQ,eAAe;YACvH,IAAIA,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,YAAY7C,qBAAqBgC,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBAC5G,iFAAiF;gBACjF,sDAAsD;gBACtDK,QAAQC,KAAK,CAAC;gBACd,MAAMzC,QAAQ4B,MAAM,YAAYO;YAClC,OAAO,IAAIU,cAAc,MAAM;gBAC7B,yFAAyF;gBACzF,iEAAiE;gBACjER,kBAAkBQ;gBAClBL,QAAQC,KAAK,CAACI,cAAc,UAAU,6EAA6E;gBACnH,MAAM7C,QAAQ4B,MAAM,YAAYO;YAClC,OAAO,IAAIS,iBAAiB,MAAM;gBAChC,kFAAkF;gBAClF,uEAAuE;gBACvE,sFAAsF;gBACtF,qFAAqF;gBACrFR,cAAcQ;gBACd,MAAMK,UAAU9C,cAAc+B,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oCAAoC,CAAC;gBACpF,MAAMjD,QAAQ4B,MAAM,YAAYO;YAClC,OAAO,IAAIa,YAAY,MAAM;gBAC3B,uFAAuF;gBACvF,wFAAwF;gBACxF,kFAAkF;gBAClF,sDAAsD;gBACtDV,iBAAiBU;gBACjB,MAAMC,UAAU9C,cAAc+B,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,sCAAsC,CAAC;gBACtF,MAAMjD,QAAQ4B,MAAM,YAAYO;YAClC,OAAO;gBACL,MAAMc,UAAU9C,cAAc+B,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oBAAoB,CAAC;gBACpEpB,SAAS;gBACT,MAAMrB,QAAQkC,KAAK,CAACf;gBACpBnC,WAAWe;gBACX,OAAOgB,mBAAmBhB,KAAKC;YACjC;QACF;QAEA,MAAMA,QAAQ0C,YAAY,CAACvB,QAAQC,MAAMrB;QAEzC,wFAAwF;QACxF,uFAAuF;QACvF,IAAIC,QAAQ2C,qBAAqB,EAAE;gBACH;YAA9B,MAAMC,gBAAgBC,QAAQ,OAAA,MAAMtD,QAAQ6B,MAAM,iCAApB,kBAAA,OAA4C;YAC1E,MAAMpB,QAAQ2C,qBAAqB,CAACxB,QAAQC,MAAMV,KAAKoC,GAAG,CAACpC,KAAKqC,GAAG,CAAC,OAAO,IAAIH,gBAAgB;QACjG;QAEA,6FAA6F;QAC7F,+FAA+F;QAC/F,wCAAwC;QACxC,MAAMI,cAAcnB,kBAAkB,AAAC,CAAA,MAAMP,QAAQ2B,UAAU,CAACpB,gBAAe,EAAGqB,MAAM,GAAG;QAC3F,MAAMC,gBAAgBrB,iBAAiB,AAAC,CAAA,MAAMR,QAAQ8B,kBAAkB,CAACtB,eAAc,EAAGoB,MAAM,GAAG;QAEnG,MAAM,EAAEA,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMhC,QAAQiC,KAAK,CAAC3B;QACzD,uIAAuI;QACvI,MAAMN,QAAQY,KAAK;QAEnB,OAAO;YAAEf;YAAQC;YAAMrB;YAAKD;YAAQoD,QAAQA,SAASF,cAAcG;YAAeE;YAAUC;QAAO;IACrG,EAAE,OAAO/C,KAAK;QACZ,MAAMe,QAAQY,KAAK;QACnB,IAAI,CAACb,QAAQ,MAAMrB,QAAQkC,KAAK,CAACf;QACjC,MAAMZ;IACR;AACF;AAEA,iGAAiG;AACjG,mFAAmF;AACnF,OAAO,eAAeiD,gBAAwBzD,GAAmB,EAAEC,OAA4B;IAC7F,MAAM,EAAEmB,MAAM,EAAEC,IAAI,EAAErB,KAAK0D,WAAW,EAAE3D,MAAM,EAAEoD,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMvC,mBAAmBhB,KAAKC;IAC3G,MAAMS,QAAQT,QAAQ0D,WAAW,CAACvC,QAAQC,MAAMqC;IAChD,0FAA0F;IAC1F7E,eAAewC,MAAMX;IACrB,OAAO;QAAEA;QAAOV,KAAK0D;QAAa3D;QAAQoD;QAAQG;QAAUC;IAAO;AACrE;AAEA,OAAO,eAAeK,SAASlD,KAAY;IACzC,MAAMmD,OAAO,MAAMnD,MAAMoD,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKE,GAAG,EAAC,EAAqBC,CAAC;AAChD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { channel } from 'node:diagnostics_channel';\nimport { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { SenseError } from '../errors.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport { classifyEmbedChange } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.ts';\nimport { lockWaitBudgetMs } from './lock-wait.ts';\nimport { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Stages } from './stages.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Per-stage wall time for this open's build pass (stages.ts), not for any narrow embed or\n  // feature invalidation that ran beside it -- on a cold build there is none, so it is the whole cost.\n  stages: Stages;\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  stages: Stages;\n}\n\n// duckdb/turso hold the cache file for their connection's whole life, so a second command waits.\n// Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.\nconst LOCK_POLL_MS = 50;\nconst lockWait = channel('sensemaking.store.lock-wait');\n\nasync function connectUnlocked<Handle>(dbPath: string, cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<{ handle: Handle; conn: Connection }> {\n  const budgetMs = lockWaitBudgetMs(cfg.baseDir);\n  const deadline = Date.now() + budgetMs;\n  for (;;) {\n    try {\n      return await dialect.connect(dbPath, cfg);\n    } catch (err) {\n      if (!dialect.isLocked?.(err as Error)) throw err;\n      // Observe a real rejected native connection, never merely the start of an async open.\n      if (lockWait.hasSubscribers) lockWait.publish({ store: cfg.store, dbPath });\n      if (Date.now() >= deadline) {\n        throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${Math.round(budgetMs / 1000)}s; wait for that command to finish, or set \"store\" to \"sqlite\" in sense.config.json, which serves concurrent commands`);\n      }\n      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));\n    }\n  }\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<ConnectResult<Handle>> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await connectUnlocked(dbPath, cfg, dialect);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): invalidate()/invalidateFeatures() below share its pool\n  // with the build() call that follows them.\n  const builder = createBuilder(conn, cfg, cfg.baseDir, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    // Set only by the preset-membership branch below; reconcile treats these paths as touched\n    // despite an unchanged stamp, for this one build() call.\n    let forcedPaths: Set<string> | undefined;\n    // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.\n    let embedInvalidate: EmbedChangeKind | undefined;\n    // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.\n    let featureToggles: FeatureToggle[] | undefined;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed\n      // preset's stored segment can't be parsed back into old include/exclude/vectors -- either\n      // way the branch below is skipped and the full rebuild stays the fallback.\n      const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.baseDir, features ?? '', changedKeys) : null;\n      // Null unless exactly one segment changed and it decomposes into a recognised embed-only\n      // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.\n      const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features ?? '', wantFeatures) : null;\n      // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed\n      // feature's stored segment can't be parsed back into old/new on-off -- either way the branch\n      // below is skipped and the full rebuild stays the fallback.\n      const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features ?? '', wantFeatures, changedKeys) : null;\n      if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (embedKind !== null) {\n        // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither\n        // case needs any file reparsed for any feature but embed itself.\n        embedInvalidate = embedKind;\n        console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (presetForced !== null) {\n        // Membership only: which files a preset covers moved, not a file's own content, a\n        // global feature, or the embed model. reconcile() already knows how to\n        // add, update, and remove a file correctly (every cross-feature cascade included); it\n        // just needs telling which unchanged files to treat as touched (forcedPaths, above).\n        forcedPaths = presetForced;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the files it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (toggles !== null) {\n        // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's\n        // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a\n        // feature turning back on, fully re-derives across every indexed file rather than\n        // trusting rows written before or during that window.\n        featureToggles = toggles;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the feature it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same\n    // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or\n    // feature-toggle-only changed-key set).\n    const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;\n    const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;\n\n    const { parsed, warnings, stages } = await builder.build(forcedPaths);\n    // Every open reconciles once and is done with it -- a watcher opens fresh per event instead of holding this connection (src/watch.ts).\n    await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed: parsed + embedParsed + featureParsed, warnings, stages };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, stages } = await connectWithDialect(cfg, dialect);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, stages };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["channel","mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","SenseError","FEATURES","createBuilder","clearCache","classifyEmbedChange","classifyFeatureToggles","isFeatureOnlyChange","lockWaitBudgetMs","forcedPresetPaths","isPresetOnlyChange","getMeta","setMeta","changedSignatureKeys","embedIdentityAdopted","signatureDiff","LOCK_POLL_MS","lockWait","connectUnlocked","dbPath","cfg","dialect","budgetMs","baseDir","deadline","Date","now","connect","err","isLocked","hasSubscribers","publish","store","Math","round","Promise","resolve","setTimeout","connectWithDialect","stateDir","recursive","filename","handle","conn","closed","builder","reconcileDialect","exec","version","features","wantFeatures","forcedPaths","embedInvalidate","featureToggles","schemaVersion","console","error","close","changedKeys","presetForced","embedKind","size","has","toggles","changed","ensureSchema","setDerivedBusyTimeout","recordedMaxMs","Number","min","max","embedParsed","invalidate","parsed","featureParsed","invalidateFeatures","warnings","stages","build","openWithDialect","resolvedCfg","createStore","docCount","stmt","prepare","get","n"],"mappings":"AAAA,SAASA,OAAO,QAAQ,2BAA2B;AACnD,SAASC,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,qBAAqB;AACjE,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,UAAU,QAAQ,aAAa;AAExC,SAASC,mBAAmB,QAAQ,mBAAmB;AAEvD,SAASC,sBAAsB,EAAEC,mBAAmB,QAAQ,qBAAqB;AACjF,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAC1E,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,aAAa,QAAQ,iBAAiB;AA8B3F,iGAAiG;AACjG,oFAAoF;AACpF,MAAMC,eAAe;AACrB,MAAMC,WAAWtB,QAAQ;AAEzB,eAAeuB,gBAAwBC,MAAc,EAAEC,GAAmB,EAAEC,OAA4B;IACtG,MAAMC,WAAWd,iBAAiBY,IAAIG,OAAO;IAC7C,MAAMC,WAAWC,KAAKC,GAAG,KAAKJ;IAC9B,OAAS;QACP,IAAI;YACF,OAAO,MAAMD,QAAQM,OAAO,CAACR,QAAQC;QACvC,EAAE,OAAOQ,KAAK;gBACPP;YAAL,IAAI,GAACA,oBAAAA,QAAQQ,QAAQ,cAAhBR,wCAAAA,uBAAAA,SAAmBO,OAAe,MAAMA;YAC7C,sFAAsF;YACtF,IAAIX,SAASa,cAAc,EAAEb,SAASc,OAAO,CAAC;gBAAEC,OAAOZ,IAAIY,KAAK;gBAAEb;YAAO;YACzE,IAAIM,KAAKC,GAAG,MAAMF,UAAU;gBAC1B,MAAM,IAAIvB,WAAW,cAAc,CAAC,2CAA2C,EAAEmB,IAAIY,KAAK,CAAC,QAAQ,EAAEb,OAAO,gCAAgC,EAAEc,KAAKC,KAAK,CAACZ,WAAW,MAAM,qHAAqH,CAAC;YAClS;YACA,MAAM,IAAIa,QAAQ,CAACC,UAAYC,WAAWD,SAASpB;QACrD;IACF;AACF;AAEA,eAAesB,mBAA2BlB,GAAmB,EAAEC,OAA4B;IACzF,MAAMkB,WAAW1C,KAAKuB,IAAIG,OAAO,EAAExB;IACnCH,UAAU2C,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMrB,SAAStB,KAAK0C,UAAUlB,QAAQoB,QAAQ;IAE9C,MAAM,EAAEC,MAAM,EAAEC,IAAI,EAAE,GAAG,MAAMzB,gBAAgBC,QAAQC,KAAKC;IAC5D,gGAAgG;IAChG,kGAAkG;IAClG,IAAIuB,SAAS;IACb,+FAA+F;IAC/F,2CAA2C;IAC3C,MAAMC,UAAU1C,cAAcwC,MAAMvB,KAAKA,IAAIG,OAAO,EAAEF,QAAQyB,gBAAgB;IAC9E,IAAI;QACF,MAAMH,KAAKI,IAAI,CAAC;QAEhB,uFAAuF;QACvF,qGAAqG;QACrG,MAAMC,UAAU,MAAMrC,QAAQgC,MAAM;QACpC,MAAMM,WAAW,MAAMtC,QAAQgC,MAAM;QACrC,MAAMO,eAAepD,iBAAiBsB,KAAKlB;QAC3C,0FAA0F;QAC1F,yDAAyD;QACzD,IAAIiD;QACJ,kGAAkG;QAClG,IAAIC;QACJ,4GAA4G;QAC5G,IAAIC;QACJ,IAAI,AAACL,YAAY,QAAQA,YAAY3B,QAAQiC,aAAa,IAAML,aAAa,QAAQA,aAAaC,cAAe;YAC/G,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAY3B,QAAQiC,aAAa,EAAE;gBACzDC,QAAQC,KAAK,CAAC;gBACd,kFAAkF;gBAClFZ,SAAS;gBACT,MAAMvB,QAAQoC,KAAK,CAACf;gBACpBtC,WAAWgB;gBACX,OAAOkB,mBAAmBlB,KAAKC;YACjC;YACA,MAAMqC,cAAc7C,qBAAqBoC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,oFAAoF;YACpF,0FAA0F;YAC1F,2EAA2E;YAC3E,MAAMS,eAAejD,mBAAmBgD,eAAejD,kBAAkBW,KAAKA,IAAIG,OAAO,EAAE0B,qBAAAA,sBAAAA,WAAY,IAAIS,eAAe;YAC1H,yFAAyF;YACzF,oFAAoF;YACpF,MAAME,YAAYF,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,WAAWzD,oBAAoB4C,qBAAAA,sBAAAA,WAAY,IAAIC,gBAAgB;YAC3H,4FAA4F;YAC5F,6FAA6F;YAC7F,4DAA4D;YAC5D,MAAMa,UAAUxD,oBAAoBmD,eAAepD,uBAAuB2C,qBAAAA,sBAAAA,WAAY,IAAIC,cAAcQ,eAAe;YACvH,IAAIA,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,YAAYhD,qBAAqBmC,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBAC5G,iFAAiF;gBACjF,sDAAsD;gBACtDK,QAAQC,KAAK,CAAC;gBACd,MAAM5C,QAAQ+B,MAAM,YAAYO;YAClC,OAAO,IAAIU,cAAc,MAAM;gBAC7B,yFAAyF;gBACzF,iEAAiE;gBACjER,kBAAkBQ;gBAClBL,QAAQC,KAAK,CAACI,cAAc,UAAU,6EAA6E;gBACnH,MAAMhD,QAAQ+B,MAAM,YAAYO;YAClC,OAAO,IAAIS,iBAAiB,MAAM;gBAChC,kFAAkF;gBAClF,uEAAuE;gBACvE,sFAAsF;gBACtF,qFAAqF;gBACrFR,cAAcQ;gBACd,MAAMK,UAAUjD,cAAckC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oCAAoC,CAAC;gBACpF,MAAMpD,QAAQ+B,MAAM,YAAYO;YAClC,OAAO,IAAIa,YAAY,MAAM;gBAC3B,uFAAuF;gBACvF,wFAAwF;gBACxF,kFAAkF;gBAClF,sDAAsD;gBACtDV,iBAAiBU;gBACjB,MAAMC,UAAUjD,cAAckC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,sCAAsC,CAAC;gBACtF,MAAMpD,QAAQ+B,MAAM,YAAYO;YAClC,OAAO;gBACL,MAAMc,UAAUjD,cAAckC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oBAAoB,CAAC;gBACpEpB,SAAS;gBACT,MAAMvB,QAAQoC,KAAK,CAACf;gBACpBtC,WAAWgB;gBACX,OAAOkB,mBAAmBlB,KAAKC;YACjC;QACF;QAEA,MAAMA,QAAQ4C,YAAY,CAACvB,QAAQC,MAAMvB;QAEzC,wFAAwF;QACxF,uFAAuF;QACvF,IAAIC,QAAQ6C,qBAAqB,EAAE;gBACH;YAA9B,MAAMC,gBAAgBC,QAAQ,OAAA,MAAMzD,QAAQgC,MAAM,iCAApB,kBAAA,OAA4C;YAC1E,MAAMtB,QAAQ6C,qBAAqB,CAACxB,QAAQC,MAAMV,KAAKoC,GAAG,CAACpC,KAAKqC,GAAG,CAAC,OAAO,IAAIH,gBAAgB;QACjG;QAEA,6FAA6F;QAC7F,+FAA+F;QAC/F,wCAAwC;QACxC,MAAMI,cAAcnB,kBAAkB,AAAC,CAAA,MAAMP,QAAQ2B,UAAU,CAACpB,gBAAe,EAAGqB,MAAM,GAAG;QAC3F,MAAMC,gBAAgBrB,iBAAiB,AAAC,CAAA,MAAMR,QAAQ8B,kBAAkB,CAACtB,eAAc,EAAGoB,MAAM,GAAG;QAEnG,MAAM,EAAEA,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMhC,QAAQiC,KAAK,CAAC3B;QACzD,uIAAuI;QACvI,MAAMN,QAAQY,KAAK;QAEnB,OAAO;YAAEf;YAAQC;YAAMvB;YAAKD;YAAQsD,QAAQA,SAASF,cAAcG;YAAeE;YAAUC;QAAO;IACrG,EAAE,OAAOjD,KAAK;QACZ,MAAMiB,QAAQY,KAAK;QACnB,IAAI,CAACb,QAAQ,MAAMvB,QAAQoC,KAAK,CAACf;QACjC,MAAMd;IACR;AACF;AAEA,iGAAiG;AACjG,mFAAmF;AACnF,OAAO,eAAemD,gBAAwB3D,GAAmB,EAAEC,OAA4B;IAC7F,MAAM,EAAEqB,MAAM,EAAEC,IAAI,EAAEvB,KAAK4D,WAAW,EAAE7D,MAAM,EAAEsD,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMvC,mBAAmBlB,KAAKC;IAC3G,MAAMW,QAAQX,QAAQ4D,WAAW,CAACvC,QAAQC,MAAMqC;IAChD,0FAA0F;IAC1FhF,eAAe2C,MAAMX;IACrB,OAAO;QAAEA;QAAOZ,KAAK4D;QAAa7D;QAAQsD;QAAQG;QAAUC;IAAO;AACrE;AAEA,OAAO,eAAeK,SAASlD,KAAY;IACzC,MAAMmD,OAAO,MAAMnD,MAAMoD,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKE,GAAG,EAAC,EAAqBC,CAAC;AAChD"}

package/dist/esm/store/shared.js

@@ -1,8 +1,7 @@
 // SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both
 // stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).
 // Rows that cannot conflict, at whatever speed this connection offers: the append path where the
-// store has one, else `conflictSql` bound the ordinary way. `conflictSql` stays the caller's own
-// guarded statement, so a store without appendRows behaves exactly as it did before.
+// store has one, else the caller's own guarded `conflictSql` statement bound the ordinary way.
 export async function appendRows(conn, table, columns, conflictSql, rows) {
     if (rows.length === 0) return;
     if (conn.appendRows) await conn.appendRows(table, columns, rows);

package/dist/esm/store/shared.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\n// Rows that cannot conflict, at whatever speed this connection offers: the append path where the\n// store has one, else `conflictSql` bound the ordinary way. `conflictSql` stays the caller's own\n// guarded statement, so a store without appendRows behaves exactly as it did before.\nexport async function appendRows(conn: Connection, table: string, columns: string[], conflictSql: string, rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  if (conn.appendRows) await conn.appendRows(table, columns, rows);\n  else await conn.runBatch(conflictSql, rows);\n}\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n\n// Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n// observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\nexport async function recordReconcileDuration(conn: Connection, ms: number): Promise<void> {\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (ms > prevMax) await setMeta(conn, 'reconcile_max_ms', String(ms));\n}\n"],"names":["appendRows","conn","table","columns","conflictSql","rows","length","runBatch","quoteIdent","name","split","join","getColumns","stmt","prepare","all","Set","map","r","getMeta","key","row","get","value","setMeta","run","recordReconcileDuration","ms","prevRaw","prevMax","Number","String"],"mappings":"AAEA,2FAA2F;AAC3F,sFAAsF;AAEtF,iGAAiG;AACjG,iGAAiG;AACjG,qFAAqF;AACrF,OAAO,eAAeA,WAAWC,IAAgB,EAAEC,KAAa,EAAEC,OAAiB,EAAEC,WAAmB,EAAEC,IAAiB;IACzH,IAAIA,KAAKC,MAAM,KAAK,GAAG;IACvB,IAAIL,KAAKD,UAAU,EAAE,MAAMC,KAAKD,UAAU,CAACE,OAAOC,SAASE;SACtD,MAAMJ,KAAKM,QAAQ,CAACH,aAAaC;AACxC;AAEA,OAAO,SAASG,WAAWC,IAAY;IACrC,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,OAAO,eAAeC,WAAWX,IAAgB;IAC/C,MAAMY,OAAO,MAAMZ,KAAKa,OAAO,CAAC;IAChC,MAAMT,OAAQ,MAAMQ,KAAKE,GAAG;IAC5B,OAAO,IAAIC,IAAIX,KAAKY,GAAG,CAAC,CAACC,IAAMA,EAAET,IAAI;AACvC;AAEA,OAAO,eAAeU,QAAQlB,IAAgB,EAAEmB,GAAW;IACzD,MAAMP,OAAO,MAAMZ,KAAKa,OAAO,CAAC;IAChC,MAAMO,MAAO,MAAMR,KAAKS,GAAG,CAACF;IAC5B,OAAOC,MAAMA,IAAIE,KAAK,GAAG;AAC3B;AAEA,OAAO,eAAeC,QAAQvB,IAAgB,EAAEmB,GAAW,EAAEG,KAAoB;IAC/E,IAAIA,UAAU,MAAM;QAClB,MAAMV,OAAO,MAAMZ,KAAKa,OAAO,CAAC;QAChC,MAAMD,KAAKY,GAAG,CAACL;QACf;IACF;IACA,MAAMP,OAAO,MAAMZ,KAAKa,OAAO,CAAC;IAChC,MAAMD,KAAKY,GAAG,CAACL,KAAKG;AACtB;AAEA,0FAA0F;AAC1F,qGAAqG;AACrG,OAAO,eAAeG,wBAAwBzB,IAAgB,EAAE0B,EAAU;IACxE,MAAMC,UAAU,MAAMT,QAAQlB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM4B,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,KAAKE,SAAS,MAAML,QAAQvB,MAAM,oBAAoB8B,OAAOJ;AACnE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\n// Rows that cannot conflict, at whatever speed this connection offers: the append path where the\n// store has one, else the caller's own guarded `conflictSql` statement bound the ordinary way.\nexport async function appendRows(conn: Connection, table: string, columns: string[], conflictSql: string, rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  if (conn.appendRows) await conn.appendRows(table, columns, rows);\n  else await conn.runBatch(conflictSql, rows);\n}\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n\n// Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n// observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\nexport async function recordReconcileDuration(conn: Connection, ms: number): Promise<void> {\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (ms > prevMax) await setMeta(conn, 'reconcile_max_ms', String(ms));\n}\n"],"names":["appendRows","conn","table","columns","conflictSql","rows","length","runBatch","quoteIdent","name","split","join","getColumns","stmt","prepare","all","Set","map","r","getMeta","key","row","get","value","setMeta","run","recordReconcileDuration","ms","prevRaw","prevMax","Number","String"],"mappings":"AAEA,2FAA2F;AAC3F,sFAAsF;AAEtF,iGAAiG;AACjG,+FAA+F;AAC/F,OAAO,eAAeA,WAAWC,IAAgB,EAAEC,KAAa,EAAEC,OAAiB,EAAEC,WAAmB,EAAEC,IAAiB;IACzH,IAAIA,KAAKC,MAAM,KAAK,GAAG;IACvB,IAAIL,KAAKD,UAAU,EAAE,MAAMC,KAAKD,UAAU,CAACE,OAAOC,SAASE;SACtD,MAAMJ,KAAKM,QAAQ,CAACH,aAAaC;AACxC;AAEA,OAAO,SAASG,WAAWC,IAAY;IACrC,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,OAAO,eAAeC,WAAWX,IAAgB;IAC/C,MAAMY,OAAO,MAAMZ,KAAKa,OAAO,CAAC;IAChC,MAAMT,OAAQ,MAAMQ,KAAKE,GAAG;IAC5B,OAAO,IAAIC,IAAIX,KAAKY,GAAG,CAAC,CAACC,IAAMA,EAAET,IAAI;AACvC;AAEA,OAAO,eAAeU,QAAQlB,IAAgB,EAAEmB,GAAW;IACzD,MAAMP,OAAO,MAAMZ,KAAKa,OAAO,CAAC;IAChC,MAAMO,MAAO,MAAMR,KAAKS,GAAG,CAACF;IAC5B,OAAOC,MAAMA,IAAIE,KAAK,GAAG;AAC3B;AAEA,OAAO,eAAeC,QAAQvB,IAAgB,EAAEmB,GAAW,EAAEG,KAAoB;IAC/E,IAAIA,UAAU,MAAM;QAClB,MAAMV,OAAO,MAAMZ,KAAKa,OAAO,CAAC;QAChC,MAAMD,KAAKY,GAAG,CAACL;QACf;IACF;IACA,MAAMP,OAAO,MAAMZ,KAAKa,OAAO,CAAC;IAChC,MAAMD,KAAKY,GAAG,CAACL,KAAKG;AACtB;AAEA,0FAA0F;AAC1F,qGAAqG;AACrG,OAAO,eAAeG,wBAAwBzB,IAAgB,EAAE0B,EAAU;IACxE,MAAMC,UAAU,MAAMT,QAAQlB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM4B,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,KAAKE,SAAS,MAAML,QAAQvB,MAAM,oBAAoB8B,OAAOJ;AACnE"}

package/dist/esm/store/sqlite/lexical.js

@@ -1,9 +1,6 @@
 import { segmentMatch } from '../../text/segment.js';
-// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc
-// (benchmark/reports/2026-08-23-0.13.2-hub-release-battery.md); past this bound, hit stays NULL for the caller's JS excerpt fallback.
-const SNIPPET_BOUND = 16384;
 const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';
-// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments. An
+// Ranked BM25 word-match query, scoped by the caller-built SQL fragments. An
 // unspaced-script run in `terms` becomes a quoted grapheme phrase against the `_seg` sidecars.
 export async function queryLexical(conn, terms, opts) {
     const { whereJoin, whereCond, scopeCond, limit } = opts;
@@ -11,7 +8,7 @@ export async function queryLexical(conn, terms, opts) {
     // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH
     // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.
     if (query.trim() === '') return [];
-    const sql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;
+    const sql = `SELECT content.path AS path FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;
     const stmt = await conn.prepare(sql);
     return await stmt.all(query);
 }

package/dist/esm/store/sqlite/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-0.13.2-hub-release-battery.md); past this bound, hit stays NULL for the caller's JS excerpt fallback.\nconst SNIPPET_BOUND = 16_384;\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n\n// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments. An\n// unspaced-script run in `terms` becomes a quoted grapheme phrase against the `_seg` sidecars.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const query = segmentMatch(terms);\n  // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH\n  // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.\n  if (query.trim() === '') return [];\n  const sql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(query)) as unknown as LexicalHit[];\n}\n"],"names":["segmentMatch","SNIPPET_BOUND","WEIGHTED_BM25","queryLexical","conn","terms","opts","whereJoin","whereCond","scopeCond","limit","query","trim","sql","stmt","prepare","all"],"mappings":"AAAA,SAASA,YAAY,QAAQ,wBAAwB;AAGrD,iFAAiF;AACjF,sIAAsI;AACtI,MAAMC,gBAAgB;AACtB,MAAMC,gBAAgB;AAEtB,0FAA0F;AAC1F,+FAA+F;AAC/F,OAAO,eAAeC,aAAaC,IAAgB,EAAEC,KAAa,EAAEC,IAAyB;IAC3F,MAAM,EAAEC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGJ;IACnD,MAAMK,QAAQX,aAAaK;IAC3B,2FAA2F;IAC3F,sGAAsG;IACtG,IAAIM,MAAMC,IAAI,OAAO,IAAI,OAAO,EAAE;IAClC,MAAMC,MAAM,CAAC,+DAA+D,EAAEZ,cAAc,+EAA+E,EAAEM,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAEP,cAAc,OAAO,EAAEQ,OAAO;IACjR,MAAMI,OAAO,MAAMV,KAAKW,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,CAACL;AACzB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n\n// Ranked BM25 word-match query, scoped by the caller-built SQL fragments. An\n// unspaced-script run in `terms` becomes a quoted grapheme phrase against the `_seg` sidecars.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const query = segmentMatch(terms);\n  // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH\n  // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.\n  if (query.trim() === '') return [];\n  const sql = `SELECT content.path AS path FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(query)) as unknown as LexicalHit[];\n}\n"],"names":["segmentMatch","WEIGHTED_BM25","queryLexical","conn","terms","opts","whereJoin","whereCond","scopeCond","limit","query","trim","sql","stmt","prepare","all"],"mappings":"AAAA,SAASA,YAAY,QAAQ,wBAAwB;AAGrD,MAAMC,gBAAgB;AAEtB,6EAA6E;AAC7E,+FAA+F;AAC/F,OAAO,eAAeC,aAAaC,IAAgB,EAAEC,KAAa,EAAEC,IAAyB;IAC3F,MAAM,EAAEC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGJ;IACnD,MAAMK,QAAQV,aAAaI;IAC3B,2FAA2F;IAC3F,sGAAsG;IACtG,IAAIM,MAAMC,IAAI,OAAO,IAAI,OAAO,EAAE;IAClC,MAAMC,MAAM,CAAC,yCAAyC,EAAEN,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAEP,cAAc,OAAO,EAAEQ,OAAO;IAC5J,MAAMI,OAAO,MAAMV,KAAKW,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,CAACL;AACzB"}

package/dist/esm/store/sqlite/store.js

@@ -5,11 +5,8 @@ import { fieldStats } from './fieldStats.js';
 import { queryLexical } from './lexical.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
 export const CAPABILITIES = new Set([
-    'phrases',
-    'snippets',
-    'lexical',
     'vectors',
-    'sql-functions'
+    'segment'
 ]);
 // Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one
 // Connection instance (conn) with the builder's own reconcile call so transaction depth is tracked against the same object everywhere.

package/dist/esm/store/sqlite/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'lexical', 'vectors', 'sql-functions']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with the builder's own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","vectors","pending","writeVectors","rows","candidates","qv","storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":"AACA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAY;IAAW;IAAW;CAAgB,EAAE;AAE7H,0FAA0F;AAC1F,uIAAuI;AACvI,OAAO,SAASC,YAAYC,EAAgB,EAAEC,IAAgB;IAC5D,OAAO;QACLC,MAAM;QACNC,cAAcN;QACd,MAAMO,MAAKC,GAAW;YACpB,MAAMJ,KAAKG,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOJ,KAAKK,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMP,KAAKM,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOrB,gBAAgBY,MAAMS;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMxB,WAAWa;iBAAO;YACtC;YACA,MAAMT,YAAWoB,OAAO,EAAEC,UAAU;gBAClC,OAAOrB,WAAWS,MAAMW,SAASC;YACnC;QACF;QACAC,SAAS;YACP,MAAMC,OAAMC,KAAK,EAAEC,IAAI;gBACrB,OAAOxB,aAAaQ,MAAMe,OAAOC;YACnC;QACF;QACAC,SAAS;YACP,MAAMC;gBACJ,OAAO5B,YAAYU;YACrB;YACA,MAAMmB,cAAaC,IAAsB;gBACvC,MAAMzB,iBAAiBK,MAAMoB;YAC/B;YACA,MAAMC,YAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;gBAC5C,OAAOhC,eAAeO,MAAMsB,IAAIC,WAAWC,OAAOC;YACpD;YACA,MAAMC,SAAQC,IAAI,EAAEX,IAAI;gBACtB,OAAOtB,YAAYM,MAAM2B,MAAMX;YACjC;YACA,MAAMY,WAAUD,IAAI;gBAClB,OAAOtC,aAAaW,MAAM2B;YAC5B;QACF;QACA,MAAME;YACJ,qFAAqF;YACrF,qFAAqF;YACrF,MAAMC,MAAM/B,GAAGM,OAAO,CAAC,uBAAuB0B,GAAG;YACjD,OAAO;gBAAEC,cAAc,GAAGF,IAAIG,OAAO,CAAC,kFAAkF,CAAC;YAAC;QAC5H;QACAC,KAAK;YACH,MAAM7B,SAAQD,GAAW;gBACvB,MAAM+B,OAAOpC,GAAGM,OAAO,CAACD;gBACxB+B,KAAKC,cAAc,CAAC,OAAO,qEAAqE;gBAChG,OAAO;oBACLzB,SAAS,IAAMwB,KAAKxB,OAAO;oBAC3B,2EAA2E;oBAC3E0B,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAKC;oBAC1B;gBACF;YACF;QACF;QACA,MAAMC;YACJxC,GAAGwC,KAAK;QACV;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['vectors', 'segment']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with the builder's own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","vectors","pending","writeVectors","rows","candidates","qv","storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":"AACA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;CAAU,EAAE;AAErF,0FAA0F;AAC1F,uIAAuI;AACvI,OAAO,SAASC,YAAYC,EAAgB,EAAEC,IAAgB;IAC5D,OAAO;QACLC,MAAM;QACNC,cAAcN;QACd,MAAMO,MAAKC,GAAW;YACpB,MAAMJ,KAAKG,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOJ,KAAKK,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMP,KAAKM,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOrB,gBAAgBY,MAAMS;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMxB,WAAWa;iBAAO;YACtC;YACA,MAAMT,YAAWoB,OAAO,EAAEC,UAAU;gBAClC,OAAOrB,WAAWS,MAAMW,SAASC;YACnC;QACF;QACAC,SAAS;YACP,MAAMC,OAAMC,KAAK,EAAEC,IAAI;gBACrB,OAAOxB,aAAaQ,MAAMe,OAAOC;YACnC;QACF;QACAC,SAAS;YACP,MAAMC;gBACJ,OAAO5B,YAAYU;YACrB;YACA,MAAMmB,cAAaC,IAAsB;gBACvC,MAAMzB,iBAAiBK,MAAMoB;YAC/B;YACA,MAAMC,YAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;gBAC5C,OAAOhC,eAAeO,MAAMsB,IAAIC,WAAWC,OAAOC;YACpD;YACA,MAAMC,SAAQC,IAAI,EAAEX,IAAI;gBACtB,OAAOtB,YAAYM,MAAM2B,MAAMX;YACjC;YACA,MAAMY,WAAUD,IAAI;gBAClB,OAAOtC,aAAaW,MAAM2B;YAC5B;QACF;QACA,MAAME;YACJ,qFAAqF;YACrF,qFAAqF;YACrF,MAAMC,MAAM/B,GAAGM,OAAO,CAAC,uBAAuB0B,GAAG;YACjD,OAAO;gBAAEC,cAAc,GAAGF,IAAIG,OAAO,CAAC,kFAAkF,CAAC;YAAC;QAC5H;QACAC,KAAK;YACH,MAAM7B,SAAQD,GAAW;gBACvB,MAAM+B,OAAOpC,GAAGM,OAAO,CAACD;gBACxB+B,KAAKC,cAAc,CAAC,OAAO,qEAAqE;gBAChG,OAAO;oBACLzB,SAAS,IAAMwB,KAAKxB,OAAO;oBAC3B,2EAA2E;oBAC3E0B,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAKC;oBAC1B;gBACF;YACF;QACF;QACA,MAAMC;YACJxC,GAAGwC,KAAK;QACV;IACF;AACF"}

package/dist/esm/store/sqlite/vectors.js

@@ -1,4 +1,4 @@
-import { asCosine, sampleEvenly } from '../vectors.js';
+import { asCosine, compareVectorScores, sampleEvenly } from '../vectors.js';
 // One runBatch call per provider batch: the caller (embed/query.ts) already batches by the
 // provider's batchCap, so this is one crossing per batch, never per row.
 export async function writeVectorBatch(conn, rows) {
@@ -13,57 +13,86 @@ export async function writeVectorBatch(conn, rows) {
 // Best chunk per file by cosine, its line range riding along: the JS loop over stored int8
 // BLOBs this store's scan strategy is (a native store scans differently, same contract).
 export async function scanCandidates(conn, qv, storeDims, fetch, allowed) {
-    const stmt = await conn.prepare('SELECT "path", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL');
+    const stmt = await conn.prepare('SELECT "path", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL ORDER BY "path", chunk');
     const rows = await stmt.all();
+    let queryNorm = 0;
+    for(let d = 0; d < Math.min(storeDims, qv.length); d++)queryNorm += qv[d] * qv[d];
+    queryNorm = Math.sqrt(queryNorm);
     const best = new Map();
     for (const row of rows){
         if (allowed && !allowed.has(row.path)) continue;
         const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));
+        // Rounding to int8 costs the stored row its unit norm, so dividing by that norm is what makes
+        // this a cosine rather than a scaled dot product. `scale` cancels out of the ratio.
         let dot = 0;
-        for(let d = 0; d < q.length; d++)dot += q[d] * qv[d];
-        const score = dot * row.scale;
+        let norm = 0;
+        const length = Math.min(q.length, qv.length);
+        for(let d = 0; d < length; d++){
+            dot += q[d] * qv[d];
+            norm += q[d] * q[d];
+        }
+        for(let d = length; d < q.length; d++)norm += q[d] * q[d];
+        const score = norm === 0 || queryNorm === 0 ? 0 : dot / (Math.sqrt(norm) * queryNorm);
         const existing = best.get(row.path);
-        if (!existing || score > existing.score) best.set(row.path, {
-            score,
-            lines: `L${row.start_line}-${row.end_line}`
-        });
+        if (!existing || score > existing.score || score === existing.score && row.chunk < existing.chunk) {
+            best.set(row.path, {
+                score,
+                chunk: row.chunk,
+                lines: `L${row.start_line}-${row.end_line}`
+            });
+        }
     }
     return [
         ...best.entries()
-    ].sort((a, b)=>b[1].score - a[1].score).slice(0, fetch).map(([path, b])=>({
+    ].map(([path, value])=>({
+            path,
+            ...value
+        })).sort(compareVectorScores).slice(0, fetch).map(({ path, lines, score })=>({
             path,
-            lines: b.lines,
-            similarity: asCosine(b.score)
+            lines,
+            similarity: asCosine(score)
         }));
 }
 // Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear
 // scan of stored vectors.
 export async function scanSimilar(conn, path, opts) {
-    const targetStmt = await conn.prepare('SELECT scale, vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk');
+    const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk');
     const targetRows = await targetStmt.all(path);
     if (targetRows.length === 0) return [];
-    const target = sampleEvenly(targetRows).map((row)=>({
-            v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength),
-            scale: row.scale
-        }));
-    const stmt = await conn.prepare('SELECT "path", scale, vector FROM embeddings WHERE vector IS NOT NULL');
+    // Each side's norm divides its own dot product, so both are taken once here rather than per pair.
+    const target = sampleEvenly(targetRows).map((row)=>{
+        const v = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
+        let norm = 0;
+        for(let d = 0; d < v.length; d++)norm += v[d] * v[d];
+        return {
+            v,
+            norm: Math.sqrt(norm)
+        };
+    });
+    const stmt = await conn.prepare('SELECT "path", vector FROM embeddings WHERE vector IS NOT NULL');
     const rows = await stmt.all();
     const best = new Map();
     for (const row of rows){
         if (row.path === path || opts.exclude.has(row.path) || opts.allowed && !opts.allowed.has(row.path)) continue;
         const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
+        let otherNorm = 0;
+        for(let d = 0; d < other.length; d++)otherNorm += other[d] * other[d];
+        otherNorm = Math.sqrt(otherNorm);
         for (const t of target){
             let dot = 0;
             const len = Math.min(t.v.length, other.length);
             for(let d = 0; d < len; d++)dot += t.v[d] * other[d];
-            const score = dot * t.scale * row.scale;
+            const score = t.norm === 0 || otherNorm === 0 ? 0 : dot / (t.norm * otherNorm);
             const existing = best.get(row.path);
             if (existing === undefined || score > existing) best.set(row.path, score);
         }
     }
     return [
         ...best.entries()
-    ].sort((a, b)=>b[1] - a[1]).slice(0, opts.k).map(([p, score])=>({
+    ].map(([p, score])=>({
+            path: p,
+            score
+        })).sort(compareVectorScores).slice(0, opts.k).map(({ path: p, score })=>({
             path: p,
             similarity: asCosine(score)
         }));

package/dist/esm/store/sqlite/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\n\n// One runBatch call per provider batch: the caller (embed/query.ts) already batches by the\n// provider's batchCap, so this is one crossing per batch, never per row.\nexport async function writeVectorBatch(conn: Connection, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await conn.runBatch(\n    'UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?',\n    rows.map((row) => [row.scale, row.vector, row.path, row.chunk])\n  );\n}\n\n// Best chunk per file by cosine, its line range riding along: the JS loop over stored int8\n// BLOBs this store's scan strategy is (a native store scans differently, same contract).\nexport async function scanCandidates(conn: Connection, qv: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const stmt = await conn.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function scanSimilar(conn: Connection, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  const targetStmt = await conn.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const target = sampleEvenly(targetRows).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const stmt = await conn.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["asCosine","sampleEvenly","writeVectorBatch","conn","rows","length","runBatch","map","row","scale","vector","path","chunk","scanCandidates","qv","storeDims","fetch","allowed","stmt","prepare","all","best","Map","has","q","Int8Array","buffer","byteOffset","Math","min","byteLength","dot","d","score","existing","get","set","lines","start_line","end_line","entries","sort","a","b","slice","similarity","scanSimilar","opts","targetStmt","targetRows","target","v","exclude","other","t","len","undefined","k","p"],"mappings":"AACA,SAASA,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AAEvD,2FAA2F;AAC3F,yEAAyE;AACzE,OAAO,eAAeC,iBAAiBC,IAAgB,EAAEC,IAAsB;IAC7E,IAAIA,KAAKC,MAAM,KAAK,GAAG;IACvB,MAAMF,KAAKG,QAAQ,CACjB,8EACAF,KAAKG,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIC,KAAK;YAAED,IAAIE,MAAM;YAAEF,IAAIG,IAAI;YAAEH,IAAII,KAAK;SAAC;AAElE;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,OAAO,eAAeC,eAAeV,IAAgB,EAAEW,EAAgB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,OAAqB;IAC9H,MAAMC,OAAO,MAAMf,KAAKgB,OAAO,CAAC;IAChC,MAAMf,OAAQ,MAAMc,KAAKE,GAAG;IAQ5B,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAMd,OAAOJ,KAAM;QACtB,IAAIa,WAAW,CAACA,QAAQM,GAAG,CAACf,IAAIG,IAAI,GAAG;QACvC,MAAMa,IAAI,IAAIC,UAAUjB,IAAIE,MAAM,CAACgB,MAAM,EAAElB,IAAIE,MAAM,CAACiB,UAAU,EAAEC,KAAKC,GAAG,CAACd,WAAWP,IAAIE,MAAM,CAACoB,UAAU;QAC3G,IAAIC,MAAM;QACV,IAAK,IAAIC,IAAI,GAAGA,IAAIR,EAAEnB,MAAM,EAAE2B,IAAKD,OAAOP,CAAC,CAACQ,EAAE,GAAGlB,EAAE,CAACkB,EAAE;QACtD,MAAMC,QAAQF,MAAMvB,IAAIC,KAAK;QAC7B,MAAMyB,WAAWb,KAAKc,GAAG,CAAC3B,IAAIG,IAAI;QAClC,IAAI,CAACuB,YAAYD,QAAQC,SAASD,KAAK,EAAEZ,KAAKe,GAAG,CAAC5B,IAAIG,IAAI,EAAE;YAAEsB;YAAOI,OAAO,CAAC,CAAC,EAAE7B,IAAI8B,UAAU,CAAC,CAAC,EAAE9B,IAAI+B,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIlB,KAAKmB,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAACV,KAAK,GAAGS,CAAC,CAAC,EAAE,CAACT,KAAK,EACtCW,KAAK,CAAC,GAAG5B,OACTT,GAAG,CAAC,CAAC,CAACI,MAAMgC,EAAE,GAAM,CAAA;YAAEhC;YAAM0B,OAAOM,EAAEN,KAAK;YAAEQ,YAAY7C,SAAS2C,EAAEV,KAAK;QAAE,CAAA;AAC/E;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAea,YAAY3C,IAAgB,EAAEQ,IAAY,EAAEoC,IAAgE;IAChI,MAAMC,aAAa,MAAM7C,KAAKgB,OAAO,CAAC;IACtC,MAAM8B,aAAc,MAAMD,WAAW5B,GAAG,CAACT;IACzC,IAAIsC,WAAW5C,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAM6C,SAASjD,aAAagD,YAAY1C,GAAG,CAAC,CAACC,MAAS,CAAA;YAAE2C,GAAG,IAAI1B,UAAUjB,IAAIE,MAAM,CAACgB,MAAM,EAAElB,IAAIE,MAAM,CAACiB,UAAU,EAAEnB,IAAIE,MAAM,CAACoB,UAAU;YAAGrB,OAAOD,IAAIC,KAAK;QAAC,CAAA;IAE5J,MAAMS,OAAO,MAAMf,KAAKgB,OAAO,CAAC;IAChC,MAAMf,OAAQ,MAAMc,KAAKE,GAAG;IAC5B,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAMd,OAAOJ,KAAM;QACtB,IAAII,IAAIG,IAAI,KAAKA,QAAQoC,KAAKK,OAAO,CAAC7B,GAAG,CAACf,IAAIG,IAAI,KAAMoC,KAAK9B,OAAO,IAAI,CAAC8B,KAAK9B,OAAO,CAACM,GAAG,CAACf,IAAIG,IAAI,GAAI;QACtG,MAAM0C,QAAQ,IAAI5B,UAAUjB,IAAIE,MAAM,CAACgB,MAAM,EAAElB,IAAIE,MAAM,CAACiB,UAAU,EAAEnB,IAAIE,MAAM,CAACoB,UAAU;QAC3F,KAAK,MAAMwB,KAAKJ,OAAQ;YACtB,IAAInB,MAAM;YACV,MAAMwB,MAAM3B,KAAKC,GAAG,CAACyB,EAAEH,CAAC,CAAC9C,MAAM,EAAEgD,MAAMhD,MAAM;YAC7C,IAAK,IAAI2B,IAAI,GAAGA,IAAIuB,KAAKvB,IAAKD,OAAOuB,EAAEH,CAAC,CAACnB,EAAE,GAAGqB,KAAK,CAACrB,EAAE;YACtD,MAAMC,QAAQF,MAAMuB,EAAE7C,KAAK,GAAGD,IAAIC,KAAK;YACvC,MAAMyB,WAAWb,KAAKc,GAAG,CAAC3B,IAAIG,IAAI;YAClC,IAAIuB,aAAasB,aAAavB,QAAQC,UAAUb,KAAKe,GAAG,CAAC5B,IAAIG,IAAI,EAAEsB;QACrE;IACF;IAEA,OAAO;WAAIZ,KAAKmB,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BE,KAAK,CAAC,GAAGG,KAAKU,CAAC,EACflD,GAAG,CAAC,CAAC,CAACmD,GAAGzB,MAAM,GAAM,CAAA;YAAEtB,MAAM+C;YAAGb,YAAY7C,SAASiC;QAAO,CAAA;AACjE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, compareVectorScores, sampleEvenly } from '../vectors.ts';\n\n// One runBatch call per provider batch: the caller (embed/query.ts) already batches by the\n// provider's batchCap, so this is one crossing per batch, never per row.\nexport async function writeVectorBatch(conn: Connection, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await conn.runBatch(\n    'UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?',\n    rows.map((row) => [row.scale, row.vector, row.path, row.chunk])\n  );\n}\n\n// Best chunk per file by cosine, its line range riding along: the JS loop over stored int8\n// BLOBs this store's scan strategy is (a native store scans differently, same contract).\nexport async function scanCandidates(conn: Connection, qv: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk, start_line, end_line, vector FROM embeddings WHERE vector IS NOT NULL ORDER BY \"path\", chunk');\n  const rows = (await stmt.all()) as Array<{\n    path: string;\n    chunk: number;\n    start_line: number;\n    end_line: number;\n    vector: Uint8Array;\n  }>;\n\n  let queryNorm = 0;\n  for (let d = 0; d < Math.min(storeDims, qv.length); d++) queryNorm += qv[d] * qv[d];\n  queryNorm = Math.sqrt(queryNorm);\n\n  const best = new Map<string, { score: number; chunk: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    // Rounding to int8 costs the stored row its unit norm, so dividing by that norm is what makes\n    // this a cosine rather than a scaled dot product. `scale` cancels out of the ratio.\n    let dot = 0;\n    let norm = 0;\n    const length = Math.min(q.length, qv.length);\n    for (let d = 0; d < length; d++) {\n      dot += q[d] * qv[d];\n      norm += q[d] * q[d];\n    }\n    for (let d = length; d < q.length; d++) norm += q[d] * q[d];\n    const score = norm === 0 || queryNorm === 0 ? 0 : dot / (Math.sqrt(norm) * queryNorm);\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score || (score === existing.score && row.chunk < existing.chunk)) {\n      best.set(row.path, { score, chunk: row.chunk, lines: `L${row.start_line}-${row.end_line}` });\n    }\n  }\n  return [...best.entries()]\n    .map(([path, value]) => ({ path, ...value }))\n    .sort(compareVectorScores)\n    .slice(0, fetch)\n    .map(({ path, lines, score }) => ({ path, lines, similarity: asCosine(score) }));\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function scanSimilar(conn: Connection, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  // Each side's norm divides its own dot product, so both are taken once here rather than per pair.\n  const target = sampleEvenly(targetRows).map((row) => {\n    const v = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    let norm = 0;\n    for (let d = 0; d < v.length; d++) norm += v[d] * v[d];\n    return { v, norm: Math.sqrt(norm) };\n  });\n\n  const stmt = await conn.prepare('SELECT \"path\", vector FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    let otherNorm = 0;\n    for (let d = 0; d < other.length; d++) otherNorm += other[d] * other[d];\n    otherNorm = Math.sqrt(otherNorm);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = t.norm === 0 || otherNorm === 0 ? 0 : dot / (t.norm * otherNorm);\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .map(([p, score]) => ({ path: p, score }))\n    .sort(compareVectorScores)\n    .slice(0, opts.k)\n    .map(({ path: p, score }) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["asCosine","compareVectorScores","sampleEvenly","writeVectorBatch","conn","rows","length","runBatch","map","row","scale","vector","path","chunk","scanCandidates","qv","storeDims","fetch","allowed","stmt","prepare","all","queryNorm","d","Math","min","sqrt","best","Map","has","q","Int8Array","buffer","byteOffset","byteLength","dot","norm","score","existing","get","set","lines","start_line","end_line","entries","value","sort","slice","similarity","scanSimilar","opts","targetStmt","targetRows","target","v","exclude","other","otherNorm","t","len","undefined","p","k"],"mappings":"AACA,SAASA,QAAQ,EAAEC,mBAAmB,EAAEC,YAAY,QAAQ,gBAAgB;AAE5E,2FAA2F;AAC3F,yEAAyE;AACzE,OAAO,eAAeC,iBAAiBC,IAAgB,EAAEC,IAAsB;IAC7E,IAAIA,KAAKC,MAAM,KAAK,GAAG;IACvB,MAAMF,KAAKG,QAAQ,CACjB,8EACAF,KAAKG,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIC,KAAK;YAAED,IAAIE,MAAM;YAAEF,IAAIG,IAAI;YAAEH,IAAII,KAAK;SAAC;AAElE;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,OAAO,eAAeC,eAAeV,IAAgB,EAAEW,EAAgB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,OAAqB;IAC9H,MAAMC,OAAO,MAAMf,KAAKgB,OAAO,CAAC;IAChC,MAAMf,OAAQ,MAAMc,KAAKE,GAAG;IAQ5B,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACT,WAAWD,GAAGT,MAAM,GAAGiB,IAAKD,aAAaP,EAAE,CAACQ,EAAE,GAAGR,EAAE,CAACQ,EAAE;IACnFD,YAAYE,KAAKE,IAAI,CAACJ;IAEtB,MAAMK,OAAO,IAAIC;IACjB,KAAK,MAAMnB,OAAOJ,KAAM;QACtB,IAAIa,WAAW,CAACA,QAAQW,GAAG,CAACpB,IAAIG,IAAI,GAAG;QACvC,MAAMkB,IAAI,IAAIC,UAAUtB,IAAIE,MAAM,CAACqB,MAAM,EAAEvB,IAAIE,MAAM,CAACsB,UAAU,EAAET,KAAKC,GAAG,CAACT,WAAWP,IAAIE,MAAM,CAACuB,UAAU;QAC3G,8FAA8F;QAC9F,oFAAoF;QACpF,IAAIC,MAAM;QACV,IAAIC,OAAO;QACX,MAAM9B,SAASkB,KAAKC,GAAG,CAACK,EAAExB,MAAM,EAAES,GAAGT,MAAM;QAC3C,IAAK,IAAIiB,IAAI,GAAGA,IAAIjB,QAAQiB,IAAK;YAC/BY,OAAOL,CAAC,CAACP,EAAE,GAAGR,EAAE,CAACQ,EAAE;YACnBa,QAAQN,CAAC,CAACP,EAAE,GAAGO,CAAC,CAACP,EAAE;QACrB;QACA,IAAK,IAAIA,IAAIjB,QAAQiB,IAAIO,EAAExB,MAAM,EAAEiB,IAAKa,QAAQN,CAAC,CAACP,EAAE,GAAGO,CAAC,CAACP,EAAE;QAC3D,MAAMc,QAAQD,SAAS,KAAKd,cAAc,IAAI,IAAIa,MAAOX,CAAAA,KAAKE,IAAI,CAACU,QAAQd,SAAQ;QACnF,MAAMgB,WAAWX,KAAKY,GAAG,CAAC9B,IAAIG,IAAI;QAClC,IAAI,CAAC0B,YAAYD,QAAQC,SAASD,KAAK,IAAKA,UAAUC,SAASD,KAAK,IAAI5B,IAAII,KAAK,GAAGyB,SAASzB,KAAK,EAAG;YACnGc,KAAKa,GAAG,CAAC/B,IAAIG,IAAI,EAAE;gBAAEyB;gBAAOxB,OAAOJ,IAAII,KAAK;gBAAE4B,OAAO,CAAC,CAAC,EAAEhC,IAAIiC,UAAU,CAAC,CAAC,EAAEjC,IAAIkC,QAAQ,EAAE;YAAC;QAC5F;IACF;IACA,OAAO;WAAIhB,KAAKiB,OAAO;KAAG,CACvBpC,GAAG,CAAC,CAAC,CAACI,MAAMiC,MAAM,GAAM,CAAA;YAAEjC;YAAM,GAAGiC,KAAK;QAAC,CAAA,GACzCC,IAAI,CAAC7C,qBACL8C,KAAK,CAAC,GAAG9B,OACTT,GAAG,CAAC,CAAC,EAAEI,IAAI,EAAE6B,KAAK,EAAEJ,KAAK,EAAE,GAAM,CAAA;YAAEzB;YAAM6B;YAAOO,YAAYhD,SAASqC;QAAO,CAAA;AACjF;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAeY,YAAY7C,IAAgB,EAAEQ,IAAY,EAAEsC,IAAgE;IAChI,MAAMC,aAAa,MAAM/C,KAAKgB,OAAO,CAAC;IACtC,MAAMgC,aAAc,MAAMD,WAAW9B,GAAG,CAACT;IACzC,IAAIwC,WAAW9C,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,kGAAkG;IAClG,MAAM+C,SAASnD,aAAakD,YAAY5C,GAAG,CAAC,CAACC;QAC3C,MAAM6C,IAAI,IAAIvB,UAAUtB,IAAIE,MAAM,CAACqB,MAAM,EAAEvB,IAAIE,MAAM,CAACsB,UAAU,EAAExB,IAAIE,MAAM,CAACuB,UAAU;QACvF,IAAIE,OAAO;QACX,IAAK,IAAIb,IAAI,GAAGA,IAAI+B,EAAEhD,MAAM,EAAEiB,IAAKa,QAAQkB,CAAC,CAAC/B,EAAE,GAAG+B,CAAC,CAAC/B,EAAE;QACtD,OAAO;YAAE+B;YAAGlB,MAAMZ,KAAKE,IAAI,CAACU;QAAM;IACpC;IAEA,MAAMjB,OAAO,MAAMf,KAAKgB,OAAO,CAAC;IAChC,MAAMf,OAAQ,MAAMc,KAAKE,GAAG;IAC5B,MAAMM,OAAO,IAAIC;IACjB,KAAK,MAAMnB,OAAOJ,KAAM;QACtB,IAAII,IAAIG,IAAI,KAAKA,QAAQsC,KAAKK,OAAO,CAAC1B,GAAG,CAACpB,IAAIG,IAAI,KAAMsC,KAAKhC,OAAO,IAAI,CAACgC,KAAKhC,OAAO,CAACW,GAAG,CAACpB,IAAIG,IAAI,GAAI;QACtG,MAAM4C,QAAQ,IAAIzB,UAAUtB,IAAIE,MAAM,CAACqB,MAAM,EAAEvB,IAAIE,MAAM,CAACsB,UAAU,EAAExB,IAAIE,MAAM,CAACuB,UAAU;QAC3F,IAAIuB,YAAY;QAChB,IAAK,IAAIlC,IAAI,GAAGA,IAAIiC,MAAMlD,MAAM,EAAEiB,IAAKkC,aAAaD,KAAK,CAACjC,EAAE,GAAGiC,KAAK,CAACjC,EAAE;QACvEkC,YAAYjC,KAAKE,IAAI,CAAC+B;QACtB,KAAK,MAAMC,KAAKL,OAAQ;YACtB,IAAIlB,MAAM;YACV,MAAMwB,MAAMnC,KAAKC,GAAG,CAACiC,EAAEJ,CAAC,CAAChD,MAAM,EAAEkD,MAAMlD,MAAM;YAC7C,IAAK,IAAIiB,IAAI,GAAGA,IAAIoC,KAAKpC,IAAKY,OAAOuB,EAAEJ,CAAC,CAAC/B,EAAE,GAAGiC,KAAK,CAACjC,EAAE;YACtD,MAAMc,QAAQqB,EAAEtB,IAAI,KAAK,KAAKqB,cAAc,IAAI,IAAItB,MAAOuB,CAAAA,EAAEtB,IAAI,GAAGqB,SAAQ;YAC5E,MAAMnB,WAAWX,KAAKY,GAAG,CAAC9B,IAAIG,IAAI;YAClC,IAAI0B,aAAasB,aAAavB,QAAQC,UAAUX,KAAKa,GAAG,CAAC/B,IAAIG,IAAI,EAAEyB;QACrE;IACF;IAEA,OAAO;WAAIV,KAAKiB,OAAO;KAAG,CACvBpC,GAAG,CAAC,CAAC,CAACqD,GAAGxB,MAAM,GAAM,CAAA;YAAEzB,MAAMiD;YAAGxB;QAAM,CAAA,GACtCS,IAAI,CAAC7C,qBACL8C,KAAK,CAAC,GAAGG,KAAKY,CAAC,EACftD,GAAG,CAAC,CAAC,EAAEI,MAAMiD,CAAC,EAAExB,KAAK,EAAE,GAAM,CAAA;YAAEzB,MAAMiD;YAAGb,YAAYhD,SAASqC;QAAO,CAAA;AACzE"}

package/dist/esm/store/turso/connection.js

@@ -3,6 +3,7 @@ import { rewriteInsert } from '../batch.js';
 import { setMeta } from '../shared.js';
 import { BEGIN_WRITE, withTransaction } from '../transaction.js';
 import { CONNECT_OPTS, tursoApi } from './native.js';
+import { rewriteFunctions } from './sql-functions.js';
 // @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%
 // faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.
 let TursoStatementWrapper = class TursoStatementWrapper {
@@ -82,7 +83,7 @@ export function createConnection(db) {
             await db.exec(sql);
         },
         async prepare (sql) {
-            return new TursoStatementWrapper(await db.prepare(sql));
+            return new TursoStatementWrapper(await db.prepare(rewriteFunctions(sql)));
         },
         // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):
         // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.
@@ -102,7 +103,7 @@ export function createConnection(db) {
                     return;
                 }
                 // Finalized here because nothing else will: open() hands back a connection the caller
-                // can hold across many batches, and the db.batch() this replaced finalized its own.
+                // can hold across many batches, and nothing else owns this statement's lifetime.
                 const stmt = await db.prepare(sql);
                 try {
                     for (const row of paramRows)await stmt.run(...row);

package/dist/esm/store/turso/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import { stat } from 'node:fs/promises';\nimport type { Database } from '@tursodatabase/database';\nimport { rewriteInsert } from '../batch.ts';\nimport { setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { CONNECT_OPTS, tursoApi } from './native.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\n// The main cache file's path, or null when unresolvable. The store holds a Database and\n// Connection, not a path (PLAN 3.52), so the path is read from the database itself.\nexport async function cacheFilePath(conn: Connection): Promise<string | null> {\n  const rows = (await (await conn.prepare('PRAGMA database_list')).all()) as Array<{ name: string; file: string }>;\n  return rows.find((r) => r.name === 'main')?.file || null;\n}\n\nexport async function fileSize(path: string): Promise<number | null> {\n  try {\n    return (await stat(path)).size;\n  } catch {\n    return null;\n  }\n}\n\n// Reclaims turso#8170's FTS space amplification (PLAN 3.41), on a throwaway connection after the\n// store's own has closed: `vacuum` doubles incremental reconcile cost merely by being enabled (3.54).\nexport async function reclaimSpace(path: string): Promise<void> {\n  try {\n    const turso = await tursoApi();\n    const db = await turso.connect(path, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental, 'vacuum'] });\n    try {\n      await db.exec('VACUUM');\n      const size = await fileSize(path);\n      // Recorded here, not by the caller: this connection is the only one still open on the file.\n      if (size !== null) await setMeta(createConnection(db), 'compact_size', String(size));\n      await checkpointWal(db);\n    } finally {\n      await db.close();\n    }\n  } catch (err) {\n    console.error(`sense: turso VACUUM failed, the cache will keep the space until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):\n    // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const rewritten = rewriteInsert(sql, paramRows.length);\n          if (rewritten) {\n            const stmt = await db.prepare(rewritten.sql);\n            try {\n              // One flat array, not spread: spreading tens of thousands of args hits a JS call-stack\n              // limit well before turso enforces any variable-count ceiling (measured; see PLAN.md).\n              await stmt.run(paramRows.flat());\n            } finally {\n              await stmt.close();\n            }\n            return;\n          }\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and the db.batch() this replaced finalized its own.\n          const stmt = await db.prepare(sql);\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["stat","rewriteInsert","setMeta","BEGIN_WRITE","withTransaction","CONNECT_OPTS","tursoApi","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","checkpointWal","db","exec","err","console","error","message","cacheFilePath","conn","rows","prepare","find","r","name","file","fileSize","path","size","reclaimSpace","turso","connect","experimental","createConnection","String","close","sql","runBatch","paramRows","length","rewritten","flat","row"],"mappings":"AAAA,SAASA,IAAI,QAAQ,mBAAmB;AAExC,SAASC,aAAa,QAAQ,cAAc;AAC5C,SAASC,OAAO,QAAQ,eAAe;AACvC,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,YAAY,EAAEC,QAAQ,QAAQ,cAAc;AAMrD,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,mGAAmG;AACnG,wGAAwG;AACxG,OAAO,eAAeQ,cAAcC,EAAY;IAC9C,IAAI;QACF,MAAMA,GAAGC,IAAI,CAAC;IAChB,EAAE,OAAOC,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,wFAAwF,EAAE,AAACF,IAAcG,OAAO,EAAE;IACnI;AACF;AAEA,wFAAwF;AACxF,oFAAoF;AACpF,OAAO,eAAeC,cAAcC,IAAgB;QAE3CC;IADP,MAAMA,OAAQ,MAAM,AAAC,CAAA,MAAMD,KAAKE,OAAO,CAAC,uBAAsB,EAAGhB,GAAG;IACpE,OAAOe,EAAAA,aAAAA,KAAKE,IAAI,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,qBAA5BJ,iCAAAA,WAAqCK,IAAI,KAAI;AACtD;AAEA,OAAO,eAAeC,SAASC,IAAY;IACzC,IAAI;QACF,OAAO,AAAC,CAAA,MAAMlC,KAAKkC,KAAI,EAAGC,IAAI;IAChC,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,iGAAiG;AACjG,sGAAsG;AACtG,OAAO,eAAeC,aAAaF,IAAY;IAC7C,IAAI;QACF,MAAMG,QAAQ,MAAM/B;QACpB,MAAMa,KAAK,MAAMkB,MAAMC,OAAO,CAACJ,MAAM;YAAE,GAAG7B,YAAY;YAAEkC,cAAc;mBAAIlC,aAAakC,YAAY;gBAAE;aAAS;QAAC;QAC/G,IAAI;YACF,MAAMpB,GAAGC,IAAI,CAAC;YACd,MAAMe,OAAO,MAAMF,SAASC;YAC5B,4FAA4F;YAC5F,IAAIC,SAAS,MAAM,MAAMjC,QAAQsC,iBAAiBrB,KAAK,gBAAgBsB,OAAON;YAC9E,MAAMjB,cAAcC;QACtB,SAAU;YACR,MAAMA,GAAGuB,KAAK;QAChB;IACF,EAAE,OAAOrB,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,8EAA8E,EAAE,AAACF,IAAcG,OAAO,EAAE;IACzH;AACF;AAEA,OAAO,SAASgB,iBAAiBrB,EAAY;IAC3C,MAAMO,OAAmB;QACvB,MAAMN,MAAKuB,GAAW;YACpB,MAAMxB,GAAGC,IAAI,CAACuB;QAChB;QACA,MAAMf,SAAQe,GAAW;YACvB,OAAO,IAAIpC,sBAAsB,MAAMY,GAAGS,OAAO,CAACe;QACpD;QACA,gGAAgG;QAChG,wGAAwG;QACxG,MAAMC,UAASD,GAAW,EAAEE,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAM1C,gBACJsB,MACA;gBACE,MAAMqB,YAAY9C,cAAc0C,KAAKE,UAAUC,MAAM;gBACrD,IAAIC,WAAW;oBACb,MAAMrC,OAAO,MAAMS,GAAGS,OAAO,CAACmB,UAAUJ,GAAG;oBAC3C,IAAI;wBACF,uFAAuF;wBACvF,uFAAuF;wBACvF,MAAMjC,KAAKF,GAAG,CAACqC,UAAUG,IAAI;oBAC/B,SAAU;wBACR,MAAMtC,KAAKgC,KAAK;oBAClB;oBACA;gBACF;gBACA,sFAAsF;gBACtF,oFAAoF;gBACpF,MAAMhC,OAAO,MAAMS,GAAGS,OAAO,CAACe;gBAC9B,IAAI;oBACF,KAAK,MAAMM,OAAOJ,UAAW,MAAMnC,KAAKF,GAAG,IAAIyC;gBACjD,SAAU;oBACR,MAAMvC,KAAKgC,KAAK;gBAClB;YACF,GACAvC;QAEJ;IACF;IACA,OAAOuB;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import { stat } from 'node:fs/promises';\nimport type { Database } from '@tursodatabase/database';\nimport { rewriteInsert } from '../batch.ts';\nimport { setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { CONNECT_OPTS, tursoApi } from './native.ts';\nimport { rewriteFunctions } from './sql-functions.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\n// The main cache file's path, or null when unresolvable. The store holds a Database and\n// Connection, not a path (PLAN 3.52), so the path is read from the database itself.\nexport async function cacheFilePath(conn: Connection): Promise<string | null> {\n  const rows = (await (await conn.prepare('PRAGMA database_list')).all()) as Array<{ name: string; file: string }>;\n  return rows.find((r) => r.name === 'main')?.file || null;\n}\n\nexport async function fileSize(path: string): Promise<number | null> {\n  try {\n    return (await stat(path)).size;\n  } catch {\n    return null;\n  }\n}\n\n// Reclaims turso#8170's FTS space amplification (PLAN 3.41), on a throwaway connection after the\n// store's own has closed: `vacuum` doubles incremental reconcile cost merely by being enabled (3.54).\nexport async function reclaimSpace(path: string): Promise<void> {\n  try {\n    const turso = await tursoApi();\n    const db = await turso.connect(path, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental, 'vacuum'] });\n    try {\n      await db.exec('VACUUM');\n      const size = await fileSize(path);\n      // Recorded here, not by the caller: this connection is the only one still open on the file.\n      if (size !== null) await setMeta(createConnection(db), 'compact_size', String(size));\n      await checkpointWal(db);\n    } finally {\n      await db.close();\n    }\n  } catch (err) {\n    console.error(`sense: turso VACUUM failed, the cache will keep the space until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(rewriteFunctions(sql)));\n    },\n    // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):\n    // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const rewritten = rewriteInsert(sql, paramRows.length);\n          if (rewritten) {\n            const stmt = await db.prepare(rewritten.sql);\n            try {\n              // One flat array, not spread: spreading tens of thousands of args hits a JS call-stack\n              // limit well before turso enforces any variable-count ceiling (measured; see PLAN.md).\n              await stmt.run(paramRows.flat());\n            } finally {\n              await stmt.close();\n            }\n            return;\n          }\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and nothing else owns this statement's lifetime.\n          const stmt = await db.prepare(sql);\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["stat","rewriteInsert","setMeta","BEGIN_WRITE","withTransaction","CONNECT_OPTS","tursoApi","rewriteFunctions","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","checkpointWal","db","exec","err","console","error","message","cacheFilePath","conn","rows","prepare","find","r","name","file","fileSize","path","size","reclaimSpace","turso","connect","experimental","createConnection","String","close","sql","runBatch","paramRows","length","rewritten","flat","row"],"mappings":"AAAA,SAASA,IAAI,QAAQ,mBAAmB;AAExC,SAASC,aAAa,QAAQ,cAAc;AAC5C,SAASC,OAAO,QAAQ,eAAe;AACvC,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,YAAY,EAAEC,QAAQ,QAAQ,cAAc;AACrD,SAASC,gBAAgB,QAAQ,qBAAqB;AAMtD,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,mGAAmG;AACnG,wGAAwG;AACxG,OAAO,eAAeQ,cAAcC,EAAY;IAC9C,IAAI;QACF,MAAMA,GAAGC,IAAI,CAAC;IAChB,EAAE,OAAOC,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,wFAAwF,EAAE,AAACF,IAAcG,OAAO,EAAE;IACnI;AACF;AAEA,wFAAwF;AACxF,oFAAoF;AACpF,OAAO,eAAeC,cAAcC,IAAgB;QAE3CC;IADP,MAAMA,OAAQ,MAAM,AAAC,CAAA,MAAMD,KAAKE,OAAO,CAAC,uBAAsB,EAAGhB,GAAG;IACpE,OAAOe,EAAAA,aAAAA,KAAKE,IAAI,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,qBAA5BJ,iCAAAA,WAAqCK,IAAI,KAAI;AACtD;AAEA,OAAO,eAAeC,SAASC,IAAY;IACzC,IAAI;QACF,OAAO,AAAC,CAAA,MAAMnC,KAAKmC,KAAI,EAAGC,IAAI;IAChC,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,iGAAiG;AACjG,sGAAsG;AACtG,OAAO,eAAeC,aAAaF,IAAY;IAC7C,IAAI;QACF,MAAMG,QAAQ,MAAMhC;QACpB,MAAMc,KAAK,MAAMkB,MAAMC,OAAO,CAACJ,MAAM;YAAE,GAAG9B,YAAY;YAAEmC,cAAc;mBAAInC,aAAamC,YAAY;gBAAE;aAAS;QAAC;QAC/G,IAAI;YACF,MAAMpB,GAAGC,IAAI,CAAC;YACd,MAAMe,OAAO,MAAMF,SAASC;YAC5B,4FAA4F;YAC5F,IAAIC,SAAS,MAAM,MAAMlC,QAAQuC,iBAAiBrB,KAAK,gBAAgBsB,OAAON;YAC9E,MAAMjB,cAAcC;QACtB,SAAU;YACR,MAAMA,GAAGuB,KAAK;QAChB;IACF,EAAE,OAAOrB,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,8EAA8E,EAAE,AAACF,IAAcG,OAAO,EAAE;IACzH;AACF;AAEA,OAAO,SAASgB,iBAAiBrB,EAAY;IAC3C,MAAMO,OAAmB;QACvB,MAAMN,MAAKuB,GAAW;YACpB,MAAMxB,GAAGC,IAAI,CAACuB;QAChB;QACA,MAAMf,SAAQe,GAAW;YACvB,OAAO,IAAIpC,sBAAsB,MAAMY,GAAGS,OAAO,CAACtB,iBAAiBqC;QACrE;QACA,gGAAgG;QAChG,wGAAwG;QACxG,MAAMC,UAASD,GAAW,EAAEE,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAM3C,gBACJuB,MACA;gBACE,MAAMqB,YAAY/C,cAAc2C,KAAKE,UAAUC,MAAM;gBACrD,IAAIC,WAAW;oBACb,MAAMrC,OAAO,MAAMS,GAAGS,OAAO,CAACmB,UAAUJ,GAAG;oBAC3C,IAAI;wBACF,uFAAuF;wBACvF,uFAAuF;wBACvF,MAAMjC,KAAKF,GAAG,CAACqC,UAAUG,IAAI;oBAC/B,SAAU;wBACR,MAAMtC,KAAKgC,KAAK;oBAClB;oBACA;gBACF;gBACA,sFAAsF;gBACtF,iFAAiF;gBACjF,MAAMhC,OAAO,MAAMS,GAAGS,OAAO,CAACe;gBAC9B,IAAI;oBACF,KAAK,MAAMM,OAAOJ,UAAW,MAAMnC,KAAKF,GAAG,IAAIyC;gBACjD,SAAU;oBACR,MAAMvC,KAAKgC,KAAK;gBAClB;YACF,GACAxC;QAEJ;IACF;IACA,OAAOwB;AACT"}

package/dist/esm/store/turso/lexical-text.d.ts

@@ -0,0 +1 @@
+export declare function stemFolded(text: string): string;

package/dist/esm/store/turso/lexical-text.js

@@ -0,0 +1,12 @@
+import { stemmer } from 'stemmer';
+import { foldForSearch, searchTokens } from '../../text/segment.js';
+export function stemFolded(text) {
+    let out = '';
+    let end = 0;
+    for (const token of searchTokens(text)){
+        out += text.slice(end, token.start).replace(/[\u0027\u2019]/gu, ' ');
+        out += token.unspaced ? token.text : stemmer(foldForSearch(token.text));
+        end = token.end;
+    }
+    return out + text.slice(end).replace(/[\u0027\u2019]/gu, ' ');
+}

package/dist/esm/store/turso/lexical-text.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical-text.ts"],"sourcesContent":["import { stemmer } from 'stemmer';\nimport { foldForSearch, searchTokens } from '../../text/segment.ts';\n\nexport function stemFolded(text: string): string {\n  let out = '';\n  let end = 0;\n  for (const token of searchTokens(text)) {\n    out += text.slice(end, token.start).replace(/[\\u0027\\u2019]/gu, ' ');\n    out += token.unspaced ? token.text : stemmer(foldForSearch(token.text));\n    end = token.end;\n  }\n  return out + text.slice(end).replace(/[\\u0027\\u2019]/gu, ' ');\n}\n"],"names":["stemmer","foldForSearch","searchTokens","stemFolded","text","out","end","token","slice","start","replace","unspaced"],"mappings":"AAAA,SAASA,OAAO,QAAQ,UAAU;AAClC,SAASC,aAAa,EAAEC,YAAY,QAAQ,wBAAwB;AAEpE,OAAO,SAASC,WAAWC,IAAY;IACrC,IAAIC,MAAM;IACV,IAAIC,MAAM;IACV,KAAK,MAAMC,SAASL,aAAaE,MAAO;QACtCC,OAAOD,KAAKI,KAAK,CAACF,KAAKC,MAAME,KAAK,EAAEC,OAAO,CAAC,oBAAoB;QAChEL,OAAOE,MAAMI,QAAQ,GAAGJ,MAAMH,IAAI,GAAGJ,QAAQC,cAAcM,MAAMH,IAAI;QACrEE,MAAMC,MAAMD,GAAG;IACjB;IACA,OAAOD,MAAMD,KAAKI,KAAK,CAACF,KAAKI,OAAO,CAAC,oBAAoB;AAC3D"}