sensemaking 0.18.4 → 0.19.1

package/dist/esm/store/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually\n  // uses for vectors. Word/lexical search has no equivalent unambiguous signal at this level\n  // (every preset defaults to wanting \"words\"), so a store lacking 'lexical' fails loudly at\n  // first use instead (see duckdb/store.ts's lexical.query()).\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { getMeta, setMeta } from './meta.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","openTurso","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","cfg","has","requireWatchConcurrency","getMeta","setMeta","clearCache","DB_FILENAME","docCount","SCHEMA_VERSION"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASJ,gBAAgBK,kBAAkB,QAAQ,mBAAmB;AAQtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcL;QAAqBM,MAAMP;IAAW;IAC9DQ,QAAQ;QAAEF,cAAcP;QAAqBQ,MAAMV;IAAW;IAC9DY,OAAO;QAAEH,cAAcH;QAAoBI,MAAML;IAAU;AAC7D;AAEA,SAASQ,SAASC,IAAe;IAC/B,MAAMC,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIhB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEe,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOhB,UAAUsB;IACvB,MAAML,QAAQF,SAASC;IAEvB,yFAAyF;IACzF,2FAA2F;IAC3F,2FAA2F;IAC3F,6DAA6D;IAC7D,IAAIjB,gBAAgBuB,QAAQ,CAACL,MAAMN,YAAY,CAACY,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACU;AACpB;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASE,wBAAwBF,GAAmB;IACzD,MAAMN,OAAOhB,UAAUsB;IACvB,IAAI,CAACP,SAASC,MAAML,YAAY,CAACY,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,OAAO,EAAEC,OAAO,QAAQ,YAAY;AAE7C,SAASC,UAAU,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","openTurso","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","cfg","has","requireWatchConcurrency","clearCache","getMeta","setMeta","DB_FILENAME","docCount","SCHEMA_VERSION"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASJ,gBAAgBK,kBAAkB,QAAQ,mBAAmB;AAQtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcL;QAAqBM,MAAMP;IAAW;IAC9DQ,QAAQ;QAAEF,cAAcP;QAAqBQ,MAAMV;IAAW;IAC9DY,OAAO;QAAEH,cAAcH;QAAoBI,MAAML;IAAU;AAC7D;AAEA,SAASQ,SAASC,IAAe;IAC/B,MAAMC,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIhB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEe,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOhB,UAAUsB;IACvB,MAAML,QAAQF,SAASC;IAEvB,8FAA8F;IAC9F,oJAAoJ;IACpJ,IAAIjB,gBAAgBuB,QAAQ,CAACL,MAAMN,YAAY,CAACY,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACU;AACpB;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASE,wBAAwBF,GAAmB;IACzD,MAAMN,OAAOhB,UAAUsB;IACvB,IAAI,CAACP,SAASC,MAAML,YAAY,CAACY,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,UAAU,QAAQ,aAAa;AACxC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAE/C,SAASC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB"}

package/dist/esm/store/native.js

@@ -3,17 +3,12 @@ import Module from 'node:module';
 import { dirname, join } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 import { SenseError } from '../errors.js';
-// install-module-linked itself is small, but a target native package can be large (@duckdb/node-
-// api's ~110MB is the current instance): deferred the same way embed/static.ts defers
-// @huggingface/tokenizers, so requiring it stays inside the already-gated branch below rather
-// than running for every store/index.ts load.
+// install-module-linked is small, but a target native package can be large (@duckdb/node-api's
+// ~110MB), so it is required lazily inside the gated branch below, not for every store/index.ts load.
 const _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;
 const _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));
-// Walks up from this file's own directory to find the real package.json, rather than a fixed
-// dot-count from __dirname (xz-compat's `../../node_modules`): this file sits at a different
-// depth under src/ during `tsds test:node` than under the built dist/{cjs,esm}/store output, so a
-// hardcoded depth is right for one and wrong for the other. Works the same way for a
-// globally-installed `sense`, whose node_modules this resolves to -- never the caller's cwd.
+// Walks up from this file's own directory rather than a fixed dot-count: it sits at a
+// different depth under src/ than under built dist/{cjs,esm}/store output.
 export function packageNodeModules() {
     let dir = _dirname;
     for(;;){
@@ -28,12 +23,8 @@ export function packageNodeModules() {
     }
 }
 const installing = new Map();
-// De-dupes concurrent installs within this process (see spawn-term's loadInk for the shape).
-// The install here is async -- the load site already awaits a dynamic import -- so a bare
-// attempted flag (xz-compat's `installationAttempted`, enough for its synchronous single-target
-// install) isn't enough on its own; concurrent callers instead await the same in-flight promise,
-// keyed by specifier and target so tests exercising several installs in one process don't share
-// one slot. A failed install is not cached, so a later call retries.
+// De-dupes concurrent installs: since the install is async, concurrent callers await the same
+// in-flight promise, keyed by specifier and target. A failed install is not cached, so a later call retries.
 function installOnce(specifier, nodeModulesPath) {
     const key = `${specifier}\n${nodeModulesPath}`;
     let promise = installing.get(key);
@@ -47,12 +38,8 @@ function installOnce(specifier, nodeModulesPath) {
     }
     return promise;
 }
-// Node caches a bare specifier's package resolution -- including a "not found" miss -- for the
-// life of the process, so retrying `import(importName)` right after installing sees the same
-// stale miss and falls through to a bogus index.js guess instead of the package's real entry
-// (verified against a real global install: the plain-specifier retry failed there, this did
-// not). Reading the freshly-installed package's own package.json and importing its entry file
-// by an absolute file: URL sidesteps that stale cache instead of re-resolving the bare name.
+// Node caches a bare specifier's resolution, including a "not found" miss, for the process
+// lifetime; importing the entry by an absolute file: URL instead of the bare name sidesteps that.
 function resolveEntryUrl(importName, nodeModulesPath) {
     var _pkg_main;
     const pkgDir = join(nodeModulesPath, ...importName.split('/'));
@@ -60,10 +47,8 @@ function resolveEntryUrl(importName, nodeModulesPath) {
     const entry = typeof pkg.exports === 'string' ? pkg.exports : (_pkg_main = pkg.main) !== null && _pkg_main !== void 0 ? _pkg_main : 'index.js';
     return pathToFileURL(join(pkgDir, entry)).href;
 }
-// Import-then-install-then-retry, over an injectable importName so the failure path (import AND
-// install both fail) can be exercised for real in tests without downloading a real store's native
-// binding. importName defaults to descriptor.pkg, which is what production always wants; the
-// thrown errors name descriptor.pkg regardless, since that is what a real user needs to install.
+// Import-then-install-then-retry, over an injectable importName so the failure path can be
+// exercised in tests without a real native binding. Thrown errors always name descriptor.pkg.
 export async function loadOrInstall(descriptor, nodeModulesPath, importName = descriptor.pkg) {
     try {
         return await import(importName);

package/dist/esm/store/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../errors.ts';\n\n// A store's optional native package, and the strings its error messages and install notice name.\nexport interface NativeDescriptor {\n  store: string;\n  pkg: string;\n  sizeHint: string;\n}\n\n// install-module-linked itself is small, but a target native package can be large (@duckdb/node-\n// api's ~110MB is the current instance): deferred the same way embed/static.ts defers\n// @huggingface/tokenizers, so requiring it stays inside the already-gated branch below rather\n// than running for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory to find the real package.json, rather than a fixed\n// dot-count from __dirname (xz-compat's `../../node_modules`): this file sits at a different\n// depth under src/ during `tsds test:node` than under the built dist/{cjs,esm}/store output, so a\n// hardcoded depth is right for one and wrong for the other. Works the same way for a\n// globally-installed `sense`, whose node_modules this resolves to -- never the caller's cwd.\nexport function packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install a native dependency into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs within this process (see spawn-term's loadInk for the shape).\n// The install here is async -- the load site already awaits a dynamic import -- so a bare\n// attempted flag (xz-compat's `installationAttempted`, enough for its synchronous single-target\n// install) isn't enough on its own; concurrent callers instead await the same in-flight promise,\n// keyed by specifier and target so tests exercising several installs in one process don't share\n// one slot. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's package resolution -- including a \"not found\" miss -- for the\n// life of the process, so retrying `import(importName)` right after installing sees the same\n// stale miss and falls through to a bogus index.js guess instead of the package's real entry\n// (verified against a real global install: the plain-specifier retry failed there, this did\n// not). Reading the freshly-installed package's own package.json and importing its entry file\n// by an absolute file: URL sidesteps that stale cache instead of re-resolving the bare name.\nfunction resolveEntryUrl(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  return pathToFileURL(join(pkgDir, entry)).href;\n}\n\n// Import-then-install-then-retry, over an injectable importName so the failure path (import AND\n// install both fail) can be exercised for real in tests without downloading a real store's native\n// binding. importName defaults to descriptor.pkg, which is what production always wants; the\n// thrown errors name descriptor.pkg regardless, since that is what a real user needs to install.\nexport async function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName: string = descriptor.pkg): Promise<T> {\n  try {\n    return (await import(importName)) as T;\n  } catch {\n    console.error(`sense: store \"${descriptor.store}\" needs ${descriptor.pkg}; installing it now (one-time download, ${descriptor.sizeHint})...`);\n    try {\n      await installOnce(importName, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" needs ${descriptor.pkg}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${descriptor.pkg}\\` yourself and try again`\n      );\n    }\n    try {\n      return (await import(resolveEntryUrl(importName, nodeModulesPath))) as T;\n    } catch (loadErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" installed ${descriptor.pkg} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${descriptor.pkg}\\` to see the underlying error`\n      );\n    }\n  }\n}\n"],"names":["existsSync","readFileSync","Module","dirname","join","fileURLToPath","pathToFileURL","SenseError","_require","require","createRequire","url","_dirname","__filename","packageNodeModules","dir","candidate","pkg","JSON","parse","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntryUrl","importName","pkgDir","split","entry","exports","main","href","loadOrInstall","descriptor","console","error","store","sizeHint","installErr","message","loadErr","process","platform","arch"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAC1C,SAASC,aAAa,EAAEC,aAAa,QAAQ,WAAW;AACxD,SAASC,UAAU,QAAQ,eAAe;AAS1C,iGAAiG;AACjG,sFAAsF;AACtF,8FAA8F;AAC9F,8CAA8C;AAC9C,MAAMC,WAAW,OAAOC,YAAY,cAAcP,OAAOQ,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAC1F,MAAMG,WAAWT,QAAQ,OAAOU,eAAe,cAAcA,aAAaR,cAAc,YAAYM,GAAG;AAEvG,6FAA6F;AAC7F,6FAA6F;AAC7F,kGAAkG;AAClG,qFAAqF;AACrF,6FAA6F;AAC7F,OAAO,SAASG;IACd,IAAIC,MAAMH;IACV,OAAS;QACP,MAAMI,YAAYZ,KAAKW,KAAK;QAC5B,IAAIf,WAAWgB,YAAY;YACzB,MAAMC,MAAMC,KAAKC,KAAK,CAAClB,aAAae,WAAW;YAC/C,IAAIC,IAAIG,IAAI,KAAK,eAAe,OAAOhB,KAAKW,KAAK;QACnD;QACA,MAAMM,SAASlB,QAAQY;QACvB,IAAIM,WAAWN,KAAK,MAAM,IAAIO,MAAM;QACpCP,MAAMM;IACR;AACF;AAEA,MAAME,aAAa,IAAIC;AAEvB,6FAA6F;AAC7F,0FAA0F;AAC1F,gGAAgG;AAChG,iGAAiG;AACjG,gGAAgG;AAChG,qEAAqE;AACrE,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,MAAMC,MAAM,GAAGF,UAAU,EAAE,EAAEC,iBAAiB;IAC9C,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,MAAME,sBAAsB,AAACvB,SAAS,yBAA0GwB,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,KAAO,GACP,CAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,6FAA6F;AAC7F,SAASQ,gBAAgBC,UAAkB,EAAEX,eAAuB;QAGHV;IAF/D,MAAMsB,SAASnC,KAAKuB,oBAAoBW,WAAWE,KAAK,CAAC;IACzD,MAAMvB,MAAMC,KAAKC,KAAK,CAAClB,aAAaG,KAAKmC,QAAQ,iBAAiB;IAClE,MAAME,QAAQ,OAAOxB,IAAIyB,OAAO,KAAK,WAAWzB,IAAIyB,OAAO,IAAIzB,YAAAA,IAAI0B,IAAI,cAAR1B,uBAAAA,YAAY;IAC3E,OAAOX,cAAcF,KAAKmC,QAAQE,QAAQG,IAAI;AAChD;AAEA,gGAAgG;AAChG,kGAAkG;AAClG,6FAA6F;AAC7F,iGAAiG;AACjG,OAAO,eAAeC,cAAiBC,UAA4B,EAAEnB,eAAuB,EAAEW,aAAqBQ,WAAW7B,GAAG;IAC/H,IAAI;QACF,OAAQ,MAAM,MAAM,CAACqB;IACvB,EAAE,OAAM;QACNS,QAAQC,KAAK,CAAC,CAAC,cAAc,EAAEF,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW7B,GAAG,CAAC,wCAAwC,EAAE6B,WAAWI,QAAQ,CAAC,IAAI,CAAC;QAC5I,IAAI;YACF,MAAMzB,YAAYa,YAAYX;QAChC,EAAE,OAAOwB,YAAY;YACnB,MAAM,IAAI5C,WACR,4BACA,CAAC,OAAO,EAAEuC,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW7B,GAAG,CAAC,0CAA0C,EAAE,AAACkC,WAAqBC,OAAO,CAAC,sKAAsK,EAAEN,WAAW7B,GAAG,CAAC,yBAAyB,CAAC;QAEnV;QACA,IAAI;YACF,OAAQ,MAAM,MAAM,CAACoB,gBAAgBC,YAAYX;QACnD,EAAE,OAAO0B,SAAS;YAChB,MAAM,IAAI9C,WACR,4BACA,CAAC,OAAO,EAAEuC,WAAWG,KAAK,CAAC,YAAY,EAAEH,WAAW7B,GAAG,CAAC,6BAA6B,EAAE,AAACoC,QAAkBD,OAAO,CAAC,8CAA8C,EAAEE,QAAQC,QAAQ,CAAC,CAAC,EAAED,QAAQE,IAAI,CAAC,uBAAuB,EAAEV,WAAW7B,GAAG,CAAC,8BAA8B,CAAC;QAE9Q;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../errors.ts';\n\n// A store's optional native package, and the strings its error messages and install notice name.\nexport interface NativeDescriptor {\n  store: string;\n  pkg: string;\n  sizeHint: string;\n}\n\n// install-module-linked is small, but a target native package can be large (@duckdb/node-api's\n// ~110MB), so it is required lazily inside the gated branch below, not for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory rather than a fixed dot-count: it sits at a\n// different depth under src/ than under built dist/{cjs,esm}/store output.\nexport function packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install a native dependency into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs: since the install is async, concurrent callers await the same\n// in-flight promise, keyed by specifier and target. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's resolution, including a \"not found\" miss, for the process\n// lifetime; importing the entry by an absolute file: URL instead of the bare name sidesteps that.\nfunction resolveEntryUrl(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  return pathToFileURL(join(pkgDir, entry)).href;\n}\n\n// Import-then-install-then-retry, over an injectable importName so the failure path can be\n// exercised in tests without a real native binding. Thrown errors always name descriptor.pkg.\nexport async function loadOrInstall<T>(descriptor: NativeDescriptor, nodeModulesPath: string, importName: string = descriptor.pkg): Promise<T> {\n  try {\n    return (await import(importName)) as T;\n  } catch {\n    console.error(`sense: store \"${descriptor.store}\" needs ${descriptor.pkg}; installing it now (one-time download, ${descriptor.sizeHint})...`);\n    try {\n      await installOnce(importName, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" needs ${descriptor.pkg}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${descriptor.pkg}\\` yourself and try again`\n      );\n    }\n    try {\n      return (await import(resolveEntryUrl(importName, nodeModulesPath))) as T;\n    } catch (loadErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"${descriptor.store}\" installed ${descriptor.pkg} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${descriptor.pkg}\\` to see the underlying error`\n      );\n    }\n  }\n}\n"],"names":["existsSync","readFileSync","Module","dirname","join","fileURLToPath","pathToFileURL","SenseError","_require","require","createRequire","url","_dirname","__filename","packageNodeModules","dir","candidate","pkg","JSON","parse","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntryUrl","importName","pkgDir","split","entry","exports","main","href","loadOrInstall","descriptor","console","error","store","sizeHint","installErr","message","loadErr","process","platform","arch"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAC1C,SAASC,aAAa,EAAEC,aAAa,QAAQ,WAAW;AACxD,SAASC,UAAU,QAAQ,eAAe;AAS1C,+FAA+F;AAC/F,sGAAsG;AACtG,MAAMC,WAAW,OAAOC,YAAY,cAAcP,OAAOQ,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAC1F,MAAMG,WAAWT,QAAQ,OAAOU,eAAe,cAAcA,aAAaR,cAAc,YAAYM,GAAG;AAEvG,sFAAsF;AACtF,2EAA2E;AAC3E,OAAO,SAASG;IACd,IAAIC,MAAMH;IACV,OAAS;QACP,MAAMI,YAAYZ,KAAKW,KAAK;QAC5B,IAAIf,WAAWgB,YAAY;YACzB,MAAMC,MAAMC,KAAKC,KAAK,CAAClB,aAAae,WAAW;YAC/C,IAAIC,IAAIG,IAAI,KAAK,eAAe,OAAOhB,KAAKW,KAAK;QACnD;QACA,MAAMM,SAASlB,QAAQY;QACvB,IAAIM,WAAWN,KAAK,MAAM,IAAIO,MAAM;QACpCP,MAAMM;IACR;AACF;AAEA,MAAME,aAAa,IAAIC;AAEvB,8FAA8F;AAC9F,6GAA6G;AAC7G,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,MAAMC,MAAM,GAAGF,UAAU,EAAE,EAAEC,iBAAiB;IAC9C,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,MAAME,sBAAsB,AAACvB,SAAS,yBAA0GwB,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,KAAO,GACP,CAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,2FAA2F;AAC3F,kGAAkG;AAClG,SAASQ,gBAAgBC,UAAkB,EAAEX,eAAuB;QAGHV;IAF/D,MAAMsB,SAASnC,KAAKuB,oBAAoBW,WAAWE,KAAK,CAAC;IACzD,MAAMvB,MAAMC,KAAKC,KAAK,CAAClB,aAAaG,KAAKmC,QAAQ,iBAAiB;IAClE,MAAME,QAAQ,OAAOxB,IAAIyB,OAAO,KAAK,WAAWzB,IAAIyB,OAAO,IAAIzB,YAAAA,IAAI0B,IAAI,cAAR1B,uBAAAA,YAAY;IAC3E,OAAOX,cAAcF,KAAKmC,QAAQE,QAAQG,IAAI;AAChD;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,OAAO,eAAeC,cAAiBC,UAA4B,EAAEnB,eAAuB,EAAEW,aAAqBQ,WAAW7B,GAAG;IAC/H,IAAI;QACF,OAAQ,MAAM,MAAM,CAACqB;IACvB,EAAE,OAAM;QACNS,QAAQC,KAAK,CAAC,CAAC,cAAc,EAAEF,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW7B,GAAG,CAAC,wCAAwC,EAAE6B,WAAWI,QAAQ,CAAC,IAAI,CAAC;QAC5I,IAAI;YACF,MAAMzB,YAAYa,YAAYX;QAChC,EAAE,OAAOwB,YAAY;YACnB,MAAM,IAAI5C,WACR,4BACA,CAAC,OAAO,EAAEuC,WAAWG,KAAK,CAAC,QAAQ,EAAEH,WAAW7B,GAAG,CAAC,0CAA0C,EAAE,AAACkC,WAAqBC,OAAO,CAAC,sKAAsK,EAAEN,WAAW7B,GAAG,CAAC,yBAAyB,CAAC;QAEnV;QACA,IAAI;YACF,OAAQ,MAAM,MAAM,CAACoB,gBAAgBC,YAAYX;QACnD,EAAE,OAAO0B,SAAS;YAChB,MAAM,IAAI9C,WACR,4BACA,CAAC,OAAO,EAAEuC,WAAWG,KAAK,CAAC,YAAY,EAAEH,WAAW7B,GAAG,CAAC,6BAA6B,EAAE,AAACoC,QAAkBD,OAAO,CAAC,8CAA8C,EAAEE,QAAQC,QAAQ,CAAC,CAAC,EAAED,QAAQE,IAAI,CAAC,uBAAuB,EAAEV,WAAW7B,GAAG,CAAC,8BAA8B,CAAC;QAE9Q;IACF;AACF"}

package/dist/esm/store/sql-functions.js

@@ -1,7 +1,6 @@
 import posix from 'node:path/posix';
 // has(field, value) / basename(path, suffix): the two sense-supplied SQL functions with no
-// engine-specific pieces, shared by sqlite/sql-functions.ts and duckdb/sql-functions.ts so each
-// store registers the identical implementation instead of a hand-copied one (DRY).
+// engine-specific pieces, registered identically by each store instead of a hand-copied one.
 // JSON-array field -> membership, string field -> substring, NULL -> false.
 export function hasImpl(field, value) {
     if (field === null || field === undefined) return 0;
@@ -17,9 +16,8 @@ export function hasImpl(field, value) {
     }
     return String(field).includes(needle) ? 1 : 0;
 }
-// Unix basename(path, suffix?): the filename, minus suffix when it ends with one. Neither
-// engine has a filename function of its own, and LIKE tricks that fake one also match folder
-// names.
+// Unix basename(path, suffix?): the filename, minus suffix when it ends with one. Neither engine
+// has a filename function of its own, and LIKE tricks that fake one also match folder names.
 export function basenameImpl(path, suffix) {
     if (path === null || path === undefined) return null;
     const name = posix.basename(String(path));

package/dist/esm/store/sql-functions.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sql-functions.ts"],"sourcesContent":["import posix from 'node:path/posix';\n\n// has(field, value) / basename(path, suffix): the two sense-supplied SQL functions with no\n// engine-specific pieces, shared by sqlite/sql-functions.ts and duckdb/sql-functions.ts so each\n// store registers the identical implementation instead of a hand-copied one (DRY).\n\n// JSON-array field -> membership, string field -> substring, NULL -> false.\nexport function hasImpl(field: unknown, value: unknown): number {\n  if (field === null || field === undefined) return 0;\n  const needle = String(value);\n  if (typeof field === 'string') {\n    if (field.startsWith('[')) {\n      try {\n        const parsed = JSON.parse(field);\n        if (Array.isArray(parsed)) return parsed.some((item) => String(item) === needle) ? 1 : 0;\n      } catch {}\n    }\n    return field.includes(needle) ? 1 : 0;\n  }\n  return String(field).includes(needle) ? 1 : 0;\n}\n\n// Unix basename(path, suffix?): the filename, minus suffix when it ends with one. Neither\n// engine has a filename function of its own, and LIKE tricks that fake one also match folder\n// names.\nexport function basenameImpl(path: unknown, suffix?: unknown): string | null {\n  if (path === null || path === undefined) return null;\n  const name = posix.basename(String(path));\n  const tail = suffix === null || suffix === undefined ? '' : String(suffix);\n  // POSIX: a suffix identical to the whole name is not removed (node's own basename strips it).\n  return tail !== '' && tail !== name && name.endsWith(tail) ? name.slice(0, -tail.length) : name;\n}\n"],"names":["posix","hasImpl","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","basenameImpl","path","suffix","name","basename","tail","endsWith","slice","length"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAEpC,2FAA2F;AAC3F,gGAAgG;AAChG,mFAAmF;AAEnF,4EAA4E;AAC5E,OAAO,SAASC,QAAQC,KAAc,EAAEC,KAAc;IACpD,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;IAClD,MAAMC,SAASC,OAAOH;IACtB,IAAI,OAAOD,UAAU,UAAU;QAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;YACzB,IAAI;gBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;gBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;YACzF,EAAE,OAAM,CAAC;QACX;QACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;IACtC;IACA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;AAC9C;AAEA,0FAA0F;AAC1F,6FAA6F;AAC7F,SAAS;AACT,OAAO,SAASW,aAAaC,IAAa,EAAEC,MAAgB;IAC1D,IAAID,SAAS,QAAQA,SAASb,WAAW,OAAO;IAChD,MAAMe,OAAOnB,MAAMoB,QAAQ,CAACd,OAAOW;IACnC,MAAMI,OAAOH,WAAW,QAAQA,WAAWd,YAAY,KAAKE,OAAOY;IACnE,8FAA8F;IAC9F,OAAOG,SAAS,MAAMA,SAASF,QAAQA,KAAKG,QAAQ,CAACD,QAAQF,KAAKI,KAAK,CAAC,GAAG,CAACF,KAAKG,MAAM,IAAIL;AAC7F"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sql-functions.ts"],"sourcesContent":["import posix from 'node:path/posix';\n\n// has(field, value) / basename(path, suffix): the two sense-supplied SQL functions with no\n// engine-specific pieces, registered identically by each store instead of a hand-copied one.\n\n// JSON-array field -> membership, string field -> substring, NULL -> false.\nexport function hasImpl(field: unknown, value: unknown): number {\n  if (field === null || field === undefined) return 0;\n  const needle = String(value);\n  if (typeof field === 'string') {\n    if (field.startsWith('[')) {\n      try {\n        const parsed = JSON.parse(field);\n        if (Array.isArray(parsed)) return parsed.some((item) => String(item) === needle) ? 1 : 0;\n      } catch {}\n    }\n    return field.includes(needle) ? 1 : 0;\n  }\n  return String(field).includes(needle) ? 1 : 0;\n}\n\n// Unix basename(path, suffix?): the filename, minus suffix when it ends with one. Neither engine\n// has a filename function of its own, and LIKE tricks that fake one also match folder names.\nexport function basenameImpl(path: unknown, suffix?: unknown): string | null {\n  if (path === null || path === undefined) return null;\n  const name = posix.basename(String(path));\n  const tail = suffix === null || suffix === undefined ? '' : String(suffix);\n  // POSIX: a suffix identical to the whole name is not removed (node's own basename strips it).\n  return tail !== '' && tail !== name && name.endsWith(tail) ? name.slice(0, -tail.length) : name;\n}\n"],"names":["posix","hasImpl","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","basenameImpl","path","suffix","name","basename","tail","endsWith","slice","length"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAEpC,2FAA2F;AAC3F,6FAA6F;AAE7F,4EAA4E;AAC5E,OAAO,SAASC,QAAQC,KAAc,EAAEC,KAAc;IACpD,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;IAClD,MAAMC,SAASC,OAAOH;IACtB,IAAI,OAAOD,UAAU,UAAU;QAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;YACzB,IAAI;gBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;gBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;YACzF,EAAE,OAAM,CAAC;QACX;QACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;IACtC;IACA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;AAC9C;AAEA,iGAAiG;AACjG,6FAA6F;AAC7F,OAAO,SAASW,aAAaC,IAAa,EAAEC,MAAgB;IAC1D,IAAID,SAAS,QAAQA,SAASb,WAAW,OAAO;IAChD,MAAMe,OAAOnB,MAAMoB,QAAQ,CAACd,OAAOW;IACnC,MAAMI,OAAOH,WAAW,QAAQA,WAAWd,YAAY,KAAKE,OAAOY;IACnE,8FAA8F;IAC9F,OAAOG,SAAS,MAAMA,SAASF,QAAQA,KAAKG,QAAQ,CAACD,QAAQF,KAAKI,KAAK,CAAC,GAAG,CAACF,KAAKG,MAAM,IAAIL;AAC7F"}

package/dist/esm/store/sqlite/connection.js

@@ -1,8 +1,6 @@
-import { withTransaction } from '../transaction.js';
-// Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every
-// store presents. Every method below does its real work synchronously and returns an already-
-// resolved promise, so reconcile's loop, the feature hooks, and runBatch never actually cross
-// an async boundary internally -- no regression against the pre-Store synchronous code.
+import { BEGIN_WRITE, withTransaction } from '../transaction.js';
+// Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every store presents. Every method does its real
+// work synchronously and returns an already-resolved promise, so reconcile's loop, the feature hooks, and runBatch never cross an async boundary internally.
 let SqliteStatement = class SqliteStatement {
     async run(...params) {
         return this.stmt.run(...params);
@@ -34,16 +32,14 @@ export function createConnection(db) {
         async prepare (sql) {
             return new SqliteStatement(db.prepare(sql));
         },
-        // Prepares once and loops synchronously: today's code, unchanged, wrapped as one async call
-        // so features share one Connection contract with DuckDB. Joins the caller's transaction when
-        // there is one (reconcile); otherwise opens its own, so a standalone batch (a vector write
-        // outside any Store.transaction()) stays atomic and commits once, not once per row.
+        // Prepares once and loops synchronously, wrapped as one async call so features share one
+        // Connection contract with DuckDB. Joins the caller's transaction when there is one (reconcile); otherwise opens its own, so a standalone batch stays atomic and commits once, not per row.
         async runBatch (sql, paramRows) {
             if (paramRows.length === 0) return;
             await withTransaction(conn, async ()=>{
                 const stmt = db.prepare(sql);
                 for (const row of paramRows)stmt.run(...row);
-            });
+            }, BEGIN_WRITE);
         }
     };
     return conn;

package/dist/esm/store/sqlite/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/connection.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every\n// store presents. Every method below does its real work synchronously and returns an already-\n// resolved promise, so reconcile's loop, the feature hooks, and runBatch never actually cross\n// an async boundary internally -- no regression against the pre-Store synchronous code.\nclass SqliteStatement implements Statement {\n  private stmt: StatementSync;\n\n  constructor(stmt: StatementSync) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...(params as Parameters<StatementSync['run']>));\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...(params as Parameters<StatementSync['get']>));\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...(params as Parameters<StatementSync['all']>));\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...(params as Parameters<StatementSync['iterate']>));\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.setReadBigInts(enabled);\n  }\n}\n\nexport function createConnection(db: DatabaseSync): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new SqliteStatement(db.prepare(sql));\n    },\n    // Prepares once and loops synchronously: today's code, unchanged, wrapped as one async call\n    // so features share one Connection contract with DuckDB. Joins the caller's transaction when\n    // there is one (reconcile); otherwise opens its own, so a standalone batch (a vector write\n    // outside any Store.transaction()) stays atomic and commits once, not once per row.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const stmt = db.prepare(sql);\n        for (const row of paramRows) stmt.run(...(row as Parameters<StatementSync['run']>));\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","SqliteStatement","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","row"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAGpD,6FAA6F;AAC7F,8FAA8F;AAC9F,8FAA8F;AAC9F,wFAAwF;AACxF,IAAA,AAAMC,kBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAKC;IAC3B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAKF;IAC3B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAKH;IAC3B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAKJ;IAC/B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACK,cAAc,CAACC;IAC3B;IA1BA,YAAYN,IAAmB,CAAE;QAC/B,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASO,iBAAiBC,EAAgB;IAC/C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpBH,GAAGE,IAAI,CAACC;QACV;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAId,gBAAgBW,GAAGI,OAAO,CAACD;QACxC;QACA,4FAA4F;QAC5F,6FAA6F;QAC7F,2FAA2F;QAC3F,oFAAoF;QACpF,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMnB,gBAAgBa,MAAM;gBAC1B,MAAMT,OAAOQ,GAAGI,OAAO,CAACD;gBACxB,KAAK,MAAMK,OAAOF,UAAWd,KAAKF,GAAG,IAAKkB;YAC5C;QACF;IACF;IACA,OAAOP;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/connection.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every store presents. Every method does its real\n// work synchronously and returns an already-resolved promise, so reconcile's loop, the feature hooks, and runBatch never cross an async boundary internally.\nclass SqliteStatement implements Statement {\n  private stmt: StatementSync;\n\n  constructor(stmt: StatementSync) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...(params as Parameters<StatementSync['run']>));\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...(params as Parameters<StatementSync['get']>));\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...(params as Parameters<StatementSync['all']>));\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...(params as Parameters<StatementSync['iterate']>));\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.setReadBigInts(enabled);\n  }\n}\n\nexport function createConnection(db: DatabaseSync): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new SqliteStatement(db.prepare(sql));\n    },\n    // Prepares once and loops synchronously, wrapped as one async call so features share one\n    // Connection contract with DuckDB. Joins the caller's transaction when there is one (reconcile); otherwise opens its own, so a standalone batch stays atomic and commits once, not per row.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const stmt = db.prepare(sql);\n          for (const row of paramRows) stmt.run(...(row as Parameters<StatementSync['run']>));\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["BEGIN_WRITE","withTransaction","SqliteStatement","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","row"],"mappings":"AACA,SAASA,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAGjE,wIAAwI;AACxI,6JAA6J;AAC7J,IAAA,AAAMC,kBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAKC;IAC3B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAKF;IAC3B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAKH;IAC3B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAKJ;IAC/B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACK,cAAc,CAACC;IAC3B;IA1BA,YAAYN,IAAmB,CAAE;QAC/B,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASO,iBAAiBC,EAAgB;IAC/C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpBH,GAAGE,IAAI,CAACC;QACV;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAId,gBAAgBW,GAAGI,OAAO,CAACD;QACxC;QACA,yFAAyF;QACzF,4LAA4L;QAC5L,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMnB,gBACJa,MACA;gBACE,MAAMT,OAAOQ,GAAGI,OAAO,CAACD;gBACxB,KAAK,MAAMK,OAAOF,UAAWd,KAAKF,GAAG,IAAKkB;YAC5C,GACArB;QAEJ;IACF;IACA,OAAOc;AACT"}

package/dist/esm/store/sqlite/fieldStats.js

@@ -1,10 +1,6 @@
 import { quoteIdent } from '../shared.js';
-// One aggregate query, one result row: COUNT + GROUP_CONCAT(DISTINCT typeof()) per column, so
-// the query stays O(columns) rather than O(rows x columns). typeof()'s output already is the
-// shared vocabulary (integer/real/text -- mapValue() never binds a boolean or blob), so no
-// mapping step is needed here (contrast duckdb/fieldStats.ts). GROUP_CONCAT(DISTINCT ...)'s
-// order is not guaranteed sorted (spike-verified: 3+ distinct types come back in insertion
-// order), so the type set is sorted in JS.
+// One aggregate query per column (COUNT + GROUP_CONCAT(DISTINCT typeof())), O(columns) not O(rows x columns);
+// typeof()'s output is already the shared vocabulary, and GROUP_CONCAT's order is unsorted, so JS sorts the type set.
 export async function fieldStats(conn, columns, scopeWhere) {
     const exprs = columns.map((name, i)=>{
         const quoted = quoteIdent(name);

package/dist/esm/store/sqlite/fieldStats.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// One aggregate query, one result row: COUNT + GROUP_CONCAT(DISTINCT typeof()) per column, so\n// the query stays O(columns) rather than O(rows x columns). typeof()'s output already is the\n// shared vocabulary (integer/real/text -- mapValue() never binds a boolean or blob), so no\n// mapping step is needed here (contrast duckdb/fieldStats.ts). GROUP_CONCAT(DISTINCT ...)'s\n// order is not guaranteed sorted (spike-verified: 3+ distinct types come back in insertion\n// order), so the type set is sorted in JS.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const exprs = columns.map((name, i) => {\n    const quoted = quoteIdent(name);\n    return `COUNT(${quoted}) AS n${i}, GROUP_CONCAT(DISTINCT typeof(${quoted})) FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n  });\n  const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n  const row = (await stmt.get()) as Record<string, number | string | null>;\n  return columns.map((name, i) => ({\n    field: name,\n    coverage: Number(row[`n${i}`] ?? 0),\n    type: [...new Set(((row[`t${i}`] as string) ?? '').split(',').filter(Boolean))].sort().join(','),\n  }));\n}\n"],"names":["quoteIdent","fieldStats","conn","columns","scopeWhere","exprs","map","name","i","quoted","stmt","prepare","join","row","get","field","coverage","Number","type","Set","split","filter","Boolean","sort"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,8FAA8F;AAC9F,6FAA6F;AAC7F,2FAA2F;AAC3F,4FAA4F;AAC5F,2FAA2F;AAC3F,2CAA2C;AAC3C,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAQF,QAAQG,GAAG,CAAC,CAACC,MAAMC;QAC/B,MAAMC,SAAST,WAAWO;QAC1B,OAAO,CAAC,MAAM,EAAEE,OAAO,MAAM,EAAED,EAAE,+BAA+B,EAAEC,OAAO,iBAAiB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;IAC5H;IACA,MAAME,OAAO,MAAMR,KAAKS,OAAO,CAAC,CAAC,OAAO,EAAEN,MAAMO,IAAI,CAAC,MAAM,kBAAkB,EAAER,YAAY;IAC3F,MAAMS,MAAO,MAAMH,KAAKI,GAAG;IAC3B,OAAOX,QAAQG,GAAG,CAAC,CAACC,MAAMC;YAEPK,OACGA;eAHW;YAC/BE,OAAOR;YACPS,UAAUC,QAAOJ,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAAgB;YACjCK,MAAM;mBAAI,IAAIC,IAAI,EAAEN,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAA2B,IAAIO,KAAK,CAAC,KAAKC,MAAM,CAACC;aAAU,CAACC,IAAI,GAAGX,IAAI,CAAC;QAC9F;;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// One aggregate query per column (COUNT + GROUP_CONCAT(DISTINCT typeof())), O(columns) not O(rows x columns);\n// typeof()'s output is already the shared vocabulary, and GROUP_CONCAT's order is unsorted, so JS sorts the type set.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const exprs = columns.map((name, i) => {\n    const quoted = quoteIdent(name);\n    return `COUNT(${quoted}) AS n${i}, GROUP_CONCAT(DISTINCT typeof(${quoted})) FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n  });\n  const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n  const row = (await stmt.get()) as Record<string, number | string | null>;\n  return columns.map((name, i) => ({\n    field: name,\n    coverage: Number(row[`n${i}`] ?? 0),\n    type: [...new Set(((row[`t${i}`] as string) ?? '').split(',').filter(Boolean))].sort().join(','),\n  }));\n}\n"],"names":["quoteIdent","fieldStats","conn","columns","scopeWhere","exprs","map","name","i","quoted","stmt","prepare","join","row","get","field","coverage","Number","type","Set","split","filter","Boolean","sort"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,8GAA8G;AAC9G,sHAAsH;AACtH,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAQF,QAAQG,GAAG,CAAC,CAACC,MAAMC;QAC/B,MAAMC,SAAST,WAAWO;QAC1B,OAAO,CAAC,MAAM,EAAEE,OAAO,MAAM,EAAED,EAAE,+BAA+B,EAAEC,OAAO,iBAAiB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;IAC5H;IACA,MAAME,OAAO,MAAMR,KAAKS,OAAO,CAAC,CAAC,OAAO,EAAEN,MAAMO,IAAI,CAAC,MAAM,kBAAkB,EAAER,YAAY;IAC3F,MAAMS,MAAO,MAAMH,KAAKI,GAAG;IAC3B,OAAOX,QAAQG,GAAG,CAAC,CAACC,MAAMC;YAEPK,OACGA;eAHW;YAC/BE,OAAOR;YACPS,UAAUC,QAAOJ,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAAgB;YACjCK,MAAM;mBAAI,IAAIC,IAAI,EAAEN,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAA2B,IAAIO,KAAK,CAAC,KAAKC,MAAM,CAACC;aAAU,CAACC,IAAI,GAAGX,IAAI,CAAC;QAC9F;;AACF"}

package/dist/esm/store/sqlite/lexical.js

@@ -1,19 +1,15 @@
 import { segmentMatch } from '../../text/segment.js';
 // snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc
-// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, hit stays NULL for
-// the caller's JS excerpt fallback.
+// (benchmark/reports/2026-08-23-hub-release-battery.md); past this bound, hit stays NULL for the caller's JS excerpt fallback.
 const SNIPPET_BOUND = 16384;
 const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';
-// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments.
-// `segmenting` mirrors the predicate that decided whether the `_seg` sidecars were populated
-// (contentTokenize(cfg) === undefined, see sqlite/store.ts): an unspaced-script run in `terms`
-// becomes a quoted grapheme phrase against those sidecars only when they exist.
+// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments. `segmenting`
+// mirrors the predicate that populated the `_seg` sidecars (contentTokenize(cfg) === undefined): an unspaced-script run in `terms` becomes a quoted grapheme phrase against them only when they exist.
 export async function queryLexical(conn, terms, opts, segmenting) {
     const { whereJoin, whereCond, scopeCond, limit } = opts;
     const query = segmenting ? segmentMatch(terms) : terms;
-    // A run of pure unspaced-script punctuation segments to nothing searchable (segmentField
-    // drops empty groups the same way): an empty MATCH string is an FTS5 syntax error, not zero
-    // rows, so this returns the zero rows directly rather than issuing it.
+    // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH
+    // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.
     if (query.trim() === '') return [];
     const sql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;
     const stmt = await conn.prepare(sql);

package/dist/esm/store/sqlite/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, hit stays NULL for\n// the caller's JS excerpt fallback.\nconst SNIPPET_BOUND = 16_384;\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n\n// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments.\n// `segmenting` mirrors the predicate that decided whether the `_seg` sidecars were populated\n// (contentTokenize(cfg) === undefined, see sqlite/store.ts): an unspaced-script run in `terms`\n// becomes a quoted grapheme phrase against those sidecars only when they exist.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, segmenting: boolean): Promise<LexicalHit[]> {\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const query = segmenting ? segmentMatch(terms) : terms;\n  // A run of pure unspaced-script punctuation segments to nothing searchable (segmentField\n  // drops empty groups the same way): an empty MATCH string is an FTS5 syntax error, not zero\n  // rows, so this returns the zero rows directly rather than issuing it.\n  if (query.trim() === '') return [];\n  const sql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(query)) as unknown as LexicalHit[];\n}\n"],"names":["segmentMatch","SNIPPET_BOUND","WEIGHTED_BM25","queryLexical","conn","terms","opts","segmenting","whereJoin","whereCond","scopeCond","limit","query","trim","sql","stmt","prepare","all"],"mappings":"AAAA,SAASA,YAAY,QAAQ,wBAAwB;AAGrD,iFAAiF;AACjF,6FAA6F;AAC7F,oCAAoC;AACpC,MAAMC,gBAAgB;AACtB,MAAMC,gBAAgB;AAEtB,uFAAuF;AACvF,6FAA6F;AAC7F,+FAA+F;AAC/F,gFAAgF;AAChF,OAAO,eAAeC,aAAaC,IAAgB,EAAEC,KAAa,EAAEC,IAAyB,EAAEC,UAAmB;IAChH,MAAM,EAAEC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAMM,QAAQL,aAAaP,aAAaK,SAASA;IACjD,yFAAyF;IACzF,4FAA4F;IAC5F,uEAAuE;IACvE,IAAIO,MAAMC,IAAI,OAAO,IAAI,OAAO,EAAE;IAClC,MAAMC,MAAM,CAAC,+DAA+D,EAAEb,cAAc,+EAA+E,EAAEO,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAER,cAAc,OAAO,EAAES,OAAO;IACjR,MAAMI,OAAO,MAAMX,KAAKY,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,CAACL;AACzB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-hub-release-battery.md); past this bound, hit stays NULL for the caller's JS excerpt fallback.\nconst SNIPPET_BOUND = 16_384;\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n\n// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments. `segmenting`\n// mirrors the predicate that populated the `_seg` sidecars (contentTokenize(cfg) === undefined): an unspaced-script run in `terms` becomes a quoted grapheme phrase against them only when they exist.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, segmenting: boolean): Promise<LexicalHit[]> {\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const query = segmenting ? segmentMatch(terms) : terms;\n  // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH\n  // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.\n  if (query.trim() === '') return [];\n  const sql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(query)) as unknown as LexicalHit[];\n}\n"],"names":["segmentMatch","SNIPPET_BOUND","WEIGHTED_BM25","queryLexical","conn","terms","opts","segmenting","whereJoin","whereCond","scopeCond","limit","query","trim","sql","stmt","prepare","all"],"mappings":"AAAA,SAASA,YAAY,QAAQ,wBAAwB;AAGrD,iFAAiF;AACjF,+HAA+H;AAC/H,MAAMC,gBAAgB;AACtB,MAAMC,gBAAgB;AAEtB,oGAAoG;AACpG,uMAAuM;AACvM,OAAO,eAAeC,aAAaC,IAAgB,EAAEC,KAAa,EAAEC,IAAyB,EAAEC,UAAmB;IAChH,MAAM,EAAEC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAMM,QAAQL,aAAaP,aAAaK,SAASA;IACjD,2FAA2F;IAC3F,sGAAsG;IACtG,IAAIO,MAAMC,IAAI,OAAO,IAAI,OAAO,EAAE;IAClC,MAAMC,MAAM,CAAC,+DAA+D,EAAEb,cAAc,+EAA+E,EAAEO,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAER,cAAc,OAAO,EAAES,OAAO;IACjR,MAAMI,OAAO,MAAMX,KAAKY,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,CAACL;AACzB"}

package/dist/esm/store/sqlite/open.d.ts

@@ -11,4 +11,3 @@ export interface OpenResult {
 }
 export declare function openSqlite(cfg: ResolvedConfig): Promise<OpenResult>;
 export declare function docCount(store: Store): Promise<number>;
-export declare function clearCache(cfg: ResolvedConfig): void;

package/dist/esm/store/sqlite/open.js

@@ -1,13 +1,15 @@
 // The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with
 // both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.
-import { mkdirSync, rmSync } from 'node:fs';
+import { mkdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 import { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.js';
+import { rekeyChunkText } from '../../embed/handoff.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
+import { clearCache } from '../cache.js';
 import { getMeta, setMeta } from '../shared.js';
-import { withTransaction } from '../transaction.js';
+import { BEGIN_WRITE, withTransaction } from '../transaction.js';
 import { createConnection } from './connection.js';
 import { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
@@ -16,16 +18,11 @@ export const DB_FILENAME = 'cache.db';
 // Cache shape version, independent of the config's own `version`. Bumping it rebuilds
 // existing trees on first query.
 export const SCHEMA_VERSION = '18';
-// Stemming is English-only, but the segmentation underneath it is what decides coverage:
-// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)
-// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how
-// such a tree picks trigram instead.
+// unicode61 splits on spaces, so a language written without them indexes a whole run as one
+// token and word search finds nothing; `content.tokenize` is how such a tree picks trigram.
 const DEFAULT_TOKENIZE = 'porter unicode61';
-// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a
-// configured value has to be concatenated. Probing a throwaway table is what makes that safe
-// and is also the whole validation: anything the linked SQLite accepts passes, anything else
-// fails here with SQLite's own message rather than against the real table. It means no table
-// of which version added which tokenizer has to be maintained.
+// FTS5 takes its tokenizer as a DDL string literal where nothing can bind, so the configured
+// value is concatenated; probing a throwaway table first is both the safety and the validation.
 function resolveTokenize(db, cfg) {
     const configured = contentTokenize(cfg);
     if (configured === undefined) return DEFAULT_TOKENIZE;
@@ -47,11 +44,8 @@ function storedTokenize(db) {
     const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);
     return m ? m[1] : null;
 }
-// The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and
-// snippet(content, 2, ...) are documented against the first three columns and keep working
-// (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced
-// runs for text that needs it, and an empty string for text that does not. Shared by
-// ensureSchema and the tokenize-only rebuild in connect(), so both create identical DDL.
+// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)
+// are documented against the first three columns. Each holds its field's exploded unspaced runs.
 async function createContentTable(conn, tokenize) {
     await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);
 }
@@ -70,6 +64,20 @@ async function ensureSchema(conn, cfg, tokenize) {
     if (await getMeta(conn, 'schema_version') === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);
     if (await getMeta(conn, 'features') === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));
 }
+// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY
+// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.
+function setJournalWal(db) {
+    const deadline = Date.now() + 5000;
+    for(;;){
+        try {
+            db.exec('PRAGMA journal_mode = WAL');
+            return;
+        } catch (err) {
+            if (Date.now() >= deadline || !/database is locked|busy/i.test(err.message)) throw err;
+            Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);
+        }
+    }
+}
 async function connect(cfg) {
     const stateDir = join(cfg.baseDir, STATE_DIR);
     mkdirSync(stateDir, {
@@ -77,16 +85,17 @@ async function connect(cfg) {
     });
     const dbPath = join(stateDir, DB_FILENAME);
     const db = new DatabaseSync(dbPath);
-    // A throw below (tokenizer probe, schema, reconcile's COLUMN_LIMIT) must release this handle,
-    // or on Windows the leaked WAL db is undeletable and scratch cleanup fails with EPERM/EBUSY.
-    // closed marks the rebuild branches that close-and-recurse, so the catch never double-closes.
+    // A throw below must release this handle, or on Windows the leaked WAL db is undeletable and
+    // scratch cleanup fails with EPERM/EBUSY. `closed` keeps the catch from double-closing.
     let closed = false;
     try {
         var _ref;
-        db.exec('PRAGMA journal_mode = WAL');
-        // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query
-        // that outwaits it still fails loudly.
+        // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive
+        // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.
         db.exec('PRAGMA busy_timeout = 30000');
+        // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy
+        // handler for it, so a concurrent cold open needs its own bounded wait.
+        setJournalWal(db);
         registerFunctions(db, contentTokenize(cfg) === undefined);
         const conn = createConnection(db);
         db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
@@ -110,20 +119,16 @@ async function connect(cfg) {
                 return connect(cfg);
             }
             const changedKeys = changedSignatureKeys(features !== null && features !== void 0 ? features : '', wantFeatures);
-            // Only the tokenizer moved: frontmatter, links, sections, and embeddings are file-derived
-            // and tokenizer-independent, so they don't need re-deriving. Everything else -- a preset
-            // edit, an embed model change -- still takes the full clear/reopen below.
+            // Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived
+            // and tokenizer-independent. Any other signature change takes the full clear/reopen below.
             if (changedKeys.size === 1 && changedKeys.has('tokenize')) {
                 console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');
-                // Drop and recreate as one transaction: a crash before COMMIT rolls back to the table
-                // that was there before (old tokenizer, still queryable), never to no table at all.
-                // IF EXISTS also makes a retry after such a crash a no-op instead of a raw SQLite error --
-                // meta.features is only updated once rebuildContentTable (below) finishes, so a retry
-                // re-enters this branch and repopulates from scratch either way.
+                // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather
+                // than to no table. IF EXISTS makes a retry after such a crash a no-op, not a raw error.
                 await withTransaction(conn, async ()=>{
                     await conn.exec('DROP TABLE IF EXISTS content');
                     await createContentTable(conn, tokenize);
-                });
+                }, BEGIN_WRITE);
                 tokenizeOnlyRebuild = true;
             } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features !== null && features !== void 0 ? features : '', wantFeatures)) {
                 // First sight of a resolved weight identity: the model itself hasn't changed, so
@@ -139,10 +144,8 @@ async function connect(cfg) {
                 return connect(cfg);
             }
         }
-        // Meta can lie after a crash between table creation and the signature write; the table's
-        // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild
-        // already recreated content with this tokenize above, so stored already matches and this
-        // guard is skipped naturally rather than needing its own case.
+        // Meta can lie after a crash between table creation and the signature write; the table's own
+        // DDL cannot, so a mismatch here rebuilds whatever meta says.
         const stored = storedTokenize(db);
         if (stored !== null && stored !== tokenize) {
             console.error('sense: cache was built with a different content tokenizer; rebuilding the index');
@@ -151,7 +154,9 @@ async function connect(cfg) {
             clearCache(cfg);
             return connect(cfg);
         }
-        await ensureSchema(conn, cfg, tokenize);
+        // One writer at a time: the feature hooks check a column then add it, so two cold opens racing
+        // here both see it missing and the second ALTER fails with a duplicate column.
+        await withTransaction(conn, ()=>ensureSchema(conn, cfg, tokenize), BEGIN_WRITE);
         let rebuildWarnings = [];
         if (tokenizeOnlyRebuild) {
             rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);
@@ -182,8 +187,11 @@ async function connect(cfg) {
 // resulting connection in the async Store interface.
 export async function openSqlite(cfg) {
     const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);
+    const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);
+    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
+    rekeyChunkText(conn, store);
     return {
-        store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir),
+        store: store,
         cfg: resolvedCfg,
         dbPath,
         parsed,
@@ -194,13 +202,3 @@ export async function docCount(store) {
     const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');
     return (await stmt.get()).n;
 }
-// Deletes the cache directory, and only that. The rebuild is not this function's job: the next
-// open() reconciles, which is what running any command already does, so a verb that bundled the
-// two described the half that was not its own. Manual reset for a doubted cache; the schema and
-// config-signature mismatches below reset themselves.
-export function clearCache(cfg) {
-    rmSync(join(cfg.baseDir, STATE_DIR), {
-        recursive: true,
-        force: true
-    });
-}