sensemaking 0.24.3 → 0.24.5

package/dist/esm/store/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { channel } from 'node:diagnostics_channel';\nimport { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { SenseError } from '../errors.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport { classifyEmbedChange } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.ts';\nimport { lockWaitBudgetMs } from './lock-wait.ts';\nimport { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Stages } from './stages.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Per-stage wall time for this open's build pass (stages.ts), not for any narrow embed or\n  // feature invalidation that ran beside it -- on a cold build there is none, so it is the whole cost.\n  stages: Stages;\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  stages: Stages;\n}\n\n// duckdb/turso hold the cache file for their connection's whole life, so a second command waits.\n// Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.\nconst LOCK_POLL_MS = 50;\nconst lockWait = channel('sensemaking.store.lock-wait');\n\nasync function connectUnlocked<Handle>(dbPath: string, cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<{ handle: Handle; conn: Connection }> {\n  const budgetMs = lockWaitBudgetMs(cfg.configDir ?? cfg.baseDir);\n  const deadline = Date.now() + budgetMs;\n  for (;;) {\n    try {\n      return await dialect.connect(dbPath, cfg);\n    } catch (err) {\n      if (!dialect.isLocked?.(err as Error)) throw err;\n      // Observe a real rejected native connection, never merely the start of an async open.\n      if (lockWait.hasSubscribers) lockWait.publish({ store: cfg.store, dbPath });\n      if (Date.now() >= deadline) {\n        throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${Math.round(budgetMs / 1000)}s; wait for that command to finish, or set \"store\" to \"sqlite\" in sense.config.json, which serves concurrent commands`);\n      }\n      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));\n    }\n  }\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<ConnectResult<Handle>> {\n  const stateDir = join(cfg.configDir ?? cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await connectUnlocked(dbPath, cfg, dialect);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): invalidate()/invalidateFeatures() below share its pool\n  // with the build() call that follows them.\n  const builder = createBuilder(conn, cfg, cfg.rootDir ?? cfg.baseDir, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    // Set only by the preset-membership branch below; reconcile treats these paths as touched\n    // despite an unchanged stamp, for this one build() call.\n    let forcedPaths: Set<string> | undefined;\n    // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.\n    let embedInvalidate: EmbedChangeKind | undefined;\n    // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.\n    let featureToggles: FeatureToggle[] | undefined;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed\n      // preset's stored segment can't be parsed back into old include/exclude/vectors -- either\n      // way the branch below is skipped and the full rebuild stays the fallback.\n      const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.rootDir ?? cfg.baseDir, features ?? '', changedKeys) : null;\n      // Null unless exactly one segment changed and it decomposes into a recognised embed-only\n      // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.\n      const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features ?? '', wantFeatures) : null;\n      // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed\n      // feature's stored segment can't be parsed back into old/new on-off -- either way the branch\n      // below is skipped and the full rebuild stays the fallback.\n      const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features ?? '', wantFeatures, changedKeys) : null;\n      if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (embedKind !== null) {\n        // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither\n        // case needs any file reparsed for any feature but embed itself.\n        embedInvalidate = embedKind;\n        console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (presetForced !== null) {\n        // Membership only: which files a preset covers moved, not a file's own content, a\n        // global feature, or the embed model. reconcile() already knows how to\n        // add, update, and remove a file correctly (every cross-feature cascade included); it\n        // just needs telling which unchanged files to treat as touched (forcedPaths, above).\n        forcedPaths = presetForced;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the files it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (toggles !== null) {\n        // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's\n        // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a\n        // feature turning back on, fully re-derives across every indexed file rather than\n        // trusting rows written before or during that window.\n        featureToggles = toggles;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the feature it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same\n    // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or\n    // feature-toggle-only changed-key set).\n    const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;\n    const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;\n\n    const { parsed, warnings, stages } = await builder.build(forcedPaths);\n    // Every open reconciles once and is done with it -- a watcher opens fresh per event instead of holding this connection (src/watch.ts).\n    await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed: parsed + embedParsed + featureParsed, warnings, stages };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  // `baseDir` was the original public open() input. Keep direct API callers working while\n  // making the two ownership paths explicit for every internal operation.\n  if (!cfg.rootDir) {\n    if (!cfg.baseDir) throw new SenseError('CONFIG_INVALID', 'open requires rootDir (or legacy baseDir)');\n    cfg.rootDir = cfg.baseDir;\n  }\n  if (!cfg.configDir) cfg.configDir = cfg.rootDir;\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, stages } = await connectWithDialect(cfg, dialect);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, stages };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["channel","mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","SenseError","FEATURES","createBuilder","clearCache","classifyEmbedChange","classifyFeatureToggles","isFeatureOnlyChange","lockWaitBudgetMs","forcedPresetPaths","isPresetOnlyChange","getMeta","setMeta","changedSignatureKeys","embedIdentityAdopted","signatureDiff","LOCK_POLL_MS","lockWait","connectUnlocked","dbPath","cfg","dialect","budgetMs","configDir","baseDir","deadline","Date","now","connect","err","isLocked","hasSubscribers","publish","store","Math","round","Promise","resolve","setTimeout","connectWithDialect","stateDir","recursive","filename","handle","conn","closed","builder","rootDir","reconcileDialect","exec","version","features","wantFeatures","forcedPaths","embedInvalidate","featureToggles","schemaVersion","console","error","close","changedKeys","presetForced","embedKind","size","has","toggles","changed","ensureSchema","setDerivedBusyTimeout","recordedMaxMs","Number","min","max","embedParsed","invalidate","parsed","featureParsed","invalidateFeatures","warnings","stages","build","openWithDialect","resolvedCfg","createStore","docCount","stmt","prepare","get","n"],"mappings":"AAAA,SAASA,OAAO,QAAQ,2BAA2B;AACnD,SAASC,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,qBAAqB;AACjE,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,UAAU,QAAQ,aAAa;AAExC,SAASC,mBAAmB,QAAQ,mBAAmB;AAEvD,SAASC,sBAAsB,EAAEC,mBAAmB,QAAQ,qBAAqB;AACjF,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAC1E,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,aAAa,QAAQ,iBAAiB;AA8B3F,iGAAiG;AACjG,oFAAoF;AACpF,MAAMC,eAAe;AACrB,MAAMC,WAAWtB,QAAQ;AAEzB,eAAeuB,gBAAwBC,MAAc,EAAEC,GAAmB,EAAEC,OAA4B;QACpED;IAAlC,MAAME,WAAWd,kBAAiBY,iBAAAA,IAAIG,SAAS,cAAbH,4BAAAA,iBAAiBA,IAAII,OAAO;IAC9D,MAAMC,WAAWC,KAAKC,GAAG,KAAKL;IAC9B,OAAS;QACP,IAAI;YACF,OAAO,MAAMD,QAAQO,OAAO,CAACT,QAAQC;QACvC,EAAE,OAAOS,KAAK;gBACPR;YAAL,IAAI,GAACA,oBAAAA,QAAQS,QAAQ,cAAhBT,wCAAAA,uBAAAA,SAAmBQ,OAAe,MAAMA;YAC7C,sFAAsF;YACtF,IAAIZ,SAASc,cAAc,EAAEd,SAASe,OAAO,CAAC;gBAAEC,OAAOb,IAAIa,KAAK;gBAAEd;YAAO;YACzE,IAAIO,KAAKC,GAAG,MAAMF,UAAU;gBAC1B,MAAM,IAAIxB,WAAW,cAAc,CAAC,2CAA2C,EAAEmB,IAAIa,KAAK,CAAC,QAAQ,EAAEd,OAAO,gCAAgC,EAAEe,KAAKC,KAAK,CAACb,WAAW,MAAM,qHAAqH,CAAC;YAClS;YACA,MAAM,IAAIc,QAAQ,CAACC,UAAYC,WAAWD,SAASrB;QACrD;IACF;AACF;AAEA,eAAeuB,mBAA2BnB,GAAmB,EAAEC,OAA4B;QACnED,gBAUmBA;IAVzC,MAAMoB,WAAW3C,MAAKuB,iBAAAA,IAAIG,SAAS,cAAbH,4BAAAA,iBAAiBA,IAAII,OAAO,EAAEzB;IACpDH,UAAU4C,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMtB,SAAStB,KAAK2C,UAAUnB,QAAQqB,QAAQ;IAE9C,MAAM,EAAEC,MAAM,EAAEC,IAAI,EAAE,GAAG,MAAM1B,gBAAgBC,QAAQC,KAAKC;IAC5D,gGAAgG;IAChG,kGAAkG;IAClG,IAAIwB,SAAS;IACb,+FAA+F;IAC/F,2CAA2C;IAC3C,MAAMC,UAAU3C,cAAcyC,MAAMxB,MAAKA,eAAAA,IAAI2B,OAAO,cAAX3B,0BAAAA,eAAeA,IAAII,OAAO,EAAEH,QAAQ2B,gBAAgB;IAC7F,IAAI;QACF,MAAMJ,KAAKK,IAAI,CAAC;QAEhB,uFAAuF;QACvF,qGAAqG;QACrG,MAAMC,UAAU,MAAMvC,QAAQiC,MAAM;QACpC,MAAMO,WAAW,MAAMxC,QAAQiC,MAAM;QACrC,MAAMQ,eAAetD,iBAAiBsB,KAAKlB;QAC3C,0FAA0F;QAC1F,yDAAyD;QACzD,IAAImD;QACJ,kGAAkG;QAClG,IAAIC;QACJ,4GAA4G;QAC5G,IAAIC;QACJ,IAAI,AAACL,YAAY,QAAQA,YAAY7B,QAAQmC,aAAa,IAAML,aAAa,QAAQA,aAAaC,cAAe;gBAejChC;YAd9E,uFAAuF;YACvF,wFAAwF;YACxF,IAAI8B,YAAY,QAAQA,YAAY7B,QAAQmC,aAAa,EAAE;gBACzDC,QAAQC,KAAK,CAAC;gBACd,kFAAkF;gBAClFb,SAAS;gBACT,MAAMxB,QAAQsC,KAAK,CAAChB;gBACpBvC,WAAWgB;gBACX,OAAOmB,mBAAmBnB,KAAKC;YACjC;YACA,MAAMuC,cAAc/C,qBAAqBsC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,oFAAoF;YACpF,0FAA0F;YAC1F,2EAA2E;YAC3E,MAAMS,eAAenD,mBAAmBkD,eAAenD,kBAAkBW,MAAKA,gBAAAA,IAAI2B,OAAO,cAAX3B,2BAAAA,gBAAeA,IAAII,OAAO,EAAE2B,qBAAAA,sBAAAA,WAAY,IAAIS,eAAe;YACzI,yFAAyF;YACzF,oFAAoF;YACpF,MAAME,YAAYF,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,WAAW3D,oBAAoB8C,qBAAAA,sBAAAA,WAAY,IAAIC,gBAAgB;YAC3H,4FAA4F;YAC5F,6FAA6F;YAC7F,4DAA4D;YAC5D,MAAMa,UAAU1D,oBAAoBqD,eAAetD,uBAAuB6C,qBAAAA,sBAAAA,WAAY,IAAIC,cAAcQ,eAAe;YACvH,IAAIA,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,YAAYlD,qBAAqBqC,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBAC5G,iFAAiF;gBACjF,sDAAsD;gBACtDK,QAAQC,KAAK,CAAC;gBACd,MAAM9C,QAAQgC,MAAM,YAAYQ;YAClC,OAAO,IAAIU,cAAc,MAAM;gBAC7B,yFAAyF;gBACzF,iEAAiE;gBACjER,kBAAkBQ;gBAClBL,QAAQC,KAAK,CAACI,cAAc,UAAU,6EAA6E;gBACnH,MAAMlD,QAAQgC,MAAM,YAAYQ;YAClC,OAAO,IAAIS,iBAAiB,MAAM;gBAChC,kFAAkF;gBAClF,uEAAuE;gBACvE,sFAAsF;gBACtF,qFAAqF;gBACrFR,cAAcQ;gBACd,MAAMK,UAAUnD,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oCAAoC,CAAC;gBACpF,MAAMtD,QAAQgC,MAAM,YAAYQ;YAClC,OAAO,IAAIa,YAAY,MAAM;gBAC3B,uFAAuF;gBACvF,wFAAwF;gBACxF,kFAAkF;gBAClF,sDAAsD;gBACtDV,iBAAiBU;gBACjB,MAAMC,UAAUnD,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,sCAAsC,CAAC;gBACtF,MAAMtD,QAAQgC,MAAM,YAAYQ;YAClC,OAAO;gBACL,MAAMc,UAAUnD,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oBAAoB,CAAC;gBACpErB,SAAS;gBACT,MAAMxB,QAAQsC,KAAK,CAAChB;gBACpBvC,WAAWgB;gBACX,OAAOmB,mBAAmBnB,KAAKC;YACjC;QACF;QAEA,MAAMA,QAAQ8C,YAAY,CAACxB,QAAQC,MAAMxB;QAEzC,wFAAwF;QACxF,uFAAuF;QACvF,IAAIC,QAAQ+C,qBAAqB,EAAE;gBACH;YAA9B,MAAMC,gBAAgBC,QAAQ,OAAA,MAAM3D,QAAQiC,MAAM,iCAApB,kBAAA,OAA4C;YAC1E,MAAMvB,QAAQ+C,qBAAqB,CAACzB,QAAQC,MAAMV,KAAKqC,GAAG,CAACrC,KAAKsC,GAAG,CAAC,OAAO,IAAIH,gBAAgB;QACjG;QAEA,6FAA6F;QAC7F,+FAA+F;QAC/F,wCAAwC;QACxC,MAAMI,cAAcnB,kBAAkB,AAAC,CAAA,MAAMR,QAAQ4B,UAAU,CAACpB,gBAAe,EAAGqB,MAAM,GAAG;QAC3F,MAAMC,gBAAgBrB,iBAAiB,AAAC,CAAA,MAAMT,QAAQ+B,kBAAkB,CAACtB,eAAc,EAAGoB,MAAM,GAAG;QAEnG,MAAM,EAAEA,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMjC,QAAQkC,KAAK,CAAC3B;QACzD,uIAAuI;QACvI,MAAMP,QAAQa,KAAK;QAEnB,OAAO;YAAEhB;YAAQC;YAAMxB;YAAKD;YAAQwD,QAAQA,SAASF,cAAcG;YAAeE;YAAUC;QAAO;IACrG,EAAE,OAAOlD,KAAK;QACZ,MAAMiB,QAAQa,KAAK;QACnB,IAAI,CAACd,QAAQ,MAAMxB,QAAQsC,KAAK,CAAChB;QACjC,MAAMd;IACR;AACF;AAEA,iGAAiG;AACjG,mFAAmF;AACnF,OAAO,eAAeoD,gBAAwB7D,GAAmB,EAAEC,OAA4B;IAC7F,wFAAwF;IACxF,wEAAwE;IACxE,IAAI,CAACD,IAAI2B,OAAO,EAAE;QAChB,IAAI,CAAC3B,IAAII,OAAO,EAAE,MAAM,IAAIvB,WAAW,kBAAkB;QACzDmB,IAAI2B,OAAO,GAAG3B,IAAII,OAAO;IAC3B;IACA,IAAI,CAACJ,IAAIG,SAAS,EAAEH,IAAIG,SAAS,GAAGH,IAAI2B,OAAO;IAC/C,MAAM,EAAEJ,MAAM,EAAEC,IAAI,EAAExB,KAAK8D,WAAW,EAAE/D,MAAM,EAAEwD,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMxC,mBAAmBnB,KAAKC;IAC3G,MAAMY,QAAQZ,QAAQ8D,WAAW,CAACxC,QAAQC,MAAMsC;IAChD,0FAA0F;IAC1FlF,eAAe4C,MAAMX;IACrB,OAAO;QAAEA;QAAOb,KAAK8D;QAAa/D;QAAQwD;QAAQG;QAAUC;IAAO;AACrE;AAEA,OAAO,eAAeK,SAASnD,KAAY;IACzC,MAAMoD,OAAO,MAAMpD,MAAMqD,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKE,GAAG,EAAC,EAAqBC,CAAC;AAChD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { channel } from 'node:diagnostics_channel';\nimport { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { SenseError } from '../errors.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport { classifyEmbedChange } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.ts';\nimport { lockWaitBudgetMs } from './lock-wait.ts';\nimport { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Stages } from './stages.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Per-stage wall time for this open's build pass (stages.ts), not for any narrow embed or\n  // feature invalidation that ran beside it -- on a cold build there is none, so it is the whole cost.\n  stages: Stages;\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  stages: Stages;\n}\n\n// duckdb/turso hold the cache file for their connection's whole life, so a second command waits.\n// Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.\nconst LOCK_POLL_MS = 50;\nconst lockWait = channel('sensemaking.store.lock-wait');\n\nasync function connectUnlocked<Handle>(dbPath: string, cfg: ResolvedConfig, configDir: string, dialect: OpenDialect<Handle>): Promise<{ handle: Handle; conn: Connection }> {\n  const budgetMs = lockWaitBudgetMs(configDir);\n  const deadline = Date.now() + budgetMs;\n  for (;;) {\n    try {\n      return await dialect.connect(dbPath, cfg);\n    } catch (err) {\n      if (!dialect.isLocked?.(err as Error)) throw err;\n      // Observe a real rejected native connection, never merely the start of an async open.\n      if (lockWait.hasSubscribers) lockWait.publish({ store: cfg.store, dbPath });\n      if (Date.now() >= deadline) {\n        throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${Math.round(budgetMs / 1000)}s; wait for that command to finish, or set \"store\" to \"sqlite\" in sense.config.json, which serves concurrent commands`);\n      }\n      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));\n    }\n  }\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<ConnectResult<Handle>> {\n  const configDir = cfg.configDir ?? cfg.baseDir;\n  const stateDir = join(configDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await connectUnlocked(dbPath, cfg, configDir, dialect);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): invalidate()/invalidateFeatures() below share its pool\n  // with the build() call that follows them.\n  const builder = createBuilder(conn, cfg, { rootDir: cfg.rootDir ?? cfg.baseDir, configDir }, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    // Set only by the preset-membership branch below; reconcile treats these paths as touched\n    // despite an unchanged stamp, for this one build() call.\n    let forcedPaths: Set<string> | undefined;\n    // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.\n    let embedInvalidate: EmbedChangeKind | undefined;\n    // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.\n    let featureToggles: FeatureToggle[] | undefined;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed\n      // preset's stored segment can't be parsed back into old include/exclude/vectors -- either\n      // way the branch below is skipped and the full rebuild stays the fallback.\n      const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.rootDir ?? cfg.baseDir, features ?? '', changedKeys) : null;\n      // Null unless exactly one segment changed and it decomposes into a recognised embed-only\n      // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.\n      const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features ?? '', wantFeatures) : null;\n      // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed\n      // feature's stored segment can't be parsed back into old/new on-off -- either way the branch\n      // below is skipped and the full rebuild stays the fallback.\n      const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features ?? '', wantFeatures, changedKeys) : null;\n      if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (embedKind !== null) {\n        // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither\n        // case needs any file reparsed for any feature but embed itself.\n        embedInvalidate = embedKind;\n        console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (presetForced !== null) {\n        // Membership only: which files a preset covers moved, not a file's own content, a\n        // global feature, or the embed model. reconcile() already knows how to\n        // add, update, and remove a file correctly (every cross-feature cascade included); it\n        // just needs telling which unchanged files to treat as touched (forcedPaths, above).\n        forcedPaths = presetForced;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the files it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (toggles !== null) {\n        // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's\n        // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a\n        // feature turning back on, fully re-derives across every indexed file rather than\n        // trusting rows written before or during that window.\n        featureToggles = toggles;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the feature it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same\n    // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or\n    // feature-toggle-only changed-key set).\n    const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;\n    const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;\n\n    const { parsed, warnings, stages } = await builder.build(forcedPaths);\n    // Every open reconciles once and is done with it -- a watcher opens fresh per event instead of holding this connection (src/watch.ts).\n    await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed: parsed + embedParsed + featureParsed, warnings, stages };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  // `baseDir` was the original public open() input. Keep direct API callers working while\n  // making the two ownership paths explicit for every internal operation.\n  if (!cfg.rootDir) {\n    if (!cfg.baseDir) throw new SenseError('CONFIG_INVALID', 'open requires rootDir (or legacy baseDir)');\n    cfg.rootDir = cfg.baseDir;\n  }\n  if (!cfg.configDir) cfg.configDir = cfg.rootDir;\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, stages } = await connectWithDialect(cfg, dialect);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, stages };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["channel","mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","SenseError","FEATURES","createBuilder","clearCache","classifyEmbedChange","classifyFeatureToggles","isFeatureOnlyChange","lockWaitBudgetMs","forcedPresetPaths","isPresetOnlyChange","getMeta","setMeta","changedSignatureKeys","embedIdentityAdopted","signatureDiff","LOCK_POLL_MS","lockWait","connectUnlocked","dbPath","cfg","configDir","dialect","budgetMs","deadline","Date","now","connect","err","isLocked","hasSubscribers","publish","store","Math","round","Promise","resolve","setTimeout","connectWithDialect","baseDir","stateDir","recursive","filename","handle","conn","closed","builder","rootDir","reconcileDialect","exec","version","features","wantFeatures","forcedPaths","embedInvalidate","featureToggles","schemaVersion","console","error","close","changedKeys","presetForced","embedKind","size","has","toggles","changed","ensureSchema","setDerivedBusyTimeout","recordedMaxMs","Number","min","max","embedParsed","invalidate","parsed","featureParsed","invalidateFeatures","warnings","stages","build","openWithDialect","resolvedCfg","createStore","docCount","stmt","prepare","get","n"],"mappings":"AAAA,SAASA,OAAO,QAAQ,2BAA2B;AACnD,SAASC,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,qBAAqB;AACjE,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,UAAU,QAAQ,aAAa;AAExC,SAASC,mBAAmB,QAAQ,mBAAmB;AAEvD,SAASC,sBAAsB,EAAEC,mBAAmB,QAAQ,qBAAqB;AACjF,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAC1E,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,aAAa,QAAQ,iBAAiB;AA8B3F,iGAAiG;AACjG,oFAAoF;AACpF,MAAMC,eAAe;AACrB,MAAMC,WAAWtB,QAAQ;AAEzB,eAAeuB,gBAAwBC,MAAc,EAAEC,GAAmB,EAAEC,SAAiB,EAAEC,OAA4B;IACzH,MAAMC,WAAWf,iBAAiBa;IAClC,MAAMG,WAAWC,KAAKC,GAAG,KAAKH;IAC9B,OAAS;QACP,IAAI;YACF,OAAO,MAAMD,QAAQK,OAAO,CAACR,QAAQC;QACvC,EAAE,OAAOQ,KAAK;gBACPN;YAAL,IAAI,GAACA,oBAAAA,QAAQO,QAAQ,cAAhBP,wCAAAA,uBAAAA,SAAmBM,OAAe,MAAMA;YAC7C,sFAAsF;YACtF,IAAIX,SAASa,cAAc,EAAEb,SAASc,OAAO,CAAC;gBAAEC,OAAOZ,IAAIY,KAAK;gBAAEb;YAAO;YACzE,IAAIM,KAAKC,GAAG,MAAMF,UAAU;gBAC1B,MAAM,IAAIvB,WAAW,cAAc,CAAC,2CAA2C,EAAEmB,IAAIY,KAAK,CAAC,QAAQ,EAAEb,OAAO,gCAAgC,EAAEc,KAAKC,KAAK,CAACX,WAAW,MAAM,qHAAqH,CAAC;YAClS;YACA,MAAM,IAAIY,QAAQ,CAACC,UAAYC,WAAWD,SAASpB;QACrD;IACF;AACF;AAEA,eAAesB,mBAA2BlB,GAAmB,EAAEE,OAA4B;QACvEF,gBAWkCA;IAXpD,MAAMC,aAAYD,iBAAAA,IAAIC,SAAS,cAAbD,4BAAAA,iBAAiBA,IAAImB,OAAO;IAC9C,MAAMC,WAAW3C,KAAKwB,WAAWtB;IACjCH,UAAU4C,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMtB,SAAStB,KAAK2C,UAAUlB,QAAQoB,QAAQ;IAE9C,MAAM,EAAEC,MAAM,EAAEC,IAAI,EAAE,GAAG,MAAM1B,gBAAgBC,QAAQC,KAAKC,WAAWC;IACvE,gGAAgG;IAChG,kGAAkG;IAClG,IAAIuB,SAAS;IACb,+FAA+F;IAC/F,2CAA2C;IAC3C,MAAMC,UAAU3C,cAAcyC,MAAMxB,KAAK;QAAE2B,OAAO,GAAE3B,eAAAA,IAAI2B,OAAO,cAAX3B,0BAAAA,eAAeA,IAAImB,OAAO;QAAElB;IAAU,GAAGC,QAAQ0B,gBAAgB;IACrH,IAAI;QACF,MAAMJ,KAAKK,IAAI,CAAC;QAEhB,uFAAuF;QACvF,qGAAqG;QACrG,MAAMC,UAAU,MAAMvC,QAAQiC,MAAM;QACpC,MAAMO,WAAW,MAAMxC,QAAQiC,MAAM;QACrC,MAAMQ,eAAetD,iBAAiBsB,KAAKlB;QAC3C,0FAA0F;QAC1F,yDAAyD;QACzD,IAAImD;QACJ,kGAAkG;QAClG,IAAIC;QACJ,4GAA4G;QAC5G,IAAIC;QACJ,IAAI,AAACL,YAAY,QAAQA,YAAY5B,QAAQkC,aAAa,IAAML,aAAa,QAAQA,aAAaC,cAAe;gBAejChC;YAd9E,uFAAuF;YACvF,wFAAwF;YACxF,IAAI8B,YAAY,QAAQA,YAAY5B,QAAQkC,aAAa,EAAE;gBACzDC,QAAQC,KAAK,CAAC;gBACd,kFAAkF;gBAClFb,SAAS;gBACT,MAAMvB,QAAQqC,KAAK,CAAChB;gBACpBvC,WAAWgB;gBACX,OAAOkB,mBAAmBlB,KAAKE;YACjC;YACA,MAAMsC,cAAc/C,qBAAqBsC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,oFAAoF;YACpF,0FAA0F;YAC1F,2EAA2E;YAC3E,MAAMS,eAAenD,mBAAmBkD,eAAenD,kBAAkBW,MAAKA,gBAAAA,IAAI2B,OAAO,cAAX3B,2BAAAA,gBAAeA,IAAImB,OAAO,EAAEY,qBAAAA,sBAAAA,WAAY,IAAIS,eAAe;YACzI,yFAAyF;YACzF,oFAAoF;YACpF,MAAME,YAAYF,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,WAAW3D,oBAAoB8C,qBAAAA,sBAAAA,WAAY,IAAIC,gBAAgB;YAC3H,4FAA4F;YAC5F,6FAA6F;YAC7F,4DAA4D;YAC5D,MAAMa,UAAU1D,oBAAoBqD,eAAetD,uBAAuB6C,qBAAAA,sBAAAA,WAAY,IAAIC,cAAcQ,eAAe;YACvH,IAAIA,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,YAAYlD,qBAAqBqC,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBAC5G,iFAAiF;gBACjF,sDAAsD;gBACtDK,QAAQC,KAAK,CAAC;gBACd,MAAM9C,QAAQgC,MAAM,YAAYQ;YAClC,OAAO,IAAIU,cAAc,MAAM;gBAC7B,yFAAyF;gBACzF,iEAAiE;gBACjER,kBAAkBQ;gBAClBL,QAAQC,KAAK,CAACI,cAAc,UAAU,6EAA6E;gBACnH,MAAMlD,QAAQgC,MAAM,YAAYQ;YAClC,OAAO,IAAIS,iBAAiB,MAAM;gBAChC,kFAAkF;gBAClF,uEAAuE;gBACvE,sFAAsF;gBACtF,qFAAqF;gBACrFR,cAAcQ;gBACd,MAAMK,UAAUnD,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oCAAoC,CAAC;gBACpF,MAAMtD,QAAQgC,MAAM,YAAYQ;YAClC,OAAO,IAAIa,YAAY,MAAM;gBAC3B,uFAAuF;gBACvF,wFAAwF;gBACxF,kFAAkF;gBAClF,sDAAsD;gBACtDV,iBAAiBU;gBACjB,MAAMC,UAAUnD,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,sCAAsC,CAAC;gBACtF,MAAMtD,QAAQgC,MAAM,YAAYQ;YAClC,OAAO;gBACL,MAAMc,UAAUnD,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oBAAoB,CAAC;gBACpErB,SAAS;gBACT,MAAMvB,QAAQqC,KAAK,CAAChB;gBACpBvC,WAAWgB;gBACX,OAAOkB,mBAAmBlB,KAAKE;YACjC;QACF;QAEA,MAAMA,QAAQ6C,YAAY,CAACxB,QAAQC,MAAMxB;QAEzC,wFAAwF;QACxF,uFAAuF;QACvF,IAAIE,QAAQ8C,qBAAqB,EAAE;gBACH;YAA9B,MAAMC,gBAAgBC,QAAQ,OAAA,MAAM3D,QAAQiC,MAAM,iCAApB,kBAAA,OAA4C;YAC1E,MAAMtB,QAAQ8C,qBAAqB,CAACzB,QAAQC,MAAMX,KAAKsC,GAAG,CAACtC,KAAKuC,GAAG,CAAC,OAAO,IAAIH,gBAAgB;QACjG;QAEA,6FAA6F;QAC7F,+FAA+F;QAC/F,wCAAwC;QACxC,MAAMI,cAAcnB,kBAAkB,AAAC,CAAA,MAAMR,QAAQ4B,UAAU,CAACpB,gBAAe,EAAGqB,MAAM,GAAG;QAC3F,MAAMC,gBAAgBrB,iBAAiB,AAAC,CAAA,MAAMT,QAAQ+B,kBAAkB,CAACtB,eAAc,EAAGoB,MAAM,GAAG;QAEnG,MAAM,EAAEA,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMjC,QAAQkC,KAAK,CAAC3B;QACzD,uIAAuI;QACvI,MAAMP,QAAQa,KAAK;QAEnB,OAAO;YAAEhB;YAAQC;YAAMxB;YAAKD;YAAQwD,QAAQA,SAASF,cAAcG;YAAeE;YAAUC;QAAO;IACrG,EAAE,OAAOnD,KAAK;QACZ,MAAMkB,QAAQa,KAAK;QACnB,IAAI,CAACd,QAAQ,MAAMvB,QAAQqC,KAAK,CAAChB;QACjC,MAAMf;IACR;AACF;AAEA,iGAAiG;AACjG,mFAAmF;AACnF,OAAO,eAAeqD,gBAAwB7D,GAAmB,EAAEE,OAA4B;IAC7F,wFAAwF;IACxF,wEAAwE;IACxE,IAAI,CAACF,IAAI2B,OAAO,EAAE;QAChB,IAAI,CAAC3B,IAAImB,OAAO,EAAE,MAAM,IAAItC,WAAW,kBAAkB;QACzDmB,IAAI2B,OAAO,GAAG3B,IAAImB,OAAO;IAC3B;IACA,IAAI,CAACnB,IAAIC,SAAS,EAAED,IAAIC,SAAS,GAAGD,IAAI2B,OAAO;IAC/C,MAAM,EAAEJ,MAAM,EAAEC,IAAI,EAAExB,KAAK8D,WAAW,EAAE/D,MAAM,EAAEwD,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMzC,mBAAmBlB,KAAKE;IAC3G,MAAMU,QAAQV,QAAQ6D,WAAW,CAACxC,QAAQC,MAAMsC;IAChD,0FAA0F;IAC1FlF,eAAe4C,MAAMZ;IACrB,OAAO;QAAEA;QAAOZ,KAAK8D;QAAa/D;QAAQwD;QAAQG;QAAUC;IAAO;AACrE;AAEA,OAAO,eAAeK,SAASpD,KAAY;IACzC,MAAMqD,OAAO,MAAMrD,MAAMsD,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKE,GAAG,EAAC,EAAqBC,CAAC;AAChD"}

package/dist/esm/store/reconcile.d.ts

@@ -3,7 +3,11 @@ import type { ParsePool } from '../scan/pool.js';
 import { type Stages } from './stages.js';
 import type { Connection, ReconcileDialect } from './types.js';
 export declare const CORE_FRONTMATTER_COLUMNS: Set<string>;
-export declare function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{
+export interface ReconcilePaths {
+    rootDir: string;
+    configDir: string;
+}
+export declare function reconcile(conn: Connection, cfg: Config, paths: ReconcilePaths, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{
     parsed: number;
     warnings: string[];
     stages: Stages;

package/dist/esm/store/reconcile.js

@@ -20,12 +20,13 @@ export const CORE_FRONTMATTER_COLUMNS = new Set([
     '_size',
     '_parse_error'
 ]);
-export async function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths) {
+export async function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
+    const { rootDir, configDir } = paths;
     const start = process.hrtime.bigint();
     const features = activeFeatures(cfg);
     const stages = stageRecorder(features.map((f)=>f.name));
     const elapsed = ()=>Number(process.hrtime.bigint() - start) / 1e6;
-    const files = await stages.time('list', ()=>listFiles(cfg, baseDir));
+    const files = await stages.time('list', ()=>listFiles(cfg, rootDir));
     const currentSet = new Set(files.map((f)=>f.relPath));
     const existingRows = await stages.time('existing', async ()=>{
         const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
@@ -180,7 +181,7 @@ export async function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths)
     }, dialect.beginMode());
     const durationMs = Date.now() - txStart;
     // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.
-    recordLockWaitMs(baseDir, durationMs);
+    recordLockWaitMs(configDir, durationMs);
     if (dialect.recordDuration) await stages.time('meta', ()=>{
         var _dialect_recordDuration;
         return (_dialect_recordDuration = dialect.recordDuration) === null || _dialect_recordDuration === void 0 ? void 0 : _dialect_recordDuration.call(dialect, conn, durationMs);

package/dist/esm/store/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { recordLockWaitMs } from './lock-wait.ts';\nimport { appendRows, getColumns, quoteIdent } from './shared.ts';\nimport { featureStage, type Stages, stageRecorder } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nexport const CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const start = process.hrtime.bigint();\n  const features = activeFeatures(cfg);\n  const stages = stageRecorder(features.map((f) => f.name));\n  const elapsed = () => Number(process.hrtime.bigint() - start) / 1e6;\n  const files = await stages.time('list', () => listFiles(cfg, baseDir));\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = await stages.time('existing', async () => {\n    const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n    return (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  });\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [], stages: stages.take(elapsed(), 0) };\n\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.\n  const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', () => reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool }));\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await stages.time('alter', () => dialect.addColumns(conn, missingColumns));\n\n      // Revalidated once the lock is held, before this process's own frontmatter upsert below:\n      // `added` came from a path read taken before this transaction's lock, so a path still\n      // called \"added\" here may already have a content row from a concurrent reconcile that\n      // committed while this one waited. One SELECT, not a DELETE per added file -- with no\n      // contention it returns the same set `existing` already ruled out, so nothing extra clears.\n      let contentTouched = touched;\n      if (added.length > 0)\n        contentTouched = await stages.time('added-recheck', async () => {\n          const currentPathsStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n          const currentPaths = new Set(((await currentPathsStmt.all()) as Array<{ path: string }>).map((r) => r.path));\n          const staleAdded = added.filter((p) => currentPaths.has(p));\n          return staleAdded.length > 0 ? [...touched, ...staleAdded] : touched;\n        });\n\n      if (parsedDocs.length > 0) {\n        const toRow = (doc: (typeof parsedDocs)[number]) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          });\n        // A path in `added` has no existing frontmatter row, so it can never conflict; the rest\n        // genuinely can, and keep the upsert.\n        await stages.time('fm-upsert', async () => {\n          const newDocs = parsedDocs.filter((d) => addedSet.has(d.relPath));\n          const updateDocs = parsedDocs.filter((d) => !addedSet.has(d.relPath));\n          await appendRows(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow));\n          if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));\n        });\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await stages.time('text-index', () => dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      await stages.time('presets', async () => {\n        if (touched.length > 0)\n          await conn.runBatch(\n            'DELETE FROM preset_files WHERE \"path\" = ?',\n            touched.map((p) => [p])\n          );\n        const presetRows: unknown[][] = [];\n        for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken\n        // before this transaction's lock, so a path this process calls \"added\" can already have\n        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.\n        await appendRows(conn, 'preset_files', ['path', 'preset'], 'INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n      });\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await stages.time('vanished', () =>\n          conn.runBatch(\n            'DELETE FROM frontmatter WHERE \"path\" = ?',\n            vanished.map((p) => [p])\n          )\n        );\n\n      // Timed per feature per hook, so link resolution and PageRank are named stages without\n      // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.\n      if (touched.length > 0) for (const feature of features) await stages.time(featureStage(feature.name, 'remove'), () => feature.remove?.(conn, touched, delta));\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await stages.time(featureStage(feature.name, 'store'), () => feature.store?.(conn, docsForFeature, delta));\n      }\n      for (const feature of features) await stages.time(featureStage(feature.name, 'after'), () => feature.afterReconcile?.(conn, delta));\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.\n  recordLockWaitMs(baseDir, durationMs);\n  if (dialect.recordDuration) await stages.time('meta', () => dialect.recordDuration?.(conn, durationMs));\n\n  return { parsed: parsedDocs.length, warnings, stages: stages.take(elapsed(), durationMs, workerParseMs) };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","recordLockWaitMs","appendRows","getColumns","quoteIdent","featureStage","stageRecorder","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","reconcile","conn","cfg","baseDir","dialect","pool","forcedPaths","start","process","hrtime","bigint","features","stages","map","f","name","elapsed","Number","files","time","currentSet","relPath","existingRows","existingStmt","prepare","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","take","seenColumns","report","docs","parsedDocs","newColumns","workerParseMs","tick","finish","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","txStart","Date","now","present","missingColumns","addColumns","contentTouched","currentPathsStmt","currentPaths","staleAdded","toRow","doc","ctimeMs","parseError","data","newDocs","updateDocs","runBatch","reconcileContent","presetRows","presetName","presets","push","feature","remove","docsForFeature","extracted","store","afterReconcile","beginMode","durationMs","recordDuration"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAE/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,UAAU,EAAEC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACjE,SAASC,YAAY,EAAeC,aAAa,QAAQ,cAAc;AACvE,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe,EAAE;AAEvG,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;IAC5J,MAAMC,QAAQC,QAAQC,MAAM,CAACC,MAAM;IACnC,MAAMC,WAAWzB,eAAegB;IAChC,MAAMU,SAAShB,cAAce,SAASE,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACvD,MAAMC,UAAU,IAAMC,OAAOT,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;IAChE,MAAMW,QAAQ,MAAMN,OAAOO,IAAI,CAAC,QAAQ,IAAM/B,UAAUc,KAAKC;IAC7D,MAAMiB,aAAa,IAAIrB,IAAImB,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IAErD,MAAMC,eAAe,MAAMV,OAAOO,IAAI,CAAC,YAAY;QACjD,MAAMI,eAAe,MAAMtB,KAAKuB,OAAO,CAAC;QACxC,OAAQ,MAAMD,aAAaE,GAAG;IAChC;IACA,MAAMC,WAAW,IAAIC,IAAIL,aAAaT,GAAG,CAAC,CAACe,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,+FAA+F;IAC/F,+FAA+F;IAC/F,gFAAgF;IAChF,MAAME,WAAWR,aAAaS,MAAM,CAAC,CAACH,IAAM,CAACR,WAAWY,GAAG,CAACJ,EAAEC,IAAI,GAAGhB,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;IAEtF,6FAA6F;IAC7F,0FAA0F;IAC1F,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACjB;;QAC9B,MAAMoB,MAAMR,SAASS,GAAG,CAACrB,EAAEO,OAAO;QAClC,OAAO,CAACa,OAAOA,IAAIE,MAAM,KAAKtB,EAAEuB,OAAO,IAAIH,IAAII,KAAK,KAAKxB,EAAEyB,IAAI,aAAKjC,wBAAAA,kCAAAA,YAAa0B,GAAG,CAAClB,EAAEO,OAAO,wCAAK;IACrG;IAEA,IAAIS,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;QAAE9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAW;IAAG;IAEzH,MAAM4B,cAAc,MAAMnD,WAAWQ;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM4C,SAAS1D,SAAS,mBAAmB8C,UAAUO,MAAM;IAC3D,wFAAwF;IACxF,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAEC,aAAa,EAAE,GAAG,MAAMrC,OAAOO,IAAI,CAAC,SAAS,IAAM7B,aAAa2C,WAAWtB,UAAUT,KAAK0C,aAAaC,OAAOK,IAAI,EAAE;YAAE7C;QAAK;IAC3KwC,OAAOM,MAAM;IACb,KAAK,MAAMC,OAAOJ,WAAYJ,YAAYS,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIV;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjGxC,QAAQmD,gBAAgB,CAACD,WAAWd,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMgB,kBAAkBF,WAAWvB,MAAM,CAAC,CAAC0B,IAAM3D,yBAAyBkC,GAAG,CAACyB,MAAM,CAACpE,iBAAiB2C,GAAG,CAACyB;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB3C,GAAG,CAACnB,YAAYiE,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB3C,GAAG,CAAC,IAAM,KAAK8C,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLzB,MAAM,CAAC,CAAC0B,IAAMA,MAAM,QACpB5C,GAAG,CAAC,CAAC4C,IAAM,GAAG/D,WAAW+D,GAAG,YAAY,EAAE/D,WAAW+D,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ3B,UAAUF,MAAM,CAAC,CAACjB,IAAM,CAACY,SAASM,GAAG,CAAClB,EAAEO,OAAO,GAAGR,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IACpF,MAAMwC,QAAwB;QAAE3C;QAAO4C,UAAUf,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO;QAAGuC;QAAO9B;IAAS;IACnG,MAAMkC,WAAW,IAAIjE,IAAI6D;IACzB,MAAMK,mBAAmBlB,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO,EAAEU,MAAM,CAAC,CAACmC,IAAM,CAACF,SAAShC,GAAG,CAACkC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAIrC;WAAamC;KAAiB;IAElD,MAAMG,UAAUC,KAAKC,GAAG;IACxB,MAAMzE,gBACJI,MACA;QACE,iGAAiG;QACjG,sFAAsF;QACtF,MAAMsE,UAAU,MAAM9E,WAAWQ;QACjC,MAAMuE,iBAAiBxB,WAAWjB,MAAM,CAAC,CAACqB,MAAQ,CAACmB,QAAQvC,GAAG,CAACoB;QAC/D,IAAIoB,eAAehC,MAAM,GAAG,GAAG,MAAM5B,OAAOO,IAAI,CAAC,SAAS,IAAMf,QAAQqE,UAAU,CAACxE,MAAMuE;QAEzF,yFAAyF;QACzF,sFAAsF;QACtF,sFAAsF;QACtF,sFAAsF;QACtF,4FAA4F;QAC5F,IAAIE,iBAAiBP;QACrB,IAAIP,MAAMpB,MAAM,GAAG,GACjBkC,iBAAiB,MAAM9D,OAAOO,IAAI,CAAC,iBAAiB;YAClD,MAAMwD,mBAAmB,MAAM1E,KAAKuB,OAAO,CAAC;YAC5C,MAAMoD,eAAe,IAAI7E,IAAI,AAAE,CAAA,MAAM4E,iBAAiBlD,GAAG,EAAC,EAA+BZ,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;YAC1G,MAAMgD,aAAajB,MAAM7B,MAAM,CAAC,CAACmC,IAAMU,aAAa5C,GAAG,CAACkC;YACxD,OAAOW,WAAWrC,MAAM,GAAG,IAAI;mBAAI2B;mBAAYU;aAAW,GAAGV;QAC/D;QAEF,IAAIpB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMsC,QAAQ,CAACC,MACbvB,gBAAgB3C,GAAG,CAAC,CAACuC;wBAOZ2B;oBANP,IAAI3B,QAAQ,QAAQ,OAAO2B,IAAI1D,OAAO;oBACtC,IAAI+B,QAAQ,UAAU,OAAO2B,IAAI1C,OAAO;oBACxC,IAAIe,QAAQ,UAAU,OAAO2B,IAAIC,OAAO;oBACxC,IAAI5B,QAAQ,SAAS,OAAO2B,IAAIxC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIa,QAAQ,gBAAgB,OAAO2B,IAAIE,UAAU;oBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAAC9B,IAAI,cAAb2B,2BAAAA,gBAAiB;gBAC1B;YACF,wFAAwF;YACxF,sCAAsC;YACtC,MAAMnE,OAAOO,IAAI,CAAC,aAAa;gBAC7B,MAAMgE,UAAUpC,WAAWhB,MAAM,CAAC,CAACgC,IAAMC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;gBAC/D,MAAM+D,aAAarC,WAAWhB,MAAM,CAAC,CAACgC,IAAM,CAACC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;gBACnE,MAAM7B,WAAWS,MAAM,eAAeuD,iBAAiBE,WAAWyB,QAAQtE,GAAG,CAACiE;gBAC9E,IAAIM,WAAW5C,MAAM,GAAG,GAAG,MAAMvC,KAAKoF,QAAQ,CAAC3B,WAAW0B,WAAWvE,GAAG,CAACiE;YAC3E;QACF;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMlE,OAAOO,IAAI,CAAC,cAAc,IAAMf,QAAQkF,gBAAgB,CAACrF,MAAMyE,gBAAgB3B,YAAYc,OAAO3D;QAExG,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMU,OAAOO,IAAI,CAAC,WAAW;YAC3B,IAAIgD,QAAQ3B,MAAM,GAAG,GACnB,MAAMvC,KAAKoF,QAAQ,CACjB,6CACAlB,QAAQtD,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;YAE1B,MAAMqB,aAA0B,EAAE;YAClC,KAAK,MAAMR,OAAOhC,WAAY,KAAK,MAAMyC,cAAcT,IAAIU,OAAO,CAAEF,WAAWG,IAAI,CAAC;gBAACX,IAAI1D,OAAO;gBAAEmE;aAAW;YAC7G,uFAAuF;YACvF,wFAAwF;YACxF,uGAAuG;YACvG,MAAMhG,WAAWS,MAAM,gBAAgB;gBAAC;gBAAQ;aAAS,EAAE,kGAAkGsF;QAC/J;QAEA,+FAA+F;QAC/F,4FAA4F;QAC5F,IAAIzD,SAASU,MAAM,GAAG,GACpB,MAAM5B,OAAOO,IAAI,CAAC,YAAY,IAC5BlB,KAAKoF,QAAQ,CACX,4CACAvD,SAASjB,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;QAI7B,uFAAuF;QACvF,0FAA0F;QAC1F,IAAIC,QAAQ3B,MAAM,GAAG,GAAG,KAAK,MAAMmD,WAAWhF,SAAU,MAAMC,OAAOO,IAAI,CAACxB,aAAagG,QAAQ5E,IAAI,EAAE,WAAW;gBAAM4E;oBAAAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,qBAAAA,SAAiB1F,MAAMkE,SAASN;;QACtJ,KAAK,MAAM8B,WAAWhF,SAAU;YAC9B,MAAMkF,iBAAiC9C,WAAWlC,GAAG,CAAC,CAACkE,MAAS,CAAA;oBAAElD,MAAMkD,IAAI1D,OAAO;oBAAEyE,WAAWf,IAAIe,SAAS,CAACH,QAAQ5E,IAAI,CAAC;gBAAC,CAAA;YAC5H,MAAMH,OAAOO,IAAI,CAACxB,aAAagG,QAAQ5E,IAAI,EAAE,UAAU;oBAAM4E;wBAAAA,iBAAAA,QAAQI,KAAK,cAAbJ,qCAAAA,oBAAAA,SAAgB1F,MAAM4F,gBAAgBhC;;QACrG;QACA,KAAK,MAAM8B,WAAWhF,SAAU,MAAMC,OAAOO,IAAI,CAACxB,aAAagG,QAAQ5E,IAAI,EAAE,UAAU;gBAAM4E;oBAAAA,0BAAAA,QAAQK,cAAc,cAAtBL,8CAAAA,6BAAAA,SAAyB1F,MAAM4D;;IAC9H,GACAzD,QAAQ6F,SAAS;IAGnB,MAAMC,aAAa7B,KAAKC,GAAG,KAAKF;IAChC,2GAA2G;IAC3G7E,iBAAiBY,SAAS+F;IAC1B,IAAI9F,QAAQ+F,cAAc,EAAE,MAAMvF,OAAOO,IAAI,CAAC,QAAQ;YAAMf;gBAAAA,0BAAAA,QAAQ+F,cAAc,cAAtB/F,8CAAAA,6BAAAA,SAAyBH,MAAMiG;;IAE3F,OAAO;QAAEzD,QAAQM,WAAWP,MAAM;QAAEE;QAAU9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAWkF,YAAYjD;IAAe;AAC1G"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { recordLockWaitMs } from './lock-wait.ts';\nimport { appendRows, getColumns, quoteIdent } from './shared.ts';\nimport { featureStage, type Stages, stageRecorder } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nexport const CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport interface ReconcilePaths {\n  rootDir: string;\n  configDir: string;\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, paths: ReconcilePaths, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const { rootDir, configDir } = paths;\n  const start = process.hrtime.bigint();\n  const features = activeFeatures(cfg);\n  const stages = stageRecorder(features.map((f) => f.name));\n  const elapsed = () => Number(process.hrtime.bigint() - start) / 1e6;\n  const files = await stages.time('list', () => listFiles(cfg, rootDir));\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = await stages.time('existing', async () => {\n    const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n    return (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  });\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [], stages: stages.take(elapsed(), 0) };\n\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.\n  const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', () => reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool }));\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await stages.time('alter', () => dialect.addColumns(conn, missingColumns));\n\n      // Revalidated once the lock is held, before this process's own frontmatter upsert below:\n      // `added` came from a path read taken before this transaction's lock, so a path still\n      // called \"added\" here may already have a content row from a concurrent reconcile that\n      // committed while this one waited. One SELECT, not a DELETE per added file -- with no\n      // contention it returns the same set `existing` already ruled out, so nothing extra clears.\n      let contentTouched = touched;\n      if (added.length > 0)\n        contentTouched = await stages.time('added-recheck', async () => {\n          const currentPathsStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n          const currentPaths = new Set(((await currentPathsStmt.all()) as Array<{ path: string }>).map((r) => r.path));\n          const staleAdded = added.filter((p) => currentPaths.has(p));\n          return staleAdded.length > 0 ? [...touched, ...staleAdded] : touched;\n        });\n\n      if (parsedDocs.length > 0) {\n        const toRow = (doc: (typeof parsedDocs)[number]) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          });\n        // A path in `added` has no existing frontmatter row, so it can never conflict; the rest\n        // genuinely can, and keep the upsert.\n        await stages.time('fm-upsert', async () => {\n          const newDocs = parsedDocs.filter((d) => addedSet.has(d.relPath));\n          const updateDocs = parsedDocs.filter((d) => !addedSet.has(d.relPath));\n          await appendRows(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow));\n          if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));\n        });\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await stages.time('text-index', () => dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      await stages.time('presets', async () => {\n        if (touched.length > 0)\n          await conn.runBatch(\n            'DELETE FROM preset_files WHERE \"path\" = ?',\n            touched.map((p) => [p])\n          );\n        const presetRows: unknown[][] = [];\n        for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken\n        // before this transaction's lock, so a path this process calls \"added\" can already have\n        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.\n        await appendRows(conn, 'preset_files', ['path', 'preset'], 'INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n      });\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await stages.time('vanished', () =>\n          conn.runBatch(\n            'DELETE FROM frontmatter WHERE \"path\" = ?',\n            vanished.map((p) => [p])\n          )\n        );\n\n      // Timed per feature per hook, so link resolution and PageRank are named stages without\n      // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.\n      if (touched.length > 0) for (const feature of features) await stages.time(featureStage(feature.name, 'remove'), () => feature.remove?.(conn, touched, delta));\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await stages.time(featureStage(feature.name, 'store'), () => feature.store?.(conn, docsForFeature, delta));\n      }\n      for (const feature of features) await stages.time(featureStage(feature.name, 'after'), () => feature.afterReconcile?.(conn, delta));\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.\n  recordLockWaitMs(configDir, durationMs);\n  if (dialect.recordDuration) await stages.time('meta', () => dialect.recordDuration?.(conn, durationMs));\n\n  return { parsed: parsedDocs.length, warnings, stages: stages.take(elapsed(), durationMs, workerParseMs) };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","recordLockWaitMs","appendRows","getColumns","quoteIdent","featureStage","stageRecorder","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","reconcile","conn","cfg","paths","dialect","pool","forcedPaths","rootDir","configDir","start","process","hrtime","bigint","features","stages","map","f","name","elapsed","Number","files","time","currentSet","relPath","existingRows","existingStmt","prepare","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","take","seenColumns","report","docs","parsedDocs","newColumns","workerParseMs","tick","finish","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","txStart","Date","now","present","missingColumns","addColumns","contentTouched","currentPathsStmt","currentPaths","staleAdded","toRow","doc","ctimeMs","parseError","data","newDocs","updateDocs","runBatch","reconcileContent","presetRows","presetName","presets","push","feature","remove","docsForFeature","extracted","store","afterReconcile","beginMode","durationMs","recordDuration"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAE/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,UAAU,EAAEC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACjE,SAASC,YAAY,EAAeC,aAAa,QAAQ,cAAc;AACvE,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe,EAAE;AAOvG,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,KAAqB,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;IAClK,MAAM,EAAEC,OAAO,EAAEC,SAAS,EAAE,GAAGL;IAC/B,MAAMM,QAAQC,QAAQC,MAAM,CAACC,MAAM;IACnC,MAAMC,WAAW3B,eAAegB;IAChC,MAAMY,SAASlB,cAAciB,SAASE,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACvD,MAAMC,UAAU,IAAMC,OAAOT,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;IAChE,MAAMW,QAAQ,MAAMN,OAAOO,IAAI,CAAC,QAAQ,IAAMjC,UAAUc,KAAKK;IAC7D,MAAMe,aAAa,IAAIvB,IAAIqB,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IAErD,MAAMC,eAAe,MAAMV,OAAOO,IAAI,CAAC,YAAY;QACjD,MAAMI,eAAe,MAAMxB,KAAKyB,OAAO,CAAC;QACxC,OAAQ,MAAMD,aAAaE,GAAG;IAChC;IACA,MAAMC,WAAW,IAAIC,IAAIL,aAAaT,GAAG,CAAC,CAACe,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,+FAA+F;IAC/F,+FAA+F;IAC/F,gFAAgF;IAChF,MAAME,WAAWR,aAAaS,MAAM,CAAC,CAACH,IAAM,CAACR,WAAWY,GAAG,CAACJ,EAAEC,IAAI,GAAGhB,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;IAEtF,6FAA6F;IAC7F,0FAA0F;IAC1F,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACjB;;QAC9B,MAAMoB,MAAMR,SAASS,GAAG,CAACrB,EAAEO,OAAO;QAClC,OAAO,CAACa,OAAOA,IAAIE,MAAM,KAAKtB,EAAEuB,OAAO,IAAIH,IAAII,KAAK,KAAKxB,EAAEyB,IAAI,aAAKnC,wBAAAA,kCAAAA,YAAa4B,GAAG,CAAClB,EAAEO,OAAO,wCAAK;IACrG;IAEA,IAAIS,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;QAAE9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAW;IAAG;IAEzH,MAAM4B,cAAc,MAAMrD,WAAWQ;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8C,SAAS5D,SAAS,mBAAmBgD,UAAUO,MAAM;IAC3D,wFAAwF;IACxF,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAEC,aAAa,EAAE,GAAG,MAAMrC,OAAOO,IAAI,CAAC,SAAS,IAAM/B,aAAa6C,WAAWtB,UAAUX,KAAK4C,aAAaC,OAAOK,IAAI,EAAE;YAAE/C;QAAK;IAC3K0C,OAAOM,MAAM;IACb,KAAK,MAAMC,OAAOJ,WAAYJ,YAAYS,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIV;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjG1C,QAAQqD,gBAAgB,CAACD,WAAWd,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMgB,kBAAkBF,WAAWvB,MAAM,CAAC,CAAC0B,IAAM7D,yBAAyBoC,GAAG,CAACyB,MAAM,CAACtE,iBAAiB6C,GAAG,CAACyB;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB3C,GAAG,CAACrB,YAAYmE,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB3C,GAAG,CAAC,IAAM,KAAK8C,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLzB,MAAM,CAAC,CAAC0B,IAAMA,MAAM,QACpB5C,GAAG,CAAC,CAAC4C,IAAM,GAAGjE,WAAWiE,GAAG,YAAY,EAAEjE,WAAWiE,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ3B,UAAUF,MAAM,CAAC,CAACjB,IAAM,CAACY,SAASM,GAAG,CAAClB,EAAEO,OAAO,GAAGR,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IACpF,MAAMwC,QAAwB;QAAE3C;QAAO4C,UAAUf,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO;QAAGuC;QAAO9B;IAAS;IACnG,MAAMkC,WAAW,IAAInE,IAAI+D;IACzB,MAAMK,mBAAmBlB,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO,EAAEU,MAAM,CAAC,CAACmC,IAAM,CAACF,SAAShC,GAAG,CAACkC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAIrC;WAAamC;KAAiB;IAElD,MAAMG,UAAUC,KAAKC,GAAG;IACxB,MAAM3E,gBACJI,MACA;QACE,iGAAiG;QACjG,sFAAsF;QACtF,MAAMwE,UAAU,MAAMhF,WAAWQ;QACjC,MAAMyE,iBAAiBxB,WAAWjB,MAAM,CAAC,CAACqB,MAAQ,CAACmB,QAAQvC,GAAG,CAACoB;QAC/D,IAAIoB,eAAehC,MAAM,GAAG,GAAG,MAAM5B,OAAOO,IAAI,CAAC,SAAS,IAAMjB,QAAQuE,UAAU,CAAC1E,MAAMyE;QAEzF,yFAAyF;QACzF,sFAAsF;QACtF,sFAAsF;QACtF,sFAAsF;QACtF,4FAA4F;QAC5F,IAAIE,iBAAiBP;QACrB,IAAIP,MAAMpB,MAAM,GAAG,GACjBkC,iBAAiB,MAAM9D,OAAOO,IAAI,CAAC,iBAAiB;YAClD,MAAMwD,mBAAmB,MAAM5E,KAAKyB,OAAO,CAAC;YAC5C,MAAMoD,eAAe,IAAI/E,IAAI,AAAE,CAAA,MAAM8E,iBAAiBlD,GAAG,EAAC,EAA+BZ,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;YAC1G,MAAMgD,aAAajB,MAAM7B,MAAM,CAAC,CAACmC,IAAMU,aAAa5C,GAAG,CAACkC;YACxD,OAAOW,WAAWrC,MAAM,GAAG,IAAI;mBAAI2B;mBAAYU;aAAW,GAAGV;QAC/D;QAEF,IAAIpB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMsC,QAAQ,CAACC,MACbvB,gBAAgB3C,GAAG,CAAC,CAACuC;wBAOZ2B;oBANP,IAAI3B,QAAQ,QAAQ,OAAO2B,IAAI1D,OAAO;oBACtC,IAAI+B,QAAQ,UAAU,OAAO2B,IAAI1C,OAAO;oBACxC,IAAIe,QAAQ,UAAU,OAAO2B,IAAIC,OAAO;oBACxC,IAAI5B,QAAQ,SAAS,OAAO2B,IAAIxC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIa,QAAQ,gBAAgB,OAAO2B,IAAIE,UAAU;oBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAAC9B,IAAI,cAAb2B,2BAAAA,gBAAiB;gBAC1B;YACF,wFAAwF;YACxF,sCAAsC;YACtC,MAAMnE,OAAOO,IAAI,CAAC,aAAa;gBAC7B,MAAMgE,UAAUpC,WAAWhB,MAAM,CAAC,CAACgC,IAAMC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;gBAC/D,MAAM+D,aAAarC,WAAWhB,MAAM,CAAC,CAACgC,IAAM,CAACC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;gBACnE,MAAM/B,WAAWS,MAAM,eAAeyD,iBAAiBE,WAAWyB,QAAQtE,GAAG,CAACiE;gBAC9E,IAAIM,WAAW5C,MAAM,GAAG,GAAG,MAAMzC,KAAKsF,QAAQ,CAAC3B,WAAW0B,WAAWvE,GAAG,CAACiE;YAC3E;QACF;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMlE,OAAOO,IAAI,CAAC,cAAc,IAAMjB,QAAQoF,gBAAgB,CAACvF,MAAM2E,gBAAgB3B,YAAYc,OAAO7D;QAExG,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMY,OAAOO,IAAI,CAAC,WAAW;YAC3B,IAAIgD,QAAQ3B,MAAM,GAAG,GACnB,MAAMzC,KAAKsF,QAAQ,CACjB,6CACAlB,QAAQtD,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;YAE1B,MAAMqB,aAA0B,EAAE;YAClC,KAAK,MAAMR,OAAOhC,WAAY,KAAK,MAAMyC,cAAcT,IAAIU,OAAO,CAAEF,WAAWG,IAAI,CAAC;gBAACX,IAAI1D,OAAO;gBAAEmE;aAAW;YAC7G,uFAAuF;YACvF,wFAAwF;YACxF,uGAAuG;YACvG,MAAMlG,WAAWS,MAAM,gBAAgB;gBAAC;gBAAQ;aAAS,EAAE,kGAAkGwF;QAC/J;QAEA,+FAA+F;QAC/F,4FAA4F;QAC5F,IAAIzD,SAASU,MAAM,GAAG,GACpB,MAAM5B,OAAOO,IAAI,CAAC,YAAY,IAC5BpB,KAAKsF,QAAQ,CACX,4CACAvD,SAASjB,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;QAI7B,uFAAuF;QACvF,0FAA0F;QAC1F,IAAIC,QAAQ3B,MAAM,GAAG,GAAG,KAAK,MAAMmD,WAAWhF,SAAU,MAAMC,OAAOO,IAAI,CAAC1B,aAAakG,QAAQ5E,IAAI,EAAE,WAAW;gBAAM4E;oBAAAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,qBAAAA,SAAiB5F,MAAMoE,SAASN;;QACtJ,KAAK,MAAM8B,WAAWhF,SAAU;YAC9B,MAAMkF,iBAAiC9C,WAAWlC,GAAG,CAAC,CAACkE,MAAS,CAAA;oBAAElD,MAAMkD,IAAI1D,OAAO;oBAAEyE,WAAWf,IAAIe,SAAS,CAACH,QAAQ5E,IAAI,CAAC;gBAAC,CAAA;YAC5H,MAAMH,OAAOO,IAAI,CAAC1B,aAAakG,QAAQ5E,IAAI,EAAE,UAAU;oBAAM4E;wBAAAA,iBAAAA,QAAQI,KAAK,cAAbJ,qCAAAA,oBAAAA,SAAgB5F,MAAM8F,gBAAgBhC;;QACrG;QACA,KAAK,MAAM8B,WAAWhF,SAAU,MAAMC,OAAOO,IAAI,CAAC1B,aAAakG,QAAQ5E,IAAI,EAAE,UAAU;gBAAM4E;oBAAAA,0BAAAA,QAAQK,cAAc,cAAtBL,8CAAAA,6BAAAA,SAAyB5F,MAAM8D;;IAC9H,GACA3D,QAAQ+F,SAAS;IAGnB,MAAMC,aAAa7B,KAAKC,GAAG,KAAKF;IAChC,2GAA2G;IAC3G/E,iBAAiBiB,WAAW4F;IAC5B,IAAIhG,QAAQiG,cAAc,EAAE,MAAMvF,OAAOO,IAAI,CAAC,QAAQ;YAAMjB;gBAAAA,0BAAAA,QAAQiG,cAAc,cAAtBjG,8CAAAA,6BAAAA,SAAyBH,MAAMmG;;IAE3F,OAAO;QAAEzD,QAAQM,WAAWP,MAAM;QAAEE;QAAU9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAWkF,YAAYjD;IAAe;AAC1G"}

package/dist/esm/store/turso/connection.js

@@ -106,7 +106,7 @@ export function createConnection(db) {
                 // can hold across many batches, and nothing else owns this statement's lifetime.
                 const stmt = await db.prepare(sql);
                 try {
-                    for (const row of paramRows)await stmt.run(...row);
+                    for (const row of paramRows)await stmt.run(row);
                 } finally{
                     await stmt.close();
                 }

package/dist/esm/store/turso/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import { stat } from 'node:fs/promises';\nimport type { Database } from '@tursodatabase/database';\nimport { rewriteInsert } from '../batch.ts';\nimport { setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { CONNECT_OPTS, tursoApi } from './native.ts';\nimport { rewriteFunctions } from './sql-functions.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\n// The main cache file's path, or null when unresolvable. The store holds a Database and\n// Connection, not a path (PLAN 3.52), so the path is read from the database itself.\nexport async function cacheFilePath(conn: Connection): Promise<string | null> {\n  const rows = (await (await conn.prepare('PRAGMA database_list')).all()) as Array<{ name: string; file: string }>;\n  return rows.find((r) => r.name === 'main')?.file || null;\n}\n\nexport async function fileSize(path: string): Promise<number | null> {\n  try {\n    return (await stat(path)).size;\n  } catch {\n    return null;\n  }\n}\n\n// Reclaims turso#8170's FTS space amplification (PLAN 3.41), on a throwaway connection after the\n// store's own has closed: `vacuum` doubles incremental reconcile cost merely by being enabled (3.54).\nexport async function reclaimSpace(path: string): Promise<void> {\n  try {\n    const turso = await tursoApi();\n    const db = await turso.connect(path, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental, 'vacuum'] });\n    try {\n      await db.exec('VACUUM');\n      const size = await fileSize(path);\n      // Recorded here, not by the caller: this connection is the only one still open on the file.\n      if (size !== null) await setMeta(createConnection(db), 'compact_size', String(size));\n      await checkpointWal(db);\n    } finally {\n      await db.close();\n    }\n  } catch (err) {\n    console.error(`sense: turso VACUUM failed, the cache will keep the space until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(rewriteFunctions(sql)));\n    },\n    // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):\n    // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const rewritten = rewriteInsert(sql, paramRows.length);\n          if (rewritten) {\n            const stmt = await db.prepare(rewritten.sql);\n            try {\n              // One flat array, not spread: spreading tens of thousands of args hits a JS call-stack\n              // limit well before turso enforces any variable-count ceiling (measured; see PLAN.md).\n              await stmt.run(paramRows.flat());\n            } finally {\n              await stmt.close();\n            }\n            return;\n          }\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and nothing else owns this statement's lifetime.\n          const stmt = await db.prepare(sql);\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["stat","rewriteInsert","setMeta","BEGIN_WRITE","withTransaction","CONNECT_OPTS","tursoApi","rewriteFunctions","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","checkpointWal","db","exec","err","console","error","message","cacheFilePath","conn","rows","prepare","find","r","name","file","fileSize","path","size","reclaimSpace","turso","connect","experimental","createConnection","String","close","sql","runBatch","paramRows","length","rewritten","flat","row"],"mappings":"AAAA,SAASA,IAAI,QAAQ,mBAAmB;AAExC,SAASC,aAAa,QAAQ,cAAc;AAC5C,SAASC,OAAO,QAAQ,eAAe;AACvC,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,YAAY,EAAEC,QAAQ,QAAQ,cAAc;AACrD,SAASC,gBAAgB,QAAQ,qBAAqB;AAMtD,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,mGAAmG;AACnG,wGAAwG;AACxG,OAAO,eAAeQ,cAAcC,EAAY;IAC9C,IAAI;QACF,MAAMA,GAAGC,IAAI,CAAC;IAChB,EAAE,OAAOC,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,wFAAwF,EAAE,AAACF,IAAcG,OAAO,EAAE;IACnI;AACF;AAEA,wFAAwF;AACxF,oFAAoF;AACpF,OAAO,eAAeC,cAAcC,IAAgB;QAE3CC;IADP,MAAMA,OAAQ,MAAM,AAAC,CAAA,MAAMD,KAAKE,OAAO,CAAC,uBAAsB,EAAGhB,GAAG;IACpE,OAAOe,EAAAA,aAAAA,KAAKE,IAAI,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,qBAA5BJ,iCAAAA,WAAqCK,IAAI,KAAI;AACtD;AAEA,OAAO,eAAeC,SAASC,IAAY;IACzC,IAAI;QACF,OAAO,AAAC,CAAA,MAAMnC,KAAKmC,KAAI,EAAGC,IAAI;IAChC,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,iGAAiG;AACjG,sGAAsG;AACtG,OAAO,eAAeC,aAAaF,IAAY;IAC7C,IAAI;QACF,MAAMG,QAAQ,MAAMhC;QACpB,MAAMc,KAAK,MAAMkB,MAAMC,OAAO,CAACJ,MAAM;YAAE,GAAG9B,YAAY;YAAEmC,cAAc;mBAAInC,aAAamC,YAAY;gBAAE;aAAS;QAAC;QAC/G,IAAI;YACF,MAAMpB,GAAGC,IAAI,CAAC;YACd,MAAMe,OAAO,MAAMF,SAASC;YAC5B,4FAA4F;YAC5F,IAAIC,SAAS,MAAM,MAAMlC,QAAQuC,iBAAiBrB,KAAK,gBAAgBsB,OAAON;YAC9E,MAAMjB,cAAcC;QACtB,SAAU;YACR,MAAMA,GAAGuB,KAAK;QAChB;IACF,EAAE,OAAOrB,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,8EAA8E,EAAE,AAACF,IAAcG,OAAO,EAAE;IACzH;AACF;AAEA,OAAO,SAASgB,iBAAiBrB,EAAY;IAC3C,MAAMO,OAAmB;QACvB,MAAMN,MAAKuB,GAAW;YACpB,MAAMxB,GAAGC,IAAI,CAACuB;QAChB;QACA,MAAMf,SAAQe,GAAW;YACvB,OAAO,IAAIpC,sBAAsB,MAAMY,GAAGS,OAAO,CAACtB,iBAAiBqC;QACrE;QACA,gGAAgG;QAChG,wGAAwG;QACxG,MAAMC,UAASD,GAAW,EAAEE,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAM3C,gBACJuB,MACA;gBACE,MAAMqB,YAAY/C,cAAc2C,KAAKE,UAAUC,MAAM;gBACrD,IAAIC,WAAW;oBACb,MAAMrC,OAAO,MAAMS,GAAGS,OAAO,CAACmB,UAAUJ,GAAG;oBAC3C,IAAI;wBACF,uFAAuF;wBACvF,uFAAuF;wBACvF,MAAMjC,KAAKF,GAAG,CAACqC,UAAUG,IAAI;oBAC/B,SAAU;wBACR,MAAMtC,KAAKgC,KAAK;oBAClB;oBACA;gBACF;gBACA,sFAAsF;gBACtF,iFAAiF;gBACjF,MAAMhC,OAAO,MAAMS,GAAGS,OAAO,CAACe;gBAC9B,IAAI;oBACF,KAAK,MAAMM,OAAOJ,UAAW,MAAMnC,KAAKF,GAAG,IAAIyC;gBACjD,SAAU;oBACR,MAAMvC,KAAKgC,KAAK;gBAClB;YACF,GACAxC;QAEJ;IACF;IACA,OAAOwB;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import { stat } from 'node:fs/promises';\nimport type { Database } from '@tursodatabase/database';\nimport { rewriteInsert } from '../batch.ts';\nimport { setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { CONNECT_OPTS, tursoApi } from './native.ts';\nimport { rewriteFunctions } from './sql-functions.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\n// The main cache file's path, or null when unresolvable. The store holds a Database and\n// Connection, not a path (PLAN 3.52), so the path is read from the database itself.\nexport async function cacheFilePath(conn: Connection): Promise<string | null> {\n  const rows = (await (await conn.prepare('PRAGMA database_list')).all()) as Array<{ name: string; file: string }>;\n  return rows.find((r) => r.name === 'main')?.file || null;\n}\n\nexport async function fileSize(path: string): Promise<number | null> {\n  try {\n    return (await stat(path)).size;\n  } catch {\n    return null;\n  }\n}\n\n// Reclaims turso#8170's FTS space amplification (PLAN 3.41), on a throwaway connection after the\n// store's own has closed: `vacuum` doubles incremental reconcile cost merely by being enabled (3.54).\nexport async function reclaimSpace(path: string): Promise<void> {\n  try {\n    const turso = await tursoApi();\n    const db = await turso.connect(path, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental, 'vacuum'] });\n    try {\n      await db.exec('VACUUM');\n      const size = await fileSize(path);\n      // Recorded here, not by the caller: this connection is the only one still open on the file.\n      if (size !== null) await setMeta(createConnection(db), 'compact_size', String(size));\n      await checkpointWal(db);\n    } finally {\n      await db.close();\n    }\n  } catch (err) {\n    console.error(`sense: turso VACUUM failed, the cache will keep the space until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(rewriteFunctions(sql)));\n    },\n    // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):\n    // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const rewritten = rewriteInsert(sql, paramRows.length);\n          if (rewritten) {\n            const stmt = await db.prepare(rewritten.sql);\n            try {\n              // One flat array, not spread: spreading tens of thousands of args hits a JS call-stack\n              // limit well before turso enforces any variable-count ceiling (measured; see PLAN.md).\n              await stmt.run(paramRows.flat());\n            } finally {\n              await stmt.close();\n            }\n            return;\n          }\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and nothing else owns this statement's lifetime.\n          const stmt = await db.prepare(sql);\n          try {\n            for (const row of paramRows) await stmt.run(row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["stat","rewriteInsert","setMeta","BEGIN_WRITE","withTransaction","CONNECT_OPTS","tursoApi","rewriteFunctions","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","checkpointWal","db","exec","err","console","error","message","cacheFilePath","conn","rows","prepare","find","r","name","file","fileSize","path","size","reclaimSpace","turso","connect","experimental","createConnection","String","close","sql","runBatch","paramRows","length","rewritten","flat","row"],"mappings":"AAAA,SAASA,IAAI,QAAQ,mBAAmB;AAExC,SAASC,aAAa,QAAQ,cAAc;AAC5C,SAASC,OAAO,QAAQ,eAAe;AACvC,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,YAAY,EAAEC,QAAQ,QAAQ,cAAc;AACrD,SAASC,gBAAgB,QAAQ,qBAAqB;AAMtD,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,mGAAmG;AACnG,wGAAwG;AACxG,OAAO,eAAeQ,cAAcC,EAAY;IAC9C,IAAI;QACF,MAAMA,GAAGC,IAAI,CAAC;IAChB,EAAE,OAAOC,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,wFAAwF,EAAE,AAACF,IAAcG,OAAO,EAAE;IACnI;AACF;AAEA,wFAAwF;AACxF,oFAAoF;AACpF,OAAO,eAAeC,cAAcC,IAAgB;QAE3CC;IADP,MAAMA,OAAQ,MAAM,AAAC,CAAA,MAAMD,KAAKE,OAAO,CAAC,uBAAsB,EAAGhB,GAAG;IACpE,OAAOe,EAAAA,aAAAA,KAAKE,IAAI,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,qBAA5BJ,iCAAAA,WAAqCK,IAAI,KAAI;AACtD;AAEA,OAAO,eAAeC,SAASC,IAAY;IACzC,IAAI;QACF,OAAO,AAAC,CAAA,MAAMnC,KAAKmC,KAAI,EAAGC,IAAI;IAChC,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,iGAAiG;AACjG,sGAAsG;AACtG,OAAO,eAAeC,aAAaF,IAAY;IAC7C,IAAI;QACF,MAAMG,QAAQ,MAAMhC;QACpB,MAAMc,KAAK,MAAMkB,MAAMC,OAAO,CAACJ,MAAM;YAAE,GAAG9B,YAAY;YAAEmC,cAAc;mBAAInC,aAAamC,YAAY;gBAAE;aAAS;QAAC;QAC/G,IAAI;YACF,MAAMpB,GAAGC,IAAI,CAAC;YACd,MAAMe,OAAO,MAAMF,SAASC;YAC5B,4FAA4F;YAC5F,IAAIC,SAAS,MAAM,MAAMlC,QAAQuC,iBAAiBrB,KAAK,gBAAgBsB,OAAON;YAC9E,MAAMjB,cAAcC;QACtB,SAAU;YACR,MAAMA,GAAGuB,KAAK;QAChB;IACF,EAAE,OAAOrB,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,8EAA8E,EAAE,AAACF,IAAcG,OAAO,EAAE;IACzH;AACF;AAEA,OAAO,SAASgB,iBAAiBrB,EAAY;IAC3C,MAAMO,OAAmB;QACvB,MAAMN,MAAKuB,GAAW;YACpB,MAAMxB,GAAGC,IAAI,CAACuB;QAChB;QACA,MAAMf,SAAQe,GAAW;YACvB,OAAO,IAAIpC,sBAAsB,MAAMY,GAAGS,OAAO,CAACtB,iBAAiBqC;QACrE;QACA,gGAAgG;QAChG,wGAAwG;QACxG,MAAMC,UAASD,GAAW,EAAEE,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAM3C,gBACJuB,MACA;gBACE,MAAMqB,YAAY/C,cAAc2C,KAAKE,UAAUC,MAAM;gBACrD,IAAIC,WAAW;oBACb,MAAMrC,OAAO,MAAMS,GAAGS,OAAO,CAACmB,UAAUJ,GAAG;oBAC3C,IAAI;wBACF,uFAAuF;wBACvF,uFAAuF;wBACvF,MAAMjC,KAAKF,GAAG,CAACqC,UAAUG,IAAI;oBAC/B,SAAU;wBACR,MAAMtC,KAAKgC,KAAK;oBAClB;oBACA;gBACF;gBACA,sFAAsF;gBACtF,iFAAiF;gBACjF,MAAMhC,OAAO,MAAMS,GAAGS,OAAO,CAACe;gBAC9B,IAAI;oBACF,KAAK,MAAMM,OAAOJ,UAAW,MAAMnC,KAAKF,GAAG,CAACyC;gBAC9C,SAAU;oBACR,MAAMvC,KAAKgC,KAAK;gBAClB;YACF,GACAxC;QAEJ;IACF;IACA,OAAOwB;AACT"}

package/dist/esm/store/turso/lexical.js

@@ -40,10 +40,14 @@ const FTS5_OPERATORS = [
     {
         label: 'column filter',
         re: /(?:^|\s)[\p{L}_]\w*\s*:/u
+    },
+    {
+        label: 'grouping operator',
+        re: /[()]/
     }
 ];
 function unsupportedOperator(terms) {
-    const withoutPhrases = terms.replace(/"[^"]*"/g, ' ');
+    const withoutPhrases = terms.replace(/"[^"]*"/g, 'phrase');
     for (const { label, re } of FTS5_OPERATORS){
         const m = withoutPhrases.match(re);
         if (m) return {

package/dist/esm/store/turso/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { stemFolded } from './lexical-text.ts';\n\nconst FIELDS = ['title_stem', 'summary_stem', 'text_stem'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\nconst AUTHORED_FIELDS = ['title', 'summary', 'text'] as const;\ntype AuthoredField = (typeof AUTHORED_FIELDS)[number];\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: SearchToken[][]; unspaced: string[]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const phrases: SearchToken[][] = [];\n  const unspaced: string[] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const tokens = searchTokens(phrase);\n    if (tokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    unspaced.push(...tokens.filter((token) => token.unspaced).map((token) => token.text));\n    phrases.push(tokens);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) unspaced.push(token.text);\n      else words.push(token.text);\n    }\n  }\n  return { words, phrases, unspaced, emptyPhrase };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n  return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: SearchToken[][], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n  const phraseParts = phrases.flatMap((phrase) => {\n    const latin = phrase.filter((token) => !token.unspaced).map((token) => stemFolded(token.text));\n    if (latin.length === 0) return [];\n    return phrase.some((token) => token.unspaced) ? latin : [latin.join(' ')];\n  });\n  const defaultParts = [...words.map(stemFolded), ...phraseParts].map((t) => `\"${t}\"`);\n  const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n  return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n  const colList = cols.map((c) => `content.${c}`).join(', ');\n  const lit = escapeFtsLiteral(query);\n  return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `snippets` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS snippet fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { words, phrases, unspaced, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase) return [];\n  const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n  if (defaultQuery === '' && ngramQuery === '') return [];\n\n  const branches: string[] = [];\n  if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n  if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n  // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n  // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC. Phrase\n  // candidates carry authored fields through the join below so the shared verifier can reject\n  // cross-field, reordered, and non-adjacent matches before applying the caller's limit.\n  const ranked = `SELECT path, SUM(score) AS score FROM (\n      ${branches.join(' UNION ALL ')}\n    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}`;\n  const phraseSelect = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const phraseJoin = phrases.length > 0 ? ' JOIN content ON content.path = sq.path' : '';\n  const sql = `SELECT sq.path, sq.score${phraseSelect} FROM (\n    ${ranked}\n  ) sq${phraseJoin} ORDER BY sq.score DESC, sq.path${phrases.length > 0 ? '' : ' LIMIT ?'}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...(phrases.length > 0 ? [] : [opts.limit]))) as Array<LexicalHit & Partial<Record<AuthoredField, string>>>;\n  // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n  // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n  if (rows.some((r) => r.path === null)) {\n    throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n  }\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => AUTHORED_FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, opts.limit).map(({ path }) => ({ path }));\n}\n"],"names":["SenseError","matchesSearchPhrase","searchTokens","stemFolded","FIELDS","NGRAM_FIELDS","AUTHORED_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","emptyPhrase","_m","inner","phrase","tokens","length","push","filter","map","text","tok","split","escapeFtsLiteral","query","buildQueries","phraseParts","flatMap","latin","some","join","defaultParts","t","unspacedParts","defaultQuery","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","queryLexical","conn","unsupported","branches","ranked","phraseSelect","phraseJoin","sql","stmt","prepare","rows","all","limit","r","path","matches","row","every","field","slice"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;AACvG,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAE5F,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,MAAMC,SAAS;IAAC;IAAc;IAAgB;CAAY;AAC1D,MAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AACnE,MAAMC,kBAAkB;IAAC;IAAS;IAAW;CAAO;AAGpD,uFAAuF;AACvF,+FAA+F;AAC/F,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,UAA2B,EAAE;IACnC,MAAMC,WAAqB,EAAE;IAC7B,IAAIC,cAAc;IAClB,MAAMV,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACU,IAAIC;QACtD,MAAMC,SAASD,MAAMP,IAAI;QACzB,MAAMS,SAASxB,aAAauB;QAC5B,IAAIC,OAAOC,MAAM,KAAK,GAAG;YACvBL,cAAc;YACd,OAAO;QACT;QACAD,SAASO,IAAI,IAAIF,OAAOG,MAAM,CAAC,CAACb,QAAUA,MAAMK,QAAQ,EAAES,GAAG,CAAC,CAACd,QAAUA,MAAMe,IAAI;QACnFX,QAAQQ,IAAI,CAACF;QACb,OAAO;IACT;IACA,KAAK,MAAMM,OAAOpB,eAAeqB,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIL,MAAM,KAAK,GAAG;QACtB,KAAK,MAAMX,SAASd,aAAa8B,KAAM;YACrC,IAAIhB,MAAMK,QAAQ,EAAEA,SAASO,IAAI,CAACZ,MAAMe,IAAI;iBACvCZ,MAAMS,IAAI,CAACZ,MAAMe,IAAI;QAC5B;IACF;IACA,OAAO;QAAEZ;QAAOC;QAASC;QAAUC;IAAY;AACjD;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASY,iBAAiBC,KAAa;IACrC,OAAOA,MAAMtB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASuB,aAAajB,KAAe,EAAEC,OAAwB,EAAEC,QAAkB;IACjF,MAAMgB,cAAcjB,QAAQkB,OAAO,CAAC,CAACb;QACnC,MAAMc,QAAQd,OAAOI,MAAM,CAAC,CAACb,QAAU,CAACA,MAAMK,QAAQ,EAAES,GAAG,CAAC,CAACd,QAAUb,WAAWa,MAAMe,IAAI;QAC5F,IAAIQ,MAAMZ,MAAM,KAAK,GAAG,OAAO,EAAE;QACjC,OAAOF,OAAOe,IAAI,CAAC,CAACxB,QAAUA,MAAMK,QAAQ,IAAIkB,QAAQ;YAACA,MAAME,IAAI,CAAC;SAAK;IAC3E;IACA,MAAMC,eAAe;WAAIvB,MAAMW,GAAG,CAAC3B;WAAgBkC;KAAY,CAACP,GAAG,CAAC,CAACa,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IACnF,MAAMC,gBAAgBvB,SAASS,GAAG,CAAC,CAACa,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IAClD,OAAO;QAAEE,cAAcH,aAAaD,IAAI,CAAC;QAAUK,YAAYF,cAAcH,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASM,UAAUC,IAAuB,EAAEb,KAAa,EAAEc,IAAyB;IAClF,MAAMC,UAAUF,KAAKlB,GAAG,CAAC,CAACqB,IAAM,CAAC,QAAQ,EAAEA,GAAG,EAAEV,IAAI,CAAC;IACrD,MAAMW,MAAMlB,iBAAiBC;IAC7B,OAAO,CAAC,yCAAyC,EAAEe,QAAQ,GAAG,EAAEE,IAAI;iBACrD,EAAEH,KAAKI,SAAS,CAAC,iBAAiB,EAAEH,QAAQ,GAAG,EAAEE,IAAI,GAAG,EAAEH,KAAKK,SAAS,CAAC,CAAC,EAAEL,KAAKM,SAAS,EAAE;AAC7G;AAEA,uFAAuF;AACvF,qMAAqM;AACrM,OAAO,eAAeC,aAAaC,IAAgB,EAAE9C,KAAa,EAAEsC,IAAyB;IAC3F,MAAMS,cAAchD,oBAAoBC;IACxC,IAAI+C,gBAAgB,MAAM;QACxB,MAAM,IAAI1D,WAAW,4BAA4B,CAAC,wCAAwC,EAAE0D,YAAYlD,KAAK,CAAC,GAAG,EAAEkD,YAAY1C,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IACzQ;IACA,MAAM,EAAEE,KAAK,EAAEC,OAAO,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ,WAAWP;IAC7D,IAAIW,aAAa,OAAO,EAAE;IAC1B,MAAM,EAAEuB,YAAY,EAAEC,UAAU,EAAE,GAAGV,aAAajB,OAAOC,SAASC;IAClE,IAAIwB,iBAAiB,MAAMC,eAAe,IAAI,OAAO,EAAE;IAEvD,MAAMa,WAAqB,EAAE;IAC7B,IAAId,iBAAiB,IAAIc,SAAS/B,IAAI,CAACmB,UAAU3C,QAAQyC,cAAcI;IACvE,IAAIH,eAAe,IAAIa,SAAS/B,IAAI,CAACmB,UAAU1C,cAAcyC,YAAYG;IAEzE,8FAA8F;IAC9F,0FAA0F;IAC1F,4FAA4F;IAC5F,uFAAuF;IACvF,MAAMW,SAAS,CAAC;MACZ,EAAED,SAASlB,IAAI,CAAC,eAAe;4CACO,EAAEkB,SAAShC,MAAM,EAAE;IAC7D,MAAMkC,eAAezC,QAAQO,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAMmC,aAAa1C,QAAQO,MAAM,GAAG,IAAI,4CAA4C;IACpF,MAAMoC,MAAM,CAAC,wBAAwB,EAAEF,aAAa;IAClD,EAAED,OAAO;MACP,EAAEE,WAAW,gCAAgC,EAAE1C,QAAQO,MAAM,GAAG,IAAI,KAAK,YAAY;IACzF,MAAMqC,OAAO,MAAMP,KAAKQ,OAAO,CAACF;IAChC,MAAMG,OAAQ,MAAMF,KAAKG,GAAG,IAAK/C,QAAQO,MAAM,GAAG,IAAI,EAAE,GAAG;QAACsB,KAAKmB,KAAK;KAAC;IACvE,+FAA+F;IAC/F,yFAAyF;IACzF,IAAIF,KAAK1B,IAAI,CAAC,CAAC6B,IAAMA,EAAEC,IAAI,KAAK,OAAO;QACrC,MAAM,IAAItE,WAAW,qBAAqB;IAC5C;IACA,MAAMuE,UAAUnD,QAAQO,MAAM,KAAK,IAAIuC,OAAOA,KAAKrC,MAAM,CAAC,CAAC2C,MAAQpD,QAAQqD,KAAK,CAAC,CAAChD,SAAWnB,gBAAgBkC,IAAI,CAAC,CAACkC;oBAA8BF;uBAApBvE,qBAAoBuE,aAAAA,GAAG,CAACE,MAAM,cAAVF,wBAAAA,aAAc,IAAI/C;;IACnK,OAAO8C,QAAQI,KAAK,CAAC,GAAG1B,KAAKmB,KAAK,EAAEtC,GAAG,CAAC,CAAC,EAAEwC,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAChE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { stemFolded } from './lexical-text.ts';\n\nconst FIELDS = ['title_stem', 'summary_stem', 'text_stem'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\nconst AUTHORED_FIELDS = ['title', 'summary', 'text'] as const;\ntype AuthoredField = (typeof AUTHORED_FIELDS)[number];\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n  { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: SearchToken[][]; unspaced: string[]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const phrases: SearchToken[][] = [];\n  const unspaced: string[] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const tokens = searchTokens(phrase);\n    if (tokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    unspaced.push(...tokens.filter((token) => token.unspaced).map((token) => token.text));\n    phrases.push(tokens);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) unspaced.push(token.text);\n      else words.push(token.text);\n    }\n  }\n  return { words, phrases, unspaced, emptyPhrase };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n  return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: SearchToken[][], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n  const phraseParts = phrases.flatMap((phrase) => {\n    const latin = phrase.filter((token) => !token.unspaced).map((token) => stemFolded(token.text));\n    if (latin.length === 0) return [];\n    return phrase.some((token) => token.unspaced) ? latin : [latin.join(' ')];\n  });\n  const defaultParts = [...words.map(stemFolded), ...phraseParts].map((t) => `\"${t}\"`);\n  const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n  return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n  const colList = cols.map((c) => `content.${c}`).join(', ');\n  const lit = escapeFtsLiteral(query);\n  return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `snippets` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS snippet fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { words, phrases, unspaced, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase) return [];\n  const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n  if (defaultQuery === '' && ngramQuery === '') return [];\n\n  const branches: string[] = [];\n  if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n  if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n  // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n  // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC. Phrase\n  // candidates carry authored fields through the join below so the shared verifier can reject\n  // cross-field, reordered, and non-adjacent matches before applying the caller's limit.\n  const ranked = `SELECT path, SUM(score) AS score FROM (\n      ${branches.join(' UNION ALL ')}\n    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}`;\n  const phraseSelect = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const phraseJoin = phrases.length > 0 ? ' JOIN content ON content.path = sq.path' : '';\n  const sql = `SELECT sq.path, sq.score${phraseSelect} FROM (\n    ${ranked}\n  ) sq${phraseJoin} ORDER BY sq.score DESC, sq.path${phrases.length > 0 ? '' : ' LIMIT ?'}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...(phrases.length > 0 ? [] : [opts.limit]))) as Array<LexicalHit & Partial<Record<AuthoredField, string>>>;\n  // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n  // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n  if (rows.some((r) => r.path === null)) {\n    throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n  }\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => AUTHORED_FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, opts.limit).map(({ path }) => ({ path }));\n}\n"],"names":["SenseError","matchesSearchPhrase","searchTokens","stemFolded","FIELDS","NGRAM_FIELDS","AUTHORED_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","emptyPhrase","_m","inner","phrase","tokens","length","push","filter","map","text","tok","split","escapeFtsLiteral","query","buildQueries","phraseParts","flatMap","latin","some","join","defaultParts","t","unspacedParts","defaultQuery","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","queryLexical","conn","unsupported","branches","ranked","phraseSelect","phraseJoin","sql","stmt","prepare","rows","all","limit","r","path","matches","row","every","field","slice"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;AACvG,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAE5F,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,MAAMC,SAAS;IAAC;IAAc;IAAgB;CAAY;AAC1D,MAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AACnE,MAAMC,kBAAkB;IAAC;IAAS;IAAW;CAAO;AAGpD,uFAAuF;AACvF,+FAA+F;AAC/F,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,UAA2B,EAAE;IACnC,MAAMC,WAAqB,EAAE;IAC7B,IAAIC,cAAc;IAClB,MAAMV,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACU,IAAIC;QACtD,MAAMC,SAASD,MAAMP,IAAI;QACzB,MAAMS,SAASxB,aAAauB;QAC5B,IAAIC,OAAOC,MAAM,KAAK,GAAG;YACvBL,cAAc;YACd,OAAO;QACT;QACAD,SAASO,IAAI,IAAIF,OAAOG,MAAM,CAAC,CAACb,QAAUA,MAAMK,QAAQ,EAAES,GAAG,CAAC,CAACd,QAAUA,MAAMe,IAAI;QACnFX,QAAQQ,IAAI,CAACF;QACb,OAAO;IACT;IACA,KAAK,MAAMM,OAAOpB,eAAeqB,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIL,MAAM,KAAK,GAAG;QACtB,KAAK,MAAMX,SAASd,aAAa8B,KAAM;YACrC,IAAIhB,MAAMK,QAAQ,EAAEA,SAASO,IAAI,CAACZ,MAAMe,IAAI;iBACvCZ,MAAMS,IAAI,CAACZ,MAAMe,IAAI;QAC5B;IACF;IACA,OAAO;QAAEZ;QAAOC;QAASC;QAAUC;IAAY;AACjD;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASY,iBAAiBC,KAAa;IACrC,OAAOA,MAAMtB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASuB,aAAajB,KAAe,EAAEC,OAAwB,EAAEC,QAAkB;IACjF,MAAMgB,cAAcjB,QAAQkB,OAAO,CAAC,CAACb;QACnC,MAAMc,QAAQd,OAAOI,MAAM,CAAC,CAACb,QAAU,CAACA,MAAMK,QAAQ,EAAES,GAAG,CAAC,CAACd,QAAUb,WAAWa,MAAMe,IAAI;QAC5F,IAAIQ,MAAMZ,MAAM,KAAK,GAAG,OAAO,EAAE;QACjC,OAAOF,OAAOe,IAAI,CAAC,CAACxB,QAAUA,MAAMK,QAAQ,IAAIkB,QAAQ;YAACA,MAAME,IAAI,CAAC;SAAK;IAC3E;IACA,MAAMC,eAAe;WAAIvB,MAAMW,GAAG,CAAC3B;WAAgBkC;KAAY,CAACP,GAAG,CAAC,CAACa,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IACnF,MAAMC,gBAAgBvB,SAASS,GAAG,CAAC,CAACa,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IAClD,OAAO;QAAEE,cAAcH,aAAaD,IAAI,CAAC;QAAUK,YAAYF,cAAcH,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASM,UAAUC,IAAuB,EAAEb,KAAa,EAAEc,IAAyB;IAClF,MAAMC,UAAUF,KAAKlB,GAAG,CAAC,CAACqB,IAAM,CAAC,QAAQ,EAAEA,GAAG,EAAEV,IAAI,CAAC;IACrD,MAAMW,MAAMlB,iBAAiBC;IAC7B,OAAO,CAAC,yCAAyC,EAAEe,QAAQ,GAAG,EAAEE,IAAI;iBACrD,EAAEH,KAAKI,SAAS,CAAC,iBAAiB,EAAEH,QAAQ,GAAG,EAAEE,IAAI,GAAG,EAAEH,KAAKK,SAAS,CAAC,CAAC,EAAEL,KAAKM,SAAS,EAAE;AAC7G;AAEA,uFAAuF;AACvF,qMAAqM;AACrM,OAAO,eAAeC,aAAaC,IAAgB,EAAE9C,KAAa,EAAEsC,IAAyB;IAC3F,MAAMS,cAAchD,oBAAoBC;IACxC,IAAI+C,gBAAgB,MAAM;QACxB,MAAM,IAAI1D,WAAW,4BAA4B,CAAC,wCAAwC,EAAE0D,YAAYlD,KAAK,CAAC,GAAG,EAAEkD,YAAY1C,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IACzQ;IACA,MAAM,EAAEE,KAAK,EAAEC,OAAO,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ,WAAWP;IAC7D,IAAIW,aAAa,OAAO,EAAE;IAC1B,MAAM,EAAEuB,YAAY,EAAEC,UAAU,EAAE,GAAGV,aAAajB,OAAOC,SAASC;IAClE,IAAIwB,iBAAiB,MAAMC,eAAe,IAAI,OAAO,EAAE;IAEvD,MAAMa,WAAqB,EAAE;IAC7B,IAAId,iBAAiB,IAAIc,SAAS/B,IAAI,CAACmB,UAAU3C,QAAQyC,cAAcI;IACvE,IAAIH,eAAe,IAAIa,SAAS/B,IAAI,CAACmB,UAAU1C,cAAcyC,YAAYG;IAEzE,8FAA8F;IAC9F,0FAA0F;IAC1F,4FAA4F;IAC5F,uFAAuF;IACvF,MAAMW,SAAS,CAAC;MACZ,EAAED,SAASlB,IAAI,CAAC,eAAe;4CACO,EAAEkB,SAAShC,MAAM,EAAE;IAC7D,MAAMkC,eAAezC,QAAQO,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAMmC,aAAa1C,QAAQO,MAAM,GAAG,IAAI,4CAA4C;IACpF,MAAMoC,MAAM,CAAC,wBAAwB,EAAEF,aAAa;IAClD,EAAED,OAAO;MACP,EAAEE,WAAW,gCAAgC,EAAE1C,QAAQO,MAAM,GAAG,IAAI,KAAK,YAAY;IACzF,MAAMqC,OAAO,MAAMP,KAAKQ,OAAO,CAACF;IAChC,MAAMG,OAAQ,MAAMF,KAAKG,GAAG,IAAK/C,QAAQO,MAAM,GAAG,IAAI,EAAE,GAAG;QAACsB,KAAKmB,KAAK;KAAC;IACvE,+FAA+F;IAC/F,yFAAyF;IACzF,IAAIF,KAAK1B,IAAI,CAAC,CAAC6B,IAAMA,EAAEC,IAAI,KAAK,OAAO;QACrC,MAAM,IAAItE,WAAW,qBAAqB;IAC5C;IACA,MAAMuE,UAAUnD,QAAQO,MAAM,KAAK,IAAIuC,OAAOA,KAAKrC,MAAM,CAAC,CAAC2C,MAAQpD,QAAQqD,KAAK,CAAC,CAAChD,SAAWnB,gBAAgBkC,IAAI,CAAC,CAACkC;oBAA8BF;uBAApBvE,qBAAoBuE,aAAAA,GAAG,CAACE,MAAM,cAAVF,wBAAAA,aAAc,IAAI/C;;IACnK,OAAO8C,QAAQI,KAAK,CAAC,GAAG1B,KAAKmB,KAAK,EAAEtC,GAAG,CAAC,CAAC,EAAEwC,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAChE"}

package/dist/esm/store/turso/reconcile.js

@@ -25,8 +25,8 @@ function ngramSidecar(text) {
 // Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with
 // no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.
 const MAX_FRONTMATTER_COLUMNS = 2000;
-// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured
-// 2026-08-30; the derivation and its bounds are pinned in this file's spec.
+// Retained policy boundary from 2026-08-30 corpus-scale measurements; the optimum after later
+// update-path changes remains pending matched corpus-scale crossover evidence.
 export const FTS_REBUILD_THRESHOLD = 250;
 // No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.
 const INSERT_CONTENT_SQL = `INSERT INTO content ("path", title, summary, text, title_stem, summary_stem, text_stem, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
@@ -58,9 +58,12 @@ export async function reconcileTursoContentWithStrategy(conn, touched, docs, str
     if (bulk) for (const name of CONTENT_FTS_NAMES)await conn.exec(`DROP INDEX IF EXISTS ${name}`);
     // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5
     // content), so vanished and reparsed docs delete in one pass.
-    if (touched.length > 0) await conn.runBatch('DELETE FROM content WHERE "path" = ?', touched.map((p)=>[
-            p
-        ]));
+    if (touched.length > 0) {
+        const placeholders = touched.map(()=>'?').join(', ');
+        await conn.runBatch(`DELETE FROM content WHERE "path" IN (${placeholders})`, [
+            touched
+        ]);
+    }
     if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));
     if (bulk) for (const ddl of CONTENT_FTS_DDL)await conn.exec(ddl);
 }

package/dist/esm/store/turso/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { quoteIdent, recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { stemFolded } from './lexical-text.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around\n// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title_stem, summary_stem, text_stem) WITH (weights = 'title_stem=10.0,summary_stem=5.0,text_stem=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\nexport type TursoFtsStrategy = 'incremental' | 'rebuild';\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_stem, summary_stem, text_stem, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, stemFolded(title), stemFolded(summary), stemFolded(text), ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nexport function tursoFtsStrategy(delta: ReconcileDelta): TursoFtsStrategy {\n  const churn = delta.reparsed.length + delta.vanished.length;\n  return delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD ? 'rebuild' : 'incremental';\n}\n\n// Kept transaction-free because the shared reconcile owns the one write transaction. The explicit\n// strategy is internal composition for real-engine diagnostics; production selects it from delta.\nexport async function reconcileTursoContentWithStrategy(conn: Connection, touched: string[], docs: ParsedDoc[], strategy: TursoFtsStrategy): Promise<void> {\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = strategy === 'rebuild';\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\nasync function reconcileTursoContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, _cfg: Config): Promise<void> {\n  await reconcileTursoContentWithStrategy(conn, touched, docs, tursoFtsStrategy(delta));\n}\n\n// turso's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  for (const name of names) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)}`);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  addColumns,\n  reconcileContent: reconcileTursoContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["SenseError","hasUnspacedRun","quoteIdent","recordReconcileDuration","BEGIN_WRITE","stemFolded","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ngramSidecar","text","MAX_FRONTMATTER_COLUMNS","FTS_REBUILD_THRESHOLD","INSERT_CONTENT_SQL","contentRow","doc","title","summary","search","relPath","tursoFtsStrategy","delta","churn","reparsed","length","vanished","files","reconcileTursoContentWithStrategy","conn","touched","docs","strategy","bulk","name","exec","runBatch","map","p","ddl","reconcileTursoContent","_cfg","addColumns","names","tursoDialect","beginMode","checkColumnLimit","count","reconcileContent","recordDuration"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,UAAU,EAAEC,uBAAuB,QAAQ,eAAe;AACnE,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,qFAAqF;AACrF,sFAAsF;AAEtF,+FAA+F;AAC/F,2FAA2F;AAC3F,iGAAiG;AACjG,uGAAuG;AACvG,OAAO,MAAMC,kBAAkB;IAC7B,CAAC,mKAAmK,CAAC;IACrK,CAAC,gMAAgM,CAAC;CACnM,CAAU;AACX,OAAO,MAAMC,oBAAoB;IAAC;IAAe;CAAoB,CAAU;AAE/E,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAOR,eAAeQ,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,MAAMC,0BAA0B;AAEhC,4FAA4F;AAC5F,4EAA4E;AAC5E,OAAO,MAAMC,wBAAwB,IAAI;AAIzC,yFAAyF;AACzF,MAAMC,qBAAqB,CAAC,qKAAqK,CAAC;AAElM,SAASC,WAAWC,GAAc;IAChC,MAAM,EAAEC,KAAK,EAAEC,OAAO,EAAEP,IAAI,EAAE,GAAGK,IAAIG,MAAM;IAC3C,OAAO;QAACH,IAAII,OAAO;QAAEH;QAAOC;QAASP;QAAMJ,WAAWU;QAAQV,WAAWW;QAAUX,WAAWI;QAAOD,aAAaO;QAAQP,aAAaQ;QAAUR,aAAaC;KAAM;AACtK;AAEA,OAAO,SAASU,iBAAiBC,KAAqB;IACpD,MAAMC,QAAQD,MAAME,QAAQ,CAACC,MAAM,GAAGH,MAAMI,QAAQ,CAACD,MAAM;IAC3D,OAAOH,MAAMK,KAAK,CAACF,MAAM,KAAK,KAAKF,QAAQV,wBAAwB,YAAY;AACjF;AAEA,kGAAkG;AAClG,kGAAkG;AAClG,OAAO,eAAee,kCAAkCC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,QAA0B;IACxI,6FAA6F;IAC7F,qDAAqD;IACrD,MAAMC,OAAOD,aAAa;IAC1B,IAAIC,MAAM,KAAK,MAAMC,QAAQzB,kBAAmB,MAAMoB,KAAKM,IAAI,CAAC,CAAC,qBAAqB,EAAED,MAAM;IAE9F,0FAA0F;IAC1F,8DAA8D;IAC9D,IAAIJ,QAAQL,MAAM,GAAG,GACnB,MAAMI,KAAKO,QAAQ,CACjB,wCACAN,QAAQO,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIP,KAAKN,MAAM,GAAG,GAAG,MAAMI,KAAKO,QAAQ,CAACtB,oBAAoBiB,KAAKM,GAAG,CAACtB;IACtE,IAAIkB,MAAM,KAAK,MAAMM,OAAO/B,gBAAiB,MAAMqB,KAAKM,IAAI,CAACI;AAC/D;AAEA,eAAeC,sBAAsBX,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAET,KAAqB,EAAEmB,IAAY;IAC9H,MAAMb,kCAAkCC,MAAMC,SAASC,MAAMV,iBAAiBC;AAChF;AAEA,yFAAyF;AACzF,eAAeoB,WAAWb,IAAgB,EAAEc,KAAe;IACzD,KAAK,MAAMT,QAAQS,MAAO,MAAMd,KAAKM,IAAI,CAAC,CAAC,mCAAmC,EAAE/B,WAAW8B,OAAO;AACpG;AAEA,OAAO,MAAMU,eAAiC;IAC5CC,WAAW,IAAMvC;IACjBwC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQnC,yBAAyB;YACnC,MAAM,IAAIV,WACR,gBACA,CAAC,uBAAuB,EAAE6C,MAAM,2DAA2D,EAAEnC,wBAAwB,gPAAgP,CAAC;QAE1W;IACF;IACA8B;IACAM,kBAAkBR;IAClBS,gBAAgB5C;AAClB,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { quoteIdent, recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { stemFolded } from './lexical-text.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around\n// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title_stem, summary_stem, text_stem) WITH (weights = 'title_stem=10.0,summary_stem=5.0,text_stem=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Retained policy boundary from 2026-08-30 corpus-scale measurements; the optimum after later\n// update-path changes remains pending matched corpus-scale crossover evidence.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\nexport type TursoFtsStrategy = 'incremental' | 'rebuild';\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_stem, summary_stem, text_stem, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, stemFolded(title), stemFolded(summary), stemFolded(text), ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nexport function tursoFtsStrategy(delta: ReconcileDelta): TursoFtsStrategy {\n  const churn = delta.reparsed.length + delta.vanished.length;\n  return delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD ? 'rebuild' : 'incremental';\n}\n\n// Kept transaction-free because the shared reconcile owns the one write transaction. The explicit\n// strategy is internal composition for real-engine diagnostics; production selects it from delta.\nexport async function reconcileTursoContentWithStrategy(conn: Connection, touched: string[], docs: ParsedDoc[], strategy: TursoFtsStrategy): Promise<void> {\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = strategy === 'rebuild';\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0) {\n    const placeholders = touched.map(() => '?').join(', ');\n    await conn.runBatch(`DELETE FROM content WHERE \"path\" IN (${placeholders})`, [touched]);\n  }\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\nasync function reconcileTursoContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, _cfg: Config): Promise<void> {\n  await reconcileTursoContentWithStrategy(conn, touched, docs, tursoFtsStrategy(delta));\n}\n\n// turso's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  for (const name of names) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)}`);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  addColumns,\n  reconcileContent: reconcileTursoContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["SenseError","hasUnspacedRun","quoteIdent","recordReconcileDuration","BEGIN_WRITE","stemFolded","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ngramSidecar","text","MAX_FRONTMATTER_COLUMNS","FTS_REBUILD_THRESHOLD","INSERT_CONTENT_SQL","contentRow","doc","title","summary","search","relPath","tursoFtsStrategy","delta","churn","reparsed","length","vanished","files","reconcileTursoContentWithStrategy","conn","touched","docs","strategy","bulk","name","exec","placeholders","map","join","runBatch","ddl","reconcileTursoContent","_cfg","addColumns","names","tursoDialect","beginMode","checkColumnLimit","count","reconcileContent","recordDuration"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,UAAU,EAAEC,uBAAuB,QAAQ,eAAe;AACnE,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,qFAAqF;AACrF,sFAAsF;AAEtF,+FAA+F;AAC/F,2FAA2F;AAC3F,iGAAiG;AACjG,uGAAuG;AACvG,OAAO,MAAMC,kBAAkB;IAC7B,CAAC,mKAAmK,CAAC;IACrK,CAAC,gMAAgM,CAAC;CACnM,CAAU;AACX,OAAO,MAAMC,oBAAoB;IAAC;IAAe;CAAoB,CAAU;AAE/E,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAOR,eAAeQ,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,MAAMC,0BAA0B;AAEhC,8FAA8F;AAC9F,+EAA+E;AAC/E,OAAO,MAAMC,wBAAwB,IAAI;AAIzC,yFAAyF;AACzF,MAAMC,qBAAqB,CAAC,qKAAqK,CAAC;AAElM,SAASC,WAAWC,GAAc;IAChC,MAAM,EAAEC,KAAK,EAAEC,OAAO,EAAEP,IAAI,EAAE,GAAGK,IAAIG,MAAM;IAC3C,OAAO;QAACH,IAAII,OAAO;QAAEH;QAAOC;QAASP;QAAMJ,WAAWU;QAAQV,WAAWW;QAAUX,WAAWI;QAAOD,aAAaO;QAAQP,aAAaQ;QAAUR,aAAaC;KAAM;AACtK;AAEA,OAAO,SAASU,iBAAiBC,KAAqB;IACpD,MAAMC,QAAQD,MAAME,QAAQ,CAACC,MAAM,GAAGH,MAAMI,QAAQ,CAACD,MAAM;IAC3D,OAAOH,MAAMK,KAAK,CAACF,MAAM,KAAK,KAAKF,QAAQV,wBAAwB,YAAY;AACjF;AAEA,kGAAkG;AAClG,kGAAkG;AAClG,OAAO,eAAee,kCAAkCC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,QAA0B;IACxI,6FAA6F;IAC7F,qDAAqD;IACrD,MAAMC,OAAOD,aAAa;IAC1B,IAAIC,MAAM,KAAK,MAAMC,QAAQzB,kBAAmB,MAAMoB,KAAKM,IAAI,CAAC,CAAC,qBAAqB,EAAED,MAAM;IAE9F,0FAA0F;IAC1F,8DAA8D;IAC9D,IAAIJ,QAAQL,MAAM,GAAG,GAAG;QACtB,MAAMW,eAAeN,QAAQO,GAAG,CAAC,IAAM,KAAKC,IAAI,CAAC;QACjD,MAAMT,KAAKU,QAAQ,CAAC,CAAC,qCAAqC,EAAEH,aAAa,CAAC,CAAC,EAAE;YAACN;SAAQ;IACxF;IACA,IAAIC,KAAKN,MAAM,GAAG,GAAG,MAAMI,KAAKU,QAAQ,CAACzB,oBAAoBiB,KAAKM,GAAG,CAACtB;IACtE,IAAIkB,MAAM,KAAK,MAAMO,OAAOhC,gBAAiB,MAAMqB,KAAKM,IAAI,CAACK;AAC/D;AAEA,eAAeC,sBAAsBZ,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAET,KAAqB,EAAEoB,IAAY;IAC9H,MAAMd,kCAAkCC,MAAMC,SAASC,MAAMV,iBAAiBC;AAChF;AAEA,yFAAyF;AACzF,eAAeqB,WAAWd,IAAgB,EAAEe,KAAe;IACzD,KAAK,MAAMV,QAAQU,MAAO,MAAMf,KAAKM,IAAI,CAAC,CAAC,mCAAmC,EAAE/B,WAAW8B,OAAO;AACpG;AAEA,OAAO,MAAMW,eAAiC;IAC5CC,WAAW,IAAMxC;IACjByC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQpC,yBAAyB;YACnC,MAAM,IAAIV,WACR,gBACA,CAAC,uBAAuB,EAAE8C,MAAM,2DAA2D,EAAEpC,wBAAwB,gPAAgP,CAAC;QAE1W;IACF;IACA+B;IACAM,kBAAkBR;IAClBS,gBAAgB7C;AAClB,EAAE"}