sensemaking 0.24.4 → 0.24.6

package/dist/esm/store/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { channel } from 'node:diagnostics_channel';\nimport { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { SenseError } from '../errors.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport { classifyEmbedChange } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.ts';\nimport { lockWaitBudgetMs } from './lock-wait.ts';\nimport { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Stages } from './stages.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Per-stage wall time for this open's build pass (stages.ts), not for any narrow embed or\n  // feature invalidation that ran beside it -- on a cold build there is none, so it is the whole cost.\n  stages: Stages;\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  stages: Stages;\n}\n\n// duckdb/turso hold the cache file for their connection's whole life, so a second command waits.\n// Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.\nconst LOCK_POLL_MS = 50;\nconst lockWait = channel('sensemaking.store.lock-wait');\n\nasync function connectUnlocked<Handle>(dbPath: string, cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<{ handle: Handle; conn: Connection }> {\n  const budgetMs = lockWaitBudgetMs(cfg.configDir ?? cfg.baseDir);\n  const deadline = Date.now() + budgetMs;\n  for (;;) {\n    try {\n      return await dialect.connect(dbPath, cfg);\n    } catch (err) {\n      if (!dialect.isLocked?.(err as Error)) throw err;\n      // Observe a real rejected native connection, never merely the start of an async open.\n      if (lockWait.hasSubscribers) lockWait.publish({ store: cfg.store, dbPath });\n      if (Date.now() >= deadline) {\n        throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${Math.round(budgetMs / 1000)}s; wait for that command to finish, or set \"store\" to \"sqlite\" in sense.config.json, which serves concurrent commands`);\n      }\n      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));\n    }\n  }\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<ConnectResult<Handle>> {\n  const stateDir = join(cfg.configDir ?? cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await connectUnlocked(dbPath, cfg, dialect);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): invalidate()/invalidateFeatures() below share its pool\n  // with the build() call that follows them.\n  const builder = createBuilder(conn, cfg, cfg.rootDir ?? cfg.baseDir, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    // Set only by the preset-membership branch below; reconcile treats these paths as touched\n    // despite an unchanged stamp, for this one build() call.\n    let forcedPaths: Set<string> | undefined;\n    // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.\n    let embedInvalidate: EmbedChangeKind | undefined;\n    // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.\n    let featureToggles: FeatureToggle[] | undefined;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed\n      // preset's stored segment can't be parsed back into old include/exclude/vectors -- either\n      // way the branch below is skipped and the full rebuild stays the fallback.\n      const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.rootDir ?? cfg.baseDir, features ?? '', changedKeys) : null;\n      // Null unless exactly one segment changed and it decomposes into a recognised embed-only\n      // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.\n      const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features ?? '', wantFeatures) : null;\n      // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed\n      // feature's stored segment can't be parsed back into old/new on-off -- either way the branch\n      // below is skipped and the full rebuild stays the fallback.\n      const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features ?? '', wantFeatures, changedKeys) : null;\n      if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (embedKind !== null) {\n        // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither\n        // case needs any file reparsed for any feature but embed itself.\n        embedInvalidate = embedKind;\n        console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (presetForced !== null) {\n        // Membership only: which files a preset covers moved, not a file's own content, a\n        // global feature, or the embed model. reconcile() already knows how to\n        // add, update, and remove a file correctly (every cross-feature cascade included); it\n        // just needs telling which unchanged files to treat as touched (forcedPaths, above).\n        forcedPaths = presetForced;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the files it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (toggles !== null) {\n        // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's\n        // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a\n        // feature turning back on, fully re-derives across every indexed file rather than\n        // trusting rows written before or during that window.\n        featureToggles = toggles;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the feature it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same\n    // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or\n    // feature-toggle-only changed-key set).\n    const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;\n    const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;\n\n    const { parsed, warnings, stages } = await builder.build(forcedPaths);\n    // Every open reconciles once and is done with it -- a watcher opens fresh per event instead of holding this connection (src/watch.ts).\n    await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed: parsed + embedParsed + featureParsed, warnings, stages };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  // `baseDir` was the original public open() input. Keep direct API callers working while\n  // making the two ownership paths explicit for every internal operation.\n  if (!cfg.rootDir) {\n    if (!cfg.baseDir) throw new SenseError('CONFIG_INVALID', 'open requires rootDir (or legacy baseDir)');\n    cfg.rootDir = cfg.baseDir;\n  }\n  if (!cfg.configDir) cfg.configDir = cfg.rootDir;\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, stages } = await connectWithDialect(cfg, dialect);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, stages };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["channel","mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","SenseError","FEATURES","createBuilder","clearCache","classifyEmbedChange","classifyFeatureToggles","isFeatureOnlyChange","lockWaitBudgetMs","forcedPresetPaths","isPresetOnlyChange","getMeta","setMeta","changedSignatureKeys","embedIdentityAdopted","signatureDiff","LOCK_POLL_MS","lockWait","connectUnlocked","dbPath","cfg","dialect","budgetMs","configDir","baseDir","deadline","Date","now","connect","err","isLocked","hasSubscribers","publish","store","Math","round","Promise","resolve","setTimeout","connectWithDialect","stateDir","recursive","filename","handle","conn","closed","builder","rootDir","reconcileDialect","exec","version","features","wantFeatures","forcedPaths","embedInvalidate","featureToggles","schemaVersion","console","error","close","changedKeys","presetForced","embedKind","size","has","toggles","changed","ensureSchema","setDerivedBusyTimeout","recordedMaxMs","Number","min","max","embedParsed","invalidate","parsed","featureParsed","invalidateFeatures","warnings","stages","build","openWithDialect","resolvedCfg","createStore","docCount","stmt","prepare","get","n"],"mappings":"AAAA,SAASA,OAAO,QAAQ,2BAA2B;AACnD,SAASC,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,qBAAqB;AACjE,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,UAAU,QAAQ,aAAa;AAExC,SAASC,mBAAmB,QAAQ,mBAAmB;AAEvD,SAASC,sBAAsB,EAAEC,mBAAmB,QAAQ,qBAAqB;AACjF,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAC1E,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,aAAa,QAAQ,iBAAiB;AA8B3F,iGAAiG;AACjG,oFAAoF;AACpF,MAAMC,eAAe;AACrB,MAAMC,WAAWtB,QAAQ;AAEzB,eAAeuB,gBAAwBC,MAAc,EAAEC,GAAmB,EAAEC,OAA4B;QACpED;IAAlC,MAAME,WAAWd,kBAAiBY,iBAAAA,IAAIG,SAAS,cAAbH,4BAAAA,iBAAiBA,IAAII,OAAO;IAC9D,MAAMC,WAAWC,KAAKC,GAAG,KAAKL;IAC9B,OAAS;QACP,IAAI;YACF,OAAO,MAAMD,QAAQO,OAAO,CAACT,QAAQC;QACvC,EAAE,OAAOS,KAAK;gBACPR;YAAL,IAAI,GAACA,oBAAAA,QAAQS,QAAQ,cAAhBT,wCAAAA,uBAAAA,SAAmBQ,OAAe,MAAMA;YAC7C,sFAAsF;YACtF,IAAIZ,SAASc,cAAc,EAAEd,SAASe,OAAO,CAAC;gBAAEC,OAAOb,IAAIa,KAAK;gBAAEd;YAAO;YACzE,IAAIO,KAAKC,GAAG,MAAMF,UAAU;gBAC1B,MAAM,IAAIxB,WAAW,cAAc,CAAC,2CAA2C,EAAEmB,IAAIa,KAAK,CAAC,QAAQ,EAAEd,OAAO,gCAAgC,EAAEe,KAAKC,KAAK,CAACb,WAAW,MAAM,qHAAqH,CAAC;YAClS;YACA,MAAM,IAAIc,QAAQ,CAACC,UAAYC,WAAWD,SAASrB;QACrD;IACF;AACF;AAEA,eAAeuB,mBAA2BnB,GAAmB,EAAEC,OAA4B;QACnED,gBAUmBA;IAVzC,MAAMoB,WAAW3C,MAAKuB,iBAAAA,IAAIG,SAAS,cAAbH,4BAAAA,iBAAiBA,IAAII,OAAO,EAAEzB;IACpDH,UAAU4C,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMtB,SAAStB,KAAK2C,UAAUnB,QAAQqB,QAAQ;IAE9C,MAAM,EAAEC,MAAM,EAAEC,IAAI,EAAE,GAAG,MAAM1B,gBAAgBC,QAAQC,KAAKC;IAC5D,gGAAgG;IAChG,kGAAkG;IAClG,IAAIwB,SAAS;IACb,+FAA+F;IAC/F,2CAA2C;IAC3C,MAAMC,UAAU3C,cAAcyC,MAAMxB,MAAKA,eAAAA,IAAI2B,OAAO,cAAX3B,0BAAAA,eAAeA,IAAII,OAAO,EAAEH,QAAQ2B,gBAAgB;IAC7F,IAAI;QACF,MAAMJ,KAAKK,IAAI,CAAC;QAEhB,uFAAuF;QACvF,qGAAqG;QACrG,MAAMC,UAAU,MAAMvC,QAAQiC,MAAM;QACpC,MAAMO,WAAW,MAAMxC,QAAQiC,MAAM;QACrC,MAAMQ,eAAetD,iBAAiBsB,KAAKlB;QAC3C,0FAA0F;QAC1F,yDAAyD;QACzD,IAAImD;QACJ,kGAAkG;QAClG,IAAIC;QACJ,4GAA4G;QAC5G,IAAIC;QACJ,IAAI,AAACL,YAAY,QAAQA,YAAY7B,QAAQmC,aAAa,IAAML,aAAa,QAAQA,aAAaC,cAAe;gBAejChC;YAd9E,uFAAuF;YACvF,wFAAwF;YACxF,IAAI8B,YAAY,QAAQA,YAAY7B,QAAQmC,aAAa,EAAE;gBACzDC,QAAQC,KAAK,CAAC;gBACd,kFAAkF;gBAClFb,SAAS;gBACT,MAAMxB,QAAQsC,KAAK,CAAChB;gBACpBvC,WAAWgB;gBACX,OAAOmB,mBAAmBnB,KAAKC;YACjC;YACA,MAAMuC,cAAc/C,qBAAqBsC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,oFAAoF;YACpF,0FAA0F;YAC1F,2EAA2E;YAC3E,MAAMS,eAAenD,mBAAmBkD,eAAenD,kBAAkBW,MAAKA,gBAAAA,IAAI2B,OAAO,cAAX3B,2BAAAA,gBAAeA,IAAII,OAAO,EAAE2B,qBAAAA,sBAAAA,WAAY,IAAIS,eAAe;YACzI,yFAAyF;YACzF,oFAAoF;YACpF,MAAME,YAAYF,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,WAAW3D,oBAAoB8C,qBAAAA,sBAAAA,WAAY,IAAIC,gBAAgB;YAC3H,4FAA4F;YAC5F,6FAA6F;YAC7F,4DAA4D;YAC5D,MAAMa,UAAU1D,oBAAoBqD,eAAetD,uBAAuB6C,qBAAAA,sBAAAA,WAAY,IAAIC,cAAcQ,eAAe;YACvH,IAAIA,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,YAAYlD,qBAAqBqC,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBAC5G,iFAAiF;gBACjF,sDAAsD;gBACtDK,QAAQC,KAAK,CAAC;gBACd,MAAM9C,QAAQgC,MAAM,YAAYQ;YAClC,OAAO,IAAIU,cAAc,MAAM;gBAC7B,yFAAyF;gBACzF,iEAAiE;gBACjER,kBAAkBQ;gBAClBL,QAAQC,KAAK,CAACI,cAAc,UAAU,6EAA6E;gBACnH,MAAMlD,QAAQgC,MAAM,YAAYQ;YAClC,OAAO,IAAIS,iBAAiB,MAAM;gBAChC,kFAAkF;gBAClF,uEAAuE;gBACvE,sFAAsF;gBACtF,qFAAqF;gBACrFR,cAAcQ;gBACd,MAAMK,UAAUnD,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oCAAoC,CAAC;gBACpF,MAAMtD,QAAQgC,MAAM,YAAYQ;YAClC,OAAO,IAAIa,YAAY,MAAM;gBAC3B,uFAAuF;gBACvF,wFAAwF;gBACxF,kFAAkF;gBAClF,sDAAsD;gBACtDV,iBAAiBU;gBACjB,MAAMC,UAAUnD,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,sCAAsC,CAAC;gBACtF,MAAMtD,QAAQgC,MAAM,YAAYQ;YAClC,OAAO;gBACL,MAAMc,UAAUnD,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oBAAoB,CAAC;gBACpErB,SAAS;gBACT,MAAMxB,QAAQsC,KAAK,CAAChB;gBACpBvC,WAAWgB;gBACX,OAAOmB,mBAAmBnB,KAAKC;YACjC;QACF;QAEA,MAAMA,QAAQ8C,YAAY,CAACxB,QAAQC,MAAMxB;QAEzC,wFAAwF;QACxF,uFAAuF;QACvF,IAAIC,QAAQ+C,qBAAqB,EAAE;gBACH;YAA9B,MAAMC,gBAAgBC,QAAQ,OAAA,MAAM3D,QAAQiC,MAAM,iCAApB,kBAAA,OAA4C;YAC1E,MAAMvB,QAAQ+C,qBAAqB,CAACzB,QAAQC,MAAMV,KAAKqC,GAAG,CAACrC,KAAKsC,GAAG,CAAC,OAAO,IAAIH,gBAAgB;QACjG;QAEA,6FAA6F;QAC7F,+FAA+F;QAC/F,wCAAwC;QACxC,MAAMI,cAAcnB,kBAAkB,AAAC,CAAA,MAAMR,QAAQ4B,UAAU,CAACpB,gBAAe,EAAGqB,MAAM,GAAG;QAC3F,MAAMC,gBAAgBrB,iBAAiB,AAAC,CAAA,MAAMT,QAAQ+B,kBAAkB,CAACtB,eAAc,EAAGoB,MAAM,GAAG;QAEnG,MAAM,EAAEA,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMjC,QAAQkC,KAAK,CAAC3B;QACzD,uIAAuI;QACvI,MAAMP,QAAQa,KAAK;QAEnB,OAAO;YAAEhB;YAAQC;YAAMxB;YAAKD;YAAQwD,QAAQA,SAASF,cAAcG;YAAeE;YAAUC;QAAO;IACrG,EAAE,OAAOlD,KAAK;QACZ,MAAMiB,QAAQa,KAAK;QACnB,IAAI,CAACd,QAAQ,MAAMxB,QAAQsC,KAAK,CAAChB;QACjC,MAAMd;IACR;AACF;AAEA,iGAAiG;AACjG,mFAAmF;AACnF,OAAO,eAAeoD,gBAAwB7D,GAAmB,EAAEC,OAA4B;IAC7F,wFAAwF;IACxF,wEAAwE;IACxE,IAAI,CAACD,IAAI2B,OAAO,EAAE;QAChB,IAAI,CAAC3B,IAAII,OAAO,EAAE,MAAM,IAAIvB,WAAW,kBAAkB;QACzDmB,IAAI2B,OAAO,GAAG3B,IAAII,OAAO;IAC3B;IACA,IAAI,CAACJ,IAAIG,SAAS,EAAEH,IAAIG,SAAS,GAAGH,IAAI2B,OAAO;IAC/C,MAAM,EAAEJ,MAAM,EAAEC,IAAI,EAAExB,KAAK8D,WAAW,EAAE/D,MAAM,EAAEwD,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMxC,mBAAmBnB,KAAKC;IAC3G,MAAMY,QAAQZ,QAAQ8D,WAAW,CAACxC,QAAQC,MAAMsC;IAChD,0FAA0F;IAC1FlF,eAAe4C,MAAMX;IACrB,OAAO;QAAEA;QAAOb,KAAK8D;QAAa/D;QAAQwD;QAAQG;QAAUC;IAAO;AACrE;AAEA,OAAO,eAAeK,SAASnD,KAAY;IACzC,MAAMoD,OAAO,MAAMpD,MAAMqD,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKE,GAAG,EAAC,EAAqBC,CAAC;AAChD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { channel } from 'node:diagnostics_channel';\nimport { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { SenseError } from '../errors.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport { classifyEmbedChange } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.ts';\nimport { lockWaitBudgetMs } from './lock-wait.ts';\nimport { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Stages } from './stages.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Per-stage wall time for this open's build pass (stages.ts), not for any narrow embed or\n  // feature invalidation that ran beside it -- on a cold build there is none, so it is the whole cost.\n  stages: Stages;\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  stages: Stages;\n}\n\n// duckdb/turso hold the cache file for their connection's whole life, so a second command waits.\n// Bounded by this tree's recorded reconcile time (lock-wait.ts), not a fixed guess.\nconst LOCK_POLL_MS = 50;\nconst lockWait = channel('sensemaking.store.lock-wait');\n\nasync function connectUnlocked<Handle>(dbPath: string, cfg: ResolvedConfig, configDir: string, dialect: OpenDialect<Handle>): Promise<{ handle: Handle; conn: Connection }> {\n  const budgetMs = lockWaitBudgetMs(configDir);\n  const deadline = Date.now() + budgetMs;\n  for (;;) {\n    try {\n      return await dialect.connect(dbPath, cfg);\n    } catch (err) {\n      if (!dialect.isLocked?.(err as Error)) throw err;\n      // Observe a real rejected native connection, never merely the start of an async open.\n      if (lockWait.hasSubscribers) lockWait.publish({ store: cfg.store, dbPath });\n      if (Date.now() >= deadline) {\n        throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${Math.round(budgetMs / 1000)}s; wait for that command to finish, or set \"store\" to \"sqlite\" in sense.config.json, which serves concurrent commands`);\n      }\n      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));\n    }\n  }\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<ConnectResult<Handle>> {\n  const configDir = cfg.configDir ?? cfg.baseDir;\n  const stateDir = join(configDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await connectUnlocked(dbPath, cfg, configDir, dialect);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): invalidate()/invalidateFeatures() below share its pool\n  // with the build() call that follows them.\n  const builder = createBuilder(conn, cfg, { rootDir: cfg.rootDir ?? cfg.baseDir, configDir }, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    // Set only by the preset-membership branch below; reconcile treats these paths as touched\n    // despite an unchanged stamp, for this one build() call.\n    let forcedPaths: Set<string> | undefined;\n    // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.\n    let embedInvalidate: EmbedChangeKind | undefined;\n    // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.\n    let featureToggles: FeatureToggle[] | undefined;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed\n      // preset's stored segment can't be parsed back into old include/exclude/vectors -- either\n      // way the branch below is skipped and the full rebuild stays the fallback.\n      const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.rootDir ?? cfg.baseDir, features ?? '', changedKeys) : null;\n      // Null unless exactly one segment changed and it decomposes into a recognised embed-only\n      // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.\n      const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features ?? '', wantFeatures) : null;\n      // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed\n      // feature's stored segment can't be parsed back into old/new on-off -- either way the branch\n      // below is skipped and the full rebuild stays the fallback.\n      const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features ?? '', wantFeatures, changedKeys) : null;\n      if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (embedKind !== null) {\n        // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither\n        // case needs any file reparsed for any feature but embed itself.\n        embedInvalidate = embedKind;\n        console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (presetForced !== null) {\n        // Membership only: which files a preset covers moved, not a file's own content, a\n        // global feature, or the embed model. reconcile() already knows how to\n        // add, update, and remove a file correctly (every cross-feature cascade included); it\n        // just needs telling which unchanged files to treat as touched (forcedPaths, above).\n        forcedPaths = presetForced;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the files it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (toggles !== null) {\n        // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's\n        // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a\n        // feature turning back on, fully re-derives across every indexed file rather than\n        // trusting rows written before or during that window.\n        featureToggles = toggles;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the feature it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect);\n      }\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same\n    // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or\n    // feature-toggle-only changed-key set).\n    const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;\n    const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;\n\n    const { parsed, warnings, stages } = await builder.build(forcedPaths);\n    // Every open reconciles once and is done with it -- a watcher opens fresh per event instead of holding this connection (src/watch.ts).\n    await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed: parsed + embedParsed + featureParsed, warnings, stages };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  // `baseDir` was the original public open() input. Keep direct API callers working while\n  // making the two ownership paths explicit for every internal operation.\n  if (!cfg.rootDir) {\n    if (!cfg.baseDir) throw new SenseError('CONFIG_INVALID', 'open requires rootDir (or legacy baseDir)');\n    cfg.rootDir = cfg.baseDir;\n  }\n  if (!cfg.configDir) cfg.configDir = cfg.rootDir;\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, stages } = await connectWithDialect(cfg, dialect);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, stages };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["channel","mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","SenseError","FEATURES","createBuilder","clearCache","classifyEmbedChange","classifyFeatureToggles","isFeatureOnlyChange","lockWaitBudgetMs","forcedPresetPaths","isPresetOnlyChange","getMeta","setMeta","changedSignatureKeys","embedIdentityAdopted","signatureDiff","LOCK_POLL_MS","lockWait","connectUnlocked","dbPath","cfg","configDir","dialect","budgetMs","deadline","Date","now","connect","err","isLocked","hasSubscribers","publish","store","Math","round","Promise","resolve","setTimeout","connectWithDialect","baseDir","stateDir","recursive","filename","handle","conn","closed","builder","rootDir","reconcileDialect","exec","version","features","wantFeatures","forcedPaths","embedInvalidate","featureToggles","schemaVersion","console","error","close","changedKeys","presetForced","embedKind","size","has","toggles","changed","ensureSchema","setDerivedBusyTimeout","recordedMaxMs","Number","min","max","embedParsed","invalidate","parsed","featureParsed","invalidateFeatures","warnings","stages","build","openWithDialect","resolvedCfg","createStore","docCount","stmt","prepare","get","n"],"mappings":"AAAA,SAASA,OAAO,QAAQ,2BAA2B;AACnD,SAASC,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,qBAAqB;AACjE,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,UAAU,QAAQ,aAAa;AAExC,SAASC,mBAAmB,QAAQ,mBAAmB;AAEvD,SAASC,sBAAsB,EAAEC,mBAAmB,QAAQ,qBAAqB;AACjF,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAC1E,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAC/C,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,aAAa,QAAQ,iBAAiB;AA8B3F,iGAAiG;AACjG,oFAAoF;AACpF,MAAMC,eAAe;AACrB,MAAMC,WAAWtB,QAAQ;AAEzB,eAAeuB,gBAAwBC,MAAc,EAAEC,GAAmB,EAAEC,SAAiB,EAAEC,OAA4B;IACzH,MAAMC,WAAWf,iBAAiBa;IAClC,MAAMG,WAAWC,KAAKC,GAAG,KAAKH;IAC9B,OAAS;QACP,IAAI;YACF,OAAO,MAAMD,QAAQK,OAAO,CAACR,QAAQC;QACvC,EAAE,OAAOQ,KAAK;gBACPN;YAAL,IAAI,GAACA,oBAAAA,QAAQO,QAAQ,cAAhBP,wCAAAA,uBAAAA,SAAmBM,OAAe,MAAMA;YAC7C,sFAAsF;YACtF,IAAIX,SAASa,cAAc,EAAEb,SAASc,OAAO,CAAC;gBAAEC,OAAOZ,IAAIY,KAAK;gBAAEb;YAAO;YACzE,IAAIM,KAAKC,GAAG,MAAMF,UAAU;gBAC1B,MAAM,IAAIvB,WAAW,cAAc,CAAC,2CAA2C,EAAEmB,IAAIY,KAAK,CAAC,QAAQ,EAAEb,OAAO,gCAAgC,EAAEc,KAAKC,KAAK,CAACX,WAAW,MAAM,qHAAqH,CAAC;YAClS;YACA,MAAM,IAAIY,QAAQ,CAACC,UAAYC,WAAWD,SAASpB;QACrD;IACF;AACF;AAEA,eAAesB,mBAA2BlB,GAAmB,EAAEE,OAA4B;QACvEF,gBAWkCA;IAXpD,MAAMC,aAAYD,iBAAAA,IAAIC,SAAS,cAAbD,4BAAAA,iBAAiBA,IAAImB,OAAO;IAC9C,MAAMC,WAAW3C,KAAKwB,WAAWtB;IACjCH,UAAU4C,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMtB,SAAStB,KAAK2C,UAAUlB,QAAQoB,QAAQ;IAE9C,MAAM,EAAEC,MAAM,EAAEC,IAAI,EAAE,GAAG,MAAM1B,gBAAgBC,QAAQC,KAAKC,WAAWC;IACvE,gGAAgG;IAChG,kGAAkG;IAClG,IAAIuB,SAAS;IACb,+FAA+F;IAC/F,2CAA2C;IAC3C,MAAMC,UAAU3C,cAAcyC,MAAMxB,KAAK;QAAE2B,OAAO,GAAE3B,eAAAA,IAAI2B,OAAO,cAAX3B,0BAAAA,eAAeA,IAAImB,OAAO;QAAElB;IAAU,GAAGC,QAAQ0B,gBAAgB;IACrH,IAAI;QACF,MAAMJ,KAAKK,IAAI,CAAC;QAEhB,uFAAuF;QACvF,qGAAqG;QACrG,MAAMC,UAAU,MAAMvC,QAAQiC,MAAM;QACpC,MAAMO,WAAW,MAAMxC,QAAQiC,MAAM;QACrC,MAAMQ,eAAetD,iBAAiBsB,KAAKlB;QAC3C,0FAA0F;QAC1F,yDAAyD;QACzD,IAAImD;QACJ,kGAAkG;QAClG,IAAIC;QACJ,4GAA4G;QAC5G,IAAIC;QACJ,IAAI,AAACL,YAAY,QAAQA,YAAY5B,QAAQkC,aAAa,IAAML,aAAa,QAAQA,aAAaC,cAAe;gBAejChC;YAd9E,uFAAuF;YACvF,wFAAwF;YACxF,IAAI8B,YAAY,QAAQA,YAAY5B,QAAQkC,aAAa,EAAE;gBACzDC,QAAQC,KAAK,CAAC;gBACd,kFAAkF;gBAClFb,SAAS;gBACT,MAAMvB,QAAQqC,KAAK,CAAChB;gBACpBvC,WAAWgB;gBACX,OAAOkB,mBAAmBlB,KAAKE;YACjC;YACA,MAAMsC,cAAc/C,qBAAqBsC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,oFAAoF;YACpF,0FAA0F;YAC1F,2EAA2E;YAC3E,MAAMS,eAAenD,mBAAmBkD,eAAenD,kBAAkBW,MAAKA,gBAAAA,IAAI2B,OAAO,cAAX3B,2BAAAA,gBAAeA,IAAImB,OAAO,EAAEY,qBAAAA,sBAAAA,WAAY,IAAIS,eAAe;YACzI,yFAAyF;YACzF,oFAAoF;YACpF,MAAME,YAAYF,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,WAAW3D,oBAAoB8C,qBAAAA,sBAAAA,WAAY,IAAIC,gBAAgB;YAC3H,4FAA4F;YAC5F,6FAA6F;YAC7F,4DAA4D;YAC5D,MAAMa,UAAU1D,oBAAoBqD,eAAetD,uBAAuB6C,qBAAAA,sBAAAA,WAAY,IAAIC,cAAcQ,eAAe;YACvH,IAAIA,YAAYG,IAAI,KAAK,KAAKH,YAAYI,GAAG,CAAC,YAAYlD,qBAAqBqC,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBAC5G,iFAAiF;gBACjF,sDAAsD;gBACtDK,QAAQC,KAAK,CAAC;gBACd,MAAM9C,QAAQgC,MAAM,YAAYQ;YAClC,OAAO,IAAIU,cAAc,MAAM;gBAC7B,yFAAyF;gBACzF,iEAAiE;gBACjER,kBAAkBQ;gBAClBL,QAAQC,KAAK,CAACI,cAAc,UAAU,6EAA6E;gBACnH,MAAMlD,QAAQgC,MAAM,YAAYQ;YAClC,OAAO,IAAIS,iBAAiB,MAAM;gBAChC,kFAAkF;gBAClF,uEAAuE;gBACvE,sFAAsF;gBACtF,qFAAqF;gBACrFR,cAAcQ;gBACd,MAAMK,UAAUnD,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oCAAoC,CAAC;gBACpF,MAAMtD,QAAQgC,MAAM,YAAYQ;YAClC,OAAO,IAAIa,YAAY,MAAM;gBAC3B,uFAAuF;gBACvF,wFAAwF;gBACxF,kFAAkF;gBAClF,sDAAsD;gBACtDV,iBAAiBU;gBACjB,MAAMC,UAAUnD,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,sCAAsC,CAAC;gBACtF,MAAMtD,QAAQgC,MAAM,YAAYQ;YAClC,OAAO;gBACL,MAAMc,UAAUnD,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CK,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEQ,QAAQ,oBAAoB,CAAC;gBACpErB,SAAS;gBACT,MAAMvB,QAAQqC,KAAK,CAAChB;gBACpBvC,WAAWgB;gBACX,OAAOkB,mBAAmBlB,KAAKE;YACjC;QACF;QAEA,MAAMA,QAAQ6C,YAAY,CAACxB,QAAQC,MAAMxB;QAEzC,wFAAwF;QACxF,uFAAuF;QACvF,IAAIE,QAAQ8C,qBAAqB,EAAE;gBACH;YAA9B,MAAMC,gBAAgBC,QAAQ,OAAA,MAAM3D,QAAQiC,MAAM,iCAApB,kBAAA,OAA4C;YAC1E,MAAMtB,QAAQ8C,qBAAqB,CAACzB,QAAQC,MAAMX,KAAKsC,GAAG,CAACtC,KAAKuC,GAAG,CAAC,OAAO,IAAIH,gBAAgB;QACjG;QAEA,6FAA6F;QAC7F,+FAA+F;QAC/F,wCAAwC;QACxC,MAAMI,cAAcnB,kBAAkB,AAAC,CAAA,MAAMR,QAAQ4B,UAAU,CAACpB,gBAAe,EAAGqB,MAAM,GAAG;QAC3F,MAAMC,gBAAgBrB,iBAAiB,AAAC,CAAA,MAAMT,QAAQ+B,kBAAkB,CAACtB,eAAc,EAAGoB,MAAM,GAAG;QAEnG,MAAM,EAAEA,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMjC,QAAQkC,KAAK,CAAC3B;QACzD,uIAAuI;QACvI,MAAMP,QAAQa,KAAK;QAEnB,OAAO;YAAEhB;YAAQC;YAAMxB;YAAKD;YAAQwD,QAAQA,SAASF,cAAcG;YAAeE;YAAUC;QAAO;IACrG,EAAE,OAAOnD,KAAK;QACZ,MAAMkB,QAAQa,KAAK;QACnB,IAAI,CAACd,QAAQ,MAAMvB,QAAQqC,KAAK,CAAChB;QACjC,MAAMf;IACR;AACF;AAEA,iGAAiG;AACjG,mFAAmF;AACnF,OAAO,eAAeqD,gBAAwB7D,GAAmB,EAAEE,OAA4B;IAC7F,wFAAwF;IACxF,wEAAwE;IACxE,IAAI,CAACF,IAAI2B,OAAO,EAAE;QAChB,IAAI,CAAC3B,IAAImB,OAAO,EAAE,MAAM,IAAItC,WAAW,kBAAkB;QACzDmB,IAAI2B,OAAO,GAAG3B,IAAImB,OAAO;IAC3B;IACA,IAAI,CAACnB,IAAIC,SAAS,EAAED,IAAIC,SAAS,GAAGD,IAAI2B,OAAO;IAC/C,MAAM,EAAEJ,MAAM,EAAEC,IAAI,EAAExB,KAAK8D,WAAW,EAAE/D,MAAM,EAAEwD,MAAM,EAAEG,QAAQ,EAAEC,MAAM,EAAE,GAAG,MAAMzC,mBAAmBlB,KAAKE;IAC3G,MAAMU,QAAQV,QAAQ6D,WAAW,CAACxC,QAAQC,MAAMsC;IAChD,0FAA0F;IAC1FlF,eAAe4C,MAAMZ;IACrB,OAAO;QAAEA;QAAOZ,KAAK8D;QAAa/D;QAAQwD;QAAQG;QAAUC;IAAO;AACrE;AAEA,OAAO,eAAeK,SAASpD,KAAY;IACzC,MAAMqD,OAAO,MAAMrD,MAAMsD,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKE,GAAG,EAAC,EAAqBC,CAAC;AAChD"}

package/dist/esm/store/reconcile.d.ts

@@ -3,7 +3,11 @@ import type { ParsePool } from '../scan/pool.js';
 import { type Stages } from './stages.js';
 import type { Connection, ReconcileDialect } from './types.js';
 export declare const CORE_FRONTMATTER_COLUMNS: Set<string>;
-export declare function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{
+export interface ReconcilePaths {
+    rootDir: string;
+    configDir: string;
+}
+export declare function reconcile(conn: Connection, cfg: Config, paths: ReconcilePaths, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{
     parsed: number;
     warnings: string[];
     stages: Stages;

package/dist/esm/store/reconcile.js

@@ -20,12 +20,13 @@ export const CORE_FRONTMATTER_COLUMNS = new Set([
     '_size',
     '_parse_error'
 ]);
-export async function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths) {
+export async function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
+    const { rootDir, configDir } = paths;
     const start = process.hrtime.bigint();
     const features = activeFeatures(cfg);
     const stages = stageRecorder(features.map((f)=>f.name));
     const elapsed = ()=>Number(process.hrtime.bigint() - start) / 1e6;
-    const files = await stages.time('list', ()=>listFiles(cfg, baseDir));
+    const files = await stages.time('list', ()=>listFiles(cfg, rootDir));
     const currentSet = new Set(files.map((f)=>f.relPath));
     const existingRows = await stages.time('existing', async ()=>{
         const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
@@ -180,7 +181,7 @@ export async function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths)
     }, dialect.beginMode());
     const durationMs = Date.now() - txStart;
     // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.
-    recordLockWaitMs(baseDir, durationMs);
+    recordLockWaitMs(configDir, durationMs);
     if (dialect.recordDuration) await stages.time('meta', ()=>{
         var _dialect_recordDuration;
         return (_dialect_recordDuration = dialect.recordDuration) === null || _dialect_recordDuration === void 0 ? void 0 : _dialect_recordDuration.call(dialect, conn, durationMs);

package/dist/esm/store/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { recordLockWaitMs } from './lock-wait.ts';\nimport { appendRows, getColumns, quoteIdent } from './shared.ts';\nimport { featureStage, type Stages, stageRecorder } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nexport const CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const start = process.hrtime.bigint();\n  const features = activeFeatures(cfg);\n  const stages = stageRecorder(features.map((f) => f.name));\n  const elapsed = () => Number(process.hrtime.bigint() - start) / 1e6;\n  const files = await stages.time('list', () => listFiles(cfg, baseDir));\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = await stages.time('existing', async () => {\n    const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n    return (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  });\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [], stages: stages.take(elapsed(), 0) };\n\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.\n  const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', () => reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool }));\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await stages.time('alter', () => dialect.addColumns(conn, missingColumns));\n\n      // Revalidated once the lock is held, before this process's own frontmatter upsert below:\n      // `added` came from a path read taken before this transaction's lock, so a path still\n      // called \"added\" here may already have a content row from a concurrent reconcile that\n      // committed while this one waited. One SELECT, not a DELETE per added file -- with no\n      // contention it returns the same set `existing` already ruled out, so nothing extra clears.\n      let contentTouched = touched;\n      if (added.length > 0)\n        contentTouched = await stages.time('added-recheck', async () => {\n          const currentPathsStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n          const currentPaths = new Set(((await currentPathsStmt.all()) as Array<{ path: string }>).map((r) => r.path));\n          const staleAdded = added.filter((p) => currentPaths.has(p));\n          return staleAdded.length > 0 ? [...touched, ...staleAdded] : touched;\n        });\n\n      if (parsedDocs.length > 0) {\n        const toRow = (doc: (typeof parsedDocs)[number]) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          });\n        // A path in `added` has no existing frontmatter row, so it can never conflict; the rest\n        // genuinely can, and keep the upsert.\n        await stages.time('fm-upsert', async () => {\n          const newDocs = parsedDocs.filter((d) => addedSet.has(d.relPath));\n          const updateDocs = parsedDocs.filter((d) => !addedSet.has(d.relPath));\n          await appendRows(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow));\n          if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));\n        });\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await stages.time('text-index', () => dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      await stages.time('presets', async () => {\n        if (touched.length > 0)\n          await conn.runBatch(\n            'DELETE FROM preset_files WHERE \"path\" = ?',\n            touched.map((p) => [p])\n          );\n        const presetRows: unknown[][] = [];\n        for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken\n        // before this transaction's lock, so a path this process calls \"added\" can already have\n        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.\n        await appendRows(conn, 'preset_files', ['path', 'preset'], 'INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n      });\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await stages.time('vanished', () =>\n          conn.runBatch(\n            'DELETE FROM frontmatter WHERE \"path\" = ?',\n            vanished.map((p) => [p])\n          )\n        );\n\n      // Timed per feature per hook, so link resolution and PageRank are named stages without\n      // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.\n      if (touched.length > 0) for (const feature of features) await stages.time(featureStage(feature.name, 'remove'), () => feature.remove?.(conn, touched, delta));\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await stages.time(featureStage(feature.name, 'store'), () => feature.store?.(conn, docsForFeature, delta));\n      }\n      for (const feature of features) await stages.time(featureStage(feature.name, 'after'), () => feature.afterReconcile?.(conn, delta));\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.\n  recordLockWaitMs(baseDir, durationMs);\n  if (dialect.recordDuration) await stages.time('meta', () => dialect.recordDuration?.(conn, durationMs));\n\n  return { parsed: parsedDocs.length, warnings, stages: stages.take(elapsed(), durationMs, workerParseMs) };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","recordLockWaitMs","appendRows","getColumns","quoteIdent","featureStage","stageRecorder","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","reconcile","conn","cfg","baseDir","dialect","pool","forcedPaths","start","process","hrtime","bigint","features","stages","map","f","name","elapsed","Number","files","time","currentSet","relPath","existingRows","existingStmt","prepare","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","take","seenColumns","report","docs","parsedDocs","newColumns","workerParseMs","tick","finish","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","txStart","Date","now","present","missingColumns","addColumns","contentTouched","currentPathsStmt","currentPaths","staleAdded","toRow","doc","ctimeMs","parseError","data","newDocs","updateDocs","runBatch","reconcileContent","presetRows","presetName","presets","push","feature","remove","docsForFeature","extracted","store","afterReconcile","beginMode","durationMs","recordDuration"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAE/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,UAAU,EAAEC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACjE,SAASC,YAAY,EAAeC,aAAa,QAAQ,cAAc;AACvE,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe,EAAE;AAEvG,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;IAC5J,MAAMC,QAAQC,QAAQC,MAAM,CAACC,MAAM;IACnC,MAAMC,WAAWzB,eAAegB;IAChC,MAAMU,SAAShB,cAAce,SAASE,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACvD,MAAMC,UAAU,IAAMC,OAAOT,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;IAChE,MAAMW,QAAQ,MAAMN,OAAOO,IAAI,CAAC,QAAQ,IAAM/B,UAAUc,KAAKC;IAC7D,MAAMiB,aAAa,IAAIrB,IAAImB,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IAErD,MAAMC,eAAe,MAAMV,OAAOO,IAAI,CAAC,YAAY;QACjD,MAAMI,eAAe,MAAMtB,KAAKuB,OAAO,CAAC;QACxC,OAAQ,MAAMD,aAAaE,GAAG;IAChC;IACA,MAAMC,WAAW,IAAIC,IAAIL,aAAaT,GAAG,CAAC,CAACe,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,+FAA+F;IAC/F,+FAA+F;IAC/F,gFAAgF;IAChF,MAAME,WAAWR,aAAaS,MAAM,CAAC,CAACH,IAAM,CAACR,WAAWY,GAAG,CAACJ,EAAEC,IAAI,GAAGhB,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;IAEtF,6FAA6F;IAC7F,0FAA0F;IAC1F,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACjB;;QAC9B,MAAMoB,MAAMR,SAASS,GAAG,CAACrB,EAAEO,OAAO;QAClC,OAAO,CAACa,OAAOA,IAAIE,MAAM,KAAKtB,EAAEuB,OAAO,IAAIH,IAAII,KAAK,KAAKxB,EAAEyB,IAAI,aAAKjC,wBAAAA,kCAAAA,YAAa0B,GAAG,CAAClB,EAAEO,OAAO,wCAAK;IACrG;IAEA,IAAIS,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;QAAE9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAW;IAAG;IAEzH,MAAM4B,cAAc,MAAMnD,WAAWQ;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM4C,SAAS1D,SAAS,mBAAmB8C,UAAUO,MAAM;IAC3D,wFAAwF;IACxF,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAEC,aAAa,EAAE,GAAG,MAAMrC,OAAOO,IAAI,CAAC,SAAS,IAAM7B,aAAa2C,WAAWtB,UAAUT,KAAK0C,aAAaC,OAAOK,IAAI,EAAE;YAAE7C;QAAK;IAC3KwC,OAAOM,MAAM;IACb,KAAK,MAAMC,OAAOJ,WAAYJ,YAAYS,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIV;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjGxC,QAAQmD,gBAAgB,CAACD,WAAWd,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMgB,kBAAkBF,WAAWvB,MAAM,CAAC,CAAC0B,IAAM3D,yBAAyBkC,GAAG,CAACyB,MAAM,CAACpE,iBAAiB2C,GAAG,CAACyB;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB3C,GAAG,CAACnB,YAAYiE,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB3C,GAAG,CAAC,IAAM,KAAK8C,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLzB,MAAM,CAAC,CAAC0B,IAAMA,MAAM,QACpB5C,GAAG,CAAC,CAAC4C,IAAM,GAAG/D,WAAW+D,GAAG,YAAY,EAAE/D,WAAW+D,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ3B,UAAUF,MAAM,CAAC,CAACjB,IAAM,CAACY,SAASM,GAAG,CAAClB,EAAEO,OAAO,GAAGR,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IACpF,MAAMwC,QAAwB;QAAE3C;QAAO4C,UAAUf,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO;QAAGuC;QAAO9B;IAAS;IACnG,MAAMkC,WAAW,IAAIjE,IAAI6D;IACzB,MAAMK,mBAAmBlB,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO,EAAEU,MAAM,CAAC,CAACmC,IAAM,CAACF,SAAShC,GAAG,CAACkC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAIrC;WAAamC;KAAiB;IAElD,MAAMG,UAAUC,KAAKC,GAAG;IACxB,MAAMzE,gBACJI,MACA;QACE,iGAAiG;QACjG,sFAAsF;QACtF,MAAMsE,UAAU,MAAM9E,WAAWQ;QACjC,MAAMuE,iBAAiBxB,WAAWjB,MAAM,CAAC,CAACqB,MAAQ,CAACmB,QAAQvC,GAAG,CAACoB;QAC/D,IAAIoB,eAAehC,MAAM,GAAG,GAAG,MAAM5B,OAAOO,IAAI,CAAC,SAAS,IAAMf,QAAQqE,UAAU,CAACxE,MAAMuE;QAEzF,yFAAyF;QACzF,sFAAsF;QACtF,sFAAsF;QACtF,sFAAsF;QACtF,4FAA4F;QAC5F,IAAIE,iBAAiBP;QACrB,IAAIP,MAAMpB,MAAM,GAAG,GACjBkC,iBAAiB,MAAM9D,OAAOO,IAAI,CAAC,iBAAiB;YAClD,MAAMwD,mBAAmB,MAAM1E,KAAKuB,OAAO,CAAC;YAC5C,MAAMoD,eAAe,IAAI7E,IAAI,AAAE,CAAA,MAAM4E,iBAAiBlD,GAAG,EAAC,EAA+BZ,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;YAC1G,MAAMgD,aAAajB,MAAM7B,MAAM,CAAC,CAACmC,IAAMU,aAAa5C,GAAG,CAACkC;YACxD,OAAOW,WAAWrC,MAAM,GAAG,IAAI;mBAAI2B;mBAAYU;aAAW,GAAGV;QAC/D;QAEF,IAAIpB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMsC,QAAQ,CAACC,MACbvB,gBAAgB3C,GAAG,CAAC,CAACuC;wBAOZ2B;oBANP,IAAI3B,QAAQ,QAAQ,OAAO2B,IAAI1D,OAAO;oBACtC,IAAI+B,QAAQ,UAAU,OAAO2B,IAAI1C,OAAO;oBACxC,IAAIe,QAAQ,UAAU,OAAO2B,IAAIC,OAAO;oBACxC,IAAI5B,QAAQ,SAAS,OAAO2B,IAAIxC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIa,QAAQ,gBAAgB,OAAO2B,IAAIE,UAAU;oBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAAC9B,IAAI,cAAb2B,2BAAAA,gBAAiB;gBAC1B;YACF,wFAAwF;YACxF,sCAAsC;YACtC,MAAMnE,OAAOO,IAAI,CAAC,aAAa;gBAC7B,MAAMgE,UAAUpC,WAAWhB,MAAM,CAAC,CAACgC,IAAMC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;gBAC/D,MAAM+D,aAAarC,WAAWhB,MAAM,CAAC,CAACgC,IAAM,CAACC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;gBACnE,MAAM7B,WAAWS,MAAM,eAAeuD,iBAAiBE,WAAWyB,QAAQtE,GAAG,CAACiE;gBAC9E,IAAIM,WAAW5C,MAAM,GAAG,GAAG,MAAMvC,KAAKoF,QAAQ,CAAC3B,WAAW0B,WAAWvE,GAAG,CAACiE;YAC3E;QACF;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMlE,OAAOO,IAAI,CAAC,cAAc,IAAMf,QAAQkF,gBAAgB,CAACrF,MAAMyE,gBAAgB3B,YAAYc,OAAO3D;QAExG,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMU,OAAOO,IAAI,CAAC,WAAW;YAC3B,IAAIgD,QAAQ3B,MAAM,GAAG,GACnB,MAAMvC,KAAKoF,QAAQ,CACjB,6CACAlB,QAAQtD,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;YAE1B,MAAMqB,aAA0B,EAAE;YAClC,KAAK,MAAMR,OAAOhC,WAAY,KAAK,MAAMyC,cAAcT,IAAIU,OAAO,CAAEF,WAAWG,IAAI,CAAC;gBAACX,IAAI1D,OAAO;gBAAEmE;aAAW;YAC7G,uFAAuF;YACvF,wFAAwF;YACxF,uGAAuG;YACvG,MAAMhG,WAAWS,MAAM,gBAAgB;gBAAC;gBAAQ;aAAS,EAAE,kGAAkGsF;QAC/J;QAEA,+FAA+F;QAC/F,4FAA4F;QAC5F,IAAIzD,SAASU,MAAM,GAAG,GACpB,MAAM5B,OAAOO,IAAI,CAAC,YAAY,IAC5BlB,KAAKoF,QAAQ,CACX,4CACAvD,SAASjB,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;QAI7B,uFAAuF;QACvF,0FAA0F;QAC1F,IAAIC,QAAQ3B,MAAM,GAAG,GAAG,KAAK,MAAMmD,WAAWhF,SAAU,MAAMC,OAAOO,IAAI,CAACxB,aAAagG,QAAQ5E,IAAI,EAAE,WAAW;gBAAM4E;oBAAAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,qBAAAA,SAAiB1F,MAAMkE,SAASN;;QACtJ,KAAK,MAAM8B,WAAWhF,SAAU;YAC9B,MAAMkF,iBAAiC9C,WAAWlC,GAAG,CAAC,CAACkE,MAAS,CAAA;oBAAElD,MAAMkD,IAAI1D,OAAO;oBAAEyE,WAAWf,IAAIe,SAAS,CAACH,QAAQ5E,IAAI,CAAC;gBAAC,CAAA;YAC5H,MAAMH,OAAOO,IAAI,CAACxB,aAAagG,QAAQ5E,IAAI,EAAE,UAAU;oBAAM4E;wBAAAA,iBAAAA,QAAQI,KAAK,cAAbJ,qCAAAA,oBAAAA,SAAgB1F,MAAM4F,gBAAgBhC;;QACrG;QACA,KAAK,MAAM8B,WAAWhF,SAAU,MAAMC,OAAOO,IAAI,CAACxB,aAAagG,QAAQ5E,IAAI,EAAE,UAAU;gBAAM4E;oBAAAA,0BAAAA,QAAQK,cAAc,cAAtBL,8CAAAA,6BAAAA,SAAyB1F,MAAM4D;;IAC9H,GACAzD,QAAQ6F,SAAS;IAGnB,MAAMC,aAAa7B,KAAKC,GAAG,KAAKF;IAChC,2GAA2G;IAC3G7E,iBAAiBY,SAAS+F;IAC1B,IAAI9F,QAAQ+F,cAAc,EAAE,MAAMvF,OAAOO,IAAI,CAAC,QAAQ;YAAMf;gBAAAA,0BAAAA,QAAQ+F,cAAc,cAAtB/F,8CAAAA,6BAAAA,SAAyBH,MAAMiG;;IAE3F,OAAO;QAAEzD,QAAQM,WAAWP,MAAM;QAAEE;QAAU9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAWkF,YAAYjD;IAAe;AAC1G"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { recordLockWaitMs } from './lock-wait.ts';\nimport { appendRows, getColumns, quoteIdent } from './shared.ts';\nimport { featureStage, type Stages, stageRecorder } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nexport const CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport interface ReconcilePaths {\n  rootDir: string;\n  configDir: string;\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, paths: ReconcilePaths, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const { rootDir, configDir } = paths;\n  const start = process.hrtime.bigint();\n  const features = activeFeatures(cfg);\n  const stages = stageRecorder(features.map((f) => f.name));\n  const elapsed = () => Number(process.hrtime.bigint() - start) / 1e6;\n  const files = await stages.time('list', () => listFiles(cfg, rootDir));\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = await stages.time('existing', async () => {\n    const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n    return (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  });\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [], stages: stages.take(elapsed(), 0) };\n\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.\n  const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', () => reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool }));\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await stages.time('alter', () => dialect.addColumns(conn, missingColumns));\n\n      // Revalidated once the lock is held, before this process's own frontmatter upsert below:\n      // `added` came from a path read taken before this transaction's lock, so a path still\n      // called \"added\" here may already have a content row from a concurrent reconcile that\n      // committed while this one waited. One SELECT, not a DELETE per added file -- with no\n      // contention it returns the same set `existing` already ruled out, so nothing extra clears.\n      let contentTouched = touched;\n      if (added.length > 0)\n        contentTouched = await stages.time('added-recheck', async () => {\n          const currentPathsStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n          const currentPaths = new Set(((await currentPathsStmt.all()) as Array<{ path: string }>).map((r) => r.path));\n          const staleAdded = added.filter((p) => currentPaths.has(p));\n          return staleAdded.length > 0 ? [...touched, ...staleAdded] : touched;\n        });\n\n      if (parsedDocs.length > 0) {\n        const toRow = (doc: (typeof parsedDocs)[number]) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          });\n        // A path in `added` has no existing frontmatter row, so it can never conflict; the rest\n        // genuinely can, and keep the upsert.\n        await stages.time('fm-upsert', async () => {\n          const newDocs = parsedDocs.filter((d) => addedSet.has(d.relPath));\n          const updateDocs = parsedDocs.filter((d) => !addedSet.has(d.relPath));\n          await appendRows(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow));\n          if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));\n        });\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await stages.time('text-index', () => dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      await stages.time('presets', async () => {\n        if (touched.length > 0)\n          await conn.runBatch(\n            'DELETE FROM preset_files WHERE \"path\" = ?',\n            touched.map((p) => [p])\n          );\n        const presetRows: unknown[][] = [];\n        for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken\n        // before this transaction's lock, so a path this process calls \"added\" can already have\n        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.\n        await appendRows(conn, 'preset_files', ['path', 'preset'], 'INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n      });\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await stages.time('vanished', () =>\n          conn.runBatch(\n            'DELETE FROM frontmatter WHERE \"path\" = ?',\n            vanished.map((p) => [p])\n          )\n        );\n\n      // Timed per feature per hook, so link resolution and PageRank are named stages without\n      // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.\n      if (touched.length > 0) for (const feature of features) await stages.time(featureStage(feature.name, 'remove'), () => feature.remove?.(conn, touched, delta));\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await stages.time(featureStage(feature.name, 'store'), () => feature.store?.(conn, docsForFeature, delta));\n      }\n      for (const feature of features) await stages.time(featureStage(feature.name, 'after'), () => feature.afterReconcile?.(conn, delta));\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.\n  recordLockWaitMs(configDir, durationMs);\n  if (dialect.recordDuration) await stages.time('meta', () => dialect.recordDuration?.(conn, durationMs));\n\n  return { parsed: parsedDocs.length, warnings, stages: stages.take(elapsed(), durationMs, workerParseMs) };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","recordLockWaitMs","appendRows","getColumns","quoteIdent","featureStage","stageRecorder","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","reconcile","conn","cfg","paths","dialect","pool","forcedPaths","rootDir","configDir","start","process","hrtime","bigint","features","stages","map","f","name","elapsed","Number","files","time","currentSet","relPath","existingRows","existingStmt","prepare","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","take","seenColumns","report","docs","parsedDocs","newColumns","workerParseMs","tick","finish","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","txStart","Date","now","present","missingColumns","addColumns","contentTouched","currentPathsStmt","currentPaths","staleAdded","toRow","doc","ctimeMs","parseError","data","newDocs","updateDocs","runBatch","reconcileContent","presetRows","presetName","presets","push","feature","remove","docsForFeature","extracted","store","afterReconcile","beginMode","durationMs","recordDuration"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAE/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,UAAU,EAAEC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACjE,SAASC,YAAY,EAAeC,aAAa,QAAQ,cAAc;AACvE,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe,EAAE;AAOvG,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,KAAqB,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;IAClK,MAAM,EAAEC,OAAO,EAAEC,SAAS,EAAE,GAAGL;IAC/B,MAAMM,QAAQC,QAAQC,MAAM,CAACC,MAAM;IACnC,MAAMC,WAAW3B,eAAegB;IAChC,MAAMY,SAASlB,cAAciB,SAASE,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACvD,MAAMC,UAAU,IAAMC,OAAOT,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;IAChE,MAAMW,QAAQ,MAAMN,OAAOO,IAAI,CAAC,QAAQ,IAAMjC,UAAUc,KAAKK;IAC7D,MAAMe,aAAa,IAAIvB,IAAIqB,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IAErD,MAAMC,eAAe,MAAMV,OAAOO,IAAI,CAAC,YAAY;QACjD,MAAMI,eAAe,MAAMxB,KAAKyB,OAAO,CAAC;QACxC,OAAQ,MAAMD,aAAaE,GAAG;IAChC;IACA,MAAMC,WAAW,IAAIC,IAAIL,aAAaT,GAAG,CAAC,CAACe,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,+FAA+F;IAC/F,+FAA+F;IAC/F,gFAAgF;IAChF,MAAME,WAAWR,aAAaS,MAAM,CAAC,CAACH,IAAM,CAACR,WAAWY,GAAG,CAACJ,EAAEC,IAAI,GAAGhB,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;IAEtF,6FAA6F;IAC7F,0FAA0F;IAC1F,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACjB;;QAC9B,MAAMoB,MAAMR,SAASS,GAAG,CAACrB,EAAEO,OAAO;QAClC,OAAO,CAACa,OAAOA,IAAIE,MAAM,KAAKtB,EAAEuB,OAAO,IAAIH,IAAII,KAAK,KAAKxB,EAAEyB,IAAI,aAAKnC,wBAAAA,kCAAAA,YAAa4B,GAAG,CAAClB,EAAEO,OAAO,wCAAK;IACrG;IAEA,IAAIS,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;QAAE9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAW;IAAG;IAEzH,MAAM4B,cAAc,MAAMrD,WAAWQ;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8C,SAAS5D,SAAS,mBAAmBgD,UAAUO,MAAM;IAC3D,wFAAwF;IACxF,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAEC,aAAa,EAAE,GAAG,MAAMrC,OAAOO,IAAI,CAAC,SAAS,IAAM/B,aAAa6C,WAAWtB,UAAUX,KAAK4C,aAAaC,OAAOK,IAAI,EAAE;YAAE/C;QAAK;IAC3K0C,OAAOM,MAAM;IACb,KAAK,MAAMC,OAAOJ,WAAYJ,YAAYS,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIV;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjG1C,QAAQqD,gBAAgB,CAACD,WAAWd,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMgB,kBAAkBF,WAAWvB,MAAM,CAAC,CAAC0B,IAAM7D,yBAAyBoC,GAAG,CAACyB,MAAM,CAACtE,iBAAiB6C,GAAG,CAACyB;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB3C,GAAG,CAACrB,YAAYmE,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB3C,GAAG,CAAC,IAAM,KAAK8C,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLzB,MAAM,CAAC,CAAC0B,IAAMA,MAAM,QACpB5C,GAAG,CAAC,CAAC4C,IAAM,GAAGjE,WAAWiE,GAAG,YAAY,EAAEjE,WAAWiE,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ3B,UAAUF,MAAM,CAAC,CAACjB,IAAM,CAACY,SAASM,GAAG,CAAClB,EAAEO,OAAO,GAAGR,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IACpF,MAAMwC,QAAwB;QAAE3C;QAAO4C,UAAUf,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO;QAAGuC;QAAO9B;IAAS;IACnG,MAAMkC,WAAW,IAAInE,IAAI+D;IACzB,MAAMK,mBAAmBlB,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO,EAAEU,MAAM,CAAC,CAACmC,IAAM,CAACF,SAAShC,GAAG,CAACkC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAIrC;WAAamC;KAAiB;IAElD,MAAMG,UAAUC,KAAKC,GAAG;IACxB,MAAM3E,gBACJI,MACA;QACE,iGAAiG;QACjG,sFAAsF;QACtF,MAAMwE,UAAU,MAAMhF,WAAWQ;QACjC,MAAMyE,iBAAiBxB,WAAWjB,MAAM,CAAC,CAACqB,MAAQ,CAACmB,QAAQvC,GAAG,CAACoB;QAC/D,IAAIoB,eAAehC,MAAM,GAAG,GAAG,MAAM5B,OAAOO,IAAI,CAAC,SAAS,IAAMjB,QAAQuE,UAAU,CAAC1E,MAAMyE;QAEzF,yFAAyF;QACzF,sFAAsF;QACtF,sFAAsF;QACtF,sFAAsF;QACtF,4FAA4F;QAC5F,IAAIE,iBAAiBP;QACrB,IAAIP,MAAMpB,MAAM,GAAG,GACjBkC,iBAAiB,MAAM9D,OAAOO,IAAI,CAAC,iBAAiB;YAClD,MAAMwD,mBAAmB,MAAM5E,KAAKyB,OAAO,CAAC;YAC5C,MAAMoD,eAAe,IAAI/E,IAAI,AAAE,CAAA,MAAM8E,iBAAiBlD,GAAG,EAAC,EAA+BZ,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;YAC1G,MAAMgD,aAAajB,MAAM7B,MAAM,CAAC,CAACmC,IAAMU,aAAa5C,GAAG,CAACkC;YACxD,OAAOW,WAAWrC,MAAM,GAAG,IAAI;mBAAI2B;mBAAYU;aAAW,GAAGV;QAC/D;QAEF,IAAIpB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMsC,QAAQ,CAACC,MACbvB,gBAAgB3C,GAAG,CAAC,CAACuC;wBAOZ2B;oBANP,IAAI3B,QAAQ,QAAQ,OAAO2B,IAAI1D,OAAO;oBACtC,IAAI+B,QAAQ,UAAU,OAAO2B,IAAI1C,OAAO;oBACxC,IAAIe,QAAQ,UAAU,OAAO2B,IAAIC,OAAO;oBACxC,IAAI5B,QAAQ,SAAS,OAAO2B,IAAIxC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIa,QAAQ,gBAAgB,OAAO2B,IAAIE,UAAU;oBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAAC9B,IAAI,cAAb2B,2BAAAA,gBAAiB;gBAC1B;YACF,wFAAwF;YACxF,sCAAsC;YACtC,MAAMnE,OAAOO,IAAI,CAAC,aAAa;gBAC7B,MAAMgE,UAAUpC,WAAWhB,MAAM,CAAC,CAACgC,IAAMC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;gBAC/D,MAAM+D,aAAarC,WAAWhB,MAAM,CAAC,CAACgC,IAAM,CAACC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;gBACnE,MAAM/B,WAAWS,MAAM,eAAeyD,iBAAiBE,WAAWyB,QAAQtE,GAAG,CAACiE;gBAC9E,IAAIM,WAAW5C,MAAM,GAAG,GAAG,MAAMzC,KAAKsF,QAAQ,CAAC3B,WAAW0B,WAAWvE,GAAG,CAACiE;YAC3E;QACF;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMlE,OAAOO,IAAI,CAAC,cAAc,IAAMjB,QAAQoF,gBAAgB,CAACvF,MAAM2E,gBAAgB3B,YAAYc,OAAO7D;QAExG,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMY,OAAOO,IAAI,CAAC,WAAW;YAC3B,IAAIgD,QAAQ3B,MAAM,GAAG,GACnB,MAAMzC,KAAKsF,QAAQ,CACjB,6CACAlB,QAAQtD,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;YAE1B,MAAMqB,aAA0B,EAAE;YAClC,KAAK,MAAMR,OAAOhC,WAAY,KAAK,MAAMyC,cAAcT,IAAIU,OAAO,CAAEF,WAAWG,IAAI,CAAC;gBAACX,IAAI1D,OAAO;gBAAEmE;aAAW;YAC7G,uFAAuF;YACvF,wFAAwF;YACxF,uGAAuG;YACvG,MAAMlG,WAAWS,MAAM,gBAAgB;gBAAC;gBAAQ;aAAS,EAAE,kGAAkGwF;QAC/J;QAEA,+FAA+F;QAC/F,4FAA4F;QAC5F,IAAIzD,SAASU,MAAM,GAAG,GACpB,MAAM5B,OAAOO,IAAI,CAAC,YAAY,IAC5BpB,KAAKsF,QAAQ,CACX,4CACAvD,SAASjB,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;QAI7B,uFAAuF;QACvF,0FAA0F;QAC1F,IAAIC,QAAQ3B,MAAM,GAAG,GAAG,KAAK,MAAMmD,WAAWhF,SAAU,MAAMC,OAAOO,IAAI,CAAC1B,aAAakG,QAAQ5E,IAAI,EAAE,WAAW;gBAAM4E;oBAAAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,qBAAAA,SAAiB5F,MAAMoE,SAASN;;QACtJ,KAAK,MAAM8B,WAAWhF,SAAU;YAC9B,MAAMkF,iBAAiC9C,WAAWlC,GAAG,CAAC,CAACkE,MAAS,CAAA;oBAAElD,MAAMkD,IAAI1D,OAAO;oBAAEyE,WAAWf,IAAIe,SAAS,CAACH,QAAQ5E,IAAI,CAAC;gBAAC,CAAA;YAC5H,MAAMH,OAAOO,IAAI,CAAC1B,aAAakG,QAAQ5E,IAAI,EAAE,UAAU;oBAAM4E;wBAAAA,iBAAAA,QAAQI,KAAK,cAAbJ,qCAAAA,oBAAAA,SAAgB5F,MAAM8F,gBAAgBhC;;QACrG;QACA,KAAK,MAAM8B,WAAWhF,SAAU,MAAMC,OAAOO,IAAI,CAAC1B,aAAakG,QAAQ5E,IAAI,EAAE,UAAU;gBAAM4E;oBAAAA,0BAAAA,QAAQK,cAAc,cAAtBL,8CAAAA,6BAAAA,SAAyB5F,MAAM8D;;IAC9H,GACA3D,QAAQ+F,SAAS;IAGnB,MAAMC,aAAa7B,KAAKC,GAAG,KAAKF;IAChC,2GAA2G;IAC3G/E,iBAAiBiB,WAAW4F;IAC5B,IAAIhG,QAAQiG,cAAc,EAAE,MAAMvF,OAAOO,IAAI,CAAC,QAAQ;YAAMjB;gBAAAA,0BAAAA,QAAQiG,cAAc,cAAtBjG,8CAAAA,6BAAAA,SAAyBH,MAAMmG;;IAE3F,OAAO;QAAEzD,QAAQM,WAAWP,MAAM;QAAEE;QAAU9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAWkF,YAAYjD;IAAe;AAC1G"}

package/dist/esm/store/turso/lexical.js

@@ -40,10 +40,14 @@ const FTS5_OPERATORS = [
     {
         label: 'column filter',
         re: /(?:^|\s)[\p{L}_]\w*\s*:/u
+    },
+    {
+        label: 'grouping operator',
+        re: /[()]/
     }
 ];
 function unsupportedOperator(terms) {
-    const withoutPhrases = terms.replace(/"[^"]*"/g, ' ');
+    const withoutPhrases = terms.replace(/"[^"]*"/g, 'phrase');
     for (const { label, re } of FTS5_OPERATORS){
         const m = withoutPhrases.match(re);
         if (m) return {

package/dist/esm/store/turso/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { stemFolded } from './lexical-text.ts';\n\nconst FIELDS = ['title_stem', 'summary_stem', 'text_stem'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\nconst AUTHORED_FIELDS = ['title', 'summary', 'text'] as const;\ntype AuthoredField = (typeof AUTHORED_FIELDS)[number];\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: SearchToken[][]; unspaced: string[]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const phrases: SearchToken[][] = [];\n  const unspaced: string[] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const tokens = searchTokens(phrase);\n    if (tokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    unspaced.push(...tokens.filter((token) => token.unspaced).map((token) => token.text));\n    phrases.push(tokens);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) unspaced.push(token.text);\n      else words.push(token.text);\n    }\n  }\n  return { words, phrases, unspaced, emptyPhrase };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n  return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: SearchToken[][], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n  const phraseParts = phrases.flatMap((phrase) => {\n    const latin = phrase.filter((token) => !token.unspaced).map((token) => stemFolded(token.text));\n    if (latin.length === 0) return [];\n    return phrase.some((token) => token.unspaced) ? latin : [latin.join(' ')];\n  });\n  const defaultParts = [...words.map(stemFolded), ...phraseParts].map((t) => `\"${t}\"`);\n  const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n  return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n  const colList = cols.map((c) => `content.${c}`).join(', ');\n  const lit = escapeFtsLiteral(query);\n  return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `snippets` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS snippet fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { words, phrases, unspaced, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase) return [];\n  const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n  if (defaultQuery === '' && ngramQuery === '') return [];\n\n  const branches: string[] = [];\n  if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n  if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n  // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n  // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC. Phrase\n  // candidates carry authored fields through the join below so the shared verifier can reject\n  // cross-field, reordered, and non-adjacent matches before applying the caller's limit.\n  const ranked = `SELECT path, SUM(score) AS score FROM (\n      ${branches.join(' UNION ALL ')}\n    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}`;\n  const phraseSelect = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const phraseJoin = phrases.length > 0 ? ' JOIN content ON content.path = sq.path' : '';\n  const sql = `SELECT sq.path, sq.score${phraseSelect} FROM (\n    ${ranked}\n  ) sq${phraseJoin} ORDER BY sq.score DESC, sq.path${phrases.length > 0 ? '' : ' LIMIT ?'}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...(phrases.length > 0 ? [] : [opts.limit]))) as Array<LexicalHit & Partial<Record<AuthoredField, string>>>;\n  // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n  // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n  if (rows.some((r) => r.path === null)) {\n    throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n  }\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => AUTHORED_FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, opts.limit).map(({ path }) => ({ path }));\n}\n"],"names":["SenseError","matchesSearchPhrase","searchTokens","stemFolded","FIELDS","NGRAM_FIELDS","AUTHORED_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","emptyPhrase","_m","inner","phrase","tokens","length","push","filter","map","text","tok","split","escapeFtsLiteral","query","buildQueries","phraseParts","flatMap","latin","some","join","defaultParts","t","unspacedParts","defaultQuery","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","queryLexical","conn","unsupported","branches","ranked","phraseSelect","phraseJoin","sql","stmt","prepare","rows","all","limit","r","path","matches","row","every","field","slice"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;AACvG,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAE5F,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,MAAMC,SAAS;IAAC;IAAc;IAAgB;CAAY;AAC1D,MAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AACnE,MAAMC,kBAAkB;IAAC;IAAS;IAAW;CAAO;AAGpD,uFAAuF;AACvF,+FAA+F;AAC/F,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,UAA2B,EAAE;IACnC,MAAMC,WAAqB,EAAE;IAC7B,IAAIC,cAAc;IAClB,MAAMV,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACU,IAAIC;QACtD,MAAMC,SAASD,MAAMP,IAAI;QACzB,MAAMS,SAASxB,aAAauB;QAC5B,IAAIC,OAAOC,MAAM,KAAK,GAAG;YACvBL,cAAc;YACd,OAAO;QACT;QACAD,SAASO,IAAI,IAAIF,OAAOG,MAAM,CAAC,CAACb,QAAUA,MAAMK,QAAQ,EAAES,GAAG,CAAC,CAACd,QAAUA,MAAMe,IAAI;QACnFX,QAAQQ,IAAI,CAACF;QACb,OAAO;IACT;IACA,KAAK,MAAMM,OAAOpB,eAAeqB,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIL,MAAM,KAAK,GAAG;QACtB,KAAK,MAAMX,SAASd,aAAa8B,KAAM;YACrC,IAAIhB,MAAMK,QAAQ,EAAEA,SAASO,IAAI,CAACZ,MAAMe,IAAI;iBACvCZ,MAAMS,IAAI,CAACZ,MAAMe,IAAI;QAC5B;IACF;IACA,OAAO;QAAEZ;QAAOC;QAASC;QAAUC;IAAY;AACjD;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASY,iBAAiBC,KAAa;IACrC,OAAOA,MAAMtB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASuB,aAAajB,KAAe,EAAEC,OAAwB,EAAEC,QAAkB;IACjF,MAAMgB,cAAcjB,QAAQkB,OAAO,CAAC,CAACb;QACnC,MAAMc,QAAQd,OAAOI,MAAM,CAAC,CAACb,QAAU,CAACA,MAAMK,QAAQ,EAAES,GAAG,CAAC,CAACd,QAAUb,WAAWa,MAAMe,IAAI;QAC5F,IAAIQ,MAAMZ,MAAM,KAAK,GAAG,OAAO,EAAE;QACjC,OAAOF,OAAOe,IAAI,CAAC,CAACxB,QAAUA,MAAMK,QAAQ,IAAIkB,QAAQ;YAACA,MAAME,IAAI,CAAC;SAAK;IAC3E;IACA,MAAMC,eAAe;WAAIvB,MAAMW,GAAG,CAAC3B;WAAgBkC;KAAY,CAACP,GAAG,CAAC,CAACa,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IACnF,MAAMC,gBAAgBvB,SAASS,GAAG,CAAC,CAACa,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IAClD,OAAO;QAAEE,cAAcH,aAAaD,IAAI,CAAC;QAAUK,YAAYF,cAAcH,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASM,UAAUC,IAAuB,EAAEb,KAAa,EAAEc,IAAyB;IAClF,MAAMC,UAAUF,KAAKlB,GAAG,CAAC,CAACqB,IAAM,CAAC,QAAQ,EAAEA,GAAG,EAAEV,IAAI,CAAC;IACrD,MAAMW,MAAMlB,iBAAiBC;IAC7B,OAAO,CAAC,yCAAyC,EAAEe,QAAQ,GAAG,EAAEE,IAAI;iBACrD,EAAEH,KAAKI,SAAS,CAAC,iBAAiB,EAAEH,QAAQ,GAAG,EAAEE,IAAI,GAAG,EAAEH,KAAKK,SAAS,CAAC,CAAC,EAAEL,KAAKM,SAAS,EAAE;AAC7G;AAEA,uFAAuF;AACvF,qMAAqM;AACrM,OAAO,eAAeC,aAAaC,IAAgB,EAAE9C,KAAa,EAAEsC,IAAyB;IAC3F,MAAMS,cAAchD,oBAAoBC;IACxC,IAAI+C,gBAAgB,MAAM;QACxB,MAAM,IAAI1D,WAAW,4BAA4B,CAAC,wCAAwC,EAAE0D,YAAYlD,KAAK,CAAC,GAAG,EAAEkD,YAAY1C,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IACzQ;IACA,MAAM,EAAEE,KAAK,EAAEC,OAAO,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ,WAAWP;IAC7D,IAAIW,aAAa,OAAO,EAAE;IAC1B,MAAM,EAAEuB,YAAY,EAAEC,UAAU,EAAE,GAAGV,aAAajB,OAAOC,SAASC;IAClE,IAAIwB,iBAAiB,MAAMC,eAAe,IAAI,OAAO,EAAE;IAEvD,MAAMa,WAAqB,EAAE;IAC7B,IAAId,iBAAiB,IAAIc,SAAS/B,IAAI,CAACmB,UAAU3C,QAAQyC,cAAcI;IACvE,IAAIH,eAAe,IAAIa,SAAS/B,IAAI,CAACmB,UAAU1C,cAAcyC,YAAYG;IAEzE,8FAA8F;IAC9F,0FAA0F;IAC1F,4FAA4F;IAC5F,uFAAuF;IACvF,MAAMW,SAAS,CAAC;MACZ,EAAED,SAASlB,IAAI,CAAC,eAAe;4CACO,EAAEkB,SAAShC,MAAM,EAAE;IAC7D,MAAMkC,eAAezC,QAAQO,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAMmC,aAAa1C,QAAQO,MAAM,GAAG,IAAI,4CAA4C;IACpF,MAAMoC,MAAM,CAAC,wBAAwB,EAAEF,aAAa;IAClD,EAAED,OAAO;MACP,EAAEE,WAAW,gCAAgC,EAAE1C,QAAQO,MAAM,GAAG,IAAI,KAAK,YAAY;IACzF,MAAMqC,OAAO,MAAMP,KAAKQ,OAAO,CAACF;IAChC,MAAMG,OAAQ,MAAMF,KAAKG,GAAG,IAAK/C,QAAQO,MAAM,GAAG,IAAI,EAAE,GAAG;QAACsB,KAAKmB,KAAK;KAAC;IACvE,+FAA+F;IAC/F,yFAAyF;IACzF,IAAIF,KAAK1B,IAAI,CAAC,CAAC6B,IAAMA,EAAEC,IAAI,KAAK,OAAO;QACrC,MAAM,IAAItE,WAAW,qBAAqB;IAC5C;IACA,MAAMuE,UAAUnD,QAAQO,MAAM,KAAK,IAAIuC,OAAOA,KAAKrC,MAAM,CAAC,CAAC2C,MAAQpD,QAAQqD,KAAK,CAAC,CAAChD,SAAWnB,gBAAgBkC,IAAI,CAAC,CAACkC;oBAA8BF;uBAApBvE,qBAAoBuE,aAAAA,GAAG,CAACE,MAAM,cAAVF,wBAAAA,aAAc,IAAI/C;;IACnK,OAAO8C,QAAQI,KAAK,CAAC,GAAG1B,KAAKmB,KAAK,EAAEtC,GAAG,CAAC,CAAC,EAAEwC,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAChE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { stemFolded } from './lexical-text.ts';\n\nconst FIELDS = ['title_stem', 'summary_stem', 'text_stem'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\nconst AUTHORED_FIELDS = ['title', 'summary', 'text'] as const;\ntype AuthoredField = (typeof AUTHORED_FIELDS)[number];\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n  { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: SearchToken[][]; unspaced: string[]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const phrases: SearchToken[][] = [];\n  const unspaced: string[] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const tokens = searchTokens(phrase);\n    if (tokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    unspaced.push(...tokens.filter((token) => token.unspaced).map((token) => token.text));\n    phrases.push(tokens);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) unspaced.push(token.text);\n      else words.push(token.text);\n    }\n  }\n  return { words, phrases, unspaced, emptyPhrase };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n  return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: SearchToken[][], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n  const phraseParts = phrases.flatMap((phrase) => {\n    const latin = phrase.filter((token) => !token.unspaced).map((token) => stemFolded(token.text));\n    if (latin.length === 0) return [];\n    return phrase.some((token) => token.unspaced) ? latin : [latin.join(' ')];\n  });\n  const defaultParts = [...words.map(stemFolded), ...phraseParts].map((t) => `\"${t}\"`);\n  const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n  return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n  const colList = cols.map((c) => `content.${c}`).join(', ');\n  const lit = escapeFtsLiteral(query);\n  return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `snippets` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS snippet fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { words, phrases, unspaced, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase) return [];\n  const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n  if (defaultQuery === '' && ngramQuery === '') return [];\n\n  const branches: string[] = [];\n  if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n  if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n  // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n  // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC. Phrase\n  // candidates carry authored fields through the join below so the shared verifier can reject\n  // cross-field, reordered, and non-adjacent matches before applying the caller's limit.\n  const ranked = `SELECT path, SUM(score) AS score FROM (\n      ${branches.join(' UNION ALL ')}\n    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}`;\n  const phraseSelect = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const phraseJoin = phrases.length > 0 ? ' JOIN content ON content.path = sq.path' : '';\n  const sql = `SELECT sq.path, sq.score${phraseSelect} FROM (\n    ${ranked}\n  ) sq${phraseJoin} ORDER BY sq.score DESC, sq.path${phrases.length > 0 ? '' : ' LIMIT ?'}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...(phrases.length > 0 ? [] : [opts.limit]))) as Array<LexicalHit & Partial<Record<AuthoredField, string>>>;\n  // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n  // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n  if (rows.some((r) => r.path === null)) {\n    throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n  }\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => AUTHORED_FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, opts.limit).map(({ path }) => ({ path }));\n}\n"],"names":["SenseError","matchesSearchPhrase","searchTokens","stemFolded","FIELDS","NGRAM_FIELDS","AUTHORED_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","emptyPhrase","_m","inner","phrase","tokens","length","push","filter","map","text","tok","split","escapeFtsLiteral","query","buildQueries","phraseParts","flatMap","latin","some","join","defaultParts","t","unspacedParts","defaultQuery","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","queryLexical","conn","unsupported","branches","ranked","phraseSelect","phraseJoin","sql","stmt","prepare","rows","all","limit","r","path","matches","row","every","field","slice"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;AACvG,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAE5F,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,MAAMC,SAAS;IAAC;IAAc;IAAgB;CAAY;AAC1D,MAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AACnE,MAAMC,kBAAkB;IAAC;IAAS;IAAW;CAAO;AAGpD,uFAAuF;AACvF,+FAA+F;AAC/F,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,UAA2B,EAAE;IACnC,MAAMC,WAAqB,EAAE;IAC7B,IAAIC,cAAc;IAClB,MAAMV,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACU,IAAIC;QACtD,MAAMC,SAASD,MAAMP,IAAI;QACzB,MAAMS,SAASxB,aAAauB;QAC5B,IAAIC,OAAOC,MAAM,KAAK,GAAG;YACvBL,cAAc;YACd,OAAO;QACT;QACAD,SAASO,IAAI,IAAIF,OAAOG,MAAM,CAAC,CAACb,QAAUA,MAAMK,QAAQ,EAAES,GAAG,CAAC,CAACd,QAAUA,MAAMe,IAAI;QACnFX,QAAQQ,IAAI,CAACF;QACb,OAAO;IACT;IACA,KAAK,MAAMM,OAAOpB,eAAeqB,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIL,MAAM,KAAK,GAAG;QACtB,KAAK,MAAMX,SAASd,aAAa8B,KAAM;YACrC,IAAIhB,MAAMK,QAAQ,EAAEA,SAASO,IAAI,CAACZ,MAAMe,IAAI;iBACvCZ,MAAMS,IAAI,CAACZ,MAAMe,IAAI;QAC5B;IACF;IACA,OAAO;QAAEZ;QAAOC;QAASC;QAAUC;IAAY;AACjD;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASY,iBAAiBC,KAAa;IACrC,OAAOA,MAAMtB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASuB,aAAajB,KAAe,EAAEC,OAAwB,EAAEC,QAAkB;IACjF,MAAMgB,cAAcjB,QAAQkB,OAAO,CAAC,CAACb;QACnC,MAAMc,QAAQd,OAAOI,MAAM,CAAC,CAACb,QAAU,CAACA,MAAMK,QAAQ,EAAES,GAAG,CAAC,CAACd,QAAUb,WAAWa,MAAMe,IAAI;QAC5F,IAAIQ,MAAMZ,MAAM,KAAK,GAAG,OAAO,EAAE;QACjC,OAAOF,OAAOe,IAAI,CAAC,CAACxB,QAAUA,MAAMK,QAAQ,IAAIkB,QAAQ;YAACA,MAAME,IAAI,CAAC;SAAK;IAC3E;IACA,MAAMC,eAAe;WAAIvB,MAAMW,GAAG,CAAC3B;WAAgBkC;KAAY,CAACP,GAAG,CAAC,CAACa,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IACnF,MAAMC,gBAAgBvB,SAASS,GAAG,CAAC,CAACa,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IAClD,OAAO;QAAEE,cAAcH,aAAaD,IAAI,CAAC;QAAUK,YAAYF,cAAcH,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASM,UAAUC,IAAuB,EAAEb,KAAa,EAAEc,IAAyB;IAClF,MAAMC,UAAUF,KAAKlB,GAAG,CAAC,CAACqB,IAAM,CAAC,QAAQ,EAAEA,GAAG,EAAEV,IAAI,CAAC;IACrD,MAAMW,MAAMlB,iBAAiBC;IAC7B,OAAO,CAAC,yCAAyC,EAAEe,QAAQ,GAAG,EAAEE,IAAI;iBACrD,EAAEH,KAAKI,SAAS,CAAC,iBAAiB,EAAEH,QAAQ,GAAG,EAAEE,IAAI,GAAG,EAAEH,KAAKK,SAAS,CAAC,CAAC,EAAEL,KAAKM,SAAS,EAAE;AAC7G;AAEA,uFAAuF;AACvF,qMAAqM;AACrM,OAAO,eAAeC,aAAaC,IAAgB,EAAE9C,KAAa,EAAEsC,IAAyB;IAC3F,MAAMS,cAAchD,oBAAoBC;IACxC,IAAI+C,gBAAgB,MAAM;QACxB,MAAM,IAAI1D,WAAW,4BAA4B,CAAC,wCAAwC,EAAE0D,YAAYlD,KAAK,CAAC,GAAG,EAAEkD,YAAY1C,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IACzQ;IACA,MAAM,EAAEE,KAAK,EAAEC,OAAO,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ,WAAWP;IAC7D,IAAIW,aAAa,OAAO,EAAE;IAC1B,MAAM,EAAEuB,YAAY,EAAEC,UAAU,EAAE,GAAGV,aAAajB,OAAOC,SAASC;IAClE,IAAIwB,iBAAiB,MAAMC,eAAe,IAAI,OAAO,EAAE;IAEvD,MAAMa,WAAqB,EAAE;IAC7B,IAAId,iBAAiB,IAAIc,SAAS/B,IAAI,CAACmB,UAAU3C,QAAQyC,cAAcI;IACvE,IAAIH,eAAe,IAAIa,SAAS/B,IAAI,CAACmB,UAAU1C,cAAcyC,YAAYG;IAEzE,8FAA8F;IAC9F,0FAA0F;IAC1F,4FAA4F;IAC5F,uFAAuF;IACvF,MAAMW,SAAS,CAAC;MACZ,EAAED,SAASlB,IAAI,CAAC,eAAe;4CACO,EAAEkB,SAAShC,MAAM,EAAE;IAC7D,MAAMkC,eAAezC,QAAQO,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAMmC,aAAa1C,QAAQO,MAAM,GAAG,IAAI,4CAA4C;IACpF,MAAMoC,MAAM,CAAC,wBAAwB,EAAEF,aAAa;IAClD,EAAED,OAAO;MACP,EAAEE,WAAW,gCAAgC,EAAE1C,QAAQO,MAAM,GAAG,IAAI,KAAK,YAAY;IACzF,MAAMqC,OAAO,MAAMP,KAAKQ,OAAO,CAACF;IAChC,MAAMG,OAAQ,MAAMF,KAAKG,GAAG,IAAK/C,QAAQO,MAAM,GAAG,IAAI,EAAE,GAAG;QAACsB,KAAKmB,KAAK;KAAC;IACvE,+FAA+F;IAC/F,yFAAyF;IACzF,IAAIF,KAAK1B,IAAI,CAAC,CAAC6B,IAAMA,EAAEC,IAAI,KAAK,OAAO;QACrC,MAAM,IAAItE,WAAW,qBAAqB;IAC5C;IACA,MAAMuE,UAAUnD,QAAQO,MAAM,KAAK,IAAIuC,OAAOA,KAAKrC,MAAM,CAAC,CAAC2C,MAAQpD,QAAQqD,KAAK,CAAC,CAAChD,SAAWnB,gBAAgBkC,IAAI,CAAC,CAACkC;oBAA8BF;uBAApBvE,qBAAoBuE,aAAAA,GAAG,CAACE,MAAM,cAAVF,wBAAAA,aAAc,IAAI/C;;IACnK,OAAO8C,QAAQI,KAAK,CAAC,GAAG1B,KAAKmB,KAAK,EAAEtC,GAAG,CAAC,CAAC,EAAEwC,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAChE"}

package/dist/esm/text/segment.js

@@ -6,7 +6,8 @@ import { stemmer } from 'stemmer';
 export const UNSPACED_SCRIPTS = '\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{scx=Khmer}\\p{scx=Lao}\\p{scx=Myanmar}';
 // A run is script BASE characters with their combining marks attached; a bare mark after a
 // Latin letter (decomposed é) never starts one.
-const RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\p{M}*)+)`, 'gu');
+const RUN_BODY = `(?:[${UNSPACED_SCRIPTS}]\\p{M}*)+`;
+const RUN = new RegExp(`(${RUN_BODY})`, 'gu');
 const HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');
 // Whether text holds a script that marks no word boundaries, the predicate every store's
 // sidecar population and query split turns on.
@@ -121,26 +122,150 @@ const QUALIFIER = /(^|[\s(])(-?)(title|summary|text)\s*:\s*$/;
 // Raw title/summary/text drop punctuation as unicode61's token separator, so `数数` and `数。数`
 // falsely match there; only the barriered `_seg` columns are safe from it, hence this fallback target.
 const SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';
+const AUTHORED_COLUMNS = '{title summary text}:';
+const QUOTED_RUN_TOKEN = new RegExp(`"(${RUN_BODY})"`, 'gu');
+const FTS_WORD = /[\p{L}\p{N}_]/u;
+function isFtsWord(char) {
+    return char !== undefined && FTS_WORD.test(char);
+}
+function quoteEnd(text, start) {
+    for(let i = start + 1; i < text.length; i++){
+        if (text[i] !== '"') continue;
+        if (text[i + 1] === '"') {
+            i++;
+            continue;
+        }
+        return i + 1;
+    }
+    return text.length;
+}
+function nearEnd(text, open) {
+    let depth = 1;
+    let quoted = false;
+    for(let i = open + 1; i < text.length; i++){
+        const char = text[i];
+        if (char === '"') {
+            if (quoted && text[i + 1] === '"') {
+                i++;
+            } else {
+                quoted = !quoted;
+            }
+        } else if (!quoted && char === '(') {
+            depth++;
+        } else if (!quoted && char === ')' && --depth === 0) {
+            return i + 1;
+        }
+    }
+    return text.length;
+}
+// Lexical routing only: SQLite remains responsible for validating the expression. The scanner
+// recognizes NEAR groups and the one FTS string governed by ^, while quoted text shields both.
+function positionalSpans(terms) {
+    const spans = [];
+    for(let i = 0; i < terms.length; i++){
+        if (terms[i] === '"') {
+            i = quoteEnd(terms, i) - 1;
+            continue;
+        }
+        if (terms.startsWith('NEAR', i) && !isFtsWord(terms[i - 1])) {
+            var _terms_open;
+            let open = i + 4;
+            while(/\s/u.test((_terms_open = terms[open]) !== null && _terms_open !== void 0 ? _terms_open : ''))open++;
+            if (terms[open] === '(') {
+                const end = nearEnd(terms, open);
+                if (HAS_RUN.test(terms.slice(open + 1, end))) spans.push({
+                    start: i,
+                    end,
+                    qualified: QUALIFIER.test(terms.slice(0, i))
+                });
+                i = end - 1;
+                continue;
+            }
+        }
+        if (terms[i] === '^' && !isFtsWord(terms[i - 1])) {
+            var _terms_operand;
+            let operand = i + 1;
+            while(/\s/u.test((_terms_operand = terms[operand]) !== null && _terms_operand !== void 0 ? _terms_operand : ''))operand++;
+            const end = terms[operand] === '"' ? quoteEnd(terms, operand) : (()=>{
+                var _terms_cursor;
+                let cursor = operand;
+                while(/[\p{L}\p{N}_]/u.test((_terms_cursor = terms[cursor]) !== null && _terms_cursor !== void 0 ? _terms_cursor : ''))cursor++;
+                return cursor;
+            })();
+            if (end > operand && HAS_RUN.test(terms.slice(operand, end))) spans.push({
+                start: i,
+                end,
+                qualified: QUALIFIER.test(terms.slice(0, i))
+            });
+            i = Math.max(i, end - 1);
+        }
+    }
+    return spans;
+}
+function routePositionalSpans(terms) {
+    const spans = positionalSpans(terms);
+    if (spans.length === 0) return terms;
+    let out = '';
+    let cursor = 0;
+    for (const span of spans){
+        out += terms.slice(cursor, span.start);
+        if (!span.qualified) out += AUTHORED_COLUMNS;
+        out += terms.slice(span.start, span.end);
+        cursor = span.end;
+    }
+    return out + terms.slice(cursor);
+}
+function inPositionalSpan(start, spans) {
+    return spans.some((span)=>start >= span.start && start < span.end);
+}
 // A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),
 // space-joined -- the same split points segmentField barriers, so query and index agree.
-function runQuery(run, columnPrefix) {
-    return splitOnPunctuation(run).map((g)=>`${columnPrefix}${g.length > 1 ? `"${g.join(' ')}"` : g[0]}`).join(' ');
+function runQuery(run, columnPrefix, quoteSingle = false) {
+    return splitOnPunctuation(run).map((g)=>`${columnPrefix}${g.length > 1 || quoteSingle ? `"${g.join(' ')}"` : g[0]}`).join(' ');
+}
+// Retarget complete quoted run tokens without interpreting the expression around them. Positional
+// spans have already been routed to authored columns, so sidecar barriers cannot change meaning.
+function quotedRunTokens(terms) {
+    const spans = positionalSpans(terms);
+    let out = '';
+    let cursor = 0;
+    for (const match of terms.matchAll(QUOTED_RUN_TOKEN)){
+        const start = match.index;
+        const run = match[1];
+        if (inPositionalSpan(start, spans) || PUNCTUATION.test(run)) continue;
+        out += terms.slice(cursor, start);
+        const qualifier = out.match(QUALIFIER);
+        if (qualifier) {
+            out = `${out.slice(0, qualifier.index)}${qualifier[1]}${runQuery(run, `${qualifier[2]}${qualifier[3]}_seg:`, true)}`;
+        } else {
+            out += runQuery(run, SIDECAR_COLUMNS, true);
+        }
+        cursor = start + match[0].length;
+    }
+    return cursor === 0 ? terms : out + terms.slice(cursor);
 }
 // Query side: each unspaced run becomes phrases of its graphemes, matching segmentField. A
 // qualifier maps to its `_seg` column; unqualified maps to all three (SIDECAR_COLUMNS).
 export function segmentMatch(terms) {
     if (!HAS_RUN.test(terms)) return terms;
+    terms = routePositionalSpans(terms);
+    terms = quotedRunTokens(terms);
+    const spans = positionalSpans(terms);
     let out = '';
     let quoted = false;
+    let offset = 0;
     const pieces = terms.split(RUN); // split keeps captured runs at odd indices
     for(let i = 0; i < pieces.length; i++){
         if (i % 2 === 0) {
             for (const ch of pieces[i])if (ch === '"') quoted = !quoted;
             out += pieces[i];
+            offset += pieces[i].length;
             continue;
         }
-        if (quoted) {
+        const protectedRun = inPositionalSpan(offset, spans);
+        if (quoted || protectedRun) {
             out += pieces[i]; // an author's phrase is matched as written
+            offset += pieces[i].length;
             continue;
         }
         const m = out.match(QUALIFIER);
@@ -149,6 +274,7 @@ export function segmentMatch(terms) {
         } else {
             out += runQuery(pieces[i], SIDECAR_COLUMNS);
         }
+        offset += pieces[i].length;
     }
     return out;
 }

package/dist/esm/text/segment.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/segment.ts"],"sourcesContent":["import { stemmer } from 'stemmer';\n\n// Word boundaries FTS5's tokenizers cannot find on their own. Contract: seg(query) must appear in\n// seg(document) for any substring query; word mode is context-dependent and breaks that, so grapheme mode (UAX #29) is used.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nexport const UNSPACED_SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// Whether text holds a script that marks no word boundaries, the predicate every store's\n// sidecar population and query split turns on.\nexport function hasUnspacedRun(text: string): boolean {\n  return HAS_RUN.test(text);\n}\n\nexport interface SearchToken {\n  text: string;\n  start: number;\n  end: number;\n  unspaced: boolean;\n}\n\n// Search tokens share the native separator contract: letters, marks, and numbers form tokens;\n// underscore, apostrophe, and hyphen separate them. Unspaced-script runs stay whole so callers\n// can apply substring semantics, while adjacent Latin text still gets its own token.\nexport function searchTokens(text: string): SearchToken[] {\n  const tokens: SearchToken[] = [];\n  for (const match of text.matchAll(/[\\p{L}\\p{N}][\\p{L}\\p{M}\\p{N}]*/gu)) {\n    const start = match.index;\n    const end = start + match[0].length;\n    let cursor = start;\n    for (const run of text.slice(start, end).matchAll(RUN)) {\n      const runStart = start + run.index;\n      if (runStart > cursor) tokens.push({ text: text.slice(cursor, runStart), start: cursor, end: runStart, unspaced: false });\n      const runEnd = runStart + run[0].length;\n      tokens.push({ text: run[0], start: runStart, end: runEnd, unspaced: true });\n      cursor = runEnd;\n    }\n    if (cursor < end) tokens.push({ text: text.slice(cursor, end), start: cursor, end, unspaced: false });\n  }\n  return tokens;\n}\n\n// Phrase membership is checked after native ranking. A CJK token matches inside one authored\n// run; Latin tokens compare folded Porter stems. Each call scans one field once per candidate.\nexport function matchesSearchPhrase(text: string, phrase: SearchToken[]): boolean {\n  const haystack = searchTokens(text);\n  if (phrase.length === 0 || haystack.length < phrase.length) return false;\n  for (let start = 0; start <= haystack.length - phrase.length; start++) {\n    let matches = true;\n    for (let i = 0; i < phrase.length; i++) {\n      const query = phrase[i];\n      const authored = haystack[start + i];\n      if (query.unspaced) {\n        if (!authored.unspaced || !authored.text.includes(query.text)) matches = false;\n      } else if (authored.unspaced || stemmer(foldForSearch(authored.text)) !== stemmer(foldForSearch(query.text))) {\n        matches = false;\n      }\n      if (!matches) break;\n    }\n    if (matches) return true;\n  }\n  return false;\n}\n\n// Character spans of unspaced-script runs in text, the same boundary the index segments on --\n// the snippet marker uses this to keep substring semantics inside a run and word/stem matching outside it.\nexport function unspacedRuns(text: string): Array<{ start: number; end: number }> {\n  return Array.from(text.matchAll(RUN), (m) => ({ start: m.index, end: m.index + m[0].length }));\n}\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built on first use and kept: construction is 6.6 ms, and a tree with no unspaced-script run\n// never segments at all, so no command pays it at module load.\nlet graphemeSegmenter: Intl.Segmenter | undefined;\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  graphemeSegmenter ??= new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n  return Array.from(graphemeSegmenter.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run. Otherwise each run explodes into\n// its graphemes, barrier-delimited from its neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// Raw title/summary/text drop punctuation as unicode61's token separator, so `数数` and `数。数`\n// falsely match there; only the barriered `_seg` columns are safe from it, hence this fallback target.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching segmentField. A\n// qualifier maps to its `_seg` column; unqualified maps to all three (SIDECAR_COLUMNS).\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n  }\n  return out;\n}\n\n// FTS tokenizers compare letters without combining marks. Keep the authored string for output and\n// offsets, but use this key when JS needs to compare a matched word to its source segment.\nexport function foldForSearch(text: string): string {\n  return text.normalize('NFD').replace(/\\p{M}/gu, '').toLowerCase();\n}\n"],"names":["stemmer","UNSPACED_SCRIPTS","RUN","RegExp","HAS_RUN","hasUnspacedRun","text","test","searchTokens","tokens","match","matchAll","start","index","end","length","cursor","run","slice","runStart","push","unspaced","runEnd","matchesSearchPhrase","phrase","haystack","matches","i","query","authored","includes","foldForSearch","unspacedRuns","Array","from","m","graphemeSegmenter","PUNCTUATION","BARRIER","graphemes","Intl","Segmenter","undefined","granularity","segment","s","splitOnPunctuation","groups","g","filter","segmentField","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","runQuery","columnPrefix","segmentMatch","terms","quoted","pieces","split","ch","normalize","toLowerCase"],"mappings":"AAAA,SAASA,OAAO,QAAQ,UAAU;AAElC,kGAAkG;AAClG,6HAA6H;AAE7H,wEAAwE;AACxE,kFAAkF;AAClF,OAAO,MAAMC,mBAAmB,wGAAwG;AACxI,2FAA2F;AAC3F,gDAAgD;AAChD,MAAMC,MAAM,IAAIC,OAAO,CAAC,KAAK,EAAEF,iBAAiB,WAAW,CAAC,EAAE;AAC9D,MAAMG,UAAU,IAAID,OAAO,CAAC,CAAC,EAAEF,iBAAiB,CAAC,CAAC,EAAE;AAEpD,yFAAyF;AACzF,+CAA+C;AAC/C,OAAO,SAASI,eAAeC,IAAY;IACzC,OAAOF,QAAQG,IAAI,CAACD;AACtB;AASA,8FAA8F;AAC9F,+FAA+F;AAC/F,qFAAqF;AACrF,OAAO,SAASE,aAAaF,IAAY;IACvC,MAAMG,SAAwB,EAAE;IAChC,KAAK,MAAMC,SAASJ,KAAKK,QAAQ,CAAC,oCAAqC;QACrE,MAAMC,QAAQF,MAAMG,KAAK;QACzB,MAAMC,MAAMF,QAAQF,KAAK,CAAC,EAAE,CAACK,MAAM;QACnC,IAAIC,SAASJ;QACb,KAAK,MAAMK,OAAOX,KAAKY,KAAK,CAACN,OAAOE,KAAKH,QAAQ,CAACT,KAAM;YACtD,MAAMiB,WAAWP,QAAQK,IAAIJ,KAAK;YAClC,IAAIM,WAAWH,QAAQP,OAAOW,IAAI,CAAC;gBAAEd,MAAMA,KAAKY,KAAK,CAACF,QAAQG;gBAAWP,OAAOI;gBAAQF,KAAKK;gBAAUE,UAAU;YAAM;YACvH,MAAMC,SAASH,WAAWF,GAAG,CAAC,EAAE,CAACF,MAAM;YACvCN,OAAOW,IAAI,CAAC;gBAAEd,MAAMW,GAAG,CAAC,EAAE;gBAAEL,OAAOO;gBAAUL,KAAKQ;gBAAQD,UAAU;YAAK;YACzEL,SAASM;QACX;QACA,IAAIN,SAASF,KAAKL,OAAOW,IAAI,CAAC;YAAEd,MAAMA,KAAKY,KAAK,CAACF,QAAQF;YAAMF,OAAOI;YAAQF;YAAKO,UAAU;QAAM;IACrG;IACA,OAAOZ;AACT;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,OAAO,SAASc,oBAAoBjB,IAAY,EAAEkB,MAAqB;IACrE,MAAMC,WAAWjB,aAAaF;IAC9B,IAAIkB,OAAOT,MAAM,KAAK,KAAKU,SAASV,MAAM,GAAGS,OAAOT,MAAM,EAAE,OAAO;IACnE,IAAK,IAAIH,QAAQ,GAAGA,SAASa,SAASV,MAAM,GAAGS,OAAOT,MAAM,EAAEH,QAAS;QACrE,IAAIc,UAAU;QACd,IAAK,IAAIC,IAAI,GAAGA,IAAIH,OAAOT,MAAM,EAAEY,IAAK;YACtC,MAAMC,QAAQJ,MAAM,CAACG,EAAE;YACvB,MAAME,WAAWJ,QAAQ,CAACb,QAAQe,EAAE;YACpC,IAAIC,MAAMP,QAAQ,EAAE;gBAClB,IAAI,CAACQ,SAASR,QAAQ,IAAI,CAACQ,SAASvB,IAAI,CAACwB,QAAQ,CAACF,MAAMtB,IAAI,GAAGoB,UAAU;YAC3E,OAAO,IAAIG,SAASR,QAAQ,IAAIrB,QAAQ+B,cAAcF,SAASvB,IAAI,OAAON,QAAQ+B,cAAcH,MAAMtB,IAAI,IAAI;gBAC5GoB,UAAU;YACZ;YACA,IAAI,CAACA,SAAS;QAChB;QACA,IAAIA,SAAS,OAAO;IACtB;IACA,OAAO;AACT;AAEA,8FAA8F;AAC9F,2GAA2G;AAC3G,OAAO,SAASM,aAAa1B,IAAY;IACvC,OAAO2B,MAAMC,IAAI,CAAC5B,KAAKK,QAAQ,CAACT,MAAM,CAACiC,IAAO,CAAA;YAAEvB,OAAOuB,EAAEtB,KAAK;YAAEC,KAAKqB,EAAEtB,KAAK,GAAGsB,CAAC,CAAC,EAAE,CAACpB,MAAM;QAAC,CAAA;AAC7F;AACA,6FAA6F;AAC7F,8FAA8F;AAC9F,+DAA+D;AAC/D,IAAIqB;AACJ,8FAA8F;AAC9F,8FAA8F;AAC9F,MAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,MAAMC,UAAU;AAEhB,SAASC,UAAUtB,GAAW;IAC5BmB,8BAAAA,+BAAAA,oBAAAA,oBAAsB,IAAII,KAAKC,SAAS,CAACC,WAAW;QAAEC,aAAa;IAAW;IAC9E,OAAOV,MAAMC,IAAI,CAACE,kBAAkBQ,OAAO,CAAC3B,MAAM,CAAC4B,IAAMA,EAAED,OAAO;AACpE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmB7B,GAAW;IACrC,MAAM8B,SAAqB;QAAC,EAAE;KAAC;IAC/B,KAAK,MAAMC,KAAKT,UAAUtB,KAAM;QAC9B,IAAIoB,YAAY9B,IAAI,CAACyC,IAAID,OAAO3B,IAAI,CAAC,EAAE;aAClC2B,MAAM,CAACA,OAAOhC,MAAM,GAAG,EAAE,CAACK,IAAI,CAAC4B;IACtC;IACA,OAAOD,OAAOE,MAAM,CAAC,CAACD,IAAMA,EAAEjC,MAAM,GAAG;AACzC;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,OAAO,SAASmC,aAAa5C,IAAY;IACvC,IAAI,CAACF,QAAQG,IAAI,CAACD,OAAO,OAAO;IAChC,MAAM6C,MAAM7C,KAAK8C,OAAO,CAAClD,KAAK,CAACe;QAC7B,MAAMoC,OAAOP,mBAAmB7B,KAC7BqC,GAAG,CAAC,CAACN,IAAMA,EAAEO,IAAI,CAAC,MAClBA,IAAI,CAAC,CAAC,CAAC,EAAEjB,QAAQ,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC,EAAEA,QAAQ,CAAC,EAAEe,KAAK,CAAC,EAAEf,QAAQ,CAAC,CAAC;IAC1C;IACA,OAAOa,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,uGAAuG;AACvG,MAAMC,kBAAkB;AAExB,4FAA4F;AAC5F,yFAAyF;AACzF,SAASC,SAAS1C,GAAW,EAAE2C,YAAoB;IACjD,OAAOd,mBAAmB7B,KACvBqC,GAAG,CAAC,CAACN,IAAM,GAAGY,eAAeZ,EAAEjC,MAAM,GAAG,IAAI,CAAC,CAAC,EAAEiC,EAAEO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAGP,CAAC,CAAC,EAAE,EAAE,EACvEO,IAAI,CAAC;AACV;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,OAAO,SAASM,aAAaC,KAAa;IACxC,IAAI,CAAC1D,QAAQG,IAAI,CAACuD,QAAQ,OAAOA;IACjC,IAAIX,MAAM;IACV,IAAIY,SAAS;IACb,MAAMC,SAASF,MAAMG,KAAK,CAAC/D,MAAM,2CAA2C;IAC5E,IAAK,IAAIyB,IAAI,GAAGA,IAAIqC,OAAOjD,MAAM,EAAEY,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;YACf,KAAK,MAAMuC,MAAMF,MAAM,CAACrC,EAAE,CAAE,IAAIuC,OAAO,KAAKH,SAAS,CAACA;YACtDZ,OAAOa,MAAM,CAACrC,EAAE;YAChB;QACF;QACA,IAAIoC,QAAQ;YACVZ,OAAOa,MAAM,CAACrC,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,MAAMQ,IAAIgB,IAAIzC,KAAK,CAAC+C;QACpB,IAAItB,GAAG;YACLgB,MAAM,GAAGA,IAAIjC,KAAK,CAAC,GAAGiB,EAAEtB,KAAK,IAAIsB,CAAC,CAAC,EAAE,GAAGwB,SAASK,MAAM,CAACrC,EAAE,EAAE,GAAGQ,CAAC,CAAC,EAAE,GAAGA,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG;QACtF,OAAO;YACLgB,OAAOQ,SAASK,MAAM,CAACrC,EAAE,EAAE+B;QAC7B;IACF;IACA,OAAOP;AACT;AAEA,kGAAkG;AAClG,2FAA2F;AAC3F,OAAO,SAASpB,cAAczB,IAAY;IACxC,OAAOA,KAAK6D,SAAS,CAAC,OAAOf,OAAO,CAAC,WAAW,IAAIgB,WAAW;AACjE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/segment.ts"],"sourcesContent":["import { stemmer } from 'stemmer';\n\n// Word boundaries FTS5's tokenizers cannot find on their own. Contract: seg(query) must appear in\n// seg(document) for any substring query; word mode is context-dependent and breaks that, so grapheme mode (UAX #29) is used.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nexport const UNSPACED_SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN_BODY = `(?:[${UNSPACED_SCRIPTS}]\\\\p{M}*)+`;\nconst RUN = new RegExp(`(${RUN_BODY})`, 'gu');\nconst HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// Whether text holds a script that marks no word boundaries, the predicate every store's\n// sidecar population and query split turns on.\nexport function hasUnspacedRun(text: string): boolean {\n  return HAS_RUN.test(text);\n}\n\nexport interface SearchToken {\n  text: string;\n  start: number;\n  end: number;\n  unspaced: boolean;\n}\n\n// Search tokens share the native separator contract: letters, marks, and numbers form tokens;\n// underscore, apostrophe, and hyphen separate them. Unspaced-script runs stay whole so callers\n// can apply substring semantics, while adjacent Latin text still gets its own token.\nexport function searchTokens(text: string): SearchToken[] {\n  const tokens: SearchToken[] = [];\n  for (const match of text.matchAll(/[\\p{L}\\p{N}][\\p{L}\\p{M}\\p{N}]*/gu)) {\n    const start = match.index;\n    const end = start + match[0].length;\n    let cursor = start;\n    for (const run of text.slice(start, end).matchAll(RUN)) {\n      const runStart = start + run.index;\n      if (runStart > cursor) tokens.push({ text: text.slice(cursor, runStart), start: cursor, end: runStart, unspaced: false });\n      const runEnd = runStart + run[0].length;\n      tokens.push({ text: run[0], start: runStart, end: runEnd, unspaced: true });\n      cursor = runEnd;\n    }\n    if (cursor < end) tokens.push({ text: text.slice(cursor, end), start: cursor, end, unspaced: false });\n  }\n  return tokens;\n}\n\n// Phrase membership is checked after native ranking. A CJK token matches inside one authored\n// run; Latin tokens compare folded Porter stems. Each call scans one field once per candidate.\nexport function matchesSearchPhrase(text: string, phrase: SearchToken[]): boolean {\n  const haystack = searchTokens(text);\n  if (phrase.length === 0 || haystack.length < phrase.length) return false;\n  for (let start = 0; start <= haystack.length - phrase.length; start++) {\n    let matches = true;\n    for (let i = 0; i < phrase.length; i++) {\n      const query = phrase[i];\n      const authored = haystack[start + i];\n      if (query.unspaced) {\n        if (!authored.unspaced || !authored.text.includes(query.text)) matches = false;\n      } else if (authored.unspaced || stemmer(foldForSearch(authored.text)) !== stemmer(foldForSearch(query.text))) {\n        matches = false;\n      }\n      if (!matches) break;\n    }\n    if (matches) return true;\n  }\n  return false;\n}\n\n// Character spans of unspaced-script runs in text, the same boundary the index segments on --\n// the snippet marker uses this to keep substring semantics inside a run and word/stem matching outside it.\nexport function unspacedRuns(text: string): Array<{ start: number; end: number }> {\n  return Array.from(text.matchAll(RUN), (m) => ({ start: m.index, end: m.index + m[0].length }));\n}\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built on first use and kept: construction is 6.6 ms, and a tree with no unspaced-script run\n// never segments at all, so no command pays it at module load.\nlet graphemeSegmenter: Intl.Segmenter | undefined;\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  graphemeSegmenter ??= new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n  return Array.from(graphemeSegmenter.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run. Otherwise each run explodes into\n// its graphemes, barrier-delimited from its neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// Raw title/summary/text drop punctuation as unicode61's token separator, so `数数` and `数。数`\n// falsely match there; only the barriered `_seg` columns are safe from it, hence this fallback target.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\nconst AUTHORED_COLUMNS = '{title summary text}:';\nconst QUOTED_RUN_TOKEN = new RegExp(`\"(${RUN_BODY})\"`, 'gu');\n\ninterface PositionalSpan {\n  start: number;\n  end: number;\n  qualified: boolean;\n}\n\nconst FTS_WORD = /[\\p{L}\\p{N}_]/u;\n\nfunction isFtsWord(char: string | undefined): boolean {\n  return char !== undefined && FTS_WORD.test(char);\n}\n\nfunction quoteEnd(text: string, start: number): number {\n  for (let i = start + 1; i < text.length; i++) {\n    if (text[i] !== '\"') continue;\n    if (text[i + 1] === '\"') {\n      i++;\n      continue;\n    }\n    return i + 1;\n  }\n  return text.length;\n}\n\nfunction nearEnd(text: string, open: number): number {\n  let depth = 1;\n  let quoted = false;\n  for (let i = open + 1; i < text.length; i++) {\n    const char = text[i];\n    if (char === '\"') {\n      if (quoted && text[i + 1] === '\"') {\n        i++;\n      } else {\n        quoted = !quoted;\n      }\n    } else if (!quoted && char === '(') {\n      depth++;\n    } else if (!quoted && char === ')' && --depth === 0) {\n      return i + 1;\n    }\n  }\n  return text.length;\n}\n\n// Lexical routing only: SQLite remains responsible for validating the expression. The scanner\n// recognizes NEAR groups and the one FTS string governed by ^, while quoted text shields both.\nfunction positionalSpans(terms: string): PositionalSpan[] {\n  const spans: PositionalSpan[] = [];\n  for (let i = 0; i < terms.length; i++) {\n    if (terms[i] === '\"') {\n      i = quoteEnd(terms, i) - 1;\n      continue;\n    }\n    if (terms.startsWith('NEAR', i) && !isFtsWord(terms[i - 1])) {\n      let open = i + 4;\n      while (/\\s/u.test(terms[open] ?? '')) open++;\n      if (terms[open] === '(') {\n        const end = nearEnd(terms, open);\n        if (HAS_RUN.test(terms.slice(open + 1, end))) spans.push({ start: i, end, qualified: QUALIFIER.test(terms.slice(0, i)) });\n        i = end - 1;\n        continue;\n      }\n    }\n    if (terms[i] === '^' && !isFtsWord(terms[i - 1])) {\n      let operand = i + 1;\n      while (/\\s/u.test(terms[operand] ?? '')) operand++;\n      const end =\n        terms[operand] === '\"'\n          ? quoteEnd(terms, operand)\n          : (() => {\n              let cursor = operand;\n              while (/[\\p{L}\\p{N}_]/u.test(terms[cursor] ?? '')) cursor++;\n              return cursor;\n            })();\n      if (end > operand && HAS_RUN.test(terms.slice(operand, end))) spans.push({ start: i, end, qualified: QUALIFIER.test(terms.slice(0, i)) });\n      i = Math.max(i, end - 1);\n    }\n  }\n  return spans;\n}\n\nfunction routePositionalSpans(terms: string): string {\n  const spans = positionalSpans(terms);\n  if (spans.length === 0) return terms;\n  let out = '';\n  let cursor = 0;\n  for (const span of spans) {\n    out += terms.slice(cursor, span.start);\n    if (!span.qualified) out += AUTHORED_COLUMNS;\n    out += terms.slice(span.start, span.end);\n    cursor = span.end;\n  }\n  return out + terms.slice(cursor);\n}\n\nfunction inPositionalSpan(start: number, spans: PositionalSpan[]): boolean {\n  return spans.some((span) => start >= span.start && start < span.end);\n}\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string, quoteSingle = false): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 || quoteSingle ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Retarget complete quoted run tokens without interpreting the expression around them. Positional\n// spans have already been routed to authored columns, so sidecar barriers cannot change meaning.\nfunction quotedRunTokens(terms: string): string {\n  const spans = positionalSpans(terms);\n  let out = '';\n  let cursor = 0;\n  for (const match of terms.matchAll(QUOTED_RUN_TOKEN)) {\n    const start = match.index;\n    const run = match[1];\n    if (inPositionalSpan(start, spans) || PUNCTUATION.test(run)) continue;\n    out += terms.slice(cursor, start);\n    const qualifier = out.match(QUALIFIER);\n    if (qualifier) {\n      out = `${out.slice(0, qualifier.index)}${qualifier[1]}${runQuery(run, `${qualifier[2]}${qualifier[3]}_seg:`, true)}`;\n    } else {\n      out += runQuery(run, SIDECAR_COLUMNS, true);\n    }\n    cursor = start + match[0].length;\n  }\n  return cursor === 0 ? terms : out + terms.slice(cursor);\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching segmentField. A\n// qualifier maps to its `_seg` column; unqualified maps to all three (SIDECAR_COLUMNS).\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  terms = routePositionalSpans(terms);\n  terms = quotedRunTokens(terms);\n  const spans = positionalSpans(terms);\n  let out = '';\n  let quoted = false;\n  let offset = 0;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      offset += pieces[i].length;\n      continue;\n    }\n    const protectedRun = inPositionalSpan(offset, spans);\n    if (quoted || protectedRun) {\n      out += pieces[i]; // an author's phrase is matched as written\n      offset += pieces[i].length;\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n    offset += pieces[i].length;\n  }\n  return out;\n}\n\n// FTS tokenizers compare letters without combining marks. Keep the authored string for output and\n// offsets, but use this key when JS needs to compare a matched word to its source segment.\nexport function foldForSearch(text: string): string {\n  return text.normalize('NFD').replace(/\\p{M}/gu, '').toLowerCase();\n}\n"],"names":["stemmer","UNSPACED_SCRIPTS","RUN_BODY","RUN","RegExp","HAS_RUN","hasUnspacedRun","text","test","searchTokens","tokens","match","matchAll","start","index","end","length","cursor","run","slice","runStart","push","unspaced","runEnd","matchesSearchPhrase","phrase","haystack","matches","i","query","authored","includes","foldForSearch","unspacedRuns","Array","from","m","graphemeSegmenter","PUNCTUATION","BARRIER","graphemes","Intl","Segmenter","undefined","granularity","segment","s","splitOnPunctuation","groups","g","filter","segmentField","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","AUTHORED_COLUMNS","QUOTED_RUN_TOKEN","FTS_WORD","isFtsWord","char","quoteEnd","nearEnd","open","depth","quoted","positionalSpans","terms","spans","startsWith","qualified","operand","Math","max","routePositionalSpans","span","inPositionalSpan","some","runQuery","columnPrefix","quoteSingle","quotedRunTokens","qualifier","segmentMatch","offset","pieces","split","ch","protectedRun","normalize","toLowerCase"],"mappings":"AAAA,SAASA,OAAO,QAAQ,UAAU;AAElC,kGAAkG;AAClG,6HAA6H;AAE7H,wEAAwE;AACxE,kFAAkF;AAClF,OAAO,MAAMC,mBAAmB,wGAAwG;AACxI,2FAA2F;AAC3F,gDAAgD;AAChD,MAAMC,WAAW,CAAC,IAAI,EAAED,iBAAiB,UAAU,CAAC;AACpD,MAAME,MAAM,IAAIC,OAAO,CAAC,CAAC,EAAEF,SAAS,CAAC,CAAC,EAAE;AACxC,MAAMG,UAAU,IAAID,OAAO,CAAC,CAAC,EAAEH,iBAAiB,CAAC,CAAC,EAAE;AAEpD,yFAAyF;AACzF,+CAA+C;AAC/C,OAAO,SAASK,eAAeC,IAAY;IACzC,OAAOF,QAAQG,IAAI,CAACD;AACtB;AASA,8FAA8F;AAC9F,+FAA+F;AAC/F,qFAAqF;AACrF,OAAO,SAASE,aAAaF,IAAY;IACvC,MAAMG,SAAwB,EAAE;IAChC,KAAK,MAAMC,SAASJ,KAAKK,QAAQ,CAAC,oCAAqC;QACrE,MAAMC,QAAQF,MAAMG,KAAK;QACzB,MAAMC,MAAMF,QAAQF,KAAK,CAAC,EAAE,CAACK,MAAM;QACnC,IAAIC,SAASJ;QACb,KAAK,MAAMK,OAAOX,KAAKY,KAAK,CAACN,OAAOE,KAAKH,QAAQ,CAACT,KAAM;YACtD,MAAMiB,WAAWP,QAAQK,IAAIJ,KAAK;YAClC,IAAIM,WAAWH,QAAQP,OAAOW,IAAI,CAAC;gBAAEd,MAAMA,KAAKY,KAAK,CAACF,QAAQG;gBAAWP,OAAOI;gBAAQF,KAAKK;gBAAUE,UAAU;YAAM;YACvH,MAAMC,SAASH,WAAWF,GAAG,CAAC,EAAE,CAACF,MAAM;YACvCN,OAAOW,IAAI,CAAC;gBAAEd,MAAMW,GAAG,CAAC,EAAE;gBAAEL,OAAOO;gBAAUL,KAAKQ;gBAAQD,UAAU;YAAK;YACzEL,SAASM;QACX;QACA,IAAIN,SAASF,KAAKL,OAAOW,IAAI,CAAC;YAAEd,MAAMA,KAAKY,KAAK,CAACF,QAAQF;YAAMF,OAAOI;YAAQF;YAAKO,UAAU;QAAM;IACrG;IACA,OAAOZ;AACT;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,OAAO,SAASc,oBAAoBjB,IAAY,EAAEkB,MAAqB;IACrE,MAAMC,WAAWjB,aAAaF;IAC9B,IAAIkB,OAAOT,MAAM,KAAK,KAAKU,SAASV,MAAM,GAAGS,OAAOT,MAAM,EAAE,OAAO;IACnE,IAAK,IAAIH,QAAQ,GAAGA,SAASa,SAASV,MAAM,GAAGS,OAAOT,MAAM,EAAEH,QAAS;QACrE,IAAIc,UAAU;QACd,IAAK,IAAIC,IAAI,GAAGA,IAAIH,OAAOT,MAAM,EAAEY,IAAK;YACtC,MAAMC,QAAQJ,MAAM,CAACG,EAAE;YACvB,MAAME,WAAWJ,QAAQ,CAACb,QAAQe,EAAE;YACpC,IAAIC,MAAMP,QAAQ,EAAE;gBAClB,IAAI,CAACQ,SAASR,QAAQ,IAAI,CAACQ,SAASvB,IAAI,CAACwB,QAAQ,CAACF,MAAMtB,IAAI,GAAGoB,UAAU;YAC3E,OAAO,IAAIG,SAASR,QAAQ,IAAItB,QAAQgC,cAAcF,SAASvB,IAAI,OAAOP,QAAQgC,cAAcH,MAAMtB,IAAI,IAAI;gBAC5GoB,UAAU;YACZ;YACA,IAAI,CAACA,SAAS;QAChB;QACA,IAAIA,SAAS,OAAO;IACtB;IACA,OAAO;AACT;AAEA,8FAA8F;AAC9F,2GAA2G;AAC3G,OAAO,SAASM,aAAa1B,IAAY;IACvC,OAAO2B,MAAMC,IAAI,CAAC5B,KAAKK,QAAQ,CAACT,MAAM,CAACiC,IAAO,CAAA;YAAEvB,OAAOuB,EAAEtB,KAAK;YAAEC,KAAKqB,EAAEtB,KAAK,GAAGsB,CAAC,CAAC,EAAE,CAACpB,MAAM;QAAC,CAAA;AAC7F;AACA,6FAA6F;AAC7F,8FAA8F;AAC9F,+DAA+D;AAC/D,IAAIqB;AACJ,8FAA8F;AAC9F,8FAA8F;AAC9F,MAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,MAAMC,UAAU;AAEhB,SAASC,UAAUtB,GAAW;IAC5BmB,8BAAAA,+BAAAA,oBAAAA,oBAAsB,IAAII,KAAKC,SAAS,CAACC,WAAW;QAAEC,aAAa;IAAW;IAC9E,OAAOV,MAAMC,IAAI,CAACE,kBAAkBQ,OAAO,CAAC3B,MAAM,CAAC4B,IAAMA,EAAED,OAAO;AACpE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmB7B,GAAW;IACrC,MAAM8B,SAAqB;QAAC,EAAE;KAAC;IAC/B,KAAK,MAAMC,KAAKT,UAAUtB,KAAM;QAC9B,IAAIoB,YAAY9B,IAAI,CAACyC,IAAID,OAAO3B,IAAI,CAAC,EAAE;aAClC2B,MAAM,CAACA,OAAOhC,MAAM,GAAG,EAAE,CAACK,IAAI,CAAC4B;IACtC;IACA,OAAOD,OAAOE,MAAM,CAAC,CAACD,IAAMA,EAAEjC,MAAM,GAAG;AACzC;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,OAAO,SAASmC,aAAa5C,IAAY;IACvC,IAAI,CAACF,QAAQG,IAAI,CAACD,OAAO,OAAO;IAChC,MAAM6C,MAAM7C,KAAK8C,OAAO,CAAClD,KAAK,CAACe;QAC7B,MAAMoC,OAAOP,mBAAmB7B,KAC7BqC,GAAG,CAAC,CAACN,IAAMA,EAAEO,IAAI,CAAC,MAClBA,IAAI,CAAC,CAAC,CAAC,EAAEjB,QAAQ,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC,EAAEA,QAAQ,CAAC,EAAEe,KAAK,CAAC,EAAEf,QAAQ,CAAC,CAAC;IAC1C;IACA,OAAOa,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,uGAAuG;AACvG,MAAMC,kBAAkB;AACxB,MAAMC,mBAAmB;AACzB,MAAMC,mBAAmB,IAAIzD,OAAO,CAAC,EAAE,EAAEF,SAAS,EAAE,CAAC,EAAE;AAQvD,MAAM4D,WAAW;AAEjB,SAASC,UAAUC,IAAwB;IACzC,OAAOA,SAASrB,aAAamB,SAAStD,IAAI,CAACwD;AAC7C;AAEA,SAASC,SAAS1D,IAAY,EAAEM,KAAa;IAC3C,IAAK,IAAIe,IAAIf,QAAQ,GAAGe,IAAIrB,KAAKS,MAAM,EAAEY,IAAK;QAC5C,IAAIrB,IAAI,CAACqB,EAAE,KAAK,KAAK;QACrB,IAAIrB,IAAI,CAACqB,IAAI,EAAE,KAAK,KAAK;YACvBA;YACA;QACF;QACA,OAAOA,IAAI;IACb;IACA,OAAOrB,KAAKS,MAAM;AACpB;AAEA,SAASkD,QAAQ3D,IAAY,EAAE4D,IAAY;IACzC,IAAIC,QAAQ;IACZ,IAAIC,SAAS;IACb,IAAK,IAAIzC,IAAIuC,OAAO,GAAGvC,IAAIrB,KAAKS,MAAM,EAAEY,IAAK;QAC3C,MAAMoC,OAAOzD,IAAI,CAACqB,EAAE;QACpB,IAAIoC,SAAS,KAAK;YAChB,IAAIK,UAAU9D,IAAI,CAACqB,IAAI,EAAE,KAAK,KAAK;gBACjCA;YACF,OAAO;gBACLyC,SAAS,CAACA;YACZ;QACF,OAAO,IAAI,CAACA,UAAUL,SAAS,KAAK;YAClCI;QACF,OAAO,IAAI,CAACC,UAAUL,SAAS,OAAO,EAAEI,UAAU,GAAG;YACnD,OAAOxC,IAAI;QACb;IACF;IACA,OAAOrB,KAAKS,MAAM;AACpB;AAEA,8FAA8F;AAC9F,+FAA+F;AAC/F,SAASsD,gBAAgBC,KAAa;IACpC,MAAMC,QAA0B,EAAE;IAClC,IAAK,IAAI5C,IAAI,GAAGA,IAAI2C,MAAMvD,MAAM,EAAEY,IAAK;QACrC,IAAI2C,KAAK,CAAC3C,EAAE,KAAK,KAAK;YACpBA,IAAIqC,SAASM,OAAO3C,KAAK;YACzB;QACF;QACA,IAAI2C,MAAME,UAAU,CAAC,QAAQ7C,MAAM,CAACmC,UAAUQ,KAAK,CAAC3C,IAAI,EAAE,GAAG;gBAEzC2C;YADlB,IAAIJ,OAAOvC,IAAI;YACf,MAAO,MAAMpB,IAAI,EAAC+D,cAAAA,KAAK,CAACJ,KAAK,cAAXI,yBAAAA,cAAe,IAAKJ;YACtC,IAAII,KAAK,CAACJ,KAAK,KAAK,KAAK;gBACvB,MAAMpD,MAAMmD,QAAQK,OAAOJ;gBAC3B,IAAI9D,QAAQG,IAAI,CAAC+D,MAAMpD,KAAK,CAACgD,OAAO,GAAGpD,OAAOyD,MAAMnD,IAAI,CAAC;oBAAER,OAAOe;oBAAGb;oBAAK2D,WAAWhB,UAAUlD,IAAI,CAAC+D,MAAMpD,KAAK,CAAC,GAAGS;gBAAI;gBACvHA,IAAIb,MAAM;gBACV;YACF;QACF;QACA,IAAIwD,KAAK,CAAC3C,EAAE,KAAK,OAAO,CAACmC,UAAUQ,KAAK,CAAC3C,IAAI,EAAE,GAAG;gBAE9B2C;YADlB,IAAII,UAAU/C,IAAI;YAClB,MAAO,MAAMpB,IAAI,EAAC+D,iBAAAA,KAAK,CAACI,QAAQ,cAAdJ,4BAAAA,iBAAkB,IAAKI;YACzC,MAAM5D,MACJwD,KAAK,CAACI,QAAQ,KAAK,MACfV,SAASM,OAAOI,WAChB,AAAC,CAAA;oBAE8BJ;gBAD7B,IAAItD,SAAS0D;gBACb,MAAO,iBAAiBnE,IAAI,EAAC+D,gBAAAA,KAAK,CAACtD,OAAO,cAAbsD,2BAAAA,gBAAiB,IAAKtD;gBACnD,OAAOA;YACT,CAAA;YACN,IAAIF,MAAM4D,WAAWtE,QAAQG,IAAI,CAAC+D,MAAMpD,KAAK,CAACwD,SAAS5D,OAAOyD,MAAMnD,IAAI,CAAC;gBAAER,OAAOe;gBAAGb;gBAAK2D,WAAWhB,UAAUlD,IAAI,CAAC+D,MAAMpD,KAAK,CAAC,GAAGS;YAAI;YACvIA,IAAIgD,KAAKC,GAAG,CAACjD,GAAGb,MAAM;QACxB;IACF;IACA,OAAOyD;AACT;AAEA,SAASM,qBAAqBP,KAAa;IACzC,MAAMC,QAAQF,gBAAgBC;IAC9B,IAAIC,MAAMxD,MAAM,KAAK,GAAG,OAAOuD;IAC/B,IAAInB,MAAM;IACV,IAAInC,SAAS;IACb,KAAK,MAAM8D,QAAQP,MAAO;QACxBpB,OAAOmB,MAAMpD,KAAK,CAACF,QAAQ8D,KAAKlE,KAAK;QACrC,IAAI,CAACkE,KAAKL,SAAS,EAAEtB,OAAOQ;QAC5BR,OAAOmB,MAAMpD,KAAK,CAAC4D,KAAKlE,KAAK,EAAEkE,KAAKhE,GAAG;QACvCE,SAAS8D,KAAKhE,GAAG;IACnB;IACA,OAAOqC,MAAMmB,MAAMpD,KAAK,CAACF;AAC3B;AAEA,SAAS+D,iBAAiBnE,KAAa,EAAE2D,KAAuB;IAC9D,OAAOA,MAAMS,IAAI,CAAC,CAACF,OAASlE,SAASkE,KAAKlE,KAAK,IAAIA,QAAQkE,KAAKhE,GAAG;AACrE;AAEA,4FAA4F;AAC5F,yFAAyF;AACzF,SAASmE,SAAShE,GAAW,EAAEiE,YAAoB,EAAEC,cAAc,KAAK;IACtE,OAAOrC,mBAAmB7B,KACvBqC,GAAG,CAAC,CAACN,IAAM,GAAGkC,eAAelC,EAAEjC,MAAM,GAAG,KAAKoE,cAAc,CAAC,CAAC,EAAEnC,EAAEO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAGP,CAAC,CAAC,EAAE,EAAE,EACtFO,IAAI,CAAC;AACV;AAEA,kGAAkG;AAClG,iGAAiG;AACjG,SAAS6B,gBAAgBd,KAAa;IACpC,MAAMC,QAAQF,gBAAgBC;IAC9B,IAAInB,MAAM;IACV,IAAInC,SAAS;IACb,KAAK,MAAMN,SAAS4D,MAAM3D,QAAQ,CAACiD,kBAAmB;QACpD,MAAMhD,QAAQF,MAAMG,KAAK;QACzB,MAAMI,MAAMP,KAAK,CAAC,EAAE;QACpB,IAAIqE,iBAAiBnE,OAAO2D,UAAUlC,YAAY9B,IAAI,CAACU,MAAM;QAC7DkC,OAAOmB,MAAMpD,KAAK,CAACF,QAAQJ;QAC3B,MAAMyE,YAAYlC,IAAIzC,KAAK,CAAC+C;QAC5B,IAAI4B,WAAW;YACblC,MAAM,GAAGA,IAAIjC,KAAK,CAAC,GAAGmE,UAAUxE,KAAK,IAAIwE,SAAS,CAAC,EAAE,GAAGJ,SAAShE,KAAK,GAAGoE,SAAS,CAAC,EAAE,GAAGA,SAAS,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO;QACtH,OAAO;YACLlC,OAAO8B,SAAShE,KAAKyC,iBAAiB;QACxC;QACA1C,SAASJ,QAAQF,KAAK,CAAC,EAAE,CAACK,MAAM;IAClC;IACA,OAAOC,WAAW,IAAIsD,QAAQnB,MAAMmB,MAAMpD,KAAK,CAACF;AAClD;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,OAAO,SAASsE,aAAahB,KAAa;IACxC,IAAI,CAAClE,QAAQG,IAAI,CAAC+D,QAAQ,OAAOA;IACjCA,QAAQO,qBAAqBP;IAC7BA,QAAQc,gBAAgBd;IACxB,MAAMC,QAAQF,gBAAgBC;IAC9B,IAAInB,MAAM;IACV,IAAIiB,SAAS;IACb,IAAImB,SAAS;IACb,MAAMC,SAASlB,MAAMmB,KAAK,CAACvF,MAAM,2CAA2C;IAC5E,IAAK,IAAIyB,IAAI,GAAGA,IAAI6D,OAAOzE,MAAM,EAAEY,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;YACf,KAAK,MAAM+D,MAAMF,MAAM,CAAC7D,EAAE,CAAE,IAAI+D,OAAO,KAAKtB,SAAS,CAACA;YACtDjB,OAAOqC,MAAM,CAAC7D,EAAE;YAChB4D,UAAUC,MAAM,CAAC7D,EAAE,CAACZ,MAAM;YAC1B;QACF;QACA,MAAM4E,eAAeZ,iBAAiBQ,QAAQhB;QAC9C,IAAIH,UAAUuB,cAAc;YAC1BxC,OAAOqC,MAAM,CAAC7D,EAAE,EAAE,2CAA2C;YAC7D4D,UAAUC,MAAM,CAAC7D,EAAE,CAACZ,MAAM;YAC1B;QACF;QACA,MAAMoB,IAAIgB,IAAIzC,KAAK,CAAC+C;QACpB,IAAItB,GAAG;YACLgB,MAAM,GAAGA,IAAIjC,KAAK,CAAC,GAAGiB,EAAEtB,KAAK,IAAIsB,CAAC,CAAC,EAAE,GAAG8C,SAASO,MAAM,CAAC7D,EAAE,EAAE,GAAGQ,CAAC,CAAC,EAAE,GAAGA,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG;QACtF,OAAO;YACLgB,OAAO8B,SAASO,MAAM,CAAC7D,EAAE,EAAE+B;QAC7B;QACA6B,UAAUC,MAAM,CAAC7D,EAAE,CAACZ,MAAM;IAC5B;IACA,OAAOoC;AACT;AAEA,kGAAkG;AAClG,2FAA2F;AAC3F,OAAO,SAASpB,cAAczB,IAAY;IACxC,OAAOA,KAAKsF,SAAS,CAAC,OAAOxC,OAAO,CAAC,WAAW,IAAIyC,WAAW;AACjE"}