sensemaking 0.18.4 → 0.19.0

package/dist/cjs/store/turso/open.js

@@ -22,10 +22,12 @@ _export(exports, {
 var _nodefs = require("node:fs");
 var _nodepath = require("node:path");
 var _indexts = require("../../config/index.js");
+var _handoffts = require("../../embed/handoff.js");
+var _typests = require("../../embed/types.js");
 var _errorsts = require("../../errors.js");
 var _indexts1 = require("../../features/index.js");
+var _cachets = require("../cache.js");
 var _sharedts = require("../shared.js");
-var _opents = require("../sqlite/open.js");
 var _connectionts = require("./connection.js");
 var _nativets = require("./native.js");
 var _reconcilets = require("./reconcile.js");
@@ -162,10 +164,9 @@ function _ts_generator(thisArg, body) {
     }
 }
 var DB_FILENAME = 'cache.turso.db';
-var SCHEMA_VERSION = '1';
-// content is a plain table (not FTS-virtual): phase 1 creates no FTS index at all -- that is
-// phase 2's job (see the plan). Untyped ALTER TABLE ADD COLUMN, like sqlite -- turso's dialect
-// accepts it with no declared type (spike-confirmed), unlike duckdb which requires VARIANT.
+var SCHEMA_VERSION = '3';
+// The ngram index is scoped to disjoint "_ngram" sidecar columns: a second index over the same
+// columns makes a bare substring match a whole word, defeating the prefix-query rejection.
 function ensureSchema(conn, cfg) {
     return _async_to_generator(function() {
         var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, feature, err;
@@ -180,66 +181,93 @@ function ensureSchema(conn, cfg) {
                     _state.sent();
                     return [
                         4,
-                        conn.exec('CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)')
+                        conn.exec('CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)')
                     ];
                 case 2:
                     _state.sent();
                     return [
                         4,
-                        conn.exec('CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))')
+                        conn.exec("CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')")
                     ];
                 case 3:
                     _state.sent();
                     return [
                         4,
-                        conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)')
+                        conn.exec("CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')")
                     ];
                 case 4:
                     _state.sent();
-                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-                    _state.label = 5;
+                    return [
+                        4,
+                        conn.exec('CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))')
+                    ];
                 case 5:
+                    _state.sent();
+                    return [
+                        4,
+                        conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)')
+                    ];
+                case 6:
+                    _state.sent();
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    _state.label = 7;
+                case 7:
                     _state.trys.push([
-                        5,
-                        10,
-                        11,
-                        12
+                        7,
+                        14,
+                        15,
+                        16
                     ]);
                     _iterator = (0, _indexts1.activeFeatures)(cfg)[Symbol.iterator]();
-                    _state.label = 6;
-                case 6:
+                    _state.label = 8;
+                case 8:
                     if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
                         3,
-                        9
+                        13
                     ];
                     feature = _step.value;
+                    if (!(feature.name === 'embed')) return [
+                        3,
+                        10
+                    ];
+                    return [
+                        4,
+                        conn.exec('CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB('.concat(_typests.STORE_DIMS, '), PRIMARY KEY ("path", chunk))'))
+                    ];
+                case 9:
+                    _state.sent();
+                    return [
+                        3,
+                        12
+                    ];
+                case 10:
                     return [
                         4,
                         feature.schema(conn)
                     ];
-                case 7:
+                case 11:
                     _state.sent();
-                    _state.label = 8;
-                case 8:
+                    _state.label = 12;
+                case 12:
                     _iteratorNormalCompletion = true;
                     return [
                         3,
-                        6
+                        8
                     ];
-                case 9:
+                case 13:
                     return [
                         3,
-                        12
+                        16
                     ];
-                case 10:
+                case 14:
                     err = _state.sent();
                     _didIteratorError = true;
                     _iteratorError = err;
                     return [
                         3,
-                        12
+                        16
                     ];
-                case 11:
+                case 15:
                     try {
                         if (!_iteratorNormalCompletion && _iterator.return != null) {
                             _iterator.return();
@@ -252,41 +280,41 @@ function ensureSchema(conn, cfg) {
                     return [
                         7
                     ];
-                case 12:
+                case 16:
                     return [
                         4,
                         (0, _sharedts.getMeta)(conn, 'schema_version')
                     ];
-                case 13:
+                case 17:
                     if (!(_state.sent() === null)) return [
                         3,
-                        15
+                        19
                     ];
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'schema_version', SCHEMA_VERSION)
                     ];
-                case 14:
+                case 18:
                     _state.sent();
-                    _state.label = 15;
-                case 15:
+                    _state.label = 19;
+                case 19:
                     return [
                         4,
                         (0, _sharedts.getMeta)(conn, 'features')
                     ];
-                case 16:
+                case 20:
                     if (!(_state.sent() === null)) return [
                         3,
-                        18
+                        22
                     ];
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'features', (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES))
                     ];
-                case 17:
+                case 21:
                     _state.sent();
-                    _state.label = 18;
-                case 18:
+                    _state.label = 22;
+                case 22:
                     return [
                         2
                     ];
@@ -337,15 +365,15 @@ function connect(cfg) {
                     return [
                         4,
                         turso.connect(dbPath, {
-                            timeout: 30000
+                            timeout: 30000,
+                            experimental: [
+                                'index_method'
+                            ]
                         })
                     ];
                 case 5:
-                    // Floored at the same 30s sqlite opens with. `timeout` is a connect-time option in this
-                    // client (no synchronous PRAGMA-at-open-time escape hatch), so the real derived value --
-                    // which depends on this cache's own recorded reconcile_max_ms, readable only once connected
-                    // -- is installed via a runtime PRAGMA below, right before the open-time reconcile, the same
-                    // lever sqlite's open() uses (spike-confirmed: raw PRAGMA busy_timeout works as a setter too).
+                    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;
+                    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).
                     db = _state.sent();
                     return [
                         3,
@@ -355,9 +383,8 @@ function connect(cfg) {
                     err1 = _state.sent();
                     throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "turso" failed to open '.concat(dbPath, ": ").concat(err1.message));
                 case 7:
-                    // A throw below (schema, reconcile's COLUMN_LIMIT) must release this handle, or the leaked WAL
-                    // makes the db file undeletable on Windows. closed marks the rebuild branch that closes and
-                    // recurses, so the catch never double-closes.
+                    // A throw below must release this handle, or the leaked WAL makes the db file undeletable on
+                    // Windows. closed guards the catch against double-closing.
                     closed = false;
                     _state.label = 8;
                 case 8:
@@ -391,11 +418,8 @@ function connect(cfg) {
                         3,
                         13
                     ];
-                    // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --
-                    // rebuild instead, same as sqlite's and duckdb's open() (see their comments for why this is
-                    // announced). No tokenize-only or embed-identity-adopted partial path (sqlite's grace for
-                    // those): turso has no content.tokenize knob to special-case, and the grace is an
-                    // optimization, not a correctness requirement -- any mismatch rebuilds, duckdb's shape.
+                    // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched
+                    // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.
                     reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
                     console.error("sense: ".concat(reason, "; rebuilding the index"));
                     closed = true;
@@ -405,7 +429,7 @@ function connect(cfg) {
                     ];
                 case 12:
                     _state.sent();
-                    (0, _opents.clearCache)(cfg);
+                    (0, _cachets.clearCache)(cfg);
                     return [
                         2,
                         connect(cfg)
@@ -473,7 +497,7 @@ function connect(cfg) {
 }
 function openTurso(cfg) {
     return _async_to_generator(function() {
-        var _ref, db, conn, resolvedCfg, dbPath, parsed, warnings;
+        var _ref, db, conn, resolvedCfg, dbPath, parsed, warnings, store;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -483,10 +507,13 @@ function openTurso(cfg) {
                     ];
                 case 1:
                     _ref = _state.sent(), db = _ref.db, conn = _ref.conn, resolvedCfg = _ref.cfg, dbPath = _ref.dbPath, parsed = _ref.parsed, warnings = _ref.warnings;
+                    store = (0, _storets.createStore)(db, conn, resolvedCfg, resolvedCfg.baseDir);
+                    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
+                    (0, _handoffts.rekeyChunkText)(conn, store);
                     return [
                         2,
                         {
-                            store: (0, _storets.createStore)(db, conn, resolvedCfg, resolvedCfg.baseDir),
+                            store: store,
                             cfg: resolvedCfg,
                             dbPath: dbPath,
                             parsed: parsed,

package/dist/cjs/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { clearCache } from '../sqlite/open.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's SCHEMA_VERSION and duckdb's -- each store's cache shape evolves\n// separately, and the store name already joins the feature signature (see featureSignature), so\n// a config's `store` key switching cleanly rebuilds rather than reusing another engine's cache.\nexport const SCHEMA_VERSION = '1';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// content is a plain table (not FTS-virtual): phase 1 creates no FTS index at all -- that is\n// phase 2's job (see the plan). Untyped ALTER TABLE ADD COLUMN, like sqlite -- turso's dialect\n// accepts it with no declared type (spike-confirmed), unlike duckdb which requires VARIANT.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n  db: Database;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never even attempt to resolve\n  // this optional dependency, so nothing in this store's module graph imports it as a value until\n  // a turso tree is actually opened (type-only imports elsewhere are erased and cost nothing\n  // either way). Installed on first use if missing (see native.ts's tursoApi).\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is a connect-time option in this\n    // client (no synchronous PRAGMA-at-open-time escape hatch), so the real derived value --\n    // which depends on this cache's own recorded reconcile_max_ms, readable only once connected\n    // -- is installed via a runtime PRAGMA below, right before the open-time reconcile, the same\n    // lever sqlite's open() uses (spike-confirmed: raw PRAGMA busy_timeout works as a setter too).\n    db = await turso.connect(dbPath, { timeout: 30_000 });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) must release this handle, or the leaked WAL\n  // makes the db file undeletable on Windows. closed marks the rebuild branch that closes and\n  // recurses, so the catch never double-closes.\n  let closed = false;\n  try {\n    const conn = createConnection(db);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's and duckdb's open() (see their comments for why this is\n      // announced). No tokenize-only or embed-identity-adopted partial path (sqlite's grace for\n      // those): turso has no content.tokenize knob to special-case, and the grace is an\n      // optimization, not a correctness requirement -- any mismatch rebuilds, duckdb's shape.\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      closed = true;\n      await db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) await db.close();\n    throw err;\n  }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  return { store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir), cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","ensureSchema","conn","cfg","feature","exec","activeFeatures","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","turso","err","db","closed","version","features","wantFeatures","reason","recordedMaxMs","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","createConnection","console","error","close","clearCache","Number","Math","min","max","reconcile","resolvedCfg","store","createStore"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAIAC;eAAAA;;QAwGSC;eAAAA;;;sBA3HI;wBACL;uBAGuB;wBACjB;wBACc;wBACR;sBACN;4BAEM;wBACO;2BACd;uBACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMF,cAAc;AAIpB,IAAMC,iBAAiB;AAU9B,6FAA6F;AAC7F,+FAA+F;AAC/F,4FAA4F;AAC5F,SAAeE,aAAaC,IAAgB,EAAEC,GAAW;;YAKlD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;oBAAsC;;wBAAMA,QAAQG,MAAM,CAACL;;;oBAArB;;;oBAAtC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACA;;wBAAMM,IAAAA,iBAAO,EAACN,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMO,IAAAA,iBAAO,EAACP,MAAM,kBAAkBH;;;oBAAtC;;;oBACjD;;wBAAMS,IAAAA,iBAAO,EAACN,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMO,IAAAA,iBAAO,EAACP,MAAM,YAAYQ,IAAAA,yBAAgB,EAACP,KAAKQ,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAWA,SAAeC,QAAQT,GAAmB;;YAClCU,UAEAC,QAMFC,OAGKC,KAKLC,IAQKD,MAMLE,QA2B4B,MAzBxBhB,MAIAiB,SACAC,UACAC,cAOEC,QAYFC,eAGuB,OAArBC,QAAQC,UAGTT;;;;oBA/DHH,WAAWa,IAAAA,cAAI,EAACvB,IAAIwB,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAAChB,UAAU;wBAAEiB,WAAW;oBAAK;oBAChChB,SAASY,IAAAA,cAAI,EAACb,UAAUf;;;;;;;;;oBAQpB;;wBAAMiC,IAAAA,kBAAQ;;;oBAAtBhB,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAegB,oBAAU,GAAE,MAAMhB;oBACrC,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACjB,IAAckB,OAAO,EAAC;;;;;;;;oBAUxH;;wBAAMnB,MAAMH,OAAO,CAACE,QAAQ;4BAAEqB,SAAS;wBAAO;;;oBALnD,wFAAwF;oBACxF,yFAAyF;oBACzF,4FAA4F;oBAC5F,6FAA6F;oBAC7F,+FAA+F;oBAC/FlB,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXlB,QAAO,MAA2B,OAAvB,AAACE,KAAckB,OAAO;;oBAEpH,+FAA+F;oBAC/F,4FAA4F;oBAC5F,8CAA8C;oBAC1ChB,SAAS;;;;;;;;;oBAELhB,OAAOkC,IAAAA,8BAAgB,EAACnB;oBAE9B;;wBAAMf,KAAKG,IAAI,CAAC;;;oBAAhB;oBAEgB;;wBAAMG,IAAAA,iBAAO,EAACN,MAAM;;;oBAA9BiB,UAAU;oBACC;;wBAAMX,IAAAA,iBAAO,EAACN,MAAM;;;oBAA/BkB,WAAW;oBACXC,eAAeX,IAAAA,yBAAgB,EAACP,KAAKQ,kBAAQ;yBAC/C,CAAA,AAACQ,YAAY,QAAQA,YAAYpB,kBAAoBqB,aAAa,QAAQA,aAAaC,YAAY,GAAnG;;;;oBACF,4FAA4F;oBAC5F,4FAA4F;oBAC5F,0FAA0F;oBAC1F,kFAAkF;oBAClF,wFAAwF;oBAClFC,SAASH,YAAY,QAAQA,YAAYpB,iBAAiB,mDAAmD;oBACnHsC,QAAQC,KAAK,CAAC,AAAC,UAAgB,OAAPhB,QAAO;oBAC/BJ,SAAS;oBACT;;wBAAMD,GAAGsB,KAAK;;;oBAAd;oBACAC,IAAAA,kBAAU,EAACrC;oBACX;;wBAAOS,QAAQT;;;oBAGjB;;wBAAMF,aAAaC,MAAMC;;;oBAAzB;oBAI8B;;wBAAMK,IAAAA,iBAAO,EAACN,MAAM;;;oBAA5CqB,gBAAgBkB;yBAAQ,OAAA,2BAAA,kBAAA,OAA4C;;oBAC1E;;wBAAMvC,KAAKG,IAAI,CAAC,AAAC,yBAA8E,OAAtDqC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIrB,gBAAgB;;;oBAAtF;oBAE6B;;wBAAMsB,IAAAA,sBAAS,EAAC3C,MAAMC,KAAKA,IAAIwB,OAAO;;;oBAAtC,QAAA,eAArBH,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;oBAEhB;;wBAAO;4BAAER,IAAAA;4BAAIf,MAAAA;4BAAMC,KAAAA;4BAAKW,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;oBAC1CT;yBACH,CAACE,QAAD;;;;oBAAS;;wBAAMD,GAAGsB,KAAK;;;oBAAd;;;oBACb,MAAMvB;;;;;;;IAEV;;AAIO,SAAehB,UAAUG,GAAmB;;YACgB,MAAzDc,IAAIf,MAAW4C,aAAahC,QAAQU,QAAQC;;;;oBAAa;;wBAAMb,QAAQT;;;oBAAd,OAAA,eAAzDc,KAAyD,KAAzDA,IAAIf,OAAqD,KAArDA,MAAW4C,cAA0C,KAA/C3C,KAAkBW,SAA6B,KAA7BA,QAAQU,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBACpD;;wBAAO;4BAAEsB,OAAOC,IAAAA,oBAAW,EAAC/B,IAAIf,MAAM4C,aAAaA,YAAYnB,OAAO;4BAAGxB,KAAK2C;4BAAahC,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACtH"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  await conn.exec(`CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`);\n  await conn.exec(`CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n  db: Database;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below must release this handle, or the leaked WAL makes the db file undeletable on\n  // Windows. closed guards the catch against double-closing.\n  let closed = false;\n  try {\n    const conn = createConnection(db);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched\n      // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      closed = true;\n      await db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) await db.close();\n    throw err;\n  }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","ensureSchema","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","turso","err","db","closed","version","features","wantFeatures","reason","recordedMaxMs","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","console","error","close","clearCache","Number","Math","min","max","reconcile","resolvedCfg","store","createStore","rekeyChunkText"],"mappings":";;;;;;;;;;;QAiBaA;eAAAA;;QAGAC;eAAAA;;QAwGSC;eAAAA;;;sBA5HI;wBACL;uBAGuB;yBACb;uBACJ;wBACA;wBACc;uBACd;wBACM;4BAEA;wBACO;2BACd;uBACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMF,cAAc;AAGpB,IAAMC,iBAAiB;AAU9B,+FAA+F;AAC/F,2FAA2F;AAC3F,SAAeE,aAAaC,IAAgB,EAAEC,GAAW;;YAOlD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBANX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,wIAAkJ,OAAXG,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBH;;;oBAAtC;;;oBACjD;;wBAAMW,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAWA,SAAeC,QAAQX,GAAmB;;YAClCY,UAEAC,QAIFC,OAGKC,KAKLC,IAKKD,MAKLE,QAwB4B,MAtBxBlB,MAIAmB,SACAC,UACAC,cAIEC,QAYFC,eAGuB,OAArBC,QAAQC,UAGTT;;;;oBAtDHH,WAAWa,IAAAA,cAAI,EAACzB,IAAI0B,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAAChB,UAAU;wBAAEiB,WAAW;oBAAK;oBAChChB,SAASY,IAAAA,cAAI,EAACb,UAAUjB;;;;;;;;;oBAMpB;;wBAAMmC,IAAAA,kBAAQ;;;oBAAtBhB,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAegB,oBAAU,GAAE,MAAMhB;oBACrC,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACjB,IAAckB,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMnB,MAAMH,OAAO,CAACE,QAAQ;4BAAEqB,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHnB,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXlB,QAAO,MAA2B,OAAvB,AAACE,KAAckB,OAAO;;oBAEpH,6FAA6F;oBAC7F,2DAA2D;oBACvDhB,SAAS;;;;;;;;;oBAELlB,OAAOqC,IAAAA,8BAAgB,EAACpB;oBAE9B;;wBAAMjB,KAAKG,IAAI,CAAC;;;oBAAhB;oBAEgB;;wBAAMK,IAAAA,iBAAO,EAACR,MAAM;;;oBAA9BmB,UAAU;oBACC;;wBAAMX,IAAAA,iBAAO,EAACR,MAAM;;;oBAA/BoB,WAAW;oBACXC,eAAeX,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;yBAC/C,CAAA,AAACQ,YAAY,QAAQA,YAAYtB,kBAAoBuB,aAAa,QAAQA,aAAaC,YAAY,GAAnG;;;;oBACF,yFAAyF;oBACzF,iGAAiG;oBAC3FC,SAASH,YAAY,QAAQA,YAAYtB,iBAAiB,mDAAmD;oBACnHyC,QAAQC,KAAK,CAAC,AAAC,UAAgB,OAAPjB,QAAO;oBAC/BJ,SAAS;oBACT;;wBAAMD,GAAGuB,KAAK;;;oBAAd;oBACAC,IAAAA,mBAAU,EAACxC;oBACX;;wBAAOW,QAAQX;;;oBAGjB;;wBAAMF,aAAaC,MAAMC;;;oBAAzB;oBAI8B;;wBAAMO,IAAAA,iBAAO,EAACR,MAAM;;;oBAA5CuB,gBAAgBmB;yBAAQ,OAAA,2BAAA,kBAAA,OAA4C;;oBAC1E;;wBAAM1C,KAAKG,IAAI,CAAC,AAAC,yBAA8E,OAAtDwC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAItB,gBAAgB;;;oBAAtF;oBAE6B;;wBAAMuB,IAAAA,sBAAS,EAAC9C,MAAMC,KAAKA,IAAI0B,OAAO;;;oBAAtC,QAAA,eAArBH,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;oBAEhB;;wBAAO;4BAAER,IAAAA;4BAAIjB,MAAAA;4BAAMC,KAAAA;4BAAKa,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;oBAC1CT;yBACH,CAACE,QAAD;;;;oBAAS;;wBAAMD,GAAGuB,KAAK;;;oBAAd;;;oBACb,MAAMxB;;;;;;;IAEV;;AAIO,SAAelB,UAAUG,GAAmB;;YACgB,MAAzDgB,IAAIjB,MAAW+C,aAAajC,QAAQU,QAAQC,UAC9CuB;;;;oBAD2D;;wBAAMpC,QAAQX;;;oBAAd,OAAA,eAAzDgB,KAAyD,KAAzDA,IAAIjB,OAAqD,KAArDA,MAAW+C,cAA0C,KAA/C9C,KAAkBa,SAA6B,KAA7BA,QAAQU,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAC9CuB,QAAQC,IAAAA,oBAAW,EAAChC,IAAIjB,MAAM+C,aAAaA,YAAYpB,OAAO;oBACpE,0FAA0F;oBAC1FuB,IAAAA,yBAAc,EAAClD,MAAMgD;oBACrB;;wBAAO;4BAAEA,OAAOA;4BAAO/C,KAAK8C;4BAAajC,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACpE"}

package/dist/cjs/store/turso/reconcile.js

@@ -13,6 +13,7 @@ var _indexts = require("../../features/index.js");
 var _progressts = require("../../output/progress.js");
 var _indexts1 = require("../../scan/index.js");
 var _reparsets = require("../../scan/reparse.js");
+var _segmentts = require("../../text/segment.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 function _array_like_to_array(arr, len) {
@@ -187,13 +188,8 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Map" || n === "Set") return Array.from(n);
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
-// Fork of sqlite/reconcile.ts (house precedent: each store owns its engine-specific SQL), not
-// duckdb's, even though duckdb's content handling is the structurally closer of the two. duckdb
-// dropped sqlite's reconcile_max_ms bookkeeping entirely, and this store's busy timeout (open.ts)
-// is derived from exactly that recorded value -- forking duckdb's file would lose it silently.
-// content itself follows duckdb's shape instead: a plain table, no rowid coupling, no FTS index
-// (phase 2's job, not reconcile's), and no `_seg` sidecars (turso's unspaced-script path is not
-// FTS5-shaped).
+// Fork of sqlite/reconcile.ts, not duckdb's, so this store keeps reconcile_max_ms bookkeeping
+// (open.ts's busy timeout derives from it). content is a plain table with "_ngram" sidecars for lexical.ts.
 var CORE_FRONTMATTER_COLUMNS = new Set([
     'path',
     '_mtime',
@@ -201,20 +197,26 @@ var CORE_FRONTMATTER_COLUMNS = new Set([
     '_size',
     '_parse_error'
 ]);
-// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with no
-// error). The real fence is a SELECT projecting more than this many result columns, which fails
-// to prepare (spike-measured, both `SELECT *` and an explicit list), so a wide frontmatter table
-// would pass every write silently and only break the first time something selects across it.
-// Enforced here, before ALTER, the same shape as sqlite's compile-time fence.
+// '' when the field has no unspaced-script run (the common case), so the ngram index carries
+// nothing for it -- same "pay for nothing when absent" shape as sqlite's segmentField sidecars.
+function ngramSidecar(text) {
+    return (0, _segmentts.hasUnspacedRun)(text) ? text : '';
+}
+// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with
+// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.
 var MAX_FRONTMATTER_COLUMNS = 2000;
 // No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.
-var INSERT_CONTENT_SQL = 'INSERT INTO content ("path", title, summary, text) VALUES (?, ?, ?, ?)';
+var INSERT_CONTENT_SQL = 'INSERT INTO content ("path", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)';
 function contentRow(doc) {
+    var _doc_search = doc.search, title = _doc_search.title, summary = _doc_search.summary, text = _doc_search.text;
     return [
         doc.relPath,
-        doc.search.title,
-        doc.search.summary,
-        doc.search.text
+        title,
+        summary,
+        text,
+        ngramSidecar(title),
+        ngramSidecar(summary),
+        ngramSidecar(text)
     ];
 }
 function reconcile(conn, cfg, baseDir) {
@@ -307,7 +309,7 @@ function reconcile(conn, cfg, baseDir) {
                     writableColumns = allColumns.filter(function(c) {
                         return CORE_FRONTMATTER_COLUMNS.has(c) || !_indexts1.RESERVED_COLUMNS.has(c);
                     });
-                    // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's identity stable across reparses.
+                    // ON CONFLICT UPDATE, rather than the delete-and-insert `OR` clause, keeps the row's identity stable across reparses.
                     insertSql = "INSERT INTO frontmatter (".concat(writableColumns.map(_sharedts.quoteIdent).join(', '), ") VALUES (").concat(writableColumns.map(function() {
                         return '?';
                     }).join(', '), ') ON CONFLICT("path") DO UPDATE SET ').concat(writableColumns.filter(function(c) {
@@ -753,10 +755,8 @@ function reconcile(conn, cfg, baseDir) {
                     ];
                 case 5:
                     _state.sent();
-                    // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the
-                    // observed max so a big watcher reconcile's lock hold is what the next open bounds its wait
-                    // against (the same mechanism sqlite's reconcile records, and the one this store's busy
-                    // timeout is derived from).
+                    // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: the observed
+                    // max is what the next open bounds its wait against.
                     durationMs = Date.now() - txStart;
                     return [
                         4,
@@ -765,7 +765,7 @@ function reconcile(conn, cfg, baseDir) {
                 case 6:
                     prevRaw = _state.sent();
                     // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)
-                    // still gets recorded instead of losing to the "nothing recorded yet" default.
+                    // gets recorded rather than losing to the "nothing recorded yet" default.
                     prevMax = prevRaw === null ? -1 : Number(prevRaw);
                     if (!(durationMs > prevMax)) return [
                         3,

package/dist/cjs/store/turso/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Fork of sqlite/reconcile.ts (house precedent: each store owns its engine-specific SQL), not\n// duckdb's, even though duckdb's content handling is the structurally closer of the two. duckdb\n// dropped sqlite's reconcile_max_ms bookkeeping entirely, and this store's busy timeout (open.ts)\n// is derived from exactly that recorded value -- forking duckdb's file would lose it silently.\n// content itself follows duckdb's shape instead: a plain table, no rowid coupling, no FTS index\n// (phase 2's job, not reconcile's), and no `_seg` sidecars (turso's unspaced-script path is not\n// FTS5-shaped).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with no\n// error). The real fence is a SELECT projecting more than this many result columns, which fails\n// to prepare (spike-measured, both `SELECT *` and an explicit list), so a wide frontmatter table\n// would pass every write silently and only break the first time something selects across it.\n// Enforced here, before ALTER, the same shape as sqlite's compile-time fence.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's identity stable across reparses.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n    // content), so vanished and reparsed docs delete in one pass.\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n  // observed max so a big watcher reconcile's lock hold is what the next open bounds its wait\n  // against (the same mechanism sqlite's reconcile records, and the one this store's busy\n  // timeout is derived from).\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["reconcile","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","conn","cfg","baseDir","files","currentSet","existingStmt","existingRows","existing","vanished","toReparse","features","seenColumns","report","parsedDocs","warnings","newColumns","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","txStart","durationMs","prevRaw","prevMax","listFiles","map","f","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","activeFeatures","getColumns","progress","reparseFiles","tick","docs","finish","add","SenseError","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","feature","rows","contentTouched","presetTouched","presetRows","presetName","removedPaths","exec","ctimeMs","parseError","data","runBatch","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","getMeta","Number","setMeta","String"],"mappings":";;;;+BAmCsBA;;;eAAAA;;;wBAlCK;uBACI;0BAEN;wBAEmB;yBACf;wBAC4B;6BACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,8FAA8F;AAC9F,gGAAgG;AAChG,kGAAkG;AAClG,+FAA+F;AAC/F,gGAAgG;AAChG,gGAAgG;AAChG,gBAAgB;AAChB,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,iGAAiG;AACjG,gGAAgG;AAChG,iGAAiG;AACjG,6FAA6F;AAC7F,8EAA8E;AAC9E,IAAMC,0BAA0B;AAEhC,yFAAyF;AACzF,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEO,SAAeX,UAAUY,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YACtEC,OACAC,YAEAC,cACAC,cACAC,UACAC,UAEAC,WAOAC,UACAC,aAIAC,QAC6C,MAArCC,YAAYC,UAAUC,YAE/B,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YASAC,iBAEAC,WAKAC,OACAC,OACAC,UACAC,kBAEAC,SA4DAC,YACAC,SAGAC;;;;oBA9GAxB,QAAQyB,IAAAA,mBAAS,EAAC3B,KAAKC;oBACvBE,aAAa,IAAId,IAAIa,MAAM0B,GAAG,CAAC,SAACC;+BAAMA,EAAEnC,OAAO;;oBAEhC;;wBAAMK,KAAK+B,OAAO,CAAC;;;oBAAlC1B,eAAe;oBACC;;wBAAMA,aAAa2B,GAAG;;;oBAAtC1B,eAAgB;oBAChBC,WAAW,IAAI0B,IAAI3B,aAAauB,GAAG,CAAC,SAACK;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBACtD1B,WAAWF,aAAa8B,MAAM,CAAC,SAACF;+BAAM,CAAC9B,WAAWiC,GAAG,CAACH,EAAEC,IAAI;uBAAGN,GAAG,CAAC,SAACK;+BAAMA,EAAEC,IAAI;;oBAEhF1B,YAAYN,MAAMiC,MAAM,CAAC,SAACN;wBAC9B,IAAMQ,MAAM/B,SAASgC,GAAG,CAACT,EAAEnC,OAAO;wBAClC,OAAO,CAAC2C,OAAOA,IAAIE,MAAM,KAAKV,EAAEW,OAAO,IAAIH,IAAII,KAAK,KAAKZ,EAAEa,IAAI;oBACjE;oBAEA,IAAInC,SAASoC,MAAM,KAAK,KAAKnC,UAAUmC,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAG/B,QAAQ;wBAAK;;oBAEhFJ,WAAWoC,IAAAA,uBAAc,EAAC7C;oBACZ;;wBAAM8C,IAAAA,oBAAU,EAAC/C;;;oBAA/BW,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAASoC,IAAAA,oBAAQ,EAAC,mBAAmBvC,UAAUmC,MAAM;oBACR;;wBAAMK,IAAAA,uBAAY,EAACxC,WAAWC,UAAUT,KAAKU,aAAaC,OAAOsC,IAAI;;;oBAArE,OAAA,eAArCrC,aAAqC,KAA3CsC,MAAkBrC,WAAyB,KAAzBA,UAAUC,aAAe,KAAfA;oBACpCH,OAAOwC,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAarC,iCAAb,6BAAA,QAAA,yBAAA;4BAAMC,MAAN;4BAAyBL,YAAY0C,GAAG,CAACrC;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGN;oBACvB,IAAIM,WAAW2B,MAAM,GAAGrD,yBAAyB;wBAC/C,MAAM,IAAI+D,oBAAU,CAClB,gBACA,AAAC,0BAAwG/D,OAA/E0B,WAAW2B,MAAM,EAAC,+DAAqF,OAAxBrD,yBAAwB;oBAErI;oBACA,0FAA0F;oBAC1F,sEAAsE;oBAChE2B,kBAAkBD,WAAWmB,MAAM,CAAC,SAACmB;+BAAMlE,yBAAyBgD,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBAC1G,uFAAuF;oBACjFpC,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBW,GAAG,CAAC4B,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FxC,OAAhFA,gBAAgBW,GAAG,CAAC;+BAAM;uBAAK6B,IAAI,CAAC,OAAM,wCAGjI,OAHuKxC,gBAClLkB,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpB1B,GAAG,CAAC,SAAC0B;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEFtC,QAAQX,UAAU2B,MAAM,CAAC,SAACN;+BAAM,CAACvB,SAAS8B,GAAG,CAACP,EAAEnC,OAAO;uBAAGkC,GAAG,CAAC,SAACC;+BAAMA,EAAEnC,OAAO;;oBAC9E0B,QAAwB;wBAAElB,OAAAA;wBAAOwD,UAAU9C,WAAWgB,GAAG,CAAC,SAAC+B;mCAAMA,EAAEjE,OAAO;;wBAAGyB,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAIhC,IAAI8B;oBACnBG,mBAAmBV,WAAWgB,GAAG,CAAC,SAAC+B;+BAAMA,EAAEjE,OAAO;uBAAEyC,MAAM,CAAC,SAACyB;+BAAM,CAACvC,SAASe,GAAG,CAACwB;;oBAEhFrC,UAAUsC,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EAAChE,MAAM;;oCA+CyCiE,iBAK7BA,yBAnDjC,2BAAA,mBAAA,gBAAA,WAAA,OAAMjD,UAGHkD,MAgBFC,gBAgBAC,eAMAC,YAC6B,4BAAA,oBAAA,iBAA9B,4BAAA,oBAAA,iBAAA,YAAA,QAAM3E,KAAwB,YAAA,QAAM4E,YAGnCC,cAC4B,4BAAA,oBAAA,iBAAA,YAAA,QAAMN,cACnC,4BAAA,oBAAA,wBAAA,YAAA,aAIA,4BAAA,oBAAA,iBAAA,YAAA,QAAMA;;;;4CAnDN,kCAAA,2BAAA;;;;;;;;;4CAAA,YAAalD;;;mDAAb,6BAAA,QAAA;;;;4CAAMC,MAAN;4CAAyB;;gDAAMhB,KAAKwE,IAAI,CAAC,AAAC,sCAAqD,OAAhBf,IAAAA,oBAAU,EAACzC;;;4CAAjE;;;4CAAzB;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAEDH,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CACIqD,OAAOrD,WAAWgB,GAAG,CAAC,SAACnC;uDAC3BwB,gBAAgBW,GAAG,CAAC,SAACb;wDAOZtB;oDANP,IAAIsB,QAAQ,QAAQ,OAAOtB,IAAIC,OAAO;oDACtC,IAAIqB,QAAQ,UAAU,OAAOtB,IAAI+C,OAAO;oDACxC,IAAIzB,QAAQ,UAAU,OAAOtB,IAAI+E,OAAO;oDACxC,IAAIzD,QAAQ,SAAS,OAAOtB,IAAIiD,IAAI;oDACpC,mFAAmF;oDACnF,IAAI3B,QAAQ,gBAAgB,OAAOtB,IAAIgF,UAAU;oDACjD,QAAOhF,gBAAAA,IAAIiF,IAAI,CAAC3D,IAAI,cAAbtB,2BAAAA,gBAAiB;gDAC1B;;4CAEF;;gDAAMM,KAAK4E,QAAQ,CAACzD,WAAW+C;;;4CAA/B;;;4CAGF,0FAA0F;4CAC1F,8DAA8D;4CACxDC,iBAAiB,AAAC,qBAAG3D,iBAAU,qBAAGe;iDACpC4C,CAAAA,eAAevB,MAAM,GAAG,CAAA,GAAxBuB;;;;4CACF;;gDAAMnE,KAAK4E,QAAQ,CACjB,wCACAT,eAAetC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF/B;;;iDAIEhD,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CAAuB;;gDAAMb,KAAK4E,QAAQ,CAACpF,oBAAoBqB,WAAWgB,GAAG,CAACpC;;;4CAAvD;;;iDAEvBe,CAAAA,SAASoC,MAAM,GAAG,CAAA,GAAlBpC;;;;4CACF;;gDAAMR,KAAK4E,QAAQ,CACjB,4CACApE,SAASqB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;4CAKF,0FAA0F;4CAC1F,kEAAkE;4CAC5DO,gBAAgB,AAAC,qBAAG5D,iBAAU,qBAAGe;iDACnC6C,CAAAA,cAAcxB,MAAM,GAAG,CAAA,GAAvBwB;;;;4CACF;;gDAAMpE,KAAK4E,QAAQ,CACjB,6CACAR,cAAcvC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF9B;;;4CAIIQ;4CAC6B,mCAAA,4BAAA,6BAA9B,mCAAA,4BAAA;;gDAAL,IAAK,aAAaxD,iCAAb,8BAAA,SAAA,0BAAA;oDAAMnB,MAAN;;wDAAyB,IAAK,aAAoBA,IAAImF,OAAO,uBAA/B,8BAAA,SAAA,0BAAA;4DAAMP,aAAN;4DAAiCD,WAAWS,IAAI;gEAAEpF,IAAIC,OAAO;gEAAE2E;;;;wDAA/D;wDAAA;;;iEAAA,8BAAA;gEAAA;;;gEAAA;sEAAA;;;;;;gDAA9B;gDAAA;;;yDAAA,8BAAA;wDAAA;;;wDAAA;8DAAA;;;;iDACDD,CAAAA,WAAWzB,MAAM,GAAG,CAAA,GAApByB;;;;4CAAuB;;gDAAMrE,KAAK4E,QAAQ,CAAC,2DAA2DP;;;4CAA/E;;;4CAErBE,eAAe,AAAC,qBAAG/D,iBAAU,qBAAGe;4CACJ,mCAAA,4BAAA;iDAA9BgD,CAAAA,aAAa3B,MAAM,GAAG,CAAA,GAAtB2B;;;;;;;;;;;;4CAA8B,aAAiB7D;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,UAAN;4CAA2B;;iDAAMA,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBjE,MAAMuE,cAAclD;;;4CAA3C;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAC7B,mCAAA,4BAAA;;;;;;;;;;oDAAM4C,SAEHA,gBADAe;;;;4DADGf,UAAN;4DACGe,iBAAiCnE,WAAWgB,GAAG,CAAC,SAACnC;uEAAS;oEAAEyC,MAAMzC,IAAIC,OAAO;oEAAEsF,WAAWvF,IAAIuF,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gEAAC;;4DAC5H;;iEAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBjE,MAAMgF,gBAAgB3D;;;4DAA5C;;;;;;4CACF;4CAHK,aAAiBX;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;4CAAA,aAAiBA;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,WAAN;4CAA2B;;iDAAMA,0BAAAA,SAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,UAAyBjE,MAAMqB;;;4CAArC;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;;;;oBArDA;oBAuDA,0FAA0F;oBAC1F,4FAA4F;oBAC5F,wFAAwF;oBACxF,4BAA4B;oBACtBI,aAAaqC,KAAKC,GAAG,KAAKvC;oBAChB;;wBAAM6D,IAAAA,iBAAO,EAACrF,MAAM;;;oBAA9B0B,UAAU;oBAChB,yFAAyF;oBACzF,+EAA+E;oBACzEC,UAAUD,YAAY,OAAO,CAAC,IAAI4D,OAAO5D;yBAC3CD,CAAAA,aAAaE,OAAM,GAAnBF;;;;oBAAsB;;wBAAM8D,IAAAA,iBAAO,EAACvF,MAAM,oBAAoBwF,OAAO/D;;;oBAA/C;;;oBAE1B;;wBAAO;4BAAEoB,QAAQhC,WAAW+B,MAAM;4BAAE9B,UAAAA;wBAAS;;;;IAC/C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Fork of sqlite/reconcile.ts, not duckdb's, so this store keeps reconcile_max_ms bookkeeping\n// (open.ts's busy timeout derives from it). content is a plain table with \"_ngram\" sidecars for lexical.ts.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE, rather than the delete-and-insert `OR` clause, keeps the row's identity stable across reparses.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n    // content), so vanished and reparsed docs delete in one pass.\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: the observed\n  // max is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // gets recorded rather than losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["reconcile","CORE_FRONTMATTER_COLUMNS","Set","ngramSidecar","text","hasUnspacedRun","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","search","title","summary","relPath","conn","cfg","baseDir","files","currentSet","existingStmt","existingRows","existing","vanished","toReparse","features","seenColumns","report","parsedDocs","warnings","newColumns","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","txStart","durationMs","prevRaw","prevMax","listFiles","map","f","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","activeFeatures","getColumns","progress","reparseFiles","tick","docs","finish","add","SenseError","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","feature","rows","contentTouched","presetTouched","presetRows","presetName","removedPaths","exec","ctimeMs","parseError","data","runBatch","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","getMeta","Number","setMeta","String"],"mappings":";;;;+BAmCsBA;;;eAAAA;;;wBAlCK;uBACI;0BAEN;wBAEmB;yBACf;yBACE;wBAC0B;6BACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,8FAA8F;AAC9F,4GAA4G;AAC5G,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAOC,IAAAA,yBAAc,EAACD,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,IAAME,0BAA0B;AAEhC,yFAAyF;AACzF,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,IAAiCA,cAAAA,IAAIC,MAAM,EAAnCC,QAAyBF,YAAzBE,OAAOC,UAAkBH,YAAlBG,SAASR,OAASK,YAATL;IACxB,OAAO;QAACK,IAAII,OAAO;QAAEF;QAAOC;QAASR;QAAMD,aAAaQ;QAAQR,aAAaS;QAAUT,aAAaC;KAAM;AAC5G;AAEO,SAAeJ,UAAUc,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YACtEC,OACAC,YAEAC,cACAC,cACAC,UACAC,UAEAC,WAOAC,UACAC,aAIAC,QAC6C,MAArCC,YAAYC,UAAUC,YAE/B,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YASAC,iBAEAC,WAKAC,OACAC,OACAC,UACAC,kBAEAC,SA0DAC,YACAC,SAGAC;;;;oBA5GAxB,QAAQyB,IAAAA,mBAAS,EAAC3B,KAAKC;oBACvBE,aAAa,IAAIhB,IAAIe,MAAM0B,GAAG,CAAC,SAACC;+BAAMA,EAAE/B,OAAO;;oBAEhC;;wBAAMC,KAAK+B,OAAO,CAAC;;;oBAAlC1B,eAAe;oBACC;;wBAAMA,aAAa2B,GAAG;;;oBAAtC1B,eAAgB;oBAChBC,WAAW,IAAI0B,IAAI3B,aAAauB,GAAG,CAAC,SAACK;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBACtD1B,WAAWF,aAAa8B,MAAM,CAAC,SAACF;+BAAM,CAAC9B,WAAWiC,GAAG,CAACH,EAAEC,IAAI;uBAAGN,GAAG,CAAC,SAACK;+BAAMA,EAAEC,IAAI;;oBAEhF1B,YAAYN,MAAMiC,MAAM,CAAC,SAACN;wBAC9B,IAAMQ,MAAM/B,SAASgC,GAAG,CAACT,EAAE/B,OAAO;wBAClC,OAAO,CAACuC,OAAOA,IAAIE,MAAM,KAAKV,EAAEW,OAAO,IAAIH,IAAII,KAAK,KAAKZ,EAAEa,IAAI;oBACjE;oBAEA,IAAInC,SAASoC,MAAM,KAAK,KAAKnC,UAAUmC,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAG/B,QAAQ;wBAAK;;oBAEhFJ,WAAWoC,IAAAA,uBAAc,EAAC7C;oBACZ;;wBAAM8C,IAAAA,oBAAU,EAAC/C;;;oBAA/BW,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAASoC,IAAAA,oBAAQ,EAAC,mBAAmBvC,UAAUmC,MAAM;oBACR;;wBAAMK,IAAAA,uBAAY,EAACxC,WAAWC,UAAUT,KAAKU,aAAaC,OAAOsC,IAAI;;;oBAArE,OAAA,eAArCrC,aAAqC,KAA3CsC,MAAkBrC,WAAyB,KAAzBA,UAAUC,aAAe,KAAfA;oBACpCH,OAAOwC,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAarC,iCAAb,6BAAA,QAAA,yBAAA;4BAAMC,MAAN;4BAAyBL,YAAY0C,GAAG,CAACrC;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGN;oBACvB,IAAIM,WAAW2B,MAAM,GAAGpD,yBAAyB;wBAC/C,MAAM,IAAI8D,oBAAU,CAClB,gBACA,AAAC,0BAAwG9D,OAA/EyB,WAAW2B,MAAM,EAAC,+DAAqF,OAAxBpD,yBAAwB;oBAErI;oBACA,0FAA0F;oBAC1F,sEAAsE;oBAChE0B,kBAAkBD,WAAWmB,MAAM,CAAC,SAACmB;+BAAMpE,yBAAyBkD,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBAC1G,sHAAsH;oBAChHpC,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBW,GAAG,CAAC4B,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FxC,OAAhFA,gBAAgBW,GAAG,CAAC;+BAAM;uBAAK6B,IAAI,CAAC,OAAM,wCAGjI,OAHuKxC,gBAClLkB,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpB1B,GAAG,CAAC,SAAC0B;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEFtC,QAAQX,UAAU2B,MAAM,CAAC,SAACN;+BAAM,CAACvB,SAAS8B,GAAG,CAACP,EAAE/B,OAAO;uBAAG8B,GAAG,CAAC,SAACC;+BAAMA,EAAE/B,OAAO;;oBAC9EsB,QAAwB;wBAAElB,OAAAA;wBAAOwD,UAAU9C,WAAWgB,GAAG,CAAC,SAAC+B;mCAAMA,EAAE7D,OAAO;;wBAAGqB,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAIlC,IAAIgC;oBACnBG,mBAAmBV,WAAWgB,GAAG,CAAC,SAAC+B;+BAAMA,EAAE7D,OAAO;uBAAEqC,MAAM,CAAC,SAACyB;+BAAM,CAACvC,SAASe,GAAG,CAACwB;;oBAEhFrC,UAAUsC,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EAAChE,MAAM;;oCA+CyCiE,iBAK7BA,yBAnDjC,2BAAA,mBAAA,gBAAA,WAAA,OAAMjD,UAGHkD,MAgBFC,gBAgBAC,eAMAC,YAC6B,4BAAA,oBAAA,iBAA9B,4BAAA,oBAAA,iBAAA,YAAA,QAAM1E,KAAwB,YAAA,QAAM2E,YAGnCC,cAC4B,4BAAA,oBAAA,iBAAA,YAAA,QAAMN,cACnC,4BAAA,oBAAA,wBAAA,YAAA,aAIA,4BAAA,oBAAA,iBAAA,YAAA,QAAMA;;;;4CAnDN,kCAAA,2BAAA;;;;;;;;;4CAAA,YAAalD;;;mDAAb,6BAAA,QAAA;;;;4CAAMC,MAAN;4CAAyB;;gDAAMhB,KAAKwE,IAAI,CAAC,AAAC,sCAAqD,OAAhBf,IAAAA,oBAAU,EAACzC;;;4CAAjE;;;4CAAzB;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAEDH,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CACIqD,OAAOrD,WAAWgB,GAAG,CAAC,SAAClC;uDAC3BuB,gBAAgBW,GAAG,CAAC,SAACb;wDAOZrB;oDANP,IAAIqB,QAAQ,QAAQ,OAAOrB,IAAII,OAAO;oDACtC,IAAIiB,QAAQ,UAAU,OAAOrB,IAAI8C,OAAO;oDACxC,IAAIzB,QAAQ,UAAU,OAAOrB,IAAI8E,OAAO;oDACxC,IAAIzD,QAAQ,SAAS,OAAOrB,IAAIgD,IAAI;oDACpC,mFAAmF;oDACnF,IAAI3B,QAAQ,gBAAgB,OAAOrB,IAAI+E,UAAU;oDACjD,QAAO/E,gBAAAA,IAAIgF,IAAI,CAAC3D,IAAI,cAAbrB,2BAAAA,gBAAiB;gDAC1B;;4CAEF;;gDAAMK,KAAK4E,QAAQ,CAACzD,WAAW+C;;;4CAA/B;;;4CAGF,0FAA0F;4CAC1F,8DAA8D;4CACxDC,iBAAiB,AAAC,qBAAG3D,iBAAU,qBAAGe;iDACpC4C,CAAAA,eAAevB,MAAM,GAAG,CAAA,GAAxBuB;;;;4CACF;;gDAAMnE,KAAK4E,QAAQ,CACjB,wCACAT,eAAetC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF/B;;;iDAIEhD,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CAAuB;;gDAAMb,KAAK4E,QAAQ,CAACnF,oBAAoBoB,WAAWgB,GAAG,CAACnC;;;4CAAvD;;;iDAEvBc,CAAAA,SAASoC,MAAM,GAAG,CAAA,GAAlBpC;;;;4CACF;;gDAAMR,KAAK4E,QAAQ,CACjB,4CACApE,SAASqB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;4CAKF,0FAA0F;4CAC1F,kEAAkE;4CAC5DO,gBAAgB,AAAC,qBAAG5D,iBAAU,qBAAGe;iDACnC6C,CAAAA,cAAcxB,MAAM,GAAG,CAAA,GAAvBwB;;;;4CACF;;gDAAMpE,KAAK4E,QAAQ,CACjB,6CACAR,cAAcvC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF9B;;;4CAIIQ;4CAC6B,mCAAA,4BAAA,6BAA9B,mCAAA,4BAAA;;gDAAL,IAAK,aAAaxD,iCAAb,8BAAA,SAAA,0BAAA;oDAAMlB,MAAN;;wDAAyB,IAAK,aAAoBA,IAAIkF,OAAO,uBAA/B,8BAAA,SAAA,0BAAA;4DAAMP,aAAN;4DAAiCD,WAAWS,IAAI;gEAAEnF,IAAII,OAAO;gEAAEuE;;;;wDAA/D;wDAAA;;;iEAAA,8BAAA;gEAAA;;;gEAAA;sEAAA;;;;;;gDAA9B;gDAAA;;;yDAAA,8BAAA;wDAAA;;;wDAAA;8DAAA;;;;iDACDD,CAAAA,WAAWzB,MAAM,GAAG,CAAA,GAApByB;;;;4CAAuB;;gDAAMrE,KAAK4E,QAAQ,CAAC,2DAA2DP;;;4CAA/E;;;4CAErBE,eAAe,AAAC,qBAAG/D,iBAAU,qBAAGe;4CACJ,mCAAA,4BAAA;iDAA9BgD,CAAAA,aAAa3B,MAAM,GAAG,CAAA,GAAtB2B;;;;;;;;;;;;4CAA8B,aAAiB7D;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,UAAN;4CAA2B;;iDAAMA,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBjE,MAAMuE,cAAclD;;;4CAA3C;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAC7B,mCAAA,4BAAA;;;;;;;;;;oDAAM4C,SAEHA,gBADAe;;;;4DADGf,UAAN;4DACGe,iBAAiCnE,WAAWgB,GAAG,CAAC,SAAClC;uEAAS;oEAAEwC,MAAMxC,IAAII,OAAO;oEAAEkF,WAAWtF,IAAIsF,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gEAAC;;4DAC5H;;iEAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBjE,MAAMgF,gBAAgB3D;;;4DAA5C;;;;;;4CACF;4CAHK,aAAiBX;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;4CAAA,aAAiBA;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,WAAN;4CAA2B;;iDAAMA,0BAAAA,SAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,UAAyBjE,MAAMqB;;;4CAArC;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;;;;oBArDA;oBAuDA,8FAA8F;oBAC9F,qDAAqD;oBAC/CI,aAAaqC,KAAKC,GAAG,KAAKvC;oBAChB;;wBAAM6D,IAAAA,iBAAO,EAACrF,MAAM;;;oBAA9B0B,UAAU;oBAChB,yFAAyF;oBACzF,0EAA0E;oBACpEC,UAAUD,YAAY,OAAO,CAAC,IAAI4D,OAAO5D;yBAC3CD,CAAAA,aAAaE,OAAM,GAAnBF;;;;oBAAsB;;wBAAM8D,IAAAA,iBAAO,EAACvF,MAAM,oBAAoBwF,OAAO/D;;;oBAA/C;;;oBAE1B;;wBAAO;4BAAEoB,QAAQhC,WAAW+B,MAAM;4BAAE9B,UAAAA;wBAAS;;;;IAC/C"}

package/dist/cjs/store/turso/store.js

@@ -16,12 +16,14 @@ _export(exports, {
         return createStore;
     }
 });
-var _errorsts = require("../../errors.js");
+var _typests = require("../../embed/types.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 var _vectorsts = require("../vectors.js");
 var _fieldStatsts = require("./fieldStats.js");
+var _lexicalts = require("./lexical.js");
 var _reconcilets = require("./reconcile.js");
+var _vectorsts1 = require("./vectors.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -360,10 +362,11 @@ function _wrap_async_generator(fn) {
         return new _async_generator(fn.apply(this, arguments));
     };
 }
-var CAPABILITIES = new Set();
-function missing(name) {
-    throw new _errorsts.SenseError('STORE_CAPABILITY_MISSING', 'store "turso" does not implement "'.concat(name, '" in this build; set "store" to "sqlite" or "duckdb" in this tree\'s config'));
-}
+var CAPABILITIES = new Set([
+    'lexical',
+    'phrases',
+    'vectors'
+]);
 function createStore(db, conn, cfg, baseDir) {
     return {
         name: 'turso',
@@ -460,50 +463,24 @@ function createStore(db, conn, cfg, baseDir) {
             }
         },
         lexical: {
-            query: function query() {
-                return _async_to_generator(function() {
-                    return _ts_generator(this, function(_state) {
-                        return [
-                            2,
-                            missing('lexical')
-                        ];
-                    });
-                })();
+            query: function query(terms, opts) {
+                return (0, _lexicalts.queryLexical)(conn, terms, opts);
             }
         },
+        // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the
+        // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.
         vectors: {
             pending: function pending() {
                 return (0, _vectorsts.pendingRows)(conn);
             },
-            writeVectors: function writeVectors() {
-                return _async_to_generator(function() {
-                    return _ts_generator(this, function(_state) {
-                        return [
-                            2,
-                            missing('vectors')
-                        ];
-                    });
-                })();
+            writeVectors: function writeVectors(rows) {
+                return (0, _vectorsts1.writeVectorBatch)(conn, _typests.STORE_DIMS, rows);
             },
-            candidates: function candidates() {
-                return _async_to_generator(function() {
-                    return _ts_generator(this, function(_state) {
-                        return [
-                            2,
-                            missing('vectors')
-                        ];
-                    });
-                })();
+            candidates: function candidates(qv, _storeDims, fetch, allowed) {
+                return (0, _vectorsts1.scanCandidates)(conn, qv, _typests.STORE_DIMS, fetch, allowed);
             },
-            similar: function similar() {
-                return _async_to_generator(function() {
-                    return _ts_generator(this, function(_state) {
-                        return [
-                            2,
-                            missing('vectors')
-                        ];
-                    });
-                })();
+            similar: function similar(path, opts) {
+                return (0, _vectorsts1.scanSimilar)(conn, _typests.STORE_DIMS, path, opts);
             },
             hasVector: function hasVector(path) {
                 return (0, _vectorsts.hasVectorRow)(conn, path);