sensemaking 0.22.0 → 0.22.2

package/dist/cjs/store/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { recordLockWaitMs } from './lock-wait.ts';\nimport { getColumns, quoteIdent } from './shared.ts';\nimport { featureStage, type Stages, stageRecorder } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nexport const CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const start = process.hrtime.bigint();\n  const features = activeFeatures(cfg);\n  const stages = stageRecorder(features.map((f) => f.name));\n  const elapsed = () => Number(process.hrtime.bigint() - start) / 1e6;\n  const files = await stages.time('list', () => listFiles(cfg, baseDir));\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = await stages.time('existing', async () => {\n    const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n    return (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  });\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [], stages: stages.take(elapsed(), 0) };\n\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.\n  const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', () => reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool }));\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await stages.time('alter', () => dialect.addColumns(conn, missingColumns));\n\n      // Revalidated once the lock is held, before this process's own frontmatter upsert below:\n      // `added` came from a path read taken before this transaction's lock, so a path still\n      // called \"added\" here may already have a content row from a concurrent reconcile that\n      // committed while this one waited. One SELECT, not a DELETE per added file -- with no\n      // contention it returns the same set `existing` already ruled out, so nothing extra clears.\n      let contentTouched = touched;\n      if (added.length > 0)\n        contentTouched = await stages.time('added-recheck', async () => {\n          const currentPathsStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n          const currentPaths = new Set(((await currentPathsStmt.all()) as Array<{ path: string }>).map((r) => r.path));\n          const staleAdded = added.filter((p) => currentPaths.has(p));\n          return staleAdded.length > 0 ? [...touched, ...staleAdded] : touched;\n        });\n\n      if (parsedDocs.length > 0) {\n        const toRow = (doc: (typeof parsedDocs)[number]) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          });\n        // A path in `added` has no existing frontmatter row, so it can never conflict: where a\n        // dialect offers a faster append-only path (insertNew), only the rest go through the upsert.\n        await stages.time('fm-upsert', async () => {\n          if (dialect.insertNew) {\n            const newDocs = parsedDocs.filter((d) => addedSet.has(d.relPath));\n            const updateDocs = parsedDocs.filter((d) => !addedSet.has(d.relPath));\n            if (newDocs.length > 0) await dialect.insertNew(conn, 'frontmatter', writableColumns, newDocs.map(toRow));\n            if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));\n          } else {\n            await conn.runBatch(insertSql, parsedDocs.map(toRow));\n          }\n        });\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await stages.time('text-index', () => dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      await stages.time('presets', async () => {\n        if (touched.length > 0)\n          await conn.runBatch(\n            'DELETE FROM preset_files WHERE \"path\" = ?',\n            touched.map((p) => [p])\n          );\n        const presetRows: unknown[][] = [];\n        for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken\n        // before this transaction's lock, so a path this process calls \"added\" can already have\n        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.\n        if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n      });\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await stages.time('vanished', () =>\n          conn.runBatch(\n            'DELETE FROM frontmatter WHERE \"path\" = ?',\n            vanished.map((p) => [p])\n          )\n        );\n\n      // Timed per feature per hook, so link resolution and PageRank are named stages without\n      // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.\n      if (touched.length > 0) for (const feature of features) await stages.time(featureStage(feature.name, 'remove'), () => feature.remove?.(conn, touched, delta));\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await stages.time(featureStage(feature.name, 'store'), () => feature.store?.(conn, docsForFeature, delta));\n      }\n      for (const feature of features) await stages.time(featureStage(feature.name, 'after'), () => feature.afterReconcile?.(conn, delta));\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.\n  recordLockWaitMs(baseDir, durationMs);\n  if (dialect.recordDuration) await stages.time('meta', () => dialect.recordDuration?.(conn, durationMs));\n\n  return { parsed: parsedDocs.length, warnings, stages: stages.take(elapsed(), durationMs, workerParseMs) };\n}\n"],"names":["CORE_FRONTMATTER_COLUMNS","reconcile","Set","conn","cfg","baseDir","dialect","pool","forcedPaths","start","features","stages","elapsed","files","currentSet","existingRows","existing","vanished","toReparse","seenColumns","report","parsedDocs","warnings","newColumns","workerParseMs","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","touched","txStart","durationMs","process","hrtime","bigint","activeFeatures","stageRecorder","map","f","name","Number","time","listFiles","relPath","existingStmt","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","take","getColumns","progress","reparseFiles","tick","docs","finish","add","checkColumnLimit","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","present","missingColumns","contentTouched","toRow","addColumns","currentPathsStmt","currentPaths","staleAdded","doc","ctimeMs","parseError","data","newDocs","updateDocs","insertNew","runBatch","reconcileContent","presetRows","presetName","presets","push","feature","featureStage","remove","docsForFeature","extracted","store","afterReconcile","beginMode","recordLockWaitMs","recordDuration"],"mappings":";;;;;;;;;;;QAqBaA;eAAAA;;QAESC;eAAAA;;;uBAtBS;0BAEN;wBACmB;yBAEf;0BACI;wBACM;wBACkB;6BACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAWzB,IAAMD,2BAA2B,IAAIE,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,SAAeD,UAAUE,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;;YACtJC,OACAC,UACAC,QACAC,SACAC,OACAC,YAEAC,cAIAC,UAIAC,UAIAC,WAOAC,aAIAC,QAE4D,MAApDC,YAAYC,UAAUC,YAAYC,eAE3C,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YAMAC,iBAGAC,WAKAC,OACAC,OACAC,UACAC,kBAGAC,SAEAC,SA2FAC;;;;oBArJA1B,QAAQ2B,QAAQC,MAAM,CAACC,MAAM;oBAC7B5B,WAAW6B,IAAAA,uBAAc,EAACnC;oBAC1BO,SAAS6B,IAAAA,uBAAa,EAAC9B,SAAS+B,GAAG,CAAC,SAACC;+BAAMA,EAAEC,IAAI;;oBACjD/B,UAAU;+BAAMgC,OAAOR,QAAQC,MAAM,CAACC,MAAM,KAAK7B,SAAS;;oBAClD;;wBAAME,OAAOkC,IAAI,CAAC,QAAQ;mCAAMC,IAAAA,mBAAS,EAAC1C,KAAKC;;;;oBAAvDQ,QAAQ;oBACRC,aAAa,IAAIZ,IAAIW,MAAM4B,GAAG,CAAC,SAACC;+BAAMA,EAAEK,OAAO;;oBAEhC;;wBAAMpC,OAAOkC,IAAI,CAAC,YAAY;;oCAC3CG;;;;4CAAe;;gDAAM7C,KAAK8C,OAAO,CAAC;;;4CAAlCD,eAAe;4CACb;;gDAAMA,aAAaE,GAAG;;;4CAA9B;;gDAAQ;;;;4BACV;;;;oBAHMnC,eAAe;oBAIfC,WAAW,IAAImC,IAAIpC,aAAa0B,GAAG,CAAC,SAACW;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBAC5D,+FAA+F;oBAC/F,+FAA+F;oBAC/F,gFAAgF;oBAC1EnC,WAAWF,aAAauC,MAAM,CAAC,SAACF;+BAAM,CAACtC,WAAWyC,GAAG,CAACH,EAAEC,IAAI;uBAAGZ,GAAG,CAAC,SAACW;+BAAMA,EAAEC,IAAI;;oBAEtF,6FAA6F;oBAC7F,0FAA0F;oBACpFnC,YAAYL,MAAMyC,MAAM,CAAC,SAACZ;;wBAC9B,IAAMc,MAAMxC,SAASyC,GAAG,CAACf,EAAEK,OAAO;wBAClC,OAAO,CAACS,OAAOA,IAAIE,MAAM,KAAKhB,EAAEiB,OAAO,IAAIH,IAAII,KAAK,KAAKlB,EAAEmB,IAAI,aAAKrD,wBAAAA,kCAAAA,YAAa+C,GAAG,CAACb,EAAEK,OAAO,wCAAK;oBACrG;oBAEA,IAAI9B,SAAS6C,MAAM,KAAK,KAAK5C,UAAU4C,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAGzC,QAAQ;4BAAMX,QAAQA,OAAOqD,IAAI,CAACpD,WAAW;wBAAG;;oBAErG;;wBAAMqD,IAAAA,oBAAU,EAAC9D;;;oBAA/BgB,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAAS8C,IAAAA,oBAAQ,EAAC,mBAAmBhD,UAAU4C,MAAM;oBAEO;;wBAAMnD,OAAOkC,IAAI,CAAC,SAAS;mCAAMsB,IAAAA,uBAAY,EAACjD,WAAWR,UAAUN,KAAKe,aAAaC,OAAOgD,IAAI,EAAE;gCAAE7D,MAAAA;4BAAK;;;;oBAAzG,OAAA,eAApDc,aAAoD,KAA1DgD,MAAkB/C,WAAwC,KAAxCA,UAAUC,aAA8B,KAA9BA,YAAYC,gBAAkB,KAAlBA;oBAChDJ,OAAOkD,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAa/C,iCAAb,6BAAA,QAAA,yBAAA;4BAAME,MAAN;4BAAyBN,YAAYoD,GAAG,CAAC9C;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGP;oBACvB,yFAAyF;oBACzF,iGAAiG;oBACjGb,QAAQkE,gBAAgB,CAAC9C,WAAWoC,MAAM;oBAC1C,0FAA0F;oBAC1F,sEAAsE;oBAChEnC,kBAAkBD,WAAW4B,MAAM,CAAC,SAACmB;+BAAMzE,yBAAyBuD,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBAC1G,yFAAyF;oBACzF,mDAAmD;oBAC7C7C,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBc,GAAG,CAACkC,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FjD,OAAhFA,gBAAgBc,GAAG,CAAC;+BAAM;uBAAKmC,IAAI,CAAC,OAAM,wCAGjI,OAHuKjD,gBAClL2B,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpBhC,GAAG,CAAC,SAACgC;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEF/C,QAAQX,UAAUoC,MAAM,CAAC,SAACZ;+BAAM,CAAC1B,SAASuC,GAAG,CAACb,EAAEK,OAAO;uBAAGN,GAAG,CAAC,SAACC;+BAAMA,EAAEK,OAAO;;oBAC9EjB,QAAwB;wBAAEjB,OAAAA;wBAAOgE,UAAUxD,WAAWoB,GAAG,CAAC,SAACqC;mCAAMA,EAAE/B,OAAO;;wBAAGlB,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAI7B,IAAI2B;oBACnBG,mBAAmBX,WAAWoB,GAAG,CAAC,SAACqC;+BAAMA,EAAE/B,OAAO;uBAAEO,MAAM,CAAC,SAACyB;+BAAM,CAAChD,SAASwB,GAAG,CAACwB;;oBACtF,+FAA+F;oBAC/F,0EAA0E;oBACpE9C,UAAU,AAAC,qBAAGhB,iBAAU,qBAAGe;oBAE3BE,UAAU8C,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EACnB/E,MACA;;oCAGQgF,SACAC,gBAQFC,gBAUIC,OAwDqB,2BAAA,mBAAA,uBAAA,WAAA,YACxB,4BAAA,oBAAA,yBAAA,YAAA,aAIA,4BAAA,oBAAA,yBAAA,YAAA;;;;4CAhFW;;gDAAMrB,IAAAA,oBAAU,EAAC9D;;;4CAA3BgF,UAAU;4CACVC,iBAAiB7D,WAAW+B,MAAM,CAAC,SAAC7B;uDAAQ,CAAC0D,QAAQ5B,GAAG,CAAC9B;;iDAC3D2D,CAAAA,eAAetB,MAAM,GAAG,CAAA,GAAxBsB;;;;4CAA2B;;gDAAMzE,OAAOkC,IAAI,CAAC,SAAS;2DAAMvC,QAAQiF,UAAU,CAACpF,MAAMiF;;;;4CAA1D;;;4CAE/B,yFAAyF;4CACzF,sFAAsF;4CACtF,sFAAsF;4CACtF,sFAAsF;4CACtF,4FAA4F;4CACxFC,iBAAiBpD;iDACjBJ,CAAAA,MAAMiC,MAAM,GAAG,CAAA,GAAfjC;;;;4CACe;;gDAAMlB,OAAOkC,IAAI,CAAC,iBAAiB;;4DAC5C2C,kBACAC,iBACAC;;;;oEAFmB;;wEAAMvF,KAAK8C,OAAO,CAAC;;;oEAAtCuC,mBAAmB;wEACAtF;oEAAM;;wEAAMsF,iBAAiBtC,GAAG;;;oEAAnDuC,eAAe,IAAA,CAAA,EAAA,MAAIvF;;wEAAM,cAA0DuC,GAAG,CAAC,SAACW;mFAAMA,EAAEC,IAAI;;sEAAC;oEACrGqC,aAAa7D,MAAMyB,MAAM,CAAC,SAACyB;+EAAMU,aAAalC,GAAG,CAACwB;;oEACxD;;wEAAOW,WAAW5B,MAAM,GAAG,IAAI,AAAC,qBAAG7B,gBAAS,qBAAGyD,eAAczD;;;;oDAC/D;;;;4CALAoD,iBAAiB;;;iDAOfhE,CAAAA,WAAWyC,MAAM,GAAG,CAAA,GAApBzC;;;;4CACIiE,QAAQ,eAACK;uDACbhE,gBAAgBc,GAAG,CAAC,SAAChB;wDAOZkE;oDANP,IAAIlE,QAAQ,QAAQ,OAAOkE,IAAI5C,OAAO;oDACtC,IAAItB,QAAQ,UAAU,OAAOkE,IAAIhC,OAAO;oDACxC,IAAIlC,QAAQ,UAAU,OAAOkE,IAAIC,OAAO;oDACxC,IAAInE,QAAQ,SAAS,OAAOkE,IAAI9B,IAAI;oDACpC,mFAAmF;oDACnF,IAAIpC,QAAQ,gBAAgB,OAAOkE,IAAIE,UAAU;oDACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAACrE,IAAI,cAAbkE,2BAAAA,gBAAiB;gDAC1B;;4CACF,uFAAuF;4CACvF,6FAA6F;4CAC7F;;gDAAMhF,OAAOkC,IAAI,CAAC,aAAa;;4DAErBkD,SACAC;;;;yEAFJ1F,QAAQ2F,SAAS,EAAjB3F;;;;oEACIyF,UAAU1E,WAAWiC,MAAM,CAAC,SAACwB;+EAAM/C,SAASwB,GAAG,CAACuB,EAAE/B,OAAO;;oEACzDiD,aAAa3E,WAAWiC,MAAM,CAAC,SAACwB;+EAAM,CAAC/C,SAASwB,GAAG,CAACuB,EAAE/B,OAAO;;yEAC/DgD,CAAAA,QAAQjC,MAAM,GAAG,CAAA,GAAjBiC;;;;oEAAoB;;wEAAMzF,QAAQ2F,SAAS,CAAC9F,MAAM,eAAewB,iBAAiBoE,QAAQtD,GAAG,CAAC6C;;;oEAA1E;;;yEACpBU,CAAAA,WAAWlC,MAAM,GAAG,CAAA,GAApBkC;;;;oEAAuB;;wEAAM7F,KAAK+F,QAAQ,CAACtE,WAAWoE,WAAWvD,GAAG,CAAC6C;;;oEAA9C;;;;;;;;oEAE3B;;wEAAMnF,KAAK+F,QAAQ,CAACtE,WAAWP,WAAWoB,GAAG,CAAC6C;;;oEAA9C;;;;;;;;oDAEJ;;;;4CATA;;;4CAYF,wFAAwF;4CACxF,2FAA2F;4CAC3F;;gDAAM3E,OAAOkC,IAAI,CAAC,cAAc;2DAAMvC,QAAQ6F,gBAAgB,CAAChG,MAAMkF,gBAAgBhE,YAAYS,OAAO1B;;;;4CAAxG;4CAEA,0FAA0F;4CAC1F,kEAAkE;4CAClE;;gDAAMO,OAAOkC,IAAI,CAAC,WAAW;;4DAMrBuD,YAC6B,2BAAA,mBAAA,gBAA9B,4BAAA,oBAAA,iBAAA,WAAA,OAAMT,KAAwB,YAAA,QAAMU;;;;yEANrCpE,CAAAA,QAAQ6B,MAAM,GAAG,CAAA,GAAjB7B;;;;oEACF;;wEAAM9B,KAAK+F,QAAQ,CACjB,6CACAjE,QAAQQ,GAAG,CAAC,SAACsC;mFAAM;gFAACA;6EAAE;;;;oEAFxB;;;oEAIIqB;oEAC6B,kCAAA,2BAAA,4BAA9B,mCAAA,4BAAA;;wEAAL,IAAK,YAAa/E,iCAAb,8BAAA,QAAA,yBAAA;4EAAMsE,MAAN;;gFAAyB,IAAK,aAAoBA,IAAIW,OAAO,uBAA/B,6BAAA,SAAA,0BAAA;oFAAMD,aAAN;oFAAiCD,WAAWG,IAAI;wFAAEZ,IAAI5C,OAAO;wFAAEsD;;;;gFAA/D;gFAAA;;;yFAAA,6BAAA;wFAAA;;;wFAAA;8FAAA;;;;;;wEAA9B;wEAAA;;;iFAAA,8BAAA;gFAAA;;;gFAAA;sFAAA;;;;yEAIDD,CAAAA,WAAWtC,MAAM,GAAG,CAAA,GAApBsC;;;;oEAAuB;;wEAAMjG,KAAK+F,QAAQ,CAAC,kGAAkGE;;;oEAAtH;;;;;;;;oDAC7B;;;;4CAZA;iDAgBInF,CAAAA,SAAS6C,MAAM,GAAG,CAAA,GAAlB7C;;;;4CACF;;gDAAMN,OAAOkC,IAAI,CAAC,YAAY;2DAC5B1C,KAAK+F,QAAQ,CACX,4CACAjF,SAASwB,GAAG,CAAC,SAACsC;+DAAM;4DAACA;yDAAE;;;;;4CAH3B;;;4CAS2B,kCAAA,2BAAA;iDAAzB9C,CAAAA,QAAQ6B,MAAM,GAAG,CAAA,GAAjB7B;;;;;;;;;;;;;oDAA+BuE;;;;4DAAAA,UAAN;4DAA2B;;gEAAM7F,OAAOkC,IAAI,CAAC4D,IAAAA,sBAAY,EAACD,QAAQ7D,IAAI,EAAE,WAAW;wEAAM6D;4EAAAA,kBAAAA,QAAQE,MAAM,cAAdF,sCAAAA,qBAAAA,SAAiBrG,MAAM8B,SAASH;;;;4DAA9F;;;;;;;4CAA3B,YAAiBpB;;;mDAAjB,6BAAA,QAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CACxB,mCAAA,4BAAA;;;;;;;;;;oDAAM8F,SACHG;;;;4DADGH,UAAN;4DACGG,iBAAiCtF,WAAWoB,GAAG,CAAC,SAACkD;uEAAS;oEAAEtC,MAAMsC,IAAI5C,OAAO;oEAAE6D,WAAWjB,IAAIiB,SAAS,CAACJ,QAAQ7D,IAAI,CAAC;gEAAC;;4DAC5H;;gEAAMhC,OAAOkC,IAAI,CAAC4D,IAAAA,sBAAY,EAACD,QAAQ7D,IAAI,EAAE,UAAU;wEAAM6D;4EAAAA,iBAAAA,QAAQK,KAAK,cAAbL,qCAAAA,oBAAAA,SAAgBrG,MAAMwG,gBAAgB7E;;;;4DAAnG;;;;;;4CACF;4CAHK,aAAiBpB;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;;oDAAM8F;;;;4DAAAA,UAAN;4DAA2B;;gEAAM7F,OAAOkC,IAAI,CAAC4D,IAAAA,sBAAY,EAACD,QAAQ7D,IAAI,EAAE,UAAU;wEAAM6D;4EAAAA,0BAAAA,QAAQM,cAAc,cAAtBN,8CAAAA,6BAAAA,SAAyBrG,MAAM2B;;;;4DAA5F;;;;;;;4CAA3B,aAAiBpB;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;2BACAJ,QAAQyG,SAAS;;;oBAvFnB;oBA0FM5E,aAAa6C,KAAKC,GAAG,KAAK/C;oBAChC,2GAA2G;oBAC3G8E,IAAAA,4BAAgB,EAAC3G,SAAS8B;yBACtB7B,QAAQ2G,cAAc,EAAtB3G;;;;oBAAwB;;wBAAMK,OAAOkC,IAAI,CAAC,QAAQ;gCAAMvC;oCAAAA,0BAAAA,QAAQ2G,cAAc,cAAtB3G,8CAAAA,6BAAAA,SAAyBH,MAAMgC;;;;oBAA/D;;;oBAE5B;;wBAAO;4BAAE4B,QAAQ1C,WAAWyC,MAAM;4BAAExC,UAAAA;4BAAUX,QAAQA,OAAOqD,IAAI,CAACpD,WAAWuB,YAAYX;wBAAe;;;;IAC1G"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { recordLockWaitMs } from './lock-wait.ts';\nimport { appendRows, getColumns, quoteIdent } from './shared.ts';\nimport { featureStage, type Stages, stageRecorder } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nexport const CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const start = process.hrtime.bigint();\n  const features = activeFeatures(cfg);\n  const stages = stageRecorder(features.map((f) => f.name));\n  const elapsed = () => Number(process.hrtime.bigint() - start) / 1e6;\n  const files = await stages.time('list', () => listFiles(cfg, baseDir));\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = await stages.time('existing', async () => {\n    const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n    return (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  });\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [], stages: stages.take(elapsed(), 0) };\n\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.\n  const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', () => reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool }));\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await stages.time('alter', () => dialect.addColumns(conn, missingColumns));\n\n      // Revalidated once the lock is held, before this process's own frontmatter upsert below:\n      // `added` came from a path read taken before this transaction's lock, so a path still\n      // called \"added\" here may already have a content row from a concurrent reconcile that\n      // committed while this one waited. One SELECT, not a DELETE per added file -- with no\n      // contention it returns the same set `existing` already ruled out, so nothing extra clears.\n      let contentTouched = touched;\n      if (added.length > 0)\n        contentTouched = await stages.time('added-recheck', async () => {\n          const currentPathsStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n          const currentPaths = new Set(((await currentPathsStmt.all()) as Array<{ path: string }>).map((r) => r.path));\n          const staleAdded = added.filter((p) => currentPaths.has(p));\n          return staleAdded.length > 0 ? [...touched, ...staleAdded] : touched;\n        });\n\n      if (parsedDocs.length > 0) {\n        const toRow = (doc: (typeof parsedDocs)[number]) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          });\n        // A path in `added` has no existing frontmatter row, so it can never conflict; the rest\n        // genuinely can, and keep the upsert.\n        await stages.time('fm-upsert', async () => {\n          const newDocs = parsedDocs.filter((d) => addedSet.has(d.relPath));\n          const updateDocs = parsedDocs.filter((d) => !addedSet.has(d.relPath));\n          await appendRows(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow));\n          if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));\n        });\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await stages.time('text-index', () => dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      await stages.time('presets', async () => {\n        if (touched.length > 0)\n          await conn.runBatch(\n            'DELETE FROM preset_files WHERE \"path\" = ?',\n            touched.map((p) => [p])\n          );\n        const presetRows: unknown[][] = [];\n        for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken\n        // before this transaction's lock, so a path this process calls \"added\" can already have\n        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.\n        await appendRows(conn, 'preset_files', ['path', 'preset'], 'INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n      });\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await stages.time('vanished', () =>\n          conn.runBatch(\n            'DELETE FROM frontmatter WHERE \"path\" = ?',\n            vanished.map((p) => [p])\n          )\n        );\n\n      // Timed per feature per hook, so link resolution and PageRank are named stages without\n      // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.\n      if (touched.length > 0) for (const feature of features) await stages.time(featureStage(feature.name, 'remove'), () => feature.remove?.(conn, touched, delta));\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await stages.time(featureStage(feature.name, 'store'), () => feature.store?.(conn, docsForFeature, delta));\n      }\n      for (const feature of features) await stages.time(featureStage(feature.name, 'after'), () => feature.afterReconcile?.(conn, delta));\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.\n  recordLockWaitMs(baseDir, durationMs);\n  if (dialect.recordDuration) await stages.time('meta', () => dialect.recordDuration?.(conn, durationMs));\n\n  return { parsed: parsedDocs.length, warnings, stages: stages.take(elapsed(), durationMs, workerParseMs) };\n}\n"],"names":["CORE_FRONTMATTER_COLUMNS","reconcile","Set","conn","cfg","baseDir","dialect","pool","forcedPaths","start","features","stages","elapsed","files","currentSet","existingRows","existing","vanished","toReparse","seenColumns","report","parsedDocs","warnings","newColumns","workerParseMs","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","touched","txStart","durationMs","process","hrtime","bigint","activeFeatures","stageRecorder","map","f","name","Number","time","listFiles","relPath","existingStmt","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","take","getColumns","progress","reparseFiles","tick","docs","finish","add","checkColumnLimit","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","present","missingColumns","contentTouched","toRow","addColumns","currentPathsStmt","currentPaths","staleAdded","doc","ctimeMs","parseError","data","newDocs","updateDocs","appendRows","runBatch","reconcileContent","presetRows","presetName","presets","push","feature","featureStage","remove","docsForFeature","extracted","store","afterReconcile","beginMode","recordLockWaitMs","recordDuration"],"mappings":";;;;;;;;;;;QAqBaA;eAAAA;;QAESC;eAAAA;;;uBAtBS;0BAEN;wBACmB;yBAEf;0BACI;wBACkB;wBACM;6BACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAWzB,IAAMD,2BAA2B,IAAIE,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,SAAeD,UAAUE,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;;YACtJC,OACAC,UACAC,QACAC,SACAC,OACAC,YAEAC,cAIAC,UAIAC,UAIAC,WAOAC,aAIAC,QAE4D,MAApDC,YAAYC,UAAUC,YAAYC,eAE3C,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YAMAC,iBAGAC,WAKAC,OACAC,OACAC,UACAC,kBAGAC,SAEAC,SAuFAC;;;;oBAjJA1B,QAAQ2B,QAAQC,MAAM,CAACC,MAAM;oBAC7B5B,WAAW6B,IAAAA,uBAAc,EAACnC;oBAC1BO,SAAS6B,IAAAA,uBAAa,EAAC9B,SAAS+B,GAAG,CAAC,SAACC;+BAAMA,EAAEC,IAAI;;oBACjD/B,UAAU;+BAAMgC,OAAOR,QAAQC,MAAM,CAACC,MAAM,KAAK7B,SAAS;;oBAClD;;wBAAME,OAAOkC,IAAI,CAAC,QAAQ;mCAAMC,IAAAA,mBAAS,EAAC1C,KAAKC;;;;oBAAvDQ,QAAQ;oBACRC,aAAa,IAAIZ,IAAIW,MAAM4B,GAAG,CAAC,SAACC;+BAAMA,EAAEK,OAAO;;oBAEhC;;wBAAMpC,OAAOkC,IAAI,CAAC,YAAY;;oCAC3CG;;;;4CAAe;;gDAAM7C,KAAK8C,OAAO,CAAC;;;4CAAlCD,eAAe;4CACb;;gDAAMA,aAAaE,GAAG;;;4CAA9B;;gDAAQ;;;;4BACV;;;;oBAHMnC,eAAe;oBAIfC,WAAW,IAAImC,IAAIpC,aAAa0B,GAAG,CAAC,SAACW;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBAC5D,+FAA+F;oBAC/F,+FAA+F;oBAC/F,gFAAgF;oBAC1EnC,WAAWF,aAAauC,MAAM,CAAC,SAACF;+BAAM,CAACtC,WAAWyC,GAAG,CAACH,EAAEC,IAAI;uBAAGZ,GAAG,CAAC,SAACW;+BAAMA,EAAEC,IAAI;;oBAEtF,6FAA6F;oBAC7F,0FAA0F;oBACpFnC,YAAYL,MAAMyC,MAAM,CAAC,SAACZ;;wBAC9B,IAAMc,MAAMxC,SAASyC,GAAG,CAACf,EAAEK,OAAO;wBAClC,OAAO,CAACS,OAAOA,IAAIE,MAAM,KAAKhB,EAAEiB,OAAO,IAAIH,IAAII,KAAK,KAAKlB,EAAEmB,IAAI,aAAKrD,wBAAAA,kCAAAA,YAAa+C,GAAG,CAACb,EAAEK,OAAO,wCAAK;oBACrG;oBAEA,IAAI9B,SAAS6C,MAAM,KAAK,KAAK5C,UAAU4C,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAGzC,QAAQ;4BAAMX,QAAQA,OAAOqD,IAAI,CAACpD,WAAW;wBAAG;;oBAErG;;wBAAMqD,IAAAA,oBAAU,EAAC9D;;;oBAA/BgB,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAAS8C,IAAAA,oBAAQ,EAAC,mBAAmBhD,UAAU4C,MAAM;oBAEO;;wBAAMnD,OAAOkC,IAAI,CAAC,SAAS;mCAAMsB,IAAAA,uBAAY,EAACjD,WAAWR,UAAUN,KAAKe,aAAaC,OAAOgD,IAAI,EAAE;gCAAE7D,MAAAA;4BAAK;;;;oBAAzG,OAAA,eAApDc,aAAoD,KAA1DgD,MAAkB/C,WAAwC,KAAxCA,UAAUC,aAA8B,KAA9BA,YAAYC,gBAAkB,KAAlBA;oBAChDJ,OAAOkD,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAa/C,iCAAb,6BAAA,QAAA,yBAAA;4BAAME,MAAN;4BAAyBN,YAAYoD,GAAG,CAAC9C;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGP;oBACvB,yFAAyF;oBACzF,iGAAiG;oBACjGb,QAAQkE,gBAAgB,CAAC9C,WAAWoC,MAAM;oBAC1C,0FAA0F;oBAC1F,sEAAsE;oBAChEnC,kBAAkBD,WAAW4B,MAAM,CAAC,SAACmB;+BAAMzE,yBAAyBuD,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBAC1G,yFAAyF;oBACzF,mDAAmD;oBAC7C7C,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBc,GAAG,CAACkC,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FjD,OAAhFA,gBAAgBc,GAAG,CAAC;+BAAM;uBAAKmC,IAAI,CAAC,OAAM,wCAGjI,OAHuKjD,gBAClL2B,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpBhC,GAAG,CAAC,SAACgC;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEF/C,QAAQX,UAAUoC,MAAM,CAAC,SAACZ;+BAAM,CAAC1B,SAASuC,GAAG,CAACb,EAAEK,OAAO;uBAAGN,GAAG,CAAC,SAACC;+BAAMA,EAAEK,OAAO;;oBAC9EjB,QAAwB;wBAAEjB,OAAAA;wBAAOgE,UAAUxD,WAAWoB,GAAG,CAAC,SAACqC;mCAAMA,EAAE/B,OAAO;;wBAAGlB,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAI7B,IAAI2B;oBACnBG,mBAAmBX,WAAWoB,GAAG,CAAC,SAACqC;+BAAMA,EAAE/B,OAAO;uBAAEO,MAAM,CAAC,SAACyB;+BAAM,CAAChD,SAASwB,GAAG,CAACwB;;oBACtF,+FAA+F;oBAC/F,0EAA0E;oBACpE9C,UAAU,AAAC,qBAAGhB,iBAAU,qBAAGe;oBAE3BE,UAAU8C,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EACnB/E,MACA;;oCAGQgF,SACAC,gBAQFC,gBAUIC,OAoDqB,2BAAA,mBAAA,uBAAA,WAAA,YACxB,4BAAA,oBAAA,yBAAA,YAAA,aAIA,4BAAA,oBAAA,yBAAA,YAAA;;;;4CA5EW;;gDAAMrB,IAAAA,oBAAU,EAAC9D;;;4CAA3BgF,UAAU;4CACVC,iBAAiB7D,WAAW+B,MAAM,CAAC,SAAC7B;uDAAQ,CAAC0D,QAAQ5B,GAAG,CAAC9B;;iDAC3D2D,CAAAA,eAAetB,MAAM,GAAG,CAAA,GAAxBsB;;;;4CAA2B;;gDAAMzE,OAAOkC,IAAI,CAAC,SAAS;2DAAMvC,QAAQiF,UAAU,CAACpF,MAAMiF;;;;4CAA1D;;;4CAE/B,yFAAyF;4CACzF,sFAAsF;4CACtF,sFAAsF;4CACtF,sFAAsF;4CACtF,4FAA4F;4CACxFC,iBAAiBpD;iDACjBJ,CAAAA,MAAMiC,MAAM,GAAG,CAAA,GAAfjC;;;;4CACe;;gDAAMlB,OAAOkC,IAAI,CAAC,iBAAiB;;4DAC5C2C,kBACAC,iBACAC;;;;oEAFmB;;wEAAMvF,KAAK8C,OAAO,CAAC;;;oEAAtCuC,mBAAmB;wEACAtF;oEAAM;;wEAAMsF,iBAAiBtC,GAAG;;;oEAAnDuC,eAAe,IAAA,CAAA,EAAA,MAAIvF;;wEAAM,cAA0DuC,GAAG,CAAC,SAACW;mFAAMA,EAAEC,IAAI;;sEAAC;oEACrGqC,aAAa7D,MAAMyB,MAAM,CAAC,SAACyB;+EAAMU,aAAalC,GAAG,CAACwB;;oEACxD;;wEAAOW,WAAW5B,MAAM,GAAG,IAAI,AAAC,qBAAG7B,gBAAS,qBAAGyD,eAAczD;;;;oDAC/D;;;;4CALAoD,iBAAiB;;;iDAOfhE,CAAAA,WAAWyC,MAAM,GAAG,CAAA,GAApBzC;;;;4CACIiE,QAAQ,eAACK;uDACbhE,gBAAgBc,GAAG,CAAC,SAAChB;wDAOZkE;oDANP,IAAIlE,QAAQ,QAAQ,OAAOkE,IAAI5C,OAAO;oDACtC,IAAItB,QAAQ,UAAU,OAAOkE,IAAIhC,OAAO;oDACxC,IAAIlC,QAAQ,UAAU,OAAOkE,IAAIC,OAAO;oDACxC,IAAInE,QAAQ,SAAS,OAAOkE,IAAI9B,IAAI;oDACpC,mFAAmF;oDACnF,IAAIpC,QAAQ,gBAAgB,OAAOkE,IAAIE,UAAU;oDACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAACrE,IAAI,cAAbkE,2BAAAA,gBAAiB;gDAC1B;;4CACF,wFAAwF;4CACxF,sCAAsC;4CACtC;;gDAAMhF,OAAOkC,IAAI,CAAC,aAAa;;4DACvBkD,SACAC;;;;oEADAD,UAAU1E,WAAWiC,MAAM,CAAC,SAACwB;+EAAM/C,SAASwB,GAAG,CAACuB,EAAE/B,OAAO;;oEACzDiD,aAAa3E,WAAWiC,MAAM,CAAC,SAACwB;+EAAM,CAAC/C,SAASwB,GAAG,CAACuB,EAAE/B,OAAO;;oEACnE;;wEAAMkD,IAAAA,oBAAU,EAAC9F,MAAM,eAAewB,iBAAiBC,WAAWmE,QAAQtD,GAAG,CAAC6C;;;oEAA9E;yEACIU,CAAAA,WAAWlC,MAAM,GAAG,CAAA,GAApBkC;;;;oEAAuB;;wEAAM7F,KAAK+F,QAAQ,CAACtE,WAAWoE,WAAWvD,GAAG,CAAC6C;;;oEAA9C;;;;;;;;oDAC7B;;;;4CALA;;;4CAQF,wFAAwF;4CACxF,2FAA2F;4CAC3F;;gDAAM3E,OAAOkC,IAAI,CAAC,cAAc;2DAAMvC,QAAQ6F,gBAAgB,CAAChG,MAAMkF,gBAAgBhE,YAAYS,OAAO1B;;;;4CAAxG;4CAEA,0FAA0F;4CAC1F,kEAAkE;4CAClE;;gDAAMO,OAAOkC,IAAI,CAAC,WAAW;;4DAMrBuD,YAC6B,2BAAA,mBAAA,gBAA9B,4BAAA,oBAAA,iBAAA,WAAA,OAAMT,KAAwB,YAAA,QAAMU;;;;yEANrCpE,CAAAA,QAAQ6B,MAAM,GAAG,CAAA,GAAjB7B;;;;oEACF;;wEAAM9B,KAAK+F,QAAQ,CACjB,6CACAjE,QAAQQ,GAAG,CAAC,SAACsC;mFAAM;gFAACA;6EAAE;;;;oEAFxB;;;oEAIIqB;oEAC6B,kCAAA,2BAAA,4BAA9B,mCAAA,4BAAA;;wEAAL,IAAK,YAAa/E,iCAAb,8BAAA,QAAA,yBAAA;4EAAMsE,MAAN;;gFAAyB,IAAK,aAAoBA,IAAIW,OAAO,uBAA/B,6BAAA,SAAA,0BAAA;oFAAMD,aAAN;oFAAiCD,WAAWG,IAAI;wFAAEZ,IAAI5C,OAAO;wFAAEsD;;;;gFAA/D;gFAAA;;;yFAAA,6BAAA;wFAAA;;;wFAAA;8FAAA;;;;;;wEAA9B;wEAAA;;;iFAAA,8BAAA;gFAAA;;;gFAAA;sFAAA;;;;oEACL,uFAAuF;oEACvF,wFAAwF;oEACxF,uGAAuG;oEACvG;;wEAAMJ,IAAAA,oBAAU,EAAC9F,MAAM;4EAAiB;4EAAQ;2EAAW,kGAAkGiG;;;oEAA7J;;;;;;oDACF;;;;4CAZA;iDAgBInF,CAAAA,SAAS6C,MAAM,GAAG,CAAA,GAAlB7C;;;;4CACF;;gDAAMN,OAAOkC,IAAI,CAAC,YAAY;2DAC5B1C,KAAK+F,QAAQ,CACX,4CACAjF,SAASwB,GAAG,CAAC,SAACsC;+DAAM;4DAACA;yDAAE;;;;;4CAH3B;;;4CAS2B,kCAAA,2BAAA;iDAAzB9C,CAAAA,QAAQ6B,MAAM,GAAG,CAAA,GAAjB7B;;;;;;;;;;;;;oDAA+BuE;;;;4DAAAA,UAAN;4DAA2B;;gEAAM7F,OAAOkC,IAAI,CAAC4D,IAAAA,sBAAY,EAACD,QAAQ7D,IAAI,EAAE,WAAW;wEAAM6D;4EAAAA,kBAAAA,QAAQE,MAAM,cAAdF,sCAAAA,qBAAAA,SAAiBrG,MAAM8B,SAASH;;;;4DAA9F;;;;;;;4CAA3B,YAAiBpB;;;mDAAjB,6BAAA,QAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CACxB,mCAAA,4BAAA;;;;;;;;;;oDAAM8F,SACHG;;;;4DADGH,UAAN;4DACGG,iBAAiCtF,WAAWoB,GAAG,CAAC,SAACkD;uEAAS;oEAAEtC,MAAMsC,IAAI5C,OAAO;oEAAE6D,WAAWjB,IAAIiB,SAAS,CAACJ,QAAQ7D,IAAI,CAAC;gEAAC;;4DAC5H;;gEAAMhC,OAAOkC,IAAI,CAAC4D,IAAAA,sBAAY,EAACD,QAAQ7D,IAAI,EAAE,UAAU;wEAAM6D;4EAAAA,iBAAAA,QAAQK,KAAK,cAAbL,qCAAAA,oBAAAA,SAAgBrG,MAAMwG,gBAAgB7E;;;;4DAAnG;;;;;;4CACF;4CAHK,aAAiBpB;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;;oDAAM8F;;;;4DAAAA,UAAN;4DAA2B;;gEAAM7F,OAAOkC,IAAI,CAAC4D,IAAAA,sBAAY,EAACD,QAAQ7D,IAAI,EAAE,UAAU;wEAAM6D;4EAAAA,0BAAAA,QAAQM,cAAc,cAAtBN,8CAAAA,6BAAAA,SAAyBrG,MAAM2B;;;;4DAA5F;;;;;;;4CAA3B,aAAiBpB;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;2BACAJ,QAAQyG,SAAS;;;oBAnFnB;oBAsFM5E,aAAa6C,KAAKC,GAAG,KAAK/C;oBAChC,2GAA2G;oBAC3G8E,IAAAA,4BAAgB,EAAC3G,SAAS8B;yBACtB7B,QAAQ2G,cAAc,EAAtB3G;;;;oBAAwB;;wBAAMK,OAAOkC,IAAI,CAAC,QAAQ;gCAAMvC;oCAAAA,0BAAAA,QAAQ2G,cAAc,cAAtB3G,8CAAAA,6BAAAA,SAAyBH,MAAMgC;;;;oBAA/D;;;oBAE5B;;wBAAO;4BAAE4B,QAAQ1C,WAAWyC,MAAM;4BAAExC,UAAAA;4BAAUX,QAAQA,OAAOqD,IAAI,CAACpD,WAAWuB,YAAYX;wBAAe;;;;IAC1G"}

package/dist/cjs/store/shared.d.cts

@@ -1,4 +1,5 @@
 import type { Connection } from './types.js';
+export declare function appendRows(conn: Connection, table: string, columns: string[], conflictSql: string, rows: unknown[][]): Promise<void>;
 export declare function quoteIdent(name: string): string;
 export declare function getColumns(conn: Connection): Promise<Set<string>>;
 export declare function getMeta(conn: Connection, key: string): Promise<string | null>;

package/dist/cjs/store/shared.d.ts

@@ -1,4 +1,5 @@
 import type { Connection } from './types.js';
+export declare function appendRows(conn: Connection, table: string, columns: string[], conflictSql: string, rows: unknown[][]): Promise<void>;
 export declare function quoteIdent(name: string): string;
 export declare function getColumns(conn: Connection): Promise<Set<string>>;
 export declare function getMeta(conn: Connection, key: string): Promise<string | null>;

package/dist/cjs/store/shared.js

@@ -9,6 +9,9 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get appendRows () {
+        return appendRows;
+    },
     get getColumns () {
         return getColumns;
     },
@@ -150,6 +153,44 @@ function _ts_generator(thisArg, body) {
         };
     }
 }
+function appendRows(conn, table, columns, conflictSql, rows) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (rows.length === 0) return [
+                        2
+                    ];
+                    if (!conn.appendRows) return [
+                        3,
+                        2
+                    ];
+                    return [
+                        4,
+                        conn.appendRows(table, columns, rows)
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        3,
+                        4
+                    ];
+                case 2:
+                    return [
+                        4,
+                        conn.runBatch(conflictSql, rows)
+                    ];
+                case 3:
+                    _state.sent();
+                    _state.label = 4;
+                case 4:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
 function quoteIdent(name) {
     return '"'.concat(name.split('"').join('""'), '"');
 }

package/dist/cjs/store/shared.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n\n// Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n// observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\nexport async function recordReconcileDuration(conn: Connection, ms: number): Promise<void> {\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (ms > prevMax) await setMeta(conn, 'reconcile_max_ms', String(ms));\n}\n"],"names":["getColumns","getMeta","quoteIdent","recordReconcileDuration","setMeta","name","split","join","conn","stmt","rows","prepare","all","Set","map","r","key","row","get","value","run","ms","prevRaw","prevMax","Number","String"],"mappings":";;;;;;;;;;;QASsBA;eAAAA;;QAMAC;eAAAA;;QAVNC;eAAAA;;QA4BMC;eAAAA;;QAZAC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAhBf,SAASF,WAAWG,IAAY;IACrC,OAAO,AAAC,IAA8B,OAA3BA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,OAAM;AACxC;AAEO,SAAeP,WAAWQ,IAAgB;;YACzCC,MACAC;;;;oBADO;;wBAAMF,KAAKG,OAAO,CAAC;;;oBAA1BF,OAAO;oBACC;;wBAAMA,KAAKG,GAAG;;;oBAAtBF,OAAQ;oBACd;;wBAAO,IAAIG,IAAIH,KAAKI,GAAG,CAAC,SAACC;mCAAMA,EAAEV,IAAI;;;;;IACvC;;AAEO,SAAeJ,QAAQO,IAAgB,EAAEQ,GAAW;;YACnDP,MACAQ;;;;oBADO;;wBAAMT,KAAKG,OAAO,CAAC;;;oBAA1BF,OAAO;oBACA;;wBAAMA,KAAKS,GAAG,CAACF;;;oBAAtBC,MAAO;oBACb;;wBAAOA,MAAMA,IAAIE,KAAK,GAAG;;;;IAC3B;;AAEO,SAAef,QAAQI,IAAgB,EAAEQ,GAAW,EAAEG,KAAoB;;YAEvEV,MAIFA;;;;yBALFU,CAAAA,UAAU,IAAG,GAAbA;;;;oBACW;;wBAAMX,KAAKG,OAAO,CAAC;;;oBAA1BF,OAAO;oBACb;;wBAAMA,KAAKW,GAAG,CAACJ;;;oBAAf;oBACA;;;;oBAEW;;wBAAMR,KAAKG,OAAO,CAAC;;;oBAA1BF,QAAO;oBACb;;wBAAMA,MAAKW,GAAG,CAACJ,KAAKG;;;oBAApB;;;;;;IACF;;AAIO,SAAehB,wBAAwBK,IAAgB,EAAEa,EAAU;;YAClEC,SAGAC;;;;oBAHU;;wBAAMtB,QAAQO,MAAM;;;oBAA9Bc,UAAU;oBAChB,yFAAyF;oBACzF,+EAA+E;oBACzEC,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;yBAC3CD,CAAAA,KAAKE,OAAM,GAAXF;;;;oBAAc;;wBAAMjB,QAAQI,MAAM,oBAAoBiB,OAAOJ;;;oBAA/C;;;;;;;;IACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\n// Rows that cannot conflict, at whatever speed this connection offers: the append path where the\n// store has one, else `conflictSql` bound the ordinary way. `conflictSql` stays the caller's own\n// guarded statement, so a store without appendRows behaves exactly as it did before.\nexport async function appendRows(conn: Connection, table: string, columns: string[], conflictSql: string, rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  if (conn.appendRows) await conn.appendRows(table, columns, rows);\n  else await conn.runBatch(conflictSql, rows);\n}\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n\n// Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n// observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\nexport async function recordReconcileDuration(conn: Connection, ms: number): Promise<void> {\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (ms > prevMax) await setMeta(conn, 'reconcile_max_ms', String(ms));\n}\n"],"names":["appendRows","getColumns","getMeta","quoteIdent","recordReconcileDuration","setMeta","conn","table","columns","conflictSql","rows","length","runBatch","name","split","join","stmt","prepare","all","Set","map","r","key","row","get","value","run","ms","prevRaw","prevMax","Number","String"],"mappings":";;;;;;;;;;;QAQsBA;eAAAA;;QAUAC;eAAAA;;QAMAC;eAAAA;;QAVNC;eAAAA;;QA4BMC;eAAAA;;QAZAC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAtBf,SAAeL,WAAWM,IAAgB,EAAEC,KAAa,EAAEC,OAAiB,EAAEC,WAAmB,EAAEC,IAAiB;;;;;oBACzH,IAAIA,KAAKC,MAAM,KAAK,GAAG;;;yBACnBL,KAAKN,UAAU,EAAfM;;;;oBAAiB;;wBAAMA,KAAKN,UAAU,CAACO,OAAOC,SAASE;;;oBAAtC;;;;;;oBAChB;;wBAAMJ,KAAKM,QAAQ,CAACH,aAAaC;;;oBAAjC;;;;;;;;IACP;;AAEO,SAASP,WAAWU,IAAY;IACrC,OAAO,AAAC,IAA8B,OAA3BA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,OAAM;AACxC;AAEO,SAAed,WAAWK,IAAgB;;YACzCU,MACAN;;;;oBADO;;wBAAMJ,KAAKW,OAAO,CAAC;;;oBAA1BD,OAAO;oBACC;;wBAAMA,KAAKE,GAAG;;;oBAAtBR,OAAQ;oBACd;;wBAAO,IAAIS,IAAIT,KAAKU,GAAG,CAAC,SAACC;mCAAMA,EAAER,IAAI;;;;;IACvC;;AAEO,SAAeX,QAAQI,IAAgB,EAAEgB,GAAW;;YACnDN,MACAO;;;;oBADO;;wBAAMjB,KAAKW,OAAO,CAAC;;;oBAA1BD,OAAO;oBACA;;wBAAMA,KAAKQ,GAAG,CAACF;;;oBAAtBC,MAAO;oBACb;;wBAAOA,MAAMA,IAAIE,KAAK,GAAG;;;;IAC3B;;AAEO,SAAepB,QAAQC,IAAgB,EAAEgB,GAAW,EAAEG,KAAoB;;YAEvET,MAIFA;;;;yBALFS,CAAAA,UAAU,IAAG,GAAbA;;;;oBACW;;wBAAMnB,KAAKW,OAAO,CAAC;;;oBAA1BD,OAAO;oBACb;;wBAAMA,KAAKU,GAAG,CAACJ;;;oBAAf;oBACA;;;;oBAEW;;wBAAMhB,KAAKW,OAAO,CAAC;;;oBAA1BD,QAAO;oBACb;;wBAAMA,MAAKU,GAAG,CAACJ,KAAKG;;;oBAApB;;;;;;IACF;;AAIO,SAAerB,wBAAwBE,IAAgB,EAAEqB,EAAU;;YAClEC,SAGAC;;;;oBAHU;;wBAAM3B,QAAQI,MAAM;;;oBAA9BsB,UAAU;oBAChB,yFAAyF;oBACzF,+EAA+E;oBACzEC,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;yBAC3CD,CAAAA,KAAKE,OAAM,GAAXF;;;;oBAAc;;wBAAMtB,QAAQC,MAAM,oBAAoByB,OAAOJ;;;oBAA/C;;;;;;;;IACpB"}

package/dist/cjs/store/turso/connection.d.cts

@@ -1,3 +1,4 @@
 import type { Database } from '@tursodatabase/database';
 import type { Connection } from '../types.js';
+export declare function checkpointWal(db: Database): Promise<void>;
 export declare function createConnection(db: Database): Connection;

package/dist/cjs/store/turso/connection.d.ts

@@ -1,3 +1,4 @@
 import type { Database } from '@tursodatabase/database';
 import type { Connection } from '../types.js';
+export declare function checkpointWal(db: Database): Promise<void>;
 export declare function createConnection(db: Database): Connection;

package/dist/cjs/store/turso/connection.js

@@ -2,9 +2,17 @@
 Object.defineProperty(exports, "__esModule", {
     value: true
 });
-Object.defineProperty(exports, "createConnection", {
-    enumerable: true,
-    get: function() {
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get checkpointWal () {
+        return checkpointWal;
+    },
+    get createConnection () {
         return createConnection;
     }
 });
@@ -350,8 +358,8 @@ function _wrap_async_generator(fn) {
         return new _async_generator(fn.apply(this, arguments));
     };
 }
-// Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this
-// wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.
+// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%
+// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.
 var TursoStatementWrapper = /*#__PURE__*/ function() {
     "use strict";
     function TursoStatementWrapper(stmt) {
@@ -431,6 +439,43 @@ var TursoStatementWrapper = /*#__PURE__*/ function() {
     };
     return TursoStatementWrapper;
 }();
+function checkpointWal(db) {
+    return _async_to_generator(function() {
+        var err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    _state.trys.push([
+                        0,
+                        2,
+                        ,
+                        3
+                    ]);
+                    return [
+                        4,
+                        db.exec('PRAGMA wal_checkpoint(TRUNCATE)')
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        3,
+                        3
+                    ];
+                case 2:
+                    err = _state.sent();
+                    console.error("sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ".concat(err.message));
+                    return [
+                        3,
+                        3
+                    ];
+                case 3:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
 function createConnection(db) {
     var conn = {
         exec: function exec(sql) {
@@ -474,8 +519,8 @@ function createConnection(db) {
                 });
             })();
         },
-        runBatch: // db.batch() is the engine's own bulk idiom: one crossing, N rows, each its own statement,
-        // no bind-variable ceiling to chunk around. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.
+        runBatch: // Prepares once and awaits run() per row: db.batch() re-prepares each statement, which cost as
+        // much as preparing per row. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.
         function runBatch(sql, paramRows) {
             return _async_to_generator(function() {
                 return _ts_generator(this, function(_state) {
@@ -488,20 +533,96 @@ function createConnection(db) {
                                 4,
                                 (0, _transactionts.withTransaction)(conn, function() {
                                     return _async_to_generator(function() {
+                                        var stmt, _stmt, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, err;
                                         return _ts_generator(this, function(_state) {
                                             switch(_state.label){
                                                 case 0:
                                                     return [
                                                         4,
-                                                        db.batch(paramRows.map(function(args) {
-                                                            return {
-                                                                sql: sql,
-                                                                args: args
-                                                            };
-                                                        }))
+                                                        db.prepare(sql)
                                                     ];
                                                 case 1:
+                                                    stmt = _state.sent();
+                                                    _state.label = 2;
+                                                case 2:
+                                                    _state.trys.push([
+                                                        2,
+                                                        ,
+                                                        11,
+                                                        13
+                                                    ]);
+                                                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                                                    _state.label = 3;
+                                                case 3:
+                                                    _state.trys.push([
+                                                        3,
+                                                        8,
+                                                        9,
+                                                        10
+                                                    ]);
+                                                    _iterator = paramRows[Symbol.iterator]();
+                                                    _state.label = 4;
+                                                case 4:
+                                                    if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                                                        3,
+                                                        7
+                                                    ];
+                                                    row = _step.value;
+                                                    return [
+                                                        4,
+                                                        (_stmt = stmt).run.apply(_stmt, _to_consumable_array(row))
+                                                    ];
+                                                case 5:
+                                                    _state.sent();
+                                                    _state.label = 6;
+                                                case 6:
+                                                    _iteratorNormalCompletion = true;
+                                                    return [
+                                                        3,
+                                                        4
+                                                    ];
+                                                case 7:
+                                                    return [
+                                                        3,
+                                                        10
+                                                    ];
+                                                case 8:
+                                                    err = _state.sent();
+                                                    _didIteratorError = true;
+                                                    _iteratorError = err;
+                                                    return [
+                                                        3,
+                                                        10
+                                                    ];
+                                                case 9:
+                                                    try {
+                                                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                                            _iterator.return();
+                                                        }
+                                                    } finally{
+                                                        if (_didIteratorError) {
+                                                            throw _iteratorError;
+                                                        }
+                                                    }
+                                                    return [
+                                                        7
+                                                    ];
+                                                case 10:
+                                                    return [
+                                                        3,
+                                                        13
+                                                    ];
+                                                case 11:
+                                                    return [
+                                                        4,
+                                                        stmt.close()
+                                                    ];
+                                                case 12:
                                                     _state.sent();
+                                                    return [
+                                                        7
+                                                    ];
+                                                case 13:
                                                     return [
                                                         2
                                                     ];

package/dist/cjs/store/turso/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this\n// wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // db.batch() is the engine's own bulk idiom: one crossing, N rows, each its own statement,\n    // no bind-variable ceiling to chunk around. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          await db.batch(paramRows.map((args) => ({ sql, args })));\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["createConnection","TursoStatementWrapper","stmt","run","params","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","db","conn","exec","sql","prepare","runBatch","paramRows","length","withTransaction","batch","map","args","BEGIN_WRITE"],"mappings":";;;;+BA0CgBA;;;eAAAA;;;6BAzC6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAO7C,+FAA+F;AAC/F,gGAAgG;AAChG,IAAA,AAAMC,sCAAN;;aAAMA,sBAGQC,IAAoB;gCAH5BD;QAIF,IAAI,CAACC,IAAI,GAAGA;;iBAJVD;IAOJ,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACC,GAAG,OAAb,YAAc,qBAAGC;;;QAC1B;;IAEA,OAAMC,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGD,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACG,GAAG,OAAb,YAAc,qBAAGD;;;QAC1B;;IAEA,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGF,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACI,GAAG,OAAb,YAAc,qBAAGF;;;QAC1B;;IAEA,OAAOG,OAEN,GAFD,SAAOA;QAAQ,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGH,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAC1B;;;;wBAAP;;uCAAA,0CAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACK,OAAO,OAAjB,YAAkB,qBAAGH;;;wBAA5B;;;;;;QACF;;IAEAI,OAAAA,OAEC,GAFDA,SAAAA;QACE,OAAO,IAAI,CAACN,IAAI,CAACM,OAAO;IAC1B;IAEAC,OAAAA,cAEC,GAFDA,SAAAA,eAAeC,OAAgB;QAC7B,IAAI,CAACR,IAAI,CAACS,YAAY,CAACD;IACzB;WA7BIT;;AAgCC,SAASD,iBAAiBY,EAAY;IAC3C,IAAMC,OAAmB;QACjBC,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMH,GAAGE,IAAI,CAACC;;;4BAAd;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;;;;gCACZd;4BAAsB;;gCAAMW,GAAGI,OAAO,CAACD;;;4BAAlD;;gCAAO,IAAA,CAAA,EAAA,MAAId;;oCAAsB;kCAAqB;;;;YACxD;;QAGMgB,UAFN,2FAA2F;QAC3F,uLAAuL;QACvL,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;;;4BAC5B;;gCAAMC,IAAAA,8BAAe,EACnBP,MACA;;;;;oDACE;;wDAAMD,GAAGS,KAAK,CAACH,UAAUI,GAAG,CAAC,SAACC;mEAAU;gEAAER,KAAAA;gEAAKQ,MAAAA;4DAAK;;;;oDAApD;;;;;;oCACF;mCACAC,0BAAW;;;4BALb;;;;;;YAOF;;IACF;IACA,OAAOX;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // Prepares once and awaits run() per row: db.batch() re-prepares each statement, which cost as\n    // much as preparing per row. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const stmt = await db.prepare(sql);\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and the db.batch() this replaced finalized its own.\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["checkpointWal","createConnection","TursoStatementWrapper","stmt","run","params","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","db","err","exec","console","error","message","conn","sql","prepare","runBatch","paramRows","length","withTransaction","row","close","BEGIN_WRITE"],"mappings":";;;;;;;;;;;QA4CsBA;eAAAA;;QAQNC;eAAAA;;;6BAnD6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAO7C,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,sCAAN;;aAAMA,sBAGQC,IAAoB;gCAH5BD;QAIF,IAAI,CAACC,IAAI,GAAGA;;iBAJVD;IAOJ,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACC,GAAG,OAAb,YAAc,qBAAGC;;;QAC1B;;IAEA,OAAMC,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGD,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACG,GAAG,OAAb,YAAc,qBAAGD;;;QAC1B;;IAEA,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGF,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACI,GAAG,OAAb,YAAc,qBAAGF;;;QAC1B;;IAEA,OAAOG,OAEN,GAFD,SAAOA;QAAQ,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGH,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAC1B;;;;wBAAP;;uCAAA,0CAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACK,OAAO,OAAjB,YAAkB,qBAAGH;;;wBAA5B;;;;;;QACF;;IAEAI,OAAAA,OAEC,GAFDA,SAAAA;QACE,OAAO,IAAI,CAACN,IAAI,CAACM,OAAO;IAC1B;IAEAC,OAAAA,cAEC,GAFDA,SAAAA,eAAeC,OAAgB;QAC7B,IAAI,CAACR,IAAI,CAACS,YAAY,CAACD;IACzB;WA7BIT;;AAkCC,SAAeF,cAAca,EAAY;;YAGrCC;;;;;;;;;;oBADP;;wBAAMD,GAAGE,IAAI,CAAC;;;oBAAd;;;;;;oBACOD;oBACPE,QAAQC,KAAK,CAAC,AAAC,2FAAiH,OAAvB,AAACH,IAAcI,OAAO;;;;;;;;;;;IAEnI;;AAEO,SAASjB,iBAAiBY,EAAY;IAC3C,IAAMM,OAAmB;QACjBJ,MAAN,SAAMA,KAAKK,GAAW;;;;;4BACpB;;gCAAMP,GAAGE,IAAI,CAACK;;;4BAAd;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;;;;gCACZlB;4BAAsB;;gCAAMW,GAAGQ,OAAO,CAACD;;;4BAAlD;;gCAAO,IAAA,CAAA,EAAA,MAAIlB;;oCAAsB;kCAAqB;;;;YACxD;;QAGMoB,UAFN,+FAA+F;QAC/F,wKAAwK;QACxK,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;;;4BAC5B;;gCAAMC,IAAAA,8BAAe,EACnBN,MACA;;4CACQhB,MAI+BA,OAA9B,2BAAA,mBAAA,gBAAA,WAAA,OAAMuB;;;;oDAJA;;wDAAMb,GAAGQ,OAAO,CAACD;;;oDAAxBjB,OAAO;;;;;;;;;oDAIN,kCAAA,2BAAA;;;;;;;;;oDAAA,YAAaoB;;;2DAAb,6BAAA,QAAA;;;;oDAAMG,MAAN;oDAAwB;;wDAAMvB,CAAAA,QAAAA,MAAKC,GAAG,OAARD,OAAS,qBAAGuB;;;oDAAlB;;;oDAAxB;;;;;;;;;;;;oDAAA;oDAAA;;;;;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;;;;;;;;;oDAEL;;wDAAMvB,KAAKwB,KAAK;;;oDAAhB;;;;;;;;;;oCAEJ;mCACAC,0BAAW;;;4BAZb;;;;;;YAcF;;IACF;IACA,OAAOT;AACT"}

package/dist/cjs/store/turso/open.js

@@ -374,9 +374,15 @@ function close(handle) {
                 case 0:
                     return [
                         4,
-                        handle.close()
+                        (0, _connectionts.checkpointWal)(handle)
                     ];
                 case 1:
+                    _state.sent();
+                    return [
+                        4,
+                        handle.close()
+                    ];
+                case 2:
                     _state.sent();
                     return [
                         2

package/dist/cjs/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","tursoOpenDialect","ensureSchema","_handle","conn","cfg","ddl","feature","exec","CONTENT_FTS_DDL","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","db","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","filename","schemaVersion","reconcileDialect","tursoDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAGAC;eAAAA;;QAuESC;eAAAA;;QAfTC;eAAAA;;;uBAxEoB;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACO;2BACM;uBAClB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,mFAAmF;AACnF,SAAeG,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;;YAGrE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaC,4BAAe;;;2BAA5B,6BAAA,QAAA;;;;oBAAMH,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBE,IAAAA,wBAAc,EAACL;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQI,IAAI,KAAK,OAAM,GAAvBJ;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXI,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAML,QAAQM,MAAM,CAACT;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,MAAMC,MAAgB;;;;;oBACnC;;wBAAMA,OAAOD,KAAK;;;oBAAlB;;;;;;IACF;;AAEA,SAAeE,sBAAsBjB,OAAiB,EAAEC,IAAgB,EAAEiB,EAAU;;;;;oBAClF;;wBAAMjB,KAAKI,IAAI,CAAC,AAAC,yBAA2B,OAAHa;;;oBAAzC;;;;;;IACF;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,OAGKC,KAKLC,IAKKD;;;;;;;;;;oBAXC;;wBAAME,IAAAA,kBAAQ;;;oBAAtBH,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAeG,oBAAU,GAAE,MAAMH;oBACrC,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACJ,IAAcK,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMN,MAAMH,OAAO,CAACC,QAAQ;4BAAES,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHN,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXN,QAAO,MAA2B,OAAvB,AAACG,KAAcK,OAAO;;oBAEpH;;wBAAO;4BAAEZ,QAAQQ;4BAAIvB,MAAM8B,IAAAA,8BAAgB,EAACP;wBAAI;;;;IAClD;;AAGO,IAAM1B,mBAA0C;IACrDkC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,yBAAY;IAC9BhB,SAAAA;IACAJ,OAAAA;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGqB,UAAU,SAAVA,SAAWb;eAAQ,iEAAiEc,IAAI,CAACd,IAAIK,OAAO;;IACpG7B,cAAAA;IACAkB,uBAAAA;IACAqB,aAAa,SAAbA,YAActB,QAAQf;eAASqC,IAAAA,oBAAW,EAACtB,QAAQf;;AACrD;AAEO,SAAeJ,UAAUK,GAAmB;;;YACjD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKJ;;;IAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { checkpointWal, createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await checkpointWal(handle);\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","tursoOpenDialect","ensureSchema","_handle","conn","cfg","ddl","feature","exec","CONTENT_FTS_DDL","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","checkpointWal","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","db","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","filename","schemaVersion","reconcileDialect","tursoDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAGAC;eAAAA;;QAwESC;eAAAA;;QAfTC;eAAAA;;;uBAzEoB;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEe;wBACR;2BACM;uBAClB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,mFAAmF;AACnF,SAAeG,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;;YAGrE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaC,4BAAe;;;2BAA5B,6BAAA,QAAA;;;;oBAAMH,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBE,IAAAA,wBAAc,EAACL;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQI,IAAI,KAAK,OAAM,GAAvBJ;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXI,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAML,QAAQM,MAAM,CAACT;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,MAAMC,MAAgB;;;;;oBACnC;;wBAAMC,IAAAA,2BAAa,EAACD;;;oBAApB;oBACA;;wBAAMA,OAAOD,KAAK;;;oBAAlB;;;;;;IACF;;AAEA,SAAeG,sBAAsBlB,OAAiB,EAAEC,IAAgB,EAAEkB,EAAU;;;;;oBAClF;;wBAAMlB,KAAKI,IAAI,CAAC,AAAC,yBAA2B,OAAHc;;;oBAAzC;;;;;;IACF;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,OAGKC,KAKLC,IAKKD;;;;;;;;;;oBAXC;;wBAAME,IAAAA,kBAAQ;;;oBAAtBH,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAeG,oBAAU,GAAE,MAAMH;oBACrC,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACJ,IAAcK,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMN,MAAMH,OAAO,CAACC,QAAQ;4BAAES,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHN,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXN,QAAO,MAA2B,OAAvB,AAACG,KAAcK,OAAO;;oBAEpH;;wBAAO;4BAAEb,QAAQS;4BAAIxB,MAAM+B,IAAAA,8BAAgB,EAACP;wBAAI;;;;IAClD;;AAGO,IAAM3B,mBAA0C;IACrDmC,UAAUtC;IACVuC,eAAetC;IACfuC,kBAAkBC,yBAAY;IAC9BhB,SAAAA;IACAL,OAAAA;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGsB,UAAU,SAAVA,SAAWb;eAAQ,iEAAiEc,IAAI,CAACd,IAAIK,OAAO;;IACpG9B,cAAAA;IACAmB,uBAAAA;IACAqB,aAAa,SAAbA,YAAcvB,QAAQf;eAASsC,IAAAA,oBAAW,EAACvB,QAAQf;;AACrD;AAEO,SAAeJ,UAAUK,GAAmB;;;YACjD;;gBAAOsC,IAAAA,uBAAe,EAACtC,KAAKJ;;;IAC9B"}

package/dist/cjs/store/turso/store.js

@@ -20,6 +20,7 @@ var _typests = require("../../embed/types.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 var _vectorsts = require("../vectors.js");
+var _connectionts = require("./connection.js");
 var _fieldStatsts = require("./fieldStats.js");
 var _lexicalts = require("./lexical.js");
 var _vectorsts1 = require("./vectors.js");
@@ -559,9 +560,15 @@ function createStore(db, conn) {
                         case 0:
                             return [
                                 4,
-                                db.close()
+                                (0, _connectionts.checkpointWal)(db)
                             ];
                         case 1:
+                            _state.sent();
+                            return [
+                                4,
+                                db.close()
+                            ];
+                        case 2:
                             _state.sent();
                             return [
                                 2

package/dist/cjs/store/turso/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QAIGC;eAAAA;;;uBAfW;wBACA;6BACK;yBAEU;4BACf;yBACE;0BACiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;CAAU;AAIvF,SAASD,YAAYE,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcN;QACRO,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMJ,KAAKG,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOJ,KAAKK,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMP,KAAKM,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACV,MAAMS;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACb;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAc,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAACd,MAAMY,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAO,SAAPA,MAAQC,OAAOC;uBAASC,IAAAA,uBAAY,EAACpB,MAAMkB,OAAOC;;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIE,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACvB;;YAC3BwB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,MAAM2B,mBAAU,EAAEF;;YAC3DG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,MAAM6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC5FE,SAAS,SAATA,QAAUC,MAAMhB;uBAASiB,IAAAA,uBAAW,EAACpC,MAAM2B,mBAAU,EAAEQ,MAAMhB;;YAC7DkB,WAAW,SAAXA,UAAYF;uBAASG,IAAAA,uBAAY,EAACtC,MAAMmC;;QAC1C;QACMI,cAAN,SAAMA;;oBAGEC;;;;4BAAc;;gCAAMzC,GAAGM,OAAO,CAAC;;;4BAAxB;;gCAAO,cAAyCoC,GAAG;;;4BAA1DD,MAAO;4BACb;;gCAAO;oCAAEE,cAAc,AAAC,GAAmB,OAAjBF,IAAIE,YAAY,EAAC;gCAAoF;;;;YACjI;;QACAC,KAAK;YACGtC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBwC;;;;gCAAO;;oCAAM7C,GAAGM,OAAO,CAACD;;;gCAAxBwC,OAAO;gCACbA,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gCACzF;;oCAAO;wCACLjC,SAAS,SAATA;mDAAegC,KAAKhC,OAAO;;wCAC3B,8DAA8D;wCAC9DkC,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDACrCH;;;;4DAAP;;2EAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAGG;;;4DAAvB;;;;;;4CACF;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;;;4BACJ;;gCAAMjD,GAAGiD,KAAK;;;4BAAd;;;;;;YACF;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { checkpointWal } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await checkpointWal(db);\n      await db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close","checkpointWal"],"mappings":";;;;;;;;;;;QAaaA;eAAAA;;QAIGC;eAAAA;;;uBAhBW;wBACA;6BACK;yBAEU;4BACZ;4BACH;yBACE;0BACiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;CAAU;AAIvF,SAASD,YAAYE,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcN;QACRO,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMJ,KAAKG,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOJ,KAAKK,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMP,KAAKM,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACV,MAAMS;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACb;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAc,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAACd,MAAMY,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAO,SAAPA,MAAQC,OAAOC;uBAASC,IAAAA,uBAAY,EAACpB,MAAMkB,OAAOC;;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIE,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACvB;;YAC3BwB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,MAAM2B,mBAAU,EAAEF;;YAC3DG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,MAAM6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC5FE,SAAS,SAATA,QAAUC,MAAMhB;uBAASiB,IAAAA,uBAAW,EAACpC,MAAM2B,mBAAU,EAAEQ,MAAMhB;;YAC7DkB,WAAW,SAAXA,UAAYF;uBAASG,IAAAA,uBAAY,EAACtC,MAAMmC;;QAC1C;QACMI,cAAN,SAAMA;;oBAGEC;;;;4BAAc;;gCAAMzC,GAAGM,OAAO,CAAC;;;4BAAxB;;gCAAO,cAAyCoC,GAAG;;;4BAA1DD,MAAO;4BACb;;gCAAO;oCAAEE,cAAc,AAAC,GAAmB,OAAjBF,IAAIE,YAAY,EAAC;gCAAoF;;;;YACjI;;QACAC,KAAK;YACGtC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBwC;;;;gCAAO;;oCAAM7C,GAAGM,OAAO,CAACD;;;gCAAxBwC,OAAO;gCACbA,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gCACzF;;oCAAO;wCACLjC,SAAS,SAATA;mDAAegC,KAAKhC,OAAO;;wCAC3B,8DAA8D;wCAC9DkC,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDACrCH;;;;4DAAP;;2EAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAGG;;;4DAAvB;;;;;;4CACF;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;;;4BACJ;;gCAAMC,IAAAA,2BAAa,EAAClD;;;4BAApB;4BACA;;gCAAMA,GAAGiD,KAAK;;;4BAAd;;;;;;YACF;;IACF;AACF"}

package/dist/cjs/store/types.d.cts

@@ -21,6 +21,7 @@ export interface Connection {
     exec(sql: string): Promise<void>;
     prepare(sql: string): Promise<Statement>;
     runBatch(sql: string, paramRows: unknown[][]): Promise<void>;
+    appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;
 }
 export interface ReconcileDialect {
     beginMode(): string;
@@ -28,7 +29,6 @@ export interface ReconcileDialect {
     addColumns(conn: Connection, names: string[]): Promise<void>;
     reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
     recordDuration?(conn: Connection, ms: number): Promise<void>;
-    insertNew?(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void>;
 }
 export interface OpenDialect<Handle> {
     filename: string;

package/dist/cjs/store/types.d.ts

@@ -21,6 +21,7 @@ export interface Connection {
     exec(sql: string): Promise<void>;
     prepare(sql: string): Promise<Statement>;
     runBatch(sql: string, paramRows: unknown[][]): Promise<void>;
+    appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;
 }
 export interface ReconcileDialect {
     beginMode(): string;
@@ -28,7 +29,6 @@ export interface ReconcileDialect {
     addColumns(conn: Connection, names: string[]): Promise<void>;
     reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
     recordDuration?(conn: Connection, ms: number): Promise<void>;
-    insertNew?(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void>;
 }
 export interface OpenDialect<Handle> {
     filename: string;