sensemaking 0.19.1 → 0.20.0

package/dist/cjs/store/sqlite/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nfunction contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      for (const col of newColumns) if (!present.has(col)) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n      // content's rowid lookup depends on its coupled frontmatter row, so the content\n      // deletes/inserts run while every path still has one -- the vanished frontmatter delete comes last.\n      if (vanished.length > 0) {\n        await conn.runBatch(\n          'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n          vanished.map((p) => [p])\n        );\n      }\n      if (reparsedExisting.length > 0) {\n        // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via\n        // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.\n        await conn.runBatch(\n          'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n          reparsedExisting.map((p) => [p])\n        );\n      }\n\n      if (parsedDocs.length > 0) {\n        const rows = parsedDocs.map((doc) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          })\n        );\n        await conn.runBatch(insertSql, rows);\n\n        // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n        // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n        const segmenting = contentTokenize(cfg) === undefined;\n        await conn.runBatch(\n          INSERT_CONTENT_SQL,\n          parsedDocs.map((doc) => contentRow(doc, segmenting))\n        );\n      }\n\n      if (vanished.length > 0)\n        await conn.runBatch(\n          'DELETE FROM frontmatter WHERE \"path\" = ?',\n          vanished.map((p) => [p])\n        );\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      const presetTouched = [...vanished, ...reparsedExisting];\n      if (presetTouched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM preset_files WHERE \"path\" = ?',\n          presetTouched.map((p) => [p])\n        );\n      const presetRows: unknown[][] = [];\n      for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n      if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n      const removedPaths = [...vanished, ...reparsedExisting];\n      if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await feature.store?.(conn, docsForFeature, delta);\n      }\n      for (const feature of features) await feature.afterReconcile?.(conn, delta);\n    },\n    BEGIN_WRITE\n  );\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(\n    conn,\n    async () => {\n      if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n    },\n    BEGIN_WRITE\n  );\n  return warnings;\n}\n"],"names":["changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","reconcile","signatureDiff","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","segmentField","conn","cfg","baseDir","files","currentSet","existingStmt","existingRows","existing","vanished","toReparse","features","seenColumns","report","parsedDocs","warnings","newColumns","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","txStart","durationMs","prevRaw","prevMax","listFiles","map","f","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","activeFeatures","getColumns","progress","reparseFiles","tick","docs","finish","add","SenseError","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","feature","present","rows","presetTouched","presetRows","presetName","removedPaths","exec","runBatch","ctimeMs","parseError","data","contentTokenize","undefined","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","BEGIN_WRITE","getMeta","Number","setMeta","String","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","key","val","keys","embedPart","find","at","indexOf","label","stmt","known","file","parseFile","fileWarnings"],"mappings":";;;;;;;;;;;QA6KgBA;eAAAA;;QAaAC;eAAAA;;QAkBMC;eAAAA;;QA7KAC;eAAAA;;QAqKNC;eAAAA;;;uBAnMgB;wBACL;wBACI;0BAEN;wBAE8B;yBAC1B;yBACA;wBAC4B;6BACZ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG7C,6FAA6F;AAC7F,4EAA4E;AAC5E,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,IAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,IAAMC,qBAAqB;AAE3B,uFAAuF;AACvF,+EAA+E;AAC/E,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IACrD,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEO,SAAeb,UAAUe,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YACtEC,OACAC,YAEAC,cACAC,cACAC,UACAC,UAEAC,WAOAC,UACAC,aAIAC,QAC6C,MAArCC,YAAYC,UAAUC,YAE/B,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YAWAC,iBAGAC,WAKAC,OACAC,OACAC,UACAC,kBAEAC,SAgFAC,YACAC,SAGAC;;;;oBArIAxB,QAAQyB,IAAAA,mBAAS,EAAC3B,KAAKC;oBACvBE,aAAa,IAAIhB,IAAIe,MAAM0B,GAAG,CAAC,SAACC;+BAAMA,EAAEpC,OAAO;;oBAEhC;;wBAAMM,KAAK+B,OAAO,CAAC;;;oBAAlC1B,eAAe;oBACC;;wBAAMA,aAAa2B,GAAG;;;oBAAtC1B,eAAgB;oBAChBC,WAAW,IAAI0B,IAAI3B,aAAauB,GAAG,CAAC,SAACK;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBACtD1B,WAAWF,aAAa8B,MAAM,CAAC,SAACF;+BAAM,CAAC9B,WAAWiC,GAAG,CAACH,EAAEC,IAAI;uBAAGN,GAAG,CAAC,SAACK;+BAAMA,EAAEC,IAAI;;oBAEhF1B,YAAYN,MAAMiC,MAAM,CAAC,SAACN;wBAC9B,IAAMQ,MAAM/B,SAASgC,GAAG,CAACT,EAAEpC,OAAO;wBAClC,OAAO,CAAC4C,OAAOA,IAAIE,MAAM,KAAKV,EAAEW,OAAO,IAAIH,IAAII,KAAK,KAAKZ,EAAEa,IAAI;oBACjE;oBAEA,IAAInC,SAASoC,MAAM,KAAK,KAAKnC,UAAUmC,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAG/B,QAAQ;wBAAK;;oBAEhFJ,WAAWoC,IAAAA,wBAAc,EAAC7C;oBACZ;;wBAAM8C,IAAAA,oBAAU,EAAC/C;;;oBAA/BW,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAASoC,IAAAA,oBAAQ,EAAC,mBAAmBvC,UAAUmC,MAAM;oBACR;;wBAAMK,IAAAA,uBAAY,EAACxC,WAAWC,UAAUT,KAAKU,aAAaC,OAAOsC,IAAI;;;oBAArE,OAAA,eAArCrC,aAAqC,KAA3CsC,MAAkBrC,WAAyB,KAAzBA,UAAUC,aAAe,KAAfA;oBACpCH,OAAOwC,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAarC,iCAAb,6BAAA,QAAA,yBAAA;4BAAMC,MAAN;4BAAyBL,YAAY0C,GAAG,CAACrC;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGN;oBACvB,uEAAuE;oBACvE,sGAAsG;oBACtG,IAAIM,WAAW2B,MAAM,GAAGvD,yBAAyB;wBAC/C,MAAM,IAAIiE,oBAAU,CAClB,gBACA,AAAC,0BAAuHjE,OAA9F4B,WAAW2B,MAAM,EAAC,8EAAoG,OAAxBvD,yBAAwB;oBAEpJ;oBACA,0FAA0F;oBAC1F,sEAAsE;oBAChE6B,kBAAkBD,WAAWmB,MAAM,CAAC,SAACmB;+BAAMpE,yBAAyBkD,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBAC1G,sFAAsF;oBACtF,gDAAgD;oBAC1CpC,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBW,GAAG,CAAC4B,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FxC,OAAhFA,gBAAgBW,GAAG,CAAC;+BAAM;uBAAK6B,IAAI,CAAC,OAAM,wCAGjI,OAHuKxC,gBAClLkB,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpB1B,GAAG,CAAC,SAAC0B;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEFtC,QAAQX,UAAU2B,MAAM,CAAC,SAACN;+BAAM,CAACvB,SAAS8B,GAAG,CAACP,EAAEpC,OAAO;uBAAGmC,GAAG,CAAC,SAACC;+BAAMA,EAAEpC,OAAO;;oBAC9E2B,QAAwB;wBAAElB,OAAAA;wBAAOwD,UAAU9C,WAAWgB,GAAG,CAAC,SAAC+B;mCAAMA,EAAElE,OAAO;;wBAAG0B,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAIlC,IAAIgC;oBACnBG,mBAAmBV,WAAWgB,GAAG,CAAC,SAAC+B;+BAAMA,EAAElE,OAAO;uBAAE0C,MAAM,CAAC,SAACyB;+BAAM,CAACvC,SAASe,GAAG,CAACwB;;oBAEhFrC,UAAUsC,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EACnBhE,MACA;;oCAiEqEiE,iBAK7BA,yBAnEhCC,SACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMlD,UAoBHmD,MAeA1E,YAeF2E,eAMAC,YAC6B,4BAAA,oBAAA,iBAA9B,4BAAA,oBAAA,iBAAA,YAAA,QAAM7E,KAAwB,YAAA,QAAM8E,YAGnCC,cAC4B,4BAAA,oBAAA,iBAAA,YAAA,QAAMN,cACnC,4BAAA,oBAAA,wBAAA,YAAA,aAIA,4BAAA,oBAAA,iBAAA,YAAA,QAAMA;;;;4CAnEK;;gDAAMlB,IAAAA,oBAAU,EAAC/C;;;4CAA3BkE,UAAU;4CACX,kCAAA,2BAAA;;;;;;;;;4CAAA,YAAanD;;;mDAAb,6BAAA,QAAA;;;;4CAAMC,MAAN;iDAA6B,CAACkD,QAAQ7B,GAAG,CAACrB,MAAb;;;;4CAAmB;;gDAAMhB,KAAKwE,IAAI,CAAC,AAAC,sCAAqD,OAAhBf,IAAAA,oBAAU,EAACzC;;;4CAAjE;;;4CAAhD;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAIDR,CAAAA,SAASoC,MAAM,GAAG,CAAA,GAAlBpC;;;;4CACF;;gDAAMR,KAAKyE,QAAQ,CACjB,sFACAjE,SAASqB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;iDAKEtC,CAAAA,iBAAiBqB,MAAM,GAAG,CAAA,GAA1BrB;;;;4CACF,6FAA6F;4CAC7F,uGAAuG;4CACvG;;gDAAMvB,KAAKyE,QAAQ,CACjB,sFACAlD,iBAAiBM,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFjC;;;iDAMEhD,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CACIsD,OAAOtD,WAAWgB,GAAG,CAAC,SAACrC;uDAC3B0B,gBAAgBW,GAAG,CAAC,SAACb;wDAOZxB;oDANP,IAAIwB,QAAQ,QAAQ,OAAOxB,IAAIE,OAAO;oDACtC,IAAIsB,QAAQ,UAAU,OAAOxB,IAAIiD,OAAO;oDACxC,IAAIzB,QAAQ,UAAU,OAAOxB,IAAIkF,OAAO;oDACxC,IAAI1D,QAAQ,SAAS,OAAOxB,IAAImD,IAAI;oDACpC,mFAAmF;oDACnF,IAAI3B,QAAQ,gBAAgB,OAAOxB,IAAImF,UAAU;oDACjD,QAAOnF,gBAAAA,IAAIoF,IAAI,CAAC5D,IAAI,cAAbxB,2BAAAA,gBAAiB;gDAC1B;;4CAEF;;gDAAMQ,KAAKyE,QAAQ,CAACtD,WAAWgD;;;4CAA/B;4CAEA,wFAAwF;4CACxF,+EAA+E;4CACzE1E,aAAaoF,IAAAA,wBAAe,EAAC5E,SAAS6E;4CAC5C;;gDAAM9E,KAAKyE,QAAQ,CACjBnF,oBACAuB,WAAWgB,GAAG,CAAC,SAACrC;2DAAQD,WAAWC,KAAKC;;;;4CAF1C;;;iDAMEe,CAAAA,SAASoC,MAAM,GAAG,CAAA,GAAlBpC;;;;4CACF;;gDAAMR,KAAKyE,QAAQ,CACjB,4CACAjE,SAASqB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;4CAKF,0FAA0F;4CAC1F,kEAAkE;4CAC5DO,gBAAgB,AAAC,qBAAG5D,iBAAU,qBAAGe;iDACnC6C,CAAAA,cAAcxB,MAAM,GAAG,CAAA,GAAvBwB;;;;4CACF;;gDAAMpE,KAAKyE,QAAQ,CACjB,6CACAL,cAAcvC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF9B;;;4CAIIQ;4CAC6B,mCAAA,4BAAA,6BAA9B,mCAAA,4BAAA;;gDAAL,IAAK,aAAaxD,iCAAb,8BAAA,SAAA,0BAAA;oDAAMrB,MAAN;;wDAAyB,IAAK,aAAoBA,IAAIuF,OAAO,uBAA/B,8BAAA,SAAA,0BAAA;4DAAMT,aAAN;4DAAiCD,WAAWW,IAAI;gEAAExF,IAAIE,OAAO;gEAAE4E;;;;wDAA/D;wDAAA;;;iEAAA,8BAAA;gEAAA;;;gEAAA;sEAAA;;;;;;gDAA9B;gDAAA;;;yDAAA,8BAAA;wDAAA;;;wDAAA;8DAAA;;;;iDACDD,CAAAA,WAAWzB,MAAM,GAAG,CAAA,GAApByB;;;;4CAAuB;;gDAAMrE,KAAKyE,QAAQ,CAAC,2DAA2DJ;;;4CAA/E;;;4CAErBE,eAAe,AAAC,qBAAG/D,iBAAU,qBAAGe;4CACJ,mCAAA,4BAAA;iDAA9BgD,CAAAA,aAAa3B,MAAM,GAAG,CAAA,GAAtB2B;;;;;;;;;;;;4CAA8B,aAAiB7D;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,UAAN;4CAA2B;;iDAAMA,kBAAAA,QAAQgB,MAAM,cAAdhB,sCAAAA,qBAAAA,SAAiBjE,MAAMuE,cAAclD;;;4CAA3C;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAC7B,mCAAA,4BAAA;;;;;;;;;;oDAAM4C,SAEHA,gBADAiB;;;;4DADGjB,UAAN;4DACGiB,iBAAiCrE,WAAWgB,GAAG,CAAC,SAACrC;uEAAS;oEAAE2C,MAAM3C,IAAIE,OAAO;oEAAEyF,WAAW3F,IAAI2F,SAAS,CAAClB,QAAQmB,IAAI,CAAC;gEAAC;;4DAC5H;;iEAAMnB,iBAAAA,QAAQoB,KAAK,cAAbpB,qCAAAA,oBAAAA,SAAgBjE,MAAMkF,gBAAgB7D;;;4DAA5C;;;;;;4CACF;4CAHK,aAAiBX;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;4CAAA,aAAiBA;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,WAAN;4CAA2B;;iDAAMA,0BAAAA,SAAQqB,cAAc,cAAtBrB,8CAAAA,6BAAAA,UAAyBjE,MAAMqB;;;4CAArC;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;2BACAkE,0BAAW;;;oBA1Eb;oBA6EA,qFAAqF;oBACrF,8GAA8G;oBACxG9D,aAAaqC,KAAKC,GAAG,KAAKvC;oBAChB;;wBAAMgE,IAAAA,iBAAO,EAACxF,MAAM;;;oBAA9B0B,UAAU;oBAChB,yFAAyF;oBACzF,+EAA+E;oBACzEC,UAAUD,YAAY,OAAO,CAAC,IAAI+D,OAAO/D;yBAC3CD,CAAAA,aAAaE,OAAM,GAAnBF;;;;oBAAsB;;wBAAMiE,IAAAA,iBAAO,EAAC1F,MAAM,oBAAoB2F,OAAOlE;;;oBAA/C;;;oBAE1B;;wBAAO;4BAAEoB,QAAQhC,WAAW+B,MAAM;4BAAE9B,UAAAA;wBAAS;;;;IAC/C;;AAIO,SAAShC,qBAAqB8G,MAAc,EAAEC,KAAa;IAChE,IAAMC,QAAQ,eAACC;eAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGxC,IAAI,CAAC,OAAOqC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;;IACxH,IAAME,QAAQ,eAACC;eAAgB,IAAInE,IAAImE,IAAIH,KAAK,CAAC,KAAKpE,GAAG,CAAC,SAACkE;mBAAS;gBAACD,MAAMC;gBAAOA;aAAK;;;IACvF,IAAMM,IAAIF,MAAMP;IAChB,IAAMU,IAAIH,MAAMN;IAChB,IAAMU,UAAU,IAAInH;QACf,kCAAA,2BAAA;;QAAL,QAAK,YAAoBkH,sBAApB,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAOE,sBAAKC;YAAW,IAAIJ,EAAE9D,GAAG,CAACiE,SAASC,KAAKF,QAAQlD,GAAG,CAACmD;;;QAA3D;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;QACA,mCAAA,4BAAA;;QAAL,QAAK,aAAaH,EAAEK,IAAI,uBAAnB,UAAA,8BAAA,SAAA,0BAAA;YAAA,IAAMF,OAAN;YAAuB,IAAI,CAACF,EAAEjE,GAAG,CAACmE,OAAMD,QAAQlD,GAAG,CAACmD;;;QAApD;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,OAAOD;AACT;AAIO,SAASxH,qBAAqB6G,MAAc,EAAEC,KAAa;IAChE,IAAMc,YAAY,mBAACP;eAAgBA,IAAIH,KAAK,CAAC,KAAKW,IAAI,CAAC,SAAC/C;mBAAMA,EAAEmC,UAAU,CAAC;;;IAC3E,IAAMM,IAAIK,UAAUf;IACpB,IAAMS,IAAIM,UAAUd;IACpB,IAAIS,MAAMxB,aAAauB,MAAMvB,WAAW,OAAO;IAC/C,IAAM+B,KAAKR,EAAES,OAAO,CAAC;IACrB,OAAOR,EAAEQ,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKR,EAAEH,KAAK,CAAC,GAAGW,QAAQP;AAClE;AAGO,SAASpH,cAAc0G,MAAc,EAAEC,KAAa;IACzD,IAAMU,UAAUzH,qBAAqB8G,QAAQC;IAC7C,IAAMkB,QAAQ,eAACP;eAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIR,UAAU,CAAC,aAAa,AAAC,WAAuB,OAAbQ,IAAIN,KAAK,CAAC,IAAG,OAAK;;IACzK,OAAOK,QAAQ5D,IAAI,KAAK,IAAI,aAAa,AAAC,qBAAG4D,SAAS1E,GAAG,CAACkF,OAAOrD,IAAI,CAAC;AACxE;AAIO,SAAe1E,oBAAoBgB,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YAChF8G,MACAC,UACA9G,OACAV,YACAqB,UACAqD,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAM+C,MAETpG,WADwCqG,YAAhC3H,KAAe4H;;;;oBAPZ;;wBAAMpH,KAAK+B,OAAO,CAAC;;;oBAA1BiF,OAAO;wBACK5H;oBAAM;;wBAAM4H,KAAKhF,GAAG;;;oBAAhCiF,QAAQ,IAAA,CAAA,EAAA,MAAI7H;;wBAAM,cAA8CyC,GAAG,CAAC,SAACK;mCAAMA,EAAEC,IAAI;;sBAAC;oBAClFhC,QAAQyB,IAAAA,mBAAS,EAAC3B,KAAKC,SAASkC,MAAM,CAAC,SAACN;+BAAMmF,MAAM5E,GAAG,CAACP,EAAEpC,OAAO;;oBACjED,aAAaoF,IAAAA,wBAAe,EAAC5E,SAAS6E;oBACtChE;oBACAqD;oBACD,kCAAA,2BAAA;;wBAAL,IAAK,YAAchE,4BAAd,6BAAA,QAAA,yBAAA,iCAAqB;4BAAf+G,OAAN;;4BACqCC,aAAAA,IAAAA,mBAAS,EAACD,OAA1C1H,MAAgC2H,WAAhC3H,KAAe4H,eAAiBD,WAA3BrG;4BACbA,CAAAA,YAAAA,UAASkE,IAAI,OAAblE,WAAc,qBAAGsG;4BACjBjD,KAAKa,IAAI,CAACzF,WAAWC,KAAKC;wBAC5B;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAKL;;wBAAMuE,IAAAA,8BAAe,EACnBhE,MACA;;;;;iDACMmE,CAAAA,KAAKvB,MAAM,GAAG,CAAA,GAAduB;;;;4CAAiB;;gDAAMnE,KAAKyE,QAAQ,CAACnF,oBAAoB6E;;;4CAAxC;;;;;;;;4BACvB;2BACAoB,0BAAW;;;oBALb;oBAOA;;wBAAOzE;;;;IACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is FTS5-virtual with \"_seg\" sidecars for lexical.ts.\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcileContent() and the tokenize-only rebuild in open.ts, so both prepare the\n// same literal instead of two copies drifting apart.\nexport const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nexport function contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta, cfg: Config): Promise<void> {\n  // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via\n  // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n      touched.map((p) => [p])\n    );\n\n  if (docs.length > 0) {\n    // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n    // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n    const segmenting = contentTokenize(cfg) === undefined;\n    await conn.runBatch(\n      INSERT_CONTENT_SQL,\n      docs.map((doc) => contentRow(doc, segmenting))\n    );\n  }\n}\n\nexport const sqliteDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    // Fence before ALTERing: SQLite's own failure past this point is a raw\n    // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  columnTypeSuffix: () => '',\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["INSERT_CONTENT_SQL","contentRow","sqliteDialect","MAX_FRONTMATTER_COLUMNS","doc","segmenting","relPath","search","title","summary","text","segmentField","reconcileContent","conn","touched","docs","_delta","cfg","length","runBatch","map","p","contentTokenize","undefined","beginMode","BEGIN_WRITE","checkColumnLimit","count","SenseError","columnTypeSuffix","recordDuration","recordReconcileDuration"],"mappings":";;;;;;;;;;;QAkBaA;eAAAA;;QAIGC;eAAAA;;QAwBHC;eAAAA;;;uBA7CmB;wBACL;yBAGE;wBACW;6BACZ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG5B,qFAAqF;AACrF,mFAAmF;AAEnF,8FAA8F;AAC9F,IAAMC,0BAA0B;AAIzB,IAAMH,qBAAqB;AAI3B,SAASC,WAAWG,GAAc,EAAEC,UAAmB;IAC5D,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,SAAeE,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB,EAAEC,GAAW;;YAYjHZ;;;;yBATJS,CAAAA,QAAQI,MAAM,GAAG,CAAA,GAAjBJ;;;;oBACF;;wBAAMD,KAAKM,QAAQ,CACjB,sFACAL,QAAQM,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAKEN,CAAAA,KAAKG,MAAM,GAAG,CAAA,GAAdH;;;;oBACF,wFAAwF;oBACxF,+EAA+E;oBACzEV,aAAaiB,IAAAA,wBAAe,EAACL,SAASM;oBAC5C;;wBAAMV,KAAKM,QAAQ,CACjBnB,oBACAe,KAAKK,GAAG,CAAC,SAAChB;mCAAQH,WAAWG,KAAKC;;;;oBAFpC;;;;;;;;IAKJ;;AAEO,IAAMH,gBAAkC;IAC7CsB,WAAW,SAAXA;eAAiBC,0BAAW;;IAC5BC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,uEAAuE;QACvE,sGAAsG;QACtG,IAAIA,QAAQxB,yBAAyB;YACnC,MAAM,IAAIyB,oBAAU,CAClB,gBACA,AAAC,0BAA2GzB,OAAlFwB,OAAM,8EAAoG,OAAxBxB,yBAAwB;QAExI;IACF;IACA0B,kBAAkB,SAAlBA;eAAwB;;IACxBjB,kBAAAA;IACAkB,gBAAgBC,iCAAuB;AACzC"}

package/dist/cjs/store/sqlite/store.d.cts

@@ -2,4 +2,4 @@ import type { DatabaseSync } from 'node:sqlite';
 import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store;
+export declare function createStore(db: DatabaseSync, conn: Connection, cfg: Config): Store;

package/dist/cjs/store/sqlite/store.d.ts

@@ -2,4 +2,4 @@ import type { DatabaseSync } from 'node:sqlite';
 import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store;
+export declare function createStore(db: DatabaseSync, conn: Connection, cfg: Config): Store;

package/dist/cjs/store/sqlite/store.js

@@ -22,7 +22,6 @@ var _transactionts = require("../transaction.js");
 var _vectorsts = require("../vectors.js");
 var _fieldStatsts = require("./fieldStats.js");
 var _lexicalts = require("./lexical.js");
-var _reconcilets = require("./reconcile.js");
 var _vectorsts1 = require("./vectors.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -367,9 +366,10 @@ var CAPABILITIES = new Set([
     'snippets',
     'watch-concurrency',
     'lexical',
-    'vectors'
+    'vectors',
+    'sql-functions'
 ]);
-function createStore(db, conn, cfg, baseDir) {
+function createStore(db, conn, cfg) {
     return {
         name: 'sqlite',
         capabilities: CAPABILITIES,
@@ -429,16 +429,6 @@ function createStore(db, conn, cfg, baseDir) {
                 });
             })();
         },
-        reconcile: function reconcile() {
-            return _async_to_generator(function() {
-                return _ts_generator(this, function(_state) {
-                    return [
-                        2,
-                        (0, _reconcilets.reconcile)(conn, cfg, baseDir)
-                    ];
-                });
-            })();
-        },
         docs: {
             columns: function columns() {
                 return _async_to_generator(function() {

package/dist/cjs/store/sqlite/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with open()'s own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","reconcile","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","contentTokenize","undefined","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","candidates","qv","storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QAIGC;eAAAA;;;uBAdgB;wBACL;6BACK;yBAEU;4BACf;yBACE;2BACH;0BACoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;CAAU;AAIxH,SAASD,YAAYE,EAAgB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC1F,OAAO;QACLC,MAAM;QACNC,cAAcR;QACRS,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMN,KAAKK,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAON,KAAKO,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMT,KAAKQ,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACZ,MAAMW;;;YAC/B;;QACME,WAAN,SAAMA;;;oBACJ;;wBAAOA,IAAAA,sBAAS,EAACb,MAAMC,KAAKC;;;YAC9B;;QACAY,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAAChB;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACMiB,YAAN,SAAMA,WAAWF,OAAO,EAAEG,UAAU;;;wBAClC;;4BAAOD,IAAAA,wBAAU,EAACjB,MAAMe,SAASG;;;gBACnC;;QACF;QACAC,SAAS;YACDC,OAAN,SAAMA,MAAMC,KAAK,EAAEC,IAAI;;;wBACrB;;4BAAOC,IAAAA,uBAAY,EAACvB,MAAMqB,OAAOC,MAAME,IAAAA,wBAAe,EAACvB,SAASwB;;;gBAClE;;QACF;QACAC,SAAS;YACDC,SAAN,SAAMA;;;wBACJ;;4BAAOC,IAAAA,sBAAW,EAAC5B;;;gBACrB;;YACM6B,cAAN,SAAMA,aAAaC,IAAsB;;;;;gCACvC;;oCAAMC,IAAAA,4BAAgB,EAAC/B,MAAM8B;;;gCAA7B;;;;;;gBACF;;YACME,YAAN,SAAMA,WAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;;;wBAC5C;;4BAAOC,IAAAA,0BAAc,EAACrC,MAAMiC,IAAIC,WAAWC,OAAOC;;;gBACpD;;YACME,SAAN,SAAMA,QAAQC,IAAI,EAAEjB,IAAI;;;wBACtB;;4BAAOkB,IAAAA,uBAAW,EAACxC,MAAMuC,MAAMjB;;;gBACjC;;YACMmB,WAAN,SAAMA,UAAUF,IAAI;;;wBAClB;;4BAAOG,IAAAA,uBAAY,EAAC1C,MAAMuC;;;gBAC5B;;QACF;QACMI,cAAN,SAAMA;;oBAGEC;;oBAFN,qFAAqF;oBACrF,qFAAqF;oBAC/EA,MAAM7C,GAAGQ,OAAO,CAAC,uBAAuBsC,GAAG;oBACjD;;wBAAO;4BAAEC,cAAc,AAAC,GAAc,OAAZF,IAAIG,OAAO,EAAC;wBAAoF;;;YAC5H;;QACAC,KAAK;YACGzC,SAAN,SAAMA,QAAQD,GAAW;;wBACjB2C;;wBAAAA,OAAOlD,GAAGQ,OAAO,CAACD;wBACxB2C,KAAKC,cAAc,CAAC,OAAO,qEAAqE;wBAChG;;4BAAO;gCACLnC,SAAS,SAATA;2CAAekC,KAAKlC,OAAO;;gCAC3B,2EAA2E;gCAC3EoC,SAAS,SAATA;oCAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;wCAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;4CACrCH;;;;oDAAP;;mEAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAIG;;;oDAAxB;;;;;;oCACF;;4BACF;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJtD,GAAGsD,KAAK;;;;;YACV;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with the builder's own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","cfg","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","contentTokenize","undefined","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","candidates","qv","storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":";;;;;;;;;;;QAWaA;eAAAA;;QAIGC;eAAAA;;;uBAbgB;wBACL;6BACK;yBAEU;4BACf;yBACE;0BACiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB;AAIzI,SAASD,YAAYE,EAAgB,EAAEC,IAAgB,EAAEC,GAAW;IACzE,OAAO;QACLC,MAAM;QACNC,cAAcP;QACRQ,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAML,KAAKI,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOL,KAAKM,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMR,KAAKO,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACX,MAAMU;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACd;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACMe,YAAN,SAAMA,WAAWF,OAAO,EAAEG,UAAU;;;wBAClC;;4BAAOD,IAAAA,wBAAU,EAACf,MAAMa,SAASG;;;gBACnC;;QACF;QACAC,SAAS;YACDC,OAAN,SAAMA,MAAMC,KAAK,EAAEC,IAAI;;;wBACrB;;4BAAOC,IAAAA,uBAAY,EAACrB,MAAMmB,OAAOC,MAAME,IAAAA,wBAAe,EAACrB,SAASsB;;;gBAClE;;QACF;QACAC,SAAS;YACDC,SAAN,SAAMA;;;wBACJ;;4BAAOC,IAAAA,sBAAW,EAAC1B;;;gBACrB;;YACM2B,cAAN,SAAMA,aAAaC,IAAsB;;;;;gCACvC;;oCAAMC,IAAAA,4BAAgB,EAAC7B,MAAM4B;;;gCAA7B;;;;;;gBACF;;YACME,YAAN,SAAMA,WAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;;;wBAC5C;;4BAAOC,IAAAA,0BAAc,EAACnC,MAAM+B,IAAIC,WAAWC,OAAOC;;;gBACpD;;YACME,SAAN,SAAMA,QAAQC,IAAI,EAAEjB,IAAI;;;wBACtB;;4BAAOkB,IAAAA,uBAAW,EAACtC,MAAMqC,MAAMjB;;;gBACjC;;YACMmB,WAAN,SAAMA,UAAUF,IAAI;;;wBAClB;;4BAAOG,IAAAA,uBAAY,EAACxC,MAAMqC;;;gBAC5B;;QACF;QACMI,cAAN,SAAMA;;oBAGEC;;oBAFN,qFAAqF;oBACrF,qFAAqF;oBAC/EA,MAAM3C,GAAGO,OAAO,CAAC,uBAAuBqC,GAAG;oBACjD;;wBAAO;4BAAEC,cAAc,AAAC,GAAc,OAAZF,IAAIG,OAAO,EAAC;wBAAoF;;;YAC5H;;QACAC,KAAK;YACGxC,SAAN,SAAMA,QAAQD,GAAW;;wBACjB0C;;wBAAAA,OAAOhD,GAAGO,OAAO,CAACD;wBACxB0C,KAAKC,cAAc,CAAC,OAAO,qEAAqE;wBAChG;;4BAAO;gCACLnC,SAAS,SAATA;2CAAekC,KAAKlC,OAAO;;gCAC3B,2EAA2E;gCAC3EoC,SAAS,SAATA;oCAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;wCAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;4CACrCH;;;;oDAAP;;mEAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAIG;;;oDAAxB;;;;;;oCACF;;4BACF;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJpD,GAAGoD,KAAK;;;;;YACV;;IACF;AACF"}

package/dist/cjs/store/turso/open.d.cts

@@ -1,14 +1,9 @@
+import type { Database } from '@tursodatabase/database';
 import type { ResolvedConfig } from '../../config/index.js';
-import type { Store } from '../types.js';
+import type { OpenResult } from '../open.js';
+import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "3";
-export interface OpenResult {
-    store: Store;
-    cfg: ResolvedConfig;
-    dbPath: string;
-    parsed: number;
-    warnings: string[];
-}
-export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
-export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
+export declare const SCHEMA_VERSION = "4";
+export type { OpenResult };
+export declare const tursoOpenDialect: OpenDialect<Database>;
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/cjs/store/turso/open.d.ts

@@ -1,14 +1,9 @@
+import type { Database } from '@tursodatabase/database';
 import type { ResolvedConfig } from '../../config/index.js';
-import type { Store } from '../types.js';
+import type { OpenResult } from '../open.js';
+import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "3";
-export interface OpenResult {
-    store: Store;
-    cfg: ResolvedConfig;
-    dbPath: string;
-    parsed: number;
-    warnings: string[];
-}
-export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
-export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
+export declare const SCHEMA_VERSION = "4";
+export type { OpenResult };
+export declare const tursoOpenDialect: OpenDialect<Database>;
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/cjs/store/turso/open.js

@@ -9,12 +9,6 @@ function _export(target, all) {
     });
 }
 _export(exports, {
-    get CONTENT_FTS_DDL () {
-        return CONTENT_FTS_DDL;
-    },
-    get CONTENT_FTS_NAMES () {
-        return CONTENT_FTS_NAMES;
-    },
     get DB_FILENAME () {
         return DB_FILENAME;
     },
@@ -23,16 +17,16 @@ _export(exports, {
     },
     get openTurso () {
         return openTurso;
+    },
+    get tursoOpenDialect () {
+        return tursoOpenDialect;
     }
 });
-var _nodefs = require("node:fs");
-var _nodepath = require("node:path");
 var _indexts = require("../../config/index.js");
-var _handoffts = require("../../embed/handoff.js");
 var _typests = require("../../embed/types.js");
 var _errorsts = require("../../errors.js");
 var _indexts1 = require("../../features/index.js");
-var _cachets = require("../cache.js");
+var _opents = require("../open.js");
 var _sharedts = require("../shared.js");
 var _connectionts = require("./connection.js");
 var _nativets = require("./native.js");
@@ -170,16 +164,10 @@ function _ts_generator(thisArg, body) {
     }
 }
 var DB_FILENAME = 'cache.turso.db';
-var SCHEMA_VERSION = '3';
-var CONTENT_FTS_DDL = [
-    "CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')",
-    "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"
-];
-var CONTENT_FTS_NAMES = [
-    'content_fts',
-    'content_fts_ngram'
-];
-function ensureSchema(conn, cfg) {
+var SCHEMA_VERSION = '4';
+// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no
+// extra native state to thread, unlike duckdb's separate instance/connection pair.
+function ensureSchema(_handle, conn, cfg) {
     return _async_to_generator(function() {
         var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, ddl, err, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, feature, err;
         return _ts_generator(this, function(_state) {
@@ -206,7 +194,7 @@ function ensureSchema(conn, cfg) {
                         9,
                         10
                     ]);
-                    _iterator = CONTENT_FTS_DDL[Symbol.iterator]();
+                    _iterator = _reconcilets.CONTENT_FTS_DDL[Symbol.iterator]();
                     _state.label = 4;
                 case 4:
                     if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
@@ -379,45 +367,74 @@ function ensureSchema(conn, cfg) {
         });
     })();
 }
-function connect(cfg) {
+function close(handle) {
     return _async_to_generator(function() {
-        var stateDir, dbPath, turso, err, db, err1, closed, _ref, conn, version, features, wantFeatures, reason, recordedMaxMs, _ref1, parsed, warnings, err2;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    stateDir = (0, _nodepath.join)(cfg.baseDir, _indexts.STATE_DIR);
-                    (0, _nodefs.mkdirSync)(stateDir, {
-                        recursive: true
-                    });
-                    dbPath = (0, _nodepath.join)(stateDir, DB_FILENAME);
-                    _state.label = 1;
+                    return [
+                        4,
+                        handle.close()
+                    ];
                 case 1:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function setDerivedBusyTimeout(_handle, conn, ms) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        conn.exec("PRAGMA busy_timeout = ".concat(ms))
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function connect(dbPath, _cfg) {
+    return _async_to_generator(function() {
+        var turso, err, db, err1;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
                     _state.trys.push([
-                        1,
-                        3,
+                        0,
+                        2,
                         ,
-                        4
+                        3
                     ]);
                     return [
                         4,
                         (0, _nativets.tursoApi)()
                     ];
-                case 2:
+                case 1:
                     turso = _state.sent();
                     return [
                         3,
-                        4
+                        3
                     ];
-                case 3:
+                case 2:
                     err = _state.sent();
                     if (_instanceof(err, _errorsts.SenseError)) throw err;
                     throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "turso" needs the '.concat(_nativets.TURSO_PACKAGE, " package (").concat(err.message, ")"));
-                case 4:
+                case 3:
                     _state.trys.push([
-                        4,
-                        6,
+                        3,
+                        5,
                         ,
-                        7
+                        6
                     ]);
                     return [
                         4,
@@ -428,156 +445,48 @@ function connect(cfg) {
                             ]
                         })
                     ];
-                case 5:
+                case 4:
                     // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;
                     // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).
                     db = _state.sent();
                     return [
                         3,
-                        7
+                        6
                     ];
-                case 6:
+                case 5:
                     err1 = _state.sent();
                     throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "turso" failed to open '.concat(dbPath, ": ").concat(err1.message));
-                case 7:
-                    // A throw below must release this handle, or the leaked WAL makes the db file undeletable on
-                    // Windows. closed guards the catch against double-closing.
-                    closed = false;
-                    _state.label = 8;
-                case 8:
-                    _state.trys.push([
-                        8,
-                        18,
-                        ,
-                        21
-                    ]);
-                    conn = (0, _connectionts.createConnection)(db);
-                    return [
-                        4,
-                        conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)')
-                    ];
-                case 9:
-                    _state.sent();
-                    return [
-                        4,
-                        (0, _sharedts.getMeta)(conn, 'schema_version')
-                    ];
-                case 10:
-                    version = _state.sent();
-                    return [
-                        4,
-                        (0, _sharedts.getMeta)(conn, 'features')
-                    ];
-                case 11:
-                    features = _state.sent();
-                    wantFeatures = (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES);
-                    if (!(version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures)) return [
-                        3,
-                        13
-                    ];
-                    // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched
-                    // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.
-                    reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
-                    console.error("sense: ".concat(reason, "; rebuilding the index"));
-                    closed = true;
-                    return [
-                        4,
-                        db.close()
-                    ];
-                case 12:
-                    _state.sent();
-                    (0, _cachets.clearCache)(cfg);
-                    return [
-                        2,
-                        connect(cfg)
-                    ];
-                case 13:
-                    return [
-                        4,
-                        ensureSchema(conn, cfg)
-                    ];
-                case 14:
-                    _state.sent();
-                    return [
-                        4,
-                        (0, _sharedts.getMeta)(conn, 'reconcile_max_ms')
-                    ];
-                case 15:
-                    recordedMaxMs = Number.apply(void 0, [
-                        (_ref = _state.sent()) !== null && _ref !== void 0 ? _ref : '0'
-                    ]);
-                    return [
-                        4,
-                        conn.exec("PRAGMA busy_timeout = ".concat(Math.min(Math.max(30000, 3 * recordedMaxMs), 600000)))
-                    ];
-                case 16:
-                    _state.sent();
-                    return [
-                        4,
-                        (0, _reconcilets.reconcile)(conn, cfg, cfg.baseDir)
-                    ];
-                case 17:
-                    _ref1 = _state.sent(), parsed = _ref1.parsed, warnings = _ref1.warnings;
+                case 6:
                     return [
                         2,
                         {
-                            db: db,
-                            conn: conn,
-                            cfg: cfg,
-                            dbPath: dbPath,
-                            parsed: parsed,
-                            warnings: warnings
+                            handle: db,
+                            conn: (0, _connectionts.createConnection)(db)
                         }
                     ];
-                case 18:
-                    err2 = _state.sent();
-                    if (!!closed) return [
-                        3,
-                        20
-                    ];
-                    return [
-                        4,
-                        db.close()
-                    ];
-                case 19:
-                    _state.sent();
-                    _state.label = 20;
-                case 20:
-                    throw err2;
-                case 21:
-                    return [
-                        2
-                    ];
             }
         });
     })();
 }
+var tursoOpenDialect = {
+    filename: DB_FILENAME,
+    schemaVersion: SCHEMA_VERSION,
+    reconcileDialect: _reconcilets.tursoDialect,
+    connect: connect,
+    close: close,
+    ensureSchema: ensureSchema,
+    setDerivedBusyTimeout: setDerivedBusyTimeout,
+    createStore: function createStore(handle, conn) {
+        return (0, _storets.createStore)(handle, conn);
+    }
+};
 function openTurso(cfg) {
     return _async_to_generator(function() {
-        var _ref, db, conn, resolvedCfg, dbPath, parsed, warnings, store;
         return _ts_generator(this, function(_state) {
-            switch(_state.label){
-                case 0:
-                    return [
-                        4,
-                        connect(cfg)
-                    ];
-                case 1:
-                    _ref = _state.sent(), db = _ref.db, conn = _ref.conn, resolvedCfg = _ref.cfg, dbPath = _ref.dbPath, parsed = _ref.parsed, warnings = _ref.warnings;
-                    store = (0, _storets.createStore)(db, conn, resolvedCfg, resolvedCfg.baseDir);
-                    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
-                    (0, _handoffts.rekeyChunkText)(conn, store);
-                    return [
-                        2,
-                        {
-                            store: store,
-                            cfg: resolvedCfg,
-                            dbPath: dbPath,
-                            parsed: parsed,
-                            warnings: warnings
-                        }
-                    ];
-            }
+            return [
+                2,
+                (0, _opents.openWithDialect)(cfg, tursoOpenDialect)
+            ];
         });
     })();
 }

package/dist/cjs/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:\n// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n  db: Database;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below must release this handle, or the leaked WAL makes the db file undeletable on\n  // Windows. closed guards the catch against double-closing.\n  let closed = false;\n  try {\n    const conn = createConnection(db);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched\n      // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      closed = true;\n      await db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) await db.close();\n    throw err;\n  }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["CONTENT_FTS_DDL","CONTENT_FTS_NAMES","DB_FILENAME","SCHEMA_VERSION","openTurso","ensureSchema","conn","cfg","ddl","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","turso","err","db","closed","version","features","wantFeatures","reason","recordedMaxMs","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","console","error","close","clearCache","Number","Math","min","max","reconcile","resolvedCfg","store","createStore","rekeyChunkText"],"mappings":";;;;;;;;;;;QAkCaA;eAAAA;;QAIAC;eAAAA;;QArBAC;eAAAA;;QAGAC;eAAAA;;QA+GSC;eAAAA;;;sBAnII;wBACL;uBAGuB;yBACb;uBACJ;wBACA;wBACc;uBACd;wBACM;4BAEA;wBACO;2BACd;uBACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMF,cAAc;AAGpB,IAAMC,iBAAiB;AAcvB,IAAMH,kBAAkB;IAC7B;IACA;CACD;AACM,IAAMC,oBAAoB;IAAC;IAAe;CAAoB;AAErE,SAAeI,aAAaC,IAAgB,EAAEC,GAAW;;YAGlD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaV;;;2BAAb,6BAAA,QAAA;;;;oBAAMQ,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBC,IAAAA,wBAAc,EAACJ;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXG,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACR;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM,kBAAkBH;;;oBAAtC;;;oBACjD;;wBAAMY,IAAAA,iBAAO,EAACT,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM,YAAYW,IAAAA,yBAAgB,EAACV,KAAKW,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAWA,SAAeC,QAAQZ,GAAmB;;YAClCa,UAEAC,QAIFC,OAGKC,KAKLC,IAKKD,MAKLE,QAwB4B,MAtBxBnB,MAIAoB,SACAC,UACAC,cAIEC,QAYFC,eAGuB,OAArBC,QAAQC,UAGTT;;;;oBAtDHH,WAAWa,IAAAA,cAAI,EAAC1B,IAAI2B,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAAChB,UAAU;wBAAEiB,WAAW;oBAAK;oBAChChB,SAASY,IAAAA,cAAI,EAACb,UAAUlB;;;;;;;;;oBAMpB;;wBAAMoC,IAAAA,kBAAQ;;;oBAAtBhB,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAegB,oBAAU,GAAE,MAAMhB;oBACrC,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACjB,IAAckB,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMnB,MAAMH,OAAO,CAACE,QAAQ;4BAAEqB,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHnB,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXlB,QAAO,MAA2B,OAAvB,AAACE,KAAckB,OAAO;;oBAEpH,6FAA6F;oBAC7F,2DAA2D;oBACvDhB,SAAS;;;;;;;;;oBAELnB,OAAOsC,IAAAA,8BAAgB,EAACpB;oBAE9B;;wBAAMlB,KAAKI,IAAI,CAAC;;;oBAAhB;oBAEgB;;wBAAMK,IAAAA,iBAAO,EAACT,MAAM;;;oBAA9BoB,UAAU;oBACC;;wBAAMX,IAAAA,iBAAO,EAACT,MAAM;;;oBAA/BqB,WAAW;oBACXC,eAAeX,IAAAA,yBAAgB,EAACV,KAAKW,kBAAQ;yBAC/C,CAAA,AAACQ,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,YAAY,GAAnG;;;;oBACF,yFAAyF;oBACzF,iGAAiG;oBAC3FC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;oBACnH0C,QAAQC,KAAK,CAAC,AAAC,UAAgB,OAAPjB,QAAO;oBAC/BJ,SAAS;oBACT;;wBAAMD,GAAGuB,KAAK;;;oBAAd;oBACAC,IAAAA,mBAAU,EAACzC;oBACX;;wBAAOY,QAAQZ;;;oBAGjB;;wBAAMF,aAAaC,MAAMC;;;oBAAzB;oBAI8B;;wBAAMQ,IAAAA,iBAAO,EAACT,MAAM;;;oBAA5CwB,gBAAgBmB;yBAAQ,OAAA,2BAAA,kBAAA,OAA4C;;oBAC1E;;wBAAM3C,KAAKI,IAAI,CAAC,AAAC,yBAA8E,OAAtDwC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAItB,gBAAgB;;;oBAAtF;oBAE6B;;wBAAMuB,IAAAA,sBAAS,EAAC/C,MAAMC,KAAKA,IAAI2B,OAAO;;;oBAAtC,QAAA,eAArBH,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;oBAEhB;;wBAAO;4BAAER,IAAAA;4BAAIlB,MAAAA;4BAAMC,KAAAA;4BAAKc,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;oBAC1CT;yBACH,CAACE,QAAD;;;;oBAAS;;wBAAMD,GAAGuB,KAAK;;;oBAAd;;;oBACb,MAAMxB;;;;;;;IAEV;;AAIO,SAAenB,UAAUG,GAAmB;;YACgB,MAAzDiB,IAAIlB,MAAWgD,aAAajC,QAAQU,QAAQC,UAC9CuB;;;;oBAD2D;;wBAAMpC,QAAQZ;;;oBAAd,OAAA,eAAzDiB,KAAyD,KAAzDA,IAAIlB,OAAqD,KAArDA,MAAWgD,cAA0C,KAA/C/C,KAAkBc,SAA6B,KAA7BA,QAAQU,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAC9CuB,QAAQC,IAAAA,oBAAW,EAAChC,IAAIlB,MAAMgD,aAAaA,YAAYpB,OAAO;oBACpE,0FAA0F;oBAC1FuB,IAAAA,yBAAc,EAACnD,MAAMiD;oBACrB;;wBAAO;4BAAEA,OAAOA;4BAAOhD,KAAK+C;4BAAajC,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACpE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","tursoOpenDialect","ensureSchema","_handle","conn","cfg","ddl","feature","exec","CONTENT_FTS_DDL","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","db","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","filename","schemaVersion","reconcileDialect","tursoDialect","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAGAC;eAAAA;;QAmESC;eAAAA;;QAXTC;eAAAA;;;uBAxEoB;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACO;2BACM;uBAClB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,mFAAmF;AACnF,SAAeG,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;;YAGrE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaC,4BAAe;;;2BAA5B,6BAAA,QAAA;;;;oBAAMH,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBE,IAAAA,wBAAc,EAACL;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQI,IAAI,KAAK,OAAM,GAAvBJ;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXI,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAML,QAAQM,MAAM,CAACT;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,MAAMC,MAAgB;;;;;oBACnC;;wBAAMA,OAAOD,KAAK;;;oBAAlB;;;;;;IACF;;AAEA,SAAeE,sBAAsBjB,OAAiB,EAAEC,IAAgB,EAAEiB,EAAU;;;;;oBAClF;;wBAAMjB,KAAKI,IAAI,CAAC,AAAC,yBAA2B,OAAHa;;;oBAAzC;;;;;;IACF;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,OAGKC,KAKLC,IAKKD;;;;;;;;;;oBAXC;;wBAAME,IAAAA,kBAAQ;;;oBAAtBH,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAeG,oBAAU,GAAE,MAAMH;oBACrC,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACJ,IAAcK,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMN,MAAMH,OAAO,CAACC,QAAQ;4BAAES,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHN,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXN,QAAO,MAA2B,OAAvB,AAACG,KAAcK,OAAO;;oBAEpH;;wBAAO;4BAAEZ,QAAQQ;4BAAIvB,MAAM8B,IAAAA,8BAAgB,EAACP;wBAAI;;;;IAClD;;AAGO,IAAM1B,mBAA0C;IACrDkC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,yBAAY;IAC9BhB,SAAAA;IACAJ,OAAAA;IACAhB,cAAAA;IACAkB,uBAAAA;IACAmB,aAAa,SAAbA,YAAcpB,QAAQf;eAASmC,IAAAA,oBAAW,EAACpB,QAAQf;;AACrD;AAEO,SAAeJ,UAAUK,GAAmB;;;YACjD;;gBAAOmC,IAAAA,uBAAe,EAACnC,KAAKJ;;;IAC9B"}

package/dist/cjs/store/turso/reconcile.d.cts

@@ -1,6 +1,5 @@
-import type { Config } from '../../config/index.js';
-import type { Connection } from '../types.js';
-export declare function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{
-    parsed: number;
-    warnings: string[];
-}>;
+import type { ReconcileDialect } from '../types.js';
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
+export declare const FTS_REBUILD_THRESHOLD = 250;
+export declare const tursoDialect: ReconcileDialect;

package/dist/cjs/store/turso/reconcile.d.ts

@@ -1,6 +1,5 @@
-import type { Config } from '../../config/index.js';
-import type { Connection } from '../types.js';
-export declare function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{
-    parsed: number;
-    warnings: string[];
-}>;
+import type { ReconcileDialect } from '../types.js';
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
+export declare const FTS_REBUILD_THRESHOLD = 250;
+export declare const tursoDialect: ReconcileDialect;