sensemaking 0.19.0 → 0.19.2

package/dist/cjs/store/sqlite/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nfunction contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    // content's rowid lookup depends on its coupled frontmatter row, so the content\n    // deletes/inserts run while every path still has one -- the vanished frontmatter delete comes last.\n    if (vanished.length > 0) {\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        vanished.map((p) => [p])\n      );\n    }\n    if (reparsedExisting.length > 0) {\n      // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via\n      // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        reparsedExisting.map((p) => [p])\n      );\n    }\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      await conn.runBatch(\n        INSERT_CONTENT_SQL,\n        parsedDocs.map((doc) => contentRow(doc, segmenting))\n      );\n    }\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(conn, async () => {\n    if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n  });\n  return warnings;\n}\n"],"names":["changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","reconcile","signatureDiff","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","segmentField","conn","cfg","baseDir","files","currentSet","existingStmt","existingRows","existing","vanished","toReparse","features","seenColumns","report","parsedDocs","warnings","newColumns","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","txStart","durationMs","prevRaw","prevMax","listFiles","map","f","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","activeFeatures","getColumns","progress","reparseFiles","tick","docs","finish","add","SenseError","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","feature","rows","presetTouched","presetRows","presetName","removedPaths","exec","runBatch","ctimeMs","parseError","data","contentTokenize","undefined","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","getMeta","Number","setMeta","String","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","key","val","keys","embedPart","find","at","indexOf","label","stmt","known","file","parseFile","fileWarnings"],"mappings":";;;;;;;;;;;QAsKgBA;eAAAA;;QAaAC;eAAAA;;QAkBMC;eAAAA;;QAtKAC;eAAAA;;QA8JNC;eAAAA;;;uBA5LgB;wBACL;wBACI;0BAEN;wBAE8B;yBAC1B;yBACA;wBAC4B;6BACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,6FAA6F;AAC7F,4EAA4E;AAC5E,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,IAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,IAAMC,qBAAqB;AAE3B,uFAAuF;AACvF,+EAA+E;AAC/E,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IACrD,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEO,SAAeb,UAAUe,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YACtEC,OACAC,YAEAC,cACAC,cACAC,UACAC,UAEAC,WAOAC,UACAC,aAIAC,QAC6C,MAArCC,YAAYC,UAAUC,YAE/B,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YAWAC,iBAGAC,WAKAC,OACAC,OACAC,UACAC,kBAEAC,SAyEAC,YACAC,SAGAC;;;;oBA9HAxB,QAAQyB,IAAAA,mBAAS,EAAC3B,KAAKC;oBACvBE,aAAa,IAAIhB,IAAIe,MAAM0B,GAAG,CAAC,SAACC;+BAAMA,EAAEpC,OAAO;;oBAEhC;;wBAAMM,KAAK+B,OAAO,CAAC;;;oBAAlC1B,eAAe;oBACC;;wBAAMA,aAAa2B,GAAG;;;oBAAtC1B,eAAgB;oBAChBC,WAAW,IAAI0B,IAAI3B,aAAauB,GAAG,CAAC,SAACK;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBACtD1B,WAAWF,aAAa8B,MAAM,CAAC,SAACF;+BAAM,CAAC9B,WAAWiC,GAAG,CAACH,EAAEC,IAAI;uBAAGN,GAAG,CAAC,SAACK;+BAAMA,EAAEC,IAAI;;oBAEhF1B,YAAYN,MAAMiC,MAAM,CAAC,SAACN;wBAC9B,IAAMQ,MAAM/B,SAASgC,GAAG,CAACT,EAAEpC,OAAO;wBAClC,OAAO,CAAC4C,OAAOA,IAAIE,MAAM,KAAKV,EAAEW,OAAO,IAAIH,IAAII,KAAK,KAAKZ,EAAEa,IAAI;oBACjE;oBAEA,IAAInC,SAASoC,MAAM,KAAK,KAAKnC,UAAUmC,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAG/B,QAAQ;wBAAK;;oBAEhFJ,WAAWoC,IAAAA,wBAAc,EAAC7C;oBACZ;;wBAAM8C,IAAAA,oBAAU,EAAC/C;;;oBAA/BW,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAASoC,IAAAA,oBAAQ,EAAC,mBAAmBvC,UAAUmC,MAAM;oBACR;;wBAAMK,IAAAA,uBAAY,EAACxC,WAAWC,UAAUT,KAAKU,aAAaC,OAAOsC,IAAI;;;oBAArE,OAAA,eAArCrC,aAAqC,KAA3CsC,MAAkBrC,WAAyB,KAAzBA,UAAUC,aAAe,KAAfA;oBACpCH,OAAOwC,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAarC,iCAAb,6BAAA,QAAA,yBAAA;4BAAMC,MAAN;4BAAyBL,YAAY0C,GAAG,CAACrC;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGN;oBACvB,uEAAuE;oBACvE,sGAAsG;oBACtG,IAAIM,WAAW2B,MAAM,GAAGvD,yBAAyB;wBAC/C,MAAM,IAAIiE,oBAAU,CAClB,gBACA,AAAC,0BAAuHjE,OAA9F4B,WAAW2B,MAAM,EAAC,8EAAoG,OAAxBvD,yBAAwB;oBAEpJ;oBACA,0FAA0F;oBAC1F,sEAAsE;oBAChE6B,kBAAkBD,WAAWmB,MAAM,CAAC,SAACmB;+BAAMpE,yBAAyBkD,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBAC1G,sFAAsF;oBACtF,gDAAgD;oBAC1CpC,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBW,GAAG,CAAC4B,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FxC,OAAhFA,gBAAgBW,GAAG,CAAC;+BAAM;uBAAK6B,IAAI,CAAC,OAAM,wCAGjI,OAHuKxC,gBAClLkB,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpB1B,GAAG,CAAC,SAAC0B;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEFtC,QAAQX,UAAU2B,MAAM,CAAC,SAACN;+BAAM,CAACvB,SAAS8B,GAAG,CAACP,EAAEpC,OAAO;uBAAGmC,GAAG,CAAC,SAACC;+BAAMA,EAAEpC,OAAO;;oBAC9E2B,QAAwB;wBAAElB,OAAAA;wBAAOwD,UAAU9C,WAAWgB,GAAG,CAAC,SAAC+B;mCAAMA,EAAElE,OAAO;;wBAAG0B,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAIlC,IAAIgC;oBACnBG,mBAAmBV,WAAWgB,GAAG,CAAC,SAAC+B;+BAAMA,EAAElE,OAAO;uBAAE0C,MAAM,CAAC,SAACyB;+BAAM,CAACvC,SAASe,GAAG,CAACwB;;oBAEhFrC,UAAUsC,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EAAChE,MAAM;;oCA8DyCiE,iBAK7BA,yBAlEjC,2BAAA,mBAAA,gBAAA,WAAA,OAAMjD,UAoBHkD,MAeAzE,YAeF0E,eAMAC,YAC6B,4BAAA,oBAAA,iBAA9B,4BAAA,oBAAA,iBAAA,YAAA,QAAM5E,KAAwB,YAAA,QAAM6E,YAGnCC,cAC4B,4BAAA,oBAAA,iBAAA,YAAA,QAAML,cACnC,4BAAA,oBAAA,wBAAA,YAAA,aAIA,4BAAA,oBAAA,iBAAA,YAAA,QAAMA;;;;4CAlEN,kCAAA,2BAAA;;;;;;;;;4CAAA,YAAalD;;;mDAAb,6BAAA,QAAA;;;;4CAAMC,MAAN;4CAAyB;;gDAAMhB,KAAKuE,IAAI,CAAC,AAAC,sCAAqD,OAAhBd,IAAAA,oBAAU,EAACzC;;;4CAAjE;;;4CAAzB;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAIDR,CAAAA,SAASoC,MAAM,GAAG,CAAA,GAAlBpC;;;;4CACF;;gDAAMR,KAAKwE,QAAQ,CACjB,sFACAhE,SAASqB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;iDAKEtC,CAAAA,iBAAiBqB,MAAM,GAAG,CAAA,GAA1BrB;;;;4CACF,6FAA6F;4CAC7F,uGAAuG;4CACvG;;gDAAMvB,KAAKwE,QAAQ,CACjB,sFACAjD,iBAAiBM,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFjC;;;iDAMEhD,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CACIqD,OAAOrD,WAAWgB,GAAG,CAAC,SAACrC;uDAC3B0B,gBAAgBW,GAAG,CAAC,SAACb;wDAOZxB;oDANP,IAAIwB,QAAQ,QAAQ,OAAOxB,IAAIE,OAAO;oDACtC,IAAIsB,QAAQ,UAAU,OAAOxB,IAAIiD,OAAO;oDACxC,IAAIzB,QAAQ,UAAU,OAAOxB,IAAIiF,OAAO;oDACxC,IAAIzD,QAAQ,SAAS,OAAOxB,IAAImD,IAAI;oDACpC,mFAAmF;oDACnF,IAAI3B,QAAQ,gBAAgB,OAAOxB,IAAIkF,UAAU;oDACjD,QAAOlF,gBAAAA,IAAImF,IAAI,CAAC3D,IAAI,cAAbxB,2BAAAA,gBAAiB;gDAC1B;;4CAEF;;gDAAMQ,KAAKwE,QAAQ,CAACrD,WAAW+C;;;4CAA/B;4CAEA,wFAAwF;4CACxF,+EAA+E;4CACzEzE,aAAamF,IAAAA,wBAAe,EAAC3E,SAAS4E;4CAC5C;;gDAAM7E,KAAKwE,QAAQ,CACjBlF,oBACAuB,WAAWgB,GAAG,CAAC,SAACrC;2DAAQD,WAAWC,KAAKC;;;;4CAF1C;;;iDAMEe,CAAAA,SAASoC,MAAM,GAAG,CAAA,GAAlBpC;;;;4CACF;;gDAAMR,KAAKwE,QAAQ,CACjB,4CACAhE,SAASqB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;4CAKF,0FAA0F;4CAC1F,kEAAkE;4CAC5DM,gBAAgB,AAAC,qBAAG3D,iBAAU,qBAAGe;iDACnC4C,CAAAA,cAAcvB,MAAM,GAAG,CAAA,GAAvBuB;;;;4CACF;;gDAAMnE,KAAKwE,QAAQ,CACjB,6CACAL,cAActC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF9B;;;4CAIIO;4CAC6B,mCAAA,4BAAA,6BAA9B,mCAAA,4BAAA;;gDAAL,IAAK,aAAavD,iCAAb,8BAAA,SAAA,0BAAA;oDAAMrB,MAAN;;wDAAyB,IAAK,aAAoBA,IAAIsF,OAAO,uBAA/B,8BAAA,SAAA,0BAAA;4DAAMT,aAAN;4DAAiCD,WAAWW,IAAI;gEAAEvF,IAAIE,OAAO;gEAAE2E;;;;wDAA/D;wDAAA;;;iEAAA,8BAAA;gEAAA;;;gEAAA;sEAAA;;;;;;gDAA9B;gDAAA;;;yDAAA,8BAAA;wDAAA;;;wDAAA;8DAAA;;;;iDACDD,CAAAA,WAAWxB,MAAM,GAAG,CAAA,GAApBwB;;;;4CAAuB;;gDAAMpE,KAAKwE,QAAQ,CAAC,2DAA2DJ;;;4CAA/E;;;4CAErBE,eAAe,AAAC,qBAAG9D,iBAAU,qBAAGe;4CACJ,mCAAA,4BAAA;iDAA9B+C,CAAAA,aAAa1B,MAAM,GAAG,CAAA,GAAtB0B;;;;;;;;;;;;4CAA8B,aAAiB5D;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,UAAN;4CAA2B;;iDAAMA,kBAAAA,QAAQe,MAAM,cAAdf,sCAAAA,qBAAAA,SAAiBjE,MAAMsE,cAAcjD;;;4CAA3C;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAC7B,mCAAA,4BAAA;;;;;;;;;;oDAAM4C,SAEHA,gBADAgB;;;;4DADGhB,UAAN;4DACGgB,iBAAiCpE,WAAWgB,GAAG,CAAC,SAACrC;uEAAS;oEAAE2C,MAAM3C,IAAIE,OAAO;oEAAEwF,WAAW1F,IAAI0F,SAAS,CAACjB,QAAQkB,IAAI,CAAC;gEAAC;;4DAC5H;;iEAAMlB,iBAAAA,QAAQmB,KAAK,cAAbnB,qCAAAA,oBAAAA,SAAgBjE,MAAMiF,gBAAgB5D;;;4DAA5C;;;;;;4CACF;4CAHK,aAAiBX;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;4CAAA,aAAiBA;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,WAAN;4CAA2B;;iDAAMA,0BAAAA,SAAQoB,cAAc,cAAtBpB,8CAAAA,6BAAAA,UAAyBjE,MAAMqB;;;4CAArC;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;;;;oBApEA;oBAsEA,qFAAqF;oBACrF,8GAA8G;oBACxGI,aAAaqC,KAAKC,GAAG,KAAKvC;oBAChB;;wBAAM8D,IAAAA,iBAAO,EAACtF,MAAM;;;oBAA9B0B,UAAU;oBAChB,yFAAyF;oBACzF,+EAA+E;oBACzEC,UAAUD,YAAY,OAAO,CAAC,IAAI6D,OAAO7D;yBAC3CD,CAAAA,aAAaE,OAAM,GAAnBF;;;;oBAAsB;;wBAAM+D,IAAAA,iBAAO,EAACxF,MAAM,oBAAoByF,OAAOhE;;;oBAA/C;;;oBAE1B;;wBAAO;4BAAEoB,QAAQhC,WAAW+B,MAAM;4BAAE9B,UAAAA;wBAAS;;;;IAC/C;;AAIO,SAAShC,qBAAqB4G,MAAc,EAAEC,KAAa;IAChE,IAAMC,QAAQ,eAACC;eAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGtC,IAAI,CAAC,OAAOmC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;;IACxH,IAAME,QAAQ,eAACC;eAAgB,IAAIjE,IAAIiE,IAAIH,KAAK,CAAC,KAAKlE,GAAG,CAAC,SAACgE;mBAAS;gBAACD,MAAMC;gBAAOA;aAAK;;;IACvF,IAAMM,IAAIF,MAAMP;IAChB,IAAMU,IAAIH,MAAMN;IAChB,IAAMU,UAAU,IAAIjH;QACf,kCAAA,2BAAA;;QAAL,QAAK,YAAoBgH,sBAApB,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAOE,sBAAKC;YAAW,IAAIJ,EAAE5D,GAAG,CAAC+D,SAASC,KAAKF,QAAQhD,GAAG,CAACiD;;;QAA3D;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;QACA,mCAAA,4BAAA;;QAAL,QAAK,aAAaH,EAAEK,IAAI,uBAAnB,UAAA,8BAAA,SAAA,0BAAA;YAAA,IAAMF,OAAN;YAAuB,IAAI,CAACF,EAAE/D,GAAG,CAACiE,OAAMD,QAAQhD,GAAG,CAACiD;;;QAApD;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,OAAOD;AACT;AAIO,SAAStH,qBAAqB2G,MAAc,EAAEC,KAAa;IAChE,IAAMc,YAAY,mBAACP;eAAgBA,IAAIH,KAAK,CAAC,KAAKW,IAAI,CAAC,SAAC7C;mBAAMA,EAAEiC,UAAU,CAAC;;;IAC3E,IAAMM,IAAIK,UAAUf;IACpB,IAAMS,IAAIM,UAAUd;IACpB,IAAIS,MAAMvB,aAAasB,MAAMtB,WAAW,OAAO;IAC/C,IAAM8B,KAAKR,EAAES,OAAO,CAAC;IACrB,OAAOR,EAAEQ,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKR,EAAEH,KAAK,CAAC,GAAGW,QAAQP;AAClE;AAGO,SAASlH,cAAcwG,MAAc,EAAEC,KAAa;IACzD,IAAMU,UAAUvH,qBAAqB4G,QAAQC;IAC7C,IAAMkB,QAAQ,eAACP;eAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIR,UAAU,CAAC,aAAa,AAAC,WAAuB,OAAbQ,IAAIN,KAAK,CAAC,IAAG,OAAK;;IACzK,OAAOK,QAAQ1D,IAAI,KAAK,IAAI,aAAa,AAAC,qBAAG0D,SAASxE,GAAG,CAACgF,OAAOnD,IAAI,CAAC;AACxE;AAIO,SAAe1E,oBAAoBgB,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YAChF4G,MACAC,UACA5G,OACAV,YACAqB,UACAoD,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAM8C,MAETlG,WADwCmG,YAAhCzH,KAAe0H;;;;oBAPZ;;wBAAMlH,KAAK+B,OAAO,CAAC;;;oBAA1B+E,OAAO;wBACK1H;oBAAM;;wBAAM0H,KAAK9E,GAAG;;;oBAAhC+E,QAAQ,IAAA,CAAA,EAAA,MAAI3H;;wBAAM,cAA8CyC,GAAG,CAAC,SAACK;mCAAMA,EAAEC,IAAI;;sBAAC;oBAClFhC,QAAQyB,IAAAA,mBAAS,EAAC3B,KAAKC,SAASkC,MAAM,CAAC,SAACN;+BAAMiF,MAAM1E,GAAG,CAACP,EAAEpC,OAAO;;oBACjED,aAAamF,IAAAA,wBAAe,EAAC3E,SAAS4E;oBACtC/D;oBACAoD;oBACD,kCAAA,2BAAA;;wBAAL,IAAK,YAAc/D,4BAAd,6BAAA,QAAA,yBAAA,iCAAqB;4BAAf6G,OAAN;;4BACqCC,aAAAA,IAAAA,mBAAS,EAACD,OAA1CxH,MAAgCyH,WAAhCzH,KAAe0H,eAAiBD,WAA3BnG;4BACbA,CAAAA,YAAAA,UAASiE,IAAI,OAAbjE,WAAc,qBAAGoG;4BACjBhD,KAAKa,IAAI,CAACxF,WAAWC,KAAKC;wBAC5B;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAKL;;wBAAMuE,IAAAA,8BAAe,EAAChE,MAAM;;;;;iDACtBkE,CAAAA,KAAKtB,MAAM,GAAG,CAAA,GAAdsB;;;;4CAAiB;;gDAAMlE,KAAKwE,QAAQ,CAAClF,oBAAoB4E;;;4CAAxC;;;;;;;;4BACvB;;;;oBAFA;oBAGA;;wBAAOpD;;;;IACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is FTS5-virtual with \"_seg\" sidecars for lexical.ts.\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcileContent() and the tokenize-only rebuild in open.ts, so both prepare the\n// same literal instead of two copies drifting apart.\nexport const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nexport function contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta, cfg: Config): Promise<void> {\n  // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via\n  // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n      touched.map((p) => [p])\n    );\n\n  if (docs.length > 0) {\n    // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n    // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n    const segmenting = contentTokenize(cfg) === undefined;\n    await conn.runBatch(\n      INSERT_CONTENT_SQL,\n      docs.map((doc) => contentRow(doc, segmenting))\n    );\n  }\n}\n\nexport const sqliteDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    // Fence before ALTERing: SQLite's own failure past this point is a raw\n    // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  columnTypeSuffix: () => '',\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["INSERT_CONTENT_SQL","contentRow","sqliteDialect","MAX_FRONTMATTER_COLUMNS","doc","segmenting","relPath","search","title","summary","text","segmentField","reconcileContent","conn","touched","docs","_delta","cfg","length","runBatch","map","p","contentTokenize","undefined","beginMode","BEGIN_WRITE","checkColumnLimit","count","SenseError","columnTypeSuffix","recordDuration","recordReconcileDuration"],"mappings":";;;;;;;;;;;QAkBaA;eAAAA;;QAIGC;eAAAA;;QAwBHC;eAAAA;;;uBA7CmB;wBACL;yBAGE;wBACW;6BACZ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG5B,qFAAqF;AACrF,mFAAmF;AAEnF,8FAA8F;AAC9F,IAAMC,0BAA0B;AAIzB,IAAMH,qBAAqB;AAI3B,SAASC,WAAWG,GAAc,EAAEC,UAAmB;IAC5D,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,SAAeE,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB,EAAEC,GAAW;;YAYjHZ;;;;yBATJS,CAAAA,QAAQI,MAAM,GAAG,CAAA,GAAjBJ;;;;oBACF;;wBAAMD,KAAKM,QAAQ,CACjB,sFACAL,QAAQM,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAKEN,CAAAA,KAAKG,MAAM,GAAG,CAAA,GAAdH;;;;oBACF,wFAAwF;oBACxF,+EAA+E;oBACzEV,aAAaiB,IAAAA,wBAAe,EAACL,SAASM;oBAC5C;;wBAAMV,KAAKM,QAAQ,CACjBnB,oBACAe,KAAKK,GAAG,CAAC,SAAChB;mCAAQH,WAAWG,KAAKC;;;;oBAFpC;;;;;;;;IAKJ;;AAEO,IAAMH,gBAAkC;IAC7CsB,WAAW,SAAXA;eAAiBC,0BAAW;;IAC5BC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,uEAAuE;QACvE,sGAAsG;QACtG,IAAIA,QAAQxB,yBAAyB;YACnC,MAAM,IAAIyB,oBAAU,CAClB,gBACA,AAAC,0BAA2GzB,OAAlFwB,OAAM,8EAAoG,OAAxBxB,yBAAwB;QAExI;IACF;IACA0B,kBAAkB,SAAlBA;eAAwB;;IACxBjB,kBAAAA;IACAkB,gBAAgBC,iCAAuB;AACzC"}

package/dist/cjs/store/sqlite/signature.d.cts

@@ -0,0 +1,6 @@
+import type { Config } from '../../config/index.js';
+import type { Connection } from '../types.js';
+export declare function changedSignatureKeys(before: string, after: string): Set<string>;
+export declare function embedIdentityAdopted(before: string, after: string): boolean;
+export declare function signatureDiff(before: string, after: string): string;
+export declare function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]>;

package/dist/cjs/store/sqlite/signature.d.ts

@@ -0,0 +1,6 @@
+import type { Config } from '../../config/index.js';
+import type { Connection } from '../types.js';
+export declare function changedSignatureKeys(before: string, after: string): Set<string>;
+export declare function embedIdentityAdopted(before: string, after: string): boolean;
+export declare function signatureDiff(before: string, after: string): string;
+export declare function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]>;

package/dist/cjs/store/sqlite/signature.js

@@ -0,0 +1,380 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get changedSignatureKeys () {
+        return changedSignatureKeys;
+    },
+    get embedIdentityAdopted () {
+        return embedIdentityAdopted;
+    },
+    get rebuildContentTable () {
+        return rebuildContentTable;
+    },
+    get signatureDiff () {
+        return signatureDiff;
+    }
+});
+var _indexts = require("../../config/index.js");
+var _indexts1 = require("../../scan/index.js");
+var _transactionts = require("../transaction.js");
+var _reconcilets = require("./reconcile.js");
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_with_holes(arr) {
+    if (Array.isArray(arr)) return arr;
+}
+function _array_without_holes(arr) {
+    if (Array.isArray(arr)) return _array_like_to_array(arr);
+}
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _iterable_to_array(iter) {
+    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
+        return Array.from(iter);
+    }
+}
+function _iterable_to_array_limit(arr, i) {
+    var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
+    if (_i == null) return;
+    var _arr = [];
+    var _n = true;
+    var _d = false;
+    var _s, _e;
+    try {
+        for(_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true){
+            _arr.push(_s.value);
+            if (i && _arr.length === i) break;
+        }
+    } catch (err) {
+        _d = true;
+        _e = err;
+    } finally{
+        try {
+            if (!_n && _i["return"] != null) _i["return"]();
+        } finally{
+            if (_d) throw _e;
+        }
+    }
+    return _arr;
+}
+function _non_iterable_rest() {
+    throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _non_iterable_spread() {
+    throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _sliced_to_array(arr, i) {
+    return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
+}
+function _to_consumable_array(arr) {
+    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+function changedSignatureKeys(before, after) {
+    var keyOf = function keyOf(part) {
+        return part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0];
+    };
+    var parse = function parse(sig) {
+        return new Map(sig.split('|').map(function(part) {
+            return [
+                keyOf(part),
+                part
+            ];
+        }));
+    };
+    var a = parse(before);
+    var b = parse(after);
+    var changed = new Set();
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = b[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var _step_value = _sliced_to_array(_step.value, 2), key = _step_value[0], val = _step_value[1];
+            if (a.get(key) !== val) changed.add(key);
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+    try {
+        for(var _iterator1 = a.keys()[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+            var key1 = _step1.value;
+            if (!b.has(key1)) changed.add(key1);
+        }
+    } catch (err) {
+        _didIteratorError1 = true;
+        _iteratorError1 = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                _iterator1.return();
+            }
+        } finally{
+            if (_didIteratorError1) {
+                throw _iteratorError1;
+            }
+        }
+    }
+    return changed;
+}
+function embedIdentityAdopted(before, after) {
+    var embedPart = function embedPart(sig) {
+        return sig.split('|').find(function(p) {
+            return p.startsWith('embed:');
+        });
+    };
+    var b = embedPart(before);
+    var a = embedPart(after);
+    if (b === undefined || a === undefined) return false;
+    var at = a.indexOf('@');
+    return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;
+}
+function signatureDiff(before, after) {
+    var changed = changedSignatureKeys(before, after);
+    var label = function label(key) {
+        return key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? 'preset "'.concat(key.slice(7), '"') : 'features';
+    };
+    return changed.size === 0 ? 'features' : _to_consumable_array(changed).map(label).join(', ');
+}
+function rebuildContentTable(conn, cfg, baseDir) {
+    return _async_to_generator(function() {
+        var stmt, known, _, files, segmenting, warnings, rows, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, _warnings, _parseFile, doc, fileWarnings;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        conn.prepare('SELECT "path" FROM frontmatter')
+                    ];
+                case 1:
+                    stmt = _state.sent();
+                    _ = Set.bind;
+                    return [
+                        4,
+                        stmt.all()
+                    ];
+                case 2:
+                    known = new (_.apply(Set, [
+                        void 0,
+                        _state.sent().map(function(r) {
+                            return r.path;
+                        })
+                    ]));
+                    files = (0, _indexts1.listFiles)(cfg, baseDir).filter(function(f) {
+                        return known.has(f.relPath);
+                    });
+                    segmenting = (0, _indexts.contentTokenize)(cfg) === undefined;
+                    warnings = [];
+                    rows = [];
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    try {
+                        for(_iterator = files[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                            file = _step.value;
+                            ;
+                            _parseFile = (0, _indexts1.parseFile)(file), doc = _parseFile.doc, fileWarnings = _parseFile.warnings;
+                            (_warnings = warnings).push.apply(_warnings, _to_consumable_array(fileWarnings));
+                            rows.push((0, _reconcilets.contentRow)(doc, segmenting));
+                        }
+                    } catch (err) {
+                        _didIteratorError = true;
+                        _iteratorError = err;
+                    } finally{
+                        try {
+                            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                _iterator.return();
+                            }
+                        } finally{
+                            if (_didIteratorError) {
+                                throw _iteratorError;
+                            }
+                        }
+                    }
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return _async_to_generator(function() {
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            if (!(rows.length > 0)) return [
+                                                3,
+                                                2
+                                            ];
+                                            return [
+                                                4,
+                                                conn.runBatch(_reconcilets.INSERT_CONTENT_SQL, rows)
+                                            ];
+                                        case 1:
+                                            _state.sent();
+                                            _state.label = 2;
+                                        case 2:
+                                            return [
+                                                2
+                                            ];
+                                    }
+                                });
+                            })();
+                        }, _transactionts.BEGIN_WRITE)
+                    ];
+                case 3:
+                    _state.sent();
+                    return [
+                        2,
+                        warnings
+                    ];
+            }
+        });
+    })();
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/sqlite/signature.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/signature.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { listFiles, parseFile } from '../../scan/index.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\nimport { contentRow, INSERT_CONTENT_SQL } from './reconcile.ts';\n\n// open()-lifecycle helpers for a config-signature change (see config.featureSignature's format:\n// global features, embed provider, tokenize, then one segment per preset). Never called from\n// reconcile() itself.\n\n// Segment keys that moved between two feature signatures.\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(\n    conn,\n    async () => {\n      if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n    },\n    BEGIN_WRITE\n  );\n  return warnings;\n}\n"],"names":["changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","signatureDiff","before","after","keyOf","part","startsWith","split","slice","join","parse","sig","Map","map","a","b","changed","Set","key","val","get","add","keys","has","embedPart","find","p","undefined","at","indexOf","label","size","conn","cfg","baseDir","stmt","known","files","segmenting","warnings","rows","file","parseFile","doc","fileWarnings","prepare","all","r","path","listFiles","filter","f","relPath","contentTokenize","push","contentRow","withTransaction","length","runBatch","INSERT_CONTENT_SQL","BEGIN_WRITE"],"mappings":";;;;;;;;;;;QAYgBA;eAAAA;;QAaAC;eAAAA;;QAkBMC;eAAAA;;QARNC;eAAAA;;;uBAlCgB;wBACK;6BACQ;2BAEE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAOxC,SAASH,qBAAqBI,MAAc,EAAEC,KAAa;IAChE,IAAMC,QAAQ,eAACC;eAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGC,IAAI,CAAC,OAAOJ,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;;IACxH,IAAMG,QAAQ,eAACC;eAAgB,IAAIC,IAAID,IAAIJ,KAAK,CAAC,KAAKM,GAAG,CAAC,SAACR;mBAAS;gBAACD,MAAMC;gBAAOA;aAAK;;;IACvF,IAAMS,IAAIJ,MAAMR;IAChB,IAAMa,IAAIL,MAAMP;IAChB,IAAMa,UAAU,IAAIC;QACf,kCAAA,2BAAA;;QAAL,QAAK,YAAoBF,sBAApB,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAOG,sBAAKC;YAAW,IAAIL,EAAEM,GAAG,CAACF,SAASC,KAAKH,QAAQK,GAAG,CAACH;;;QAA3D;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;QACA,mCAAA,4BAAA;;QAAL,QAAK,aAAaJ,EAAEQ,IAAI,uBAAnB,UAAA,8BAAA,SAAA,0BAAA;YAAA,IAAMJ,OAAN;YAAuB,IAAI,CAACH,EAAEQ,GAAG,CAACL,OAAMF,QAAQK,GAAG,CAACH;;;QAApD;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,OAAOF;AACT;AAIO,SAASjB,qBAAqBG,MAAc,EAAEC,KAAa;IAChE,IAAMqB,YAAY,mBAACb;eAAgBA,IAAIJ,KAAK,CAAC,KAAKkB,IAAI,CAAC,SAACC;mBAAMA,EAAEpB,UAAU,CAAC;;;IAC3E,IAAMS,IAAIS,UAAUtB;IACpB,IAAMY,IAAIU,UAAUrB;IACpB,IAAIY,MAAMY,aAAab,MAAMa,WAAW,OAAO;IAC/C,IAAMC,KAAKd,EAAEe,OAAO,CAAC;IACrB,OAAOd,EAAEc,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKd,EAAEN,KAAK,CAAC,GAAGoB,QAAQb;AAClE;AAGO,SAASd,cAAcC,MAAc,EAAEC,KAAa;IACzD,IAAMa,UAAUlB,qBAAqBI,QAAQC;IAC7C,IAAM2B,QAAQ,eAACZ;eAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIZ,UAAU,CAAC,aAAa,AAAC,WAAuB,OAAbY,IAAIV,KAAK,CAAC,IAAG,OAAK;;IACzK,OAAOQ,QAAQe,IAAI,KAAK,IAAI,aAAa,AAAC,qBAAGf,SAASH,GAAG,CAACiB,OAAOrB,IAAI,CAAC;AACxE;AAIO,SAAeT,oBAAoBgC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YAChFC,MACAC,UACAC,OACAC,YACAC,UACAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,MAETF,WADwCG,YAAhCC,KAAeC;;;;oBAPZ;;wBAAMZ,KAAKa,OAAO,CAAC;;;oBAA1BV,OAAO;wBACKlB;oBAAM;;wBAAMkB,KAAKW,GAAG;;;oBAAhCV,QAAQ,IAAA,CAAA,EAAA,MAAInB;;wBAAM,cAA8CJ,GAAG,CAAC,SAACkC;mCAAMA,EAAEC,IAAI;;sBAAC;oBAClFX,QAAQY,IAAAA,mBAAS,EAAChB,KAAKC,SAASgB,MAAM,CAAC,SAACC;+BAAMf,MAAMb,GAAG,CAAC4B,EAAEC,OAAO;;oBACjEd,aAAae,IAAAA,wBAAe,EAACpB,SAASN;oBACtCY;oBACAC;oBACD,kCAAA,2BAAA;;wBAAL,IAAK,YAAcH,4BAAd,6BAAA,QAAA,yBAAA,iCAAqB;4BAAfI,OAAN;;4BACqCC,aAAAA,IAAAA,mBAAS,EAACD,OAA1CE,MAAgCD,WAAhCC,KAAeC,eAAiBF,WAA3BH;4BACbA,CAAAA,YAAAA,UAASe,IAAI,OAAbf,WAAc,qBAAGK;4BACjBJ,KAAKc,IAAI,CAACC,IAAAA,uBAAU,EAACZ,KAAKL;wBAC5B;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAKL;;wBAAMkB,IAAAA,8BAAe,EACnBxB,MACA;;;;;iDACMQ,CAAAA,KAAKiB,MAAM,GAAG,CAAA,GAAdjB;;;;4CAAiB;;gDAAMR,KAAK0B,QAAQ,CAACC,+BAAkB,EAAEnB;;;4CAAxC;;;;;;;;4BACvB;2BACAoB,0BAAW;;;oBALb;oBAOA;;wBAAOrB;;;;IACT"}

package/dist/cjs/store/sqlite/store.js

@@ -17,12 +17,13 @@ _export(exports, {
     }
 });
 var _indexts = require("../../config/index.js");
+var _reconcilets = require("../reconcile.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 var _vectorsts = require("../vectors.js");
 var _fieldStatsts = require("./fieldStats.js");
 var _lexicalts = require("./lexical.js");
-var _reconcilets = require("./reconcile.js");
+var _reconcilets1 = require("./reconcile.js");
 var _vectorsts1 = require("./vectors.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -367,7 +368,8 @@ var CAPABILITIES = new Set([
     'snippets',
     'watch-concurrency',
     'lexical',
-    'vectors'
+    'vectors',
+    'sql-functions'
 ]);
 function createStore(db, conn, cfg, baseDir) {
     return {
@@ -434,7 +436,7 @@ function createStore(db, conn, cfg, baseDir) {
                 return _ts_generator(this, function(_state) {
                     return [
                         2,
-                        (0, _reconcilets.reconcile)(conn, cfg, baseDir)
+                        (0, _reconcilets.reconcile)(conn, cfg, baseDir, _reconcilets1.sqliteDialect)
                     ];
                 });
             })();

package/dist/cjs/store/sqlite/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with open()'s own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","reconcile","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","contentTokenize","undefined","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","candidates","qv","storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QAIGC;eAAAA;;;uBAdgB;wBACL;6BACK;yBAEU;4BACf;yBACE;2BACH;0BACoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;CAAU;AAIxH,SAASD,YAAYE,EAAgB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC1F,OAAO;QACLC,MAAM;QACNC,cAAcR;QACRS,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMN,KAAKK,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAON,KAAKO,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMT,KAAKQ,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACZ,MAAMW;;;YAC/B;;QACME,WAAN,SAAMA;;;oBACJ;;wBAAOA,IAAAA,sBAAS,EAACb,MAAMC,KAAKC;;;YAC9B;;QACAY,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAAChB;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACMiB,YAAN,SAAMA,WAAWF,OAAO,EAAEG,UAAU;;;wBAClC;;4BAAOD,IAAAA,wBAAU,EAACjB,MAAMe,SAASG;;;gBACnC;;QACF;QACAC,SAAS;YACDC,OAAN,SAAMA,MAAMC,KAAK,EAAEC,IAAI;;;wBACrB;;4BAAOC,IAAAA,uBAAY,EAACvB,MAAMqB,OAAOC,MAAME,IAAAA,wBAAe,EAACvB,SAASwB;;;gBAClE;;QACF;QACAC,SAAS;YACDC,SAAN,SAAMA;;;wBACJ;;4BAAOC,IAAAA,sBAAW,EAAC5B;;;gBACrB;;YACM6B,cAAN,SAAMA,aAAaC,IAAsB;;;;;gCACvC;;oCAAMC,IAAAA,4BAAgB,EAAC/B,MAAM8B;;;gCAA7B;;;;;;gBACF;;YACME,YAAN,SAAMA,WAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;;;wBAC5C;;4BAAOC,IAAAA,0BAAc,EAACrC,MAAMiC,IAAIC,WAAWC,OAAOC;;;gBACpD;;YACME,SAAN,SAAMA,QAAQC,IAAI,EAAEjB,IAAI;;;wBACtB;;4BAAOkB,IAAAA,uBAAW,EAACxC,MAAMuC,MAAMjB;;;gBACjC;;YACMmB,WAAN,SAAMA,UAAUF,IAAI;;;wBAClB;;4BAAOG,IAAAA,uBAAY,EAAC1C,MAAMuC;;;gBAC5B;;QACF;QACMI,cAAN,SAAMA;;oBAGEC;;oBAFN,qFAAqF;oBACrF,qFAAqF;oBAC/EA,MAAM7C,GAAGQ,OAAO,CAAC,uBAAuBsC,GAAG;oBACjD;;wBAAO;4BAAEC,cAAc,AAAC,GAAc,OAAZF,IAAIG,OAAO,EAAC;wBAAoF;;;YAC5H;;QACAC,KAAK;YACGzC,SAAN,SAAMA,QAAQD,GAAW;;wBACjB2C;;wBAAAA,OAAOlD,GAAGQ,OAAO,CAACD;wBACxB2C,KAAKC,cAAc,CAAC,OAAO,qEAAqE;wBAChG;;4BAAO;gCACLnC,SAAS,SAATA;2CAAekC,KAAKlC,OAAO;;gCAC3B,2EAA2E;gCAC3EoC,SAAS,SAATA;oCAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;wCAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;4CACrCH;;;;oDAAP;;mEAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAIG;;;oDAAxB;;;;;;oCACF;;4BACF;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJtD,GAAGsD,KAAK;;;;;YACV;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with open()'s own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir, sqliteDialect);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","reconcile","sqliteDialect","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","contentTokenize","undefined","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","candidates","qv","storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":";;;;;;;;;;;QAaaA;eAAAA;;QAIGC;eAAAA;;;uBAfgB;2BACN;wBACC;6BACK;yBAEU;4BACf;yBACE;4BACC;0BACgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB;AAIzI,SAASD,YAAYE,EAAgB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC1F,OAAO;QACLC,MAAM;QACNC,cAAcR;QACRS,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMN,KAAKK,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAON,KAAKO,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMT,KAAKQ,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACZ,MAAMW;;;YAC/B;;QACME,WAAN,SAAMA;;;oBACJ;;wBAAOA,IAAAA,sBAAS,EAACb,MAAMC,KAAKC,SAASY,2BAAa;;;YACpD;;QACAC,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACjB;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACMkB,YAAN,SAAMA,WAAWF,OAAO,EAAEG,UAAU;;;wBAClC;;4BAAOD,IAAAA,wBAAU,EAAClB,MAAMgB,SAASG;;;gBACnC;;QACF;QACAC,SAAS;YACDC,OAAN,SAAMA,MAAMC,KAAK,EAAEC,IAAI;;;wBACrB;;4BAAOC,IAAAA,uBAAY,EAACxB,MAAMsB,OAAOC,MAAME,IAAAA,wBAAe,EAACxB,SAASyB;;;gBAClE;;QACF;QACAC,SAAS;YACDC,SAAN,SAAMA;;;wBACJ;;4BAAOC,IAAAA,sBAAW,EAAC7B;;;gBACrB;;YACM8B,cAAN,SAAMA,aAAaC,IAAsB;;;;;gCACvC;;oCAAMC,IAAAA,4BAAgB,EAAChC,MAAM+B;;;gCAA7B;;;;;;gBACF;;YACME,YAAN,SAAMA,WAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;;;wBAC5C;;4BAAOC,IAAAA,0BAAc,EAACtC,MAAMkC,IAAIC,WAAWC,OAAOC;;;gBACpD;;YACME,SAAN,SAAMA,QAAQC,IAAI,EAAEjB,IAAI;;;wBACtB;;4BAAOkB,IAAAA,uBAAW,EAACzC,MAAMwC,MAAMjB;;;gBACjC;;YACMmB,WAAN,SAAMA,UAAUF,IAAI;;;wBAClB;;4BAAOG,IAAAA,uBAAY,EAAC3C,MAAMwC;;;gBAC5B;;QACF;QACMI,cAAN,SAAMA;;oBAGEC;;oBAFN,qFAAqF;oBACrF,qFAAqF;oBAC/EA,MAAM9C,GAAGQ,OAAO,CAAC,uBAAuBuC,GAAG;oBACjD;;wBAAO;4BAAEC,cAAc,AAAC,GAAc,OAAZF,IAAIG,OAAO,EAAC;wBAAoF;;;YAC5H;;QACAC,KAAK;YACG1C,SAAN,SAAMA,QAAQD,GAAW;;wBACjB4C;;wBAAAA,OAAOnD,GAAGQ,OAAO,CAACD;wBACxB4C,KAAKC,cAAc,CAAC,OAAO,qEAAqE;wBAChG;;4BAAO;gCACLnC,SAAS,SAATA;2CAAekC,KAAKlC,OAAO;;gCAC3B,2EAA2E;gCAC3EoC,SAAS,SAATA;oCAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;wCAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;4CACrCH;;;;oDAAP;;mEAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAIG;;;oDAAxB;;;;;;oCACF;;4BACF;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJvD,GAAGuD,KAAK;;;;;YACV;;IACF;AACF"}

package/dist/cjs/store/transaction.d.cts

@@ -1,5 +1,6 @@
 interface ExecConnection {
     exec(sql: string): Promise<void>;
 }
-export declare function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>): Promise<T>;
+export declare const BEGIN_WRITE = "BEGIN IMMEDIATE";
+export declare function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>, begin?: string): Promise<T>;
 export {};

package/dist/cjs/store/transaction.d.ts

@@ -1,5 +1,6 @@
 interface ExecConnection {
     exec(sql: string): Promise<void>;
 }
-export declare function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>): Promise<T>;
+export declare const BEGIN_WRITE = "BEGIN IMMEDIATE";
+export declare function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>, begin?: string): Promise<T>;
 export {};

package/dist/cjs/store/transaction.js

@@ -4,9 +4,17 @@
 Object.defineProperty(exports, "__esModule", {
     value: true
 });
-Object.defineProperty(exports, "withTransaction", {
-    enumerable: true,
-    get: function() {
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get BEGIN_WRITE () {
+        return BEGIN_WRITE;
+    },
+    get withTransaction () {
         return withTransaction;
     }
 });
@@ -149,7 +157,7 @@ function stateFor(conn) {
     }
     return state;
 }
-function enter(conn, state) {
+function enter(conn, state, begin) {
     return _async_to_generator(function() {
         return _ts_generator(this, function(_state) {
             switch(_state.label){
@@ -160,7 +168,7 @@ function enter(conn, state) {
                     ];
                     return [
                         4,
-                        conn.exec('BEGIN')
+                        conn.exec(begin)
                     ];
                 case 1:
                     _state.sent();
@@ -237,7 +245,9 @@ function leaveErr(conn, state) {
         });
     })();
 }
+var BEGIN_WRITE = 'BEGIN IMMEDIATE';
 function withTransaction(conn, fn) {
+    var begin = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : 'BEGIN';
     return _async_to_generator(function() {
         var state, result, err;
         return _ts_generator(this, function(_state) {
@@ -246,7 +256,7 @@ function withTransaction(conn, fn) {
                     state = stateFor(conn);
                     return [
                         4,
-                        enter(conn, state)
+                        enter(conn, state, begin)
                     ];
                 case 1:
                     _state.sent();