sensemaking 0.18.2 → 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/dist/cjs/cli/status.js +32 -31
  2. package/dist/cjs/cli/status.js.map +1 -1
  3. package/dist/cjs/commands/map.js +50 -67
  4. package/dist/cjs/commands/map.js.map +1 -1
  5. package/dist/cjs/config/access.d.cts +2 -2
  6. package/dist/cjs/config/access.d.ts +2 -2
  7. package/dist/cjs/config/access.js.map +1 -1
  8. package/dist/cjs/config/index.d.cts +1 -1
  9. package/dist/cjs/config/index.d.ts +1 -1
  10. package/dist/cjs/config/index.js.map +1 -1
  11. package/dist/cjs/config/load.js +3 -2
  12. package/dist/cjs/config/load.js.map +1 -1
  13. package/dist/cjs/config/types.d.cts +2 -1
  14. package/dist/cjs/config/types.d.ts +2 -1
  15. package/dist/cjs/config/types.js.map +1 -1
  16. package/dist/cjs/config/validate.js +3 -0
  17. package/dist/cjs/config/validate.js.map +1 -1
  18. package/dist/cjs/embed/identity.js +3 -2
  19. package/dist/cjs/embed/identity.js.map +1 -1
  20. package/dist/cjs/embed/store.js +3 -3
  21. package/dist/cjs/embed/store.js.map +1 -1
  22. package/dist/cjs/index.d.cts +2 -2
  23. package/dist/cjs/index.d.ts +2 -2
  24. package/dist/cjs/index.js.map +1 -1
  25. package/dist/cjs/lib/atomic-write.d.cts +1 -0
  26. package/dist/cjs/lib/atomic-write.d.ts +1 -0
  27. package/dist/cjs/lib/atomic-write.js +26 -0
  28. package/dist/cjs/lib/atomic-write.js.map +1 -0
  29. package/dist/cjs/store/duckdb/batch.js +3 -2
  30. package/dist/cjs/store/duckdb/batch.js.map +1 -1
  31. package/dist/cjs/store/duckdb/connection.js +12 -0
  32. package/dist/cjs/store/duckdb/connection.js.map +1 -1
  33. package/dist/cjs/store/duckdb/fieldStats.d.cts +2 -0
  34. package/dist/cjs/store/duckdb/fieldStats.d.ts +2 -0
  35. package/dist/cjs/store/duckdb/fieldStats.js +219 -0
  36. package/dist/cjs/store/duckdb/fieldStats.js.map +1 -0
  37. package/dist/cjs/store/duckdb/lexical.js +16 -9
  38. package/dist/cjs/store/duckdb/lexical.js.map +1 -1
  39. package/dist/cjs/store/duckdb/native.d.cts +0 -1
  40. package/dist/cjs/store/duckdb/native.d.ts +0 -1
  41. package/dist/cjs/store/duckdb/native.js +7 -336
  42. package/dist/cjs/store/duckdb/native.js.map +1 -1
  43. package/dist/cjs/store/duckdb/sql-functions.js +1 -1
  44. package/dist/cjs/store/duckdb/sql-functions.js.map +1 -1
  45. package/dist/cjs/store/duckdb/store.js +16 -1
  46. package/dist/cjs/store/duckdb/store.js.map +1 -1
  47. package/dist/cjs/store/duckdb/vectors.js +5 -17
  48. package/dist/cjs/store/duckdb/vectors.js.map +1 -1
  49. package/dist/cjs/store/index.js +6 -0
  50. package/dist/cjs/store/index.js.map +1 -1
  51. package/dist/cjs/store/native.d.cts +7 -0
  52. package/dist/cjs/store/native.d.ts +7 -0
  53. package/dist/cjs/store/native.js +350 -0
  54. package/dist/cjs/store/native.js.map +1 -0
  55. package/dist/cjs/store/sqlite/fieldStats.d.cts +2 -0
  56. package/dist/cjs/store/sqlite/fieldStats.d.ts +2 -0
  57. package/dist/cjs/store/sqlite/fieldStats.js +201 -0
  58. package/dist/cjs/store/sqlite/fieldStats.js.map +1 -0
  59. package/dist/cjs/store/sqlite/open.js +82 -29
  60. package/dist/cjs/store/sqlite/open.js.map +1 -1
  61. package/dist/cjs/store/sqlite/store.js +32 -4
  62. package/dist/cjs/store/sqlite/store.js.map +1 -1
  63. package/dist/cjs/store/sqlite/vectors.d.cts +0 -5
  64. package/dist/cjs/store/sqlite/vectors.d.ts +0 -5
  65. package/dist/cjs/store/sqlite/vectors.js +5 -71
  66. package/dist/cjs/store/sqlite/vectors.js.map +1 -1
  67. package/dist/cjs/store/turso/connection.d.cts +3 -0
  68. package/dist/cjs/store/turso/connection.d.ts +3 -0
  69. package/dist/cjs/store/turso/connection.js +529 -0
  70. package/dist/cjs/store/turso/connection.js.map +1 -0
  71. package/dist/cjs/store/turso/fieldStats.d.cts +2 -0
  72. package/dist/cjs/store/turso/fieldStats.d.ts +2 -0
  73. package/dist/cjs/store/turso/fieldStats.js +201 -0
  74. package/dist/cjs/store/turso/fieldStats.js.map +1 -0
  75. package/dist/cjs/store/turso/native.d.cts +2 -0
  76. package/dist/cjs/store/turso/native.d.ts +2 -0
  77. package/dist/cjs/store/turso/native.js +31 -0
  78. package/dist/cjs/store/turso/native.js.map +1 -0
  79. package/dist/cjs/store/turso/open.d.cts +12 -0
  80. package/dist/cjs/store/turso/open.d.ts +12 -0
  81. package/dist/cjs/store/turso/open.js +500 -0
  82. package/dist/cjs/store/turso/open.js.map +1 -0
  83. package/dist/cjs/store/turso/reconcile.d.cts +6 -0
  84. package/dist/cjs/store/turso/reconcile.d.ts +6 -0
  85. package/dist/cjs/store/turso/reconcile.js +793 -0
  86. package/dist/cjs/store/turso/reconcile.js.map +1 -0
  87. package/dist/cjs/store/turso/store.d.cts +5 -0
  88. package/dist/cjs/store/turso/store.d.ts +5 -0
  89. package/dist/cjs/store/turso/store.js +609 -0
  90. package/dist/cjs/store/turso/store.js.map +1 -0
  91. package/dist/cjs/store/types.d.cts +9 -1
  92. package/dist/cjs/store/types.d.ts +9 -1
  93. package/dist/cjs/store/types.js +0 -7
  94. package/dist/cjs/store/types.js.map +1 -1
  95. package/dist/cjs/store/vectors.d.cts +9 -0
  96. package/dist/cjs/store/vectors.d.ts +9 -0
  97. package/dist/cjs/store/vectors.js +215 -0
  98. package/dist/cjs/store/vectors.js.map +1 -0
  99. package/dist/esm/cli/status.js +5 -7
  100. package/dist/esm/cli/status.js.map +1 -1
  101. package/dist/esm/commands/map.js +7 -19
  102. package/dist/esm/commands/map.js.map +1 -1
  103. package/dist/esm/config/access.d.ts +2 -2
  104. package/dist/esm/config/access.js.map +1 -1
  105. package/dist/esm/config/index.d.ts +1 -1
  106. package/dist/esm/config/index.js.map +1 -1
  107. package/dist/esm/config/load.js +4 -3
  108. package/dist/esm/config/load.js.map +1 -1
  109. package/dist/esm/config/types.d.ts +2 -1
  110. package/dist/esm/config/types.js.map +1 -1
  111. package/dist/esm/config/validate.js +3 -0
  112. package/dist/esm/config/validate.js.map +1 -1
  113. package/dist/esm/embed/identity.js +4 -3
  114. package/dist/esm/embed/identity.js.map +1 -1
  115. package/dist/esm/embed/store.js +3 -3
  116. package/dist/esm/embed/store.js.map +1 -1
  117. package/dist/esm/index.d.ts +2 -2
  118. package/dist/esm/index.js.map +1 -1
  119. package/dist/esm/lib/atomic-write.d.ts +1 -0
  120. package/dist/esm/lib/atomic-write.js +17 -0
  121. package/dist/esm/lib/atomic-write.js.map +1 -0
  122. package/dist/esm/store/duckdb/batch.js +3 -2
  123. package/dist/esm/store/duckdb/batch.js.map +1 -1
  124. package/dist/esm/store/duckdb/connection.js +12 -0
  125. package/dist/esm/store/duckdb/connection.js.map +1 -1
  126. package/dist/esm/store/duckdb/fieldStats.d.ts +2 -0
  127. package/dist/esm/store/duckdb/fieldStats.js +39 -0
  128. package/dist/esm/store/duckdb/fieldStats.js.map +1 -0
  129. package/dist/esm/store/duckdb/lexical.js +19 -9
  130. package/dist/esm/store/duckdb/lexical.js.map +1 -1
  131. package/dist/esm/store/duckdb/native.d.ts +0 -1
  132. package/dist/esm/store/duckdb/native.js +7 -84
  133. package/dist/esm/store/duckdb/native.js.map +1 -1
  134. package/dist/esm/store/duckdb/sql-functions.js +1 -1
  135. package/dist/esm/store/duckdb/sql-functions.js.map +1 -1
  136. package/dist/esm/store/duckdb/store.js +8 -5
  137. package/dist/esm/store/duckdb/store.js.map +1 -1
  138. package/dist/esm/store/duckdb/vectors.js +2 -12
  139. package/dist/esm/store/duckdb/vectors.js.map +1 -1
  140. package/dist/esm/store/index.js +6 -0
  141. package/dist/esm/store/index.js.map +1 -1
  142. package/dist/esm/store/native.d.ts +7 -0
  143. package/dist/esm/store/native.js +83 -0
  144. package/dist/esm/store/native.js.map +1 -0
  145. package/dist/esm/store/sqlite/fieldStats.d.ts +2 -0
  146. package/dist/esm/store/sqlite/fieldStats.js +25 -0
  147. package/dist/esm/store/sqlite/fieldStats.js.map +1 -0
  148. package/dist/esm/store/sqlite/open.js +21 -8
  149. package/dist/esm/store/sqlite/open.js.map +1 -1
  150. package/dist/esm/store/sqlite/store.js +14 -1
  151. package/dist/esm/store/sqlite/store.js.map +1 -1
  152. package/dist/esm/store/sqlite/vectors.d.ts +0 -5
  153. package/dist/esm/store/sqlite/vectors.js +2 -22
  154. package/dist/esm/store/sqlite/vectors.js.map +1 -1
  155. package/dist/esm/store/turso/connection.d.ts +3 -0
  156. package/dist/esm/store/turso/connection.js +52 -0
  157. package/dist/esm/store/turso/connection.js.map +1 -0
  158. package/dist/esm/store/turso/fieldStats.d.ts +2 -0
  159. package/dist/esm/store/turso/fieldStats.js +24 -0
  160. package/dist/esm/store/turso/fieldStats.js.map +1 -0
  161. package/dist/esm/store/turso/native.d.ts +2 -0
  162. package/dist/esm/store/turso/native.js +19 -0
  163. package/dist/esm/store/turso/native.js.map +1 -0
  164. package/dist/esm/store/turso/open.d.ts +12 -0
  165. package/dist/esm/store/turso/open.js +113 -0
  166. package/dist/esm/store/turso/open.js.map +1 -0
  167. package/dist/esm/store/turso/reconcile.d.ts +6 -0
  168. package/dist/esm/store/turso/reconcile.js +158 -0
  169. package/dist/esm/store/turso/reconcile.js.map +1 -0
  170. package/dist/esm/store/turso/store.d.ts +5 -0
  171. package/dist/esm/store/turso/store.js +93 -0
  172. package/dist/esm/store/turso/store.js.map +1 -0
  173. package/dist/esm/store/types.d.ts +9 -1
  174. package/dist/esm/store/types.js +0 -7
  175. package/dist/esm/store/types.js.map +1 -1
  176. package/dist/esm/store/vectors.d.ts +9 -0
  177. package/dist/esm/store/vectors.js +26 -0
  178. package/dist/esm/store/vectors.js.map +1 -0
  179. package/package.json +2 -1
@@ -1,8 +1,4 @@
1
1
  import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
2
- export declare function pendingRows(conn: Connection): Promise<Array<{
3
- path: string;
4
- chunk: number;
5
- }>>;
6
2
  export declare function writeVectorBatch(conn: Connection, rows: VectorWriteRow[]): Promise<void>;
7
3
  export declare function scanCandidates(conn: Connection, qv: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
8
4
  export declare function scanSimilar(conn: Connection, path: string, opts: {
@@ -10,4 +6,3 @@ export declare function scanSimilar(conn: Connection, path: string, opts: {
10
6
  allowed?: Set<string>;
11
7
  k: number;
12
8
  }): Promise<VectorSimilar[]>;
13
- export declare function hasVectorRow(conn: Connection, path: string): Promise<boolean>;
@@ -1,14 +1,4 @@
1
- // Seed chunks that participate in a `related` scan; see similar() for the measurement.
2
- const TARGET_CHUNK_CAP = 16;
3
- // Dequantised int8 dot products land a little either side of a true cosine, so an identical
4
- // pair prints 1.001 and undermines a column whose whole job is being a bounded number.
5
- function asCosine(score) {
6
- return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
7
- }
8
- export async function pendingRows(conn) {
9
- const stmt = await conn.prepare('SELECT "path", chunk FROM embeddings WHERE vector IS NULL ORDER BY "path", chunk');
10
- return await stmt.all();
11
- }
1
+ import { asCosine, sampleEvenly } from '../vectors.js';
12
2
  // One runBatch call per provider batch: the caller (embed/query.ts) already batches by the
13
3
  // provider's batchCap, so this is one crossing per batch, never per row.
14
4
  export async function writeVectorBatch(conn, rows) {
@@ -49,13 +39,10 @@ export async function scanCandidates(conn, qv, storeDims, fetch, allowed) {
49
39
  // Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear
50
40
  // scan of stored vectors.
51
41
  export async function scanSimilar(conn, path, opts) {
52
- // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus
53
- // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.
54
42
  const targetStmt = await conn.prepare('SELECT scale, vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk');
55
43
  const targetRows = await targetStmt.all(path);
56
44
  if (targetRows.length === 0) return [];
57
- const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));
58
- const target = targetRows.filter((_, i)=>i % step === 0).map((row)=>({
45
+ const target = sampleEvenly(targetRows).map((row)=>({
59
46
  v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength),
60
47
  scale: row.scale
61
48
  }));
@@ -81,10 +68,3 @@ export async function scanSimilar(conn, path, opts) {
81
68
  similarity: asCosine(score)
82
69
  }));
83
70
  }
84
- // Whether a note has any chunk with a vector. Distinguishes "nothing is near this note" from
85
- // "this note has no text", which look the same in an empty result.
86
- export async function hasVectorRow(conn, path) {
87
- const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE "path" = ? AND vector IS NOT NULL LIMIT 1');
88
- const row = await stmt.get(path);
89
- return row !== undefined;
90
- }
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\n\n// Seed chunks that participate in a `related` scan; see similar() for the measurement.\nconst TARGET_CHUNK_CAP = 16;\n\n// Dequantised int8 dot products land a little either side of a true cosine, so an identical\n// pair prints 1.001 and undermines a column whose whole job is being a bounded number.\nfunction asCosine(score: number): number {\n return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\nexport async function pendingRows(conn: Connection): Promise<Array<{ path: string; chunk: number }>> {\n const stmt = await conn.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n return (await stmt.all()) as Array<{ path: string; chunk: number }>;\n}\n\n// One runBatch call per provider batch: the caller (embed/query.ts) already batches by the\n// provider's batchCap, so this is one crossing per batch, never per row.\nexport async function writeVectorBatch(conn: Connection, rows: VectorWriteRow[]): Promise<void> {\n if (rows.length === 0) return;\n await conn.runBatch(\n 'UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?',\n rows.map((row) => [row.scale, row.vector, row.path, row.chunk])\n );\n}\n\n// Best chunk per file by cosine, its line range riding along: the JS loop over stored int8\n// BLOBs this store's scan strategy is (a native store scans differently, same contract).\nexport async function scanCandidates(conn: Connection, qv: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n const stmt = await conn.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL');\n const rows = (await stmt.all()) as Array<{\n path: string;\n start_line: number;\n end_line: number;\n scale: number;\n vector: Uint8Array;\n }>;\n\n const best = new Map<string, { score: number; lines: string }>();\n for (const row of rows) {\n if (allowed && !allowed.has(row.path)) continue;\n const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n let dot = 0;\n for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n const score = dot * row.scale;\n const existing = best.get(row.path);\n if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n }\n return [...best.entries()]\n .sort((a, b) => b[1].score - a[1].score)\n .slice(0, fetch)\n .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function scanSimilar(conn: Connection, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus\n // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.\n const targetStmt = await conn.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n const targetRows = (await targetStmt.all(path)) as Array<{ scale: number; vector: Uint8Array }>;\n if (targetRows.length === 0) return [];\n const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n const target = targetRows.filter((_, i) => i % step === 0).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n const stmt = await conn.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL');\n const rows = (await stmt.all()) as Array<{ path: string; scale: number; vector: Uint8Array }>;\n const best = new Map<string, number>();\n for (const row of rows) {\n if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n for (const t of target) {\n let dot = 0;\n const len = Math.min(t.v.length, other.length);\n for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n const score = dot * t.scale * row.scale;\n const existing = best.get(row.path);\n if (existing === undefined || score > existing) best.set(row.path, score);\n }\n }\n\n return [...best.entries()]\n .sort((a, b) => b[1] - a[1])\n .slice(0, opts.k)\n .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasVectorRow(conn: Connection, path: string): Promise<boolean> {\n const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1');\n const row = (await stmt.get(path)) as { ok: number } | undefined;\n return row !== undefined;\n}\n"],"names":["TARGET_CHUNK_CAP","asCosine","score","Math","round","min","max","pendingRows","conn","stmt","prepare","all","writeVectorBatch","rows","length","runBatch","map","row","scale","vector","path","chunk","scanCandidates","qv","storeDims","fetch","allowed","best","Map","has","q","Int8Array","buffer","byteOffset","byteLength","dot","d","existing","get","set","lines","start_line","end_line","entries","sort","a","b","slice","similarity","scanSimilar","opts","targetStmt","targetRows","step","ceil","target","filter","_","i","v","exclude","other","t","len","undefined","k","p","hasVectorRow"],"mappings":"AAEA,uFAAuF;AACvF,MAAMA,mBAAmB;AAEzB,4FAA4F;AAC5F,uFAAuF;AACvF,SAASC,SAASC,KAAa;IAC7B,OAAOC,KAAKC,KAAK,CAACD,KAAKE,GAAG,CAAC,GAAGF,KAAKG,GAAG,CAAC,CAAC,GAAGJ,UAAU,QAAQ;AAC/D;AAEA,OAAO,eAAeK,YAAYC,IAAgB;IAChD,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,OAAQ,MAAMD,KAAKE,GAAG;AACxB;AAEA,2FAA2F;AAC3F,yEAAyE;AACzE,OAAO,eAAeC,iBAAiBJ,IAAgB,EAAEK,IAAsB;IAC7E,IAAIA,KAAKC,MAAM,KAAK,GAAG;IACvB,MAAMN,KAAKO,QAAQ,CACjB,8EACAF,KAAKG,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIC,KAAK;YAAED,IAAIE,MAAM;YAAEF,IAAIG,IAAI;YAAEH,IAAII,KAAK;SAAC;AAElE;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,OAAO,eAAeC,eAAed,IAAgB,EAAEe,EAAgB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,OAAqB;IAC9H,MAAMjB,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMG,OAAQ,MAAMJ,KAAKE,GAAG;IAQ5B,MAAMgB,OAAO,IAAIC;IACjB,KAAK,MAAMX,OAAOJ,KAAM;QACtB,IAAIa,WAAW,CAACA,QAAQG,GAAG,CAACZ,IAAIG,IAAI,GAAG;QACvC,MAAMU,IAAI,IAAIC,UAAUd,IAAIE,MAAM,CAACa,MAAM,EAAEf,IAAIE,MAAM,CAACc,UAAU,EAAE9B,KAAKE,GAAG,CAACmB,WAAWP,IAAIE,MAAM,CAACe,UAAU;QAC3G,IAAIC,MAAM;QACV,IAAK,IAAIC,IAAI,GAAGA,IAAIN,EAAEhB,MAAM,EAAEsB,IAAKD,OAAOL,CAAC,CAACM,EAAE,GAAGb,EAAE,CAACa,EAAE;QACtD,MAAMlC,QAAQiC,MAAMlB,IAAIC,KAAK;QAC7B,MAAMmB,WAAWV,KAAKW,GAAG,CAACrB,IAAIG,IAAI;QAClC,IAAI,CAACiB,YAAYnC,QAAQmC,SAASnC,KAAK,EAAEyB,KAAKY,GAAG,CAACtB,IAAIG,IAAI,EAAE;YAAElB;YAAOsC,OAAO,CAAC,CAAC,EAAEvB,IAAIwB,UAAU,CAAC,CAAC,EAAExB,IAAIyB,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIf,KAAKgB,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAAC5C,KAAK,GAAG2C,CAAC,CAAC,EAAE,CAAC3C,KAAK,EACtC6C,KAAK,CAAC,GAAGtB,OACTT,GAAG,CAAC,CAAC,CAACI,MAAM0B,EAAE,GAAM,CAAA;YAAE1B;YAAMoB,OAAOM,EAAEN,KAAK;YAAEQ,YAAY/C,SAAS6C,EAAE5C,KAAK;QAAE,CAAA;AAC/E;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAe+C,YAAYzC,IAAgB,EAAEY,IAAY,EAAE8B,IAAgE;IAChI,0FAA0F;IAC1F,qFAAqF;IACrF,MAAMC,aAAa,MAAM3C,KAAKE,OAAO,CAAC;IACtC,MAAM0C,aAAc,MAAMD,WAAWxC,GAAG,CAACS;IACzC,IAAIgC,WAAWtC,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMuC,OAAOlD,KAAKG,GAAG,CAAC,GAAGH,KAAKmD,IAAI,CAACF,WAAWtC,MAAM,GAAGd;IACvD,MAAMuD,SAASH,WAAWI,MAAM,CAAC,CAACC,GAAGC,IAAMA,IAAIL,SAAS,GAAGrC,GAAG,CAAC,CAACC,MAAS,CAAA;YAAE0C,GAAG,IAAI5B,UAAUd,IAAIE,MAAM,CAACa,MAAM,EAAEf,IAAIE,MAAM,CAACc,UAAU,EAAEhB,IAAIE,MAAM,CAACe,UAAU;YAAGhB,OAAOD,IAAIC,KAAK;QAAC,CAAA;IAE/K,MAAMT,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMG,OAAQ,MAAMJ,KAAKE,GAAG;IAC5B,MAAMgB,OAAO,IAAIC;IACjB,KAAK,MAAMX,OAAOJ,KAAM;QACtB,IAAII,IAAIG,IAAI,KAAKA,QAAQ8B,KAAKU,OAAO,CAAC/B,GAAG,CAACZ,IAAIG,IAAI,KAAM8B,KAAKxB,OAAO,IAAI,CAACwB,KAAKxB,OAAO,CAACG,GAAG,CAACZ,IAAIG,IAAI,GAAI;QACtG,MAAMyC,QAAQ,IAAI9B,UAAUd,IAAIE,MAAM,CAACa,MAAM,EAAEf,IAAIE,MAAM,CAACc,UAAU,EAAEhB,IAAIE,MAAM,CAACe,UAAU;QAC3F,KAAK,MAAM4B,KAAKP,OAAQ;YACtB,IAAIpB,MAAM;YACV,MAAM4B,MAAM5D,KAAKE,GAAG,CAACyD,EAAEH,CAAC,CAAC7C,MAAM,EAAE+C,MAAM/C,MAAM;YAC7C,IAAK,IAAIsB,IAAI,GAAGA,IAAI2B,KAAK3B,IAAKD,OAAO2B,EAAEH,CAAC,CAACvB,EAAE,GAAGyB,KAAK,CAACzB,EAAE;YACtD,MAAMlC,QAAQiC,MAAM2B,EAAE5C,KAAK,GAAGD,IAAIC,KAAK;YACvC,MAAMmB,WAAWV,KAAKW,GAAG,CAACrB,IAAIG,IAAI;YAClC,IAAIiB,aAAa2B,aAAa9D,QAAQmC,UAAUV,KAAKY,GAAG,CAACtB,IAAIG,IAAI,EAAElB;QACrE;IACF;IAEA,OAAO;WAAIyB,KAAKgB,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BE,KAAK,CAAC,GAAGG,KAAKe,CAAC,EACfjD,GAAG,CAAC,CAAC,CAACkD,GAAGhE,MAAM,GAAM,CAAA;YAAEkB,MAAM8C;YAAGlB,YAAY/C,SAASC;QAAO,CAAA;AACjE;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAeiE,aAAa3D,IAAgB,EAAEY,IAAY;IAC/D,MAAMX,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMO,MAAO,MAAMR,KAAK6B,GAAG,CAAClB;IAC5B,OAAOH,QAAQ+C;AACjB"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/vectors.ts"],"sourcesContent":["import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\n\n// One runBatch call per provider batch: the caller (embed/query.ts) already batches by the\n// provider's batchCap, so this is one crossing per batch, never per row.\nexport async function writeVectorBatch(conn: Connection, rows: VectorWriteRow[]): Promise<void> {\n if (rows.length === 0) return;\n await conn.runBatch(\n 'UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?',\n rows.map((row) => [row.scale, row.vector, row.path, row.chunk])\n );\n}\n\n// Best chunk per file by cosine, its line range riding along: the JS loop over stored int8\n// BLOBs this store's scan strategy is (a native store scans differently, same contract).\nexport async function scanCandidates(conn: Connection, qv: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n const stmt = await conn.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL');\n const rows = (await stmt.all()) as Array<{\n path: string;\n start_line: number;\n end_line: number;\n scale: number;\n vector: Uint8Array;\n }>;\n\n const best = new Map<string, { score: number; lines: string }>();\n for (const row of rows) {\n if (allowed && !allowed.has(row.path)) continue;\n const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n let dot = 0;\n for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n const score = dot * row.scale;\n const existing = best.get(row.path);\n if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n }\n return [...best.entries()]\n .sort((a, b) => b[1].score - a[1].score)\n .slice(0, fetch)\n .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function scanSimilar(conn: Connection, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n const targetStmt = await conn.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n const targetRows = (await targetStmt.all(path)) as Array<{ scale: number; vector: Uint8Array }>;\n if (targetRows.length === 0) return [];\n const target = sampleEvenly(targetRows).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n const stmt = await conn.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL');\n const rows = (await stmt.all()) as Array<{ path: string; scale: number; vector: Uint8Array }>;\n const best = new Map<string, number>();\n for (const row of rows) {\n if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n for (const t of target) {\n let dot = 0;\n const len = Math.min(t.v.length, other.length);\n for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n const score = dot * t.scale * row.scale;\n const existing = best.get(row.path);\n if (existing === undefined || score > existing) best.set(row.path, score);\n }\n }\n\n return [...best.entries()]\n .sort((a, b) => b[1] - a[1])\n .slice(0, opts.k)\n .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["asCosine","sampleEvenly","writeVectorBatch","conn","rows","length","runBatch","map","row","scale","vector","path","chunk","scanCandidates","qv","storeDims","fetch","allowed","stmt","prepare","all","best","Map","has","q","Int8Array","buffer","byteOffset","Math","min","byteLength","dot","d","score","existing","get","set","lines","start_line","end_line","entries","sort","a","b","slice","similarity","scanSimilar","opts","targetStmt","targetRows","target","v","exclude","other","t","len","undefined","k","p"],"mappings":"AACA,SAASA,QAAQ,EAAEC,YAAY,QAAQ,gBAAgB;AAEvD,2FAA2F;AAC3F,yEAAyE;AACzE,OAAO,eAAeC,iBAAiBC,IAAgB,EAAEC,IAAsB;IAC7E,IAAIA,KAAKC,MAAM,KAAK,GAAG;IACvB,MAAMF,KAAKG,QAAQ,CACjB,8EACAF,KAAKG,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIC,KAAK;YAAED,IAAIE,MAAM;YAAEF,IAAIG,IAAI;YAAEH,IAAII,KAAK;SAAC;AAElE;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,OAAO,eAAeC,eAAeV,IAAgB,EAAEW,EAAgB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,OAAqB;IAC9H,MAAMC,OAAO,MAAMf,KAAKgB,OAAO,CAAC;IAChC,MAAMf,OAAQ,MAAMc,KAAKE,GAAG;IAQ5B,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAMd,OAAOJ,KAAM;QACtB,IAAIa,WAAW,CAACA,QAAQM,GAAG,CAACf,IAAIG,IAAI,GAAG;QACvC,MAAMa,IAAI,IAAIC,UAAUjB,IAAIE,MAAM,CAACgB,MAAM,EAAElB,IAAIE,MAAM,CAACiB,UAAU,EAAEC,KAAKC,GAAG,CAACd,WAAWP,IAAIE,MAAM,CAACoB,UAAU;QAC3G,IAAIC,MAAM;QACV,IAAK,IAAIC,IAAI,GAAGA,IAAIR,EAAEnB,MAAM,EAAE2B,IAAKD,OAAOP,CAAC,CAACQ,EAAE,GAAGlB,EAAE,CAACkB,EAAE;QACtD,MAAMC,QAAQF,MAAMvB,IAAIC,KAAK;QAC7B,MAAMyB,WAAWb,KAAKc,GAAG,CAAC3B,IAAIG,IAAI;QAClC,IAAI,CAACuB,YAAYD,QAAQC,SAASD,KAAK,EAAEZ,KAAKe,GAAG,CAAC5B,IAAIG,IAAI,EAAE;YAAEsB;YAAOI,OAAO,CAAC,CAAC,EAAE7B,IAAI8B,UAAU,CAAC,CAAC,EAAE9B,IAAI+B,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIlB,KAAKmB,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAACV,KAAK,GAAGS,CAAC,CAAC,EAAE,CAACT,KAAK,EACtCW,KAAK,CAAC,GAAG5B,OACTT,GAAG,CAAC,CAAC,CAACI,MAAMgC,EAAE,GAAM,CAAA;YAAEhC;YAAM0B,OAAOM,EAAEN,KAAK;YAAEQ,YAAY7C,SAAS2C,EAAEV,KAAK;QAAE,CAAA;AAC/E;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAea,YAAY3C,IAAgB,EAAEQ,IAAY,EAAEoC,IAAgE;IAChI,MAAMC,aAAa,MAAM7C,KAAKgB,OAAO,CAAC;IACtC,MAAM8B,aAAc,MAAMD,WAAW5B,GAAG,CAACT;IACzC,IAAIsC,WAAW5C,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAM6C,SAASjD,aAAagD,YAAY1C,GAAG,CAAC,CAACC,MAAS,CAAA;YAAE2C,GAAG,IAAI1B,UAAUjB,IAAIE,MAAM,CAACgB,MAAM,EAAElB,IAAIE,MAAM,CAACiB,UAAU,EAAEnB,IAAIE,MAAM,CAACoB,UAAU;YAAGrB,OAAOD,IAAIC,KAAK;QAAC,CAAA;IAE5J,MAAMS,OAAO,MAAMf,KAAKgB,OAAO,CAAC;IAChC,MAAMf,OAAQ,MAAMc,KAAKE,GAAG;IAC5B,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAMd,OAAOJ,KAAM;QACtB,IAAII,IAAIG,IAAI,KAAKA,QAAQoC,KAAKK,OAAO,CAAC7B,GAAG,CAACf,IAAIG,IAAI,KAAMoC,KAAK9B,OAAO,IAAI,CAAC8B,KAAK9B,OAAO,CAACM,GAAG,CAACf,IAAIG,IAAI,GAAI;QACtG,MAAM0C,QAAQ,IAAI5B,UAAUjB,IAAIE,MAAM,CAACgB,MAAM,EAAElB,IAAIE,MAAM,CAACiB,UAAU,EAAEnB,IAAIE,MAAM,CAACoB,UAAU;QAC3F,KAAK,MAAMwB,KAAKJ,OAAQ;YACtB,IAAInB,MAAM;YACV,MAAMwB,MAAM3B,KAAKC,GAAG,CAACyB,EAAEH,CAAC,CAAC9C,MAAM,EAAEgD,MAAMhD,MAAM;YAC7C,IAAK,IAAI2B,IAAI,GAAGA,IAAIuB,KAAKvB,IAAKD,OAAOuB,EAAEH,CAAC,CAACnB,EAAE,GAAGqB,KAAK,CAACrB,EAAE;YACtD,MAAMC,QAAQF,MAAMuB,EAAE7C,KAAK,GAAGD,IAAIC,KAAK;YACvC,MAAMyB,WAAWb,KAAKc,GAAG,CAAC3B,IAAIG,IAAI;YAClC,IAAIuB,aAAasB,aAAavB,QAAQC,UAAUb,KAAKe,GAAG,CAAC5B,IAAIG,IAAI,EAAEsB;QACrE;IACF;IAEA,OAAO;WAAIZ,KAAKmB,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BE,KAAK,CAAC,GAAGG,KAAKU,CAAC,EACflD,GAAG,CAAC,CAAC,CAACmD,GAAGzB,MAAM,GAAM,CAAA;YAAEtB,MAAM+C;YAAGb,YAAY7C,SAASiC;QAAO,CAAA;AACjE"}
@@ -0,0 +1,3 @@
1
+ import type { Database } from '@tursodatabase/database';
2
+ import type { Connection } from '../types.js';
3
+ export declare function createConnection(db: Database): Connection;
@@ -0,0 +1,52 @@
1
+ import { withTransaction } from '../transaction.js';
2
+ // Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this
3
+ // wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.
4
+ let TursoStatementWrapper = class TursoStatementWrapper {
5
+ async run(...params) {
6
+ return this.stmt.run(...params);
7
+ }
8
+ async get(...params) {
9
+ return this.stmt.get(...params);
10
+ }
11
+ async all(...params) {
12
+ return this.stmt.all(...params);
13
+ }
14
+ async *iterate(...params) {
15
+ yield* this.stmt.iterate(...params);
16
+ }
17
+ columns() {
18
+ return this.stmt.columns();
19
+ }
20
+ setReadBigInts(enabled) {
21
+ this.stmt.safeIntegers(enabled);
22
+ }
23
+ constructor(stmt){
24
+ this.stmt = stmt;
25
+ }
26
+ };
27
+ export function createConnection(db) {
28
+ const conn = {
29
+ async exec (sql) {
30
+ await db.exec(sql);
31
+ },
32
+ async prepare (sql) {
33
+ return new TursoStatementWrapper(await db.prepare(sql));
34
+ },
35
+ // db.batch() is the engine's own bulk idiom: one crossing, N rows, each row its own
36
+ // statement (no 32k bind-variable ceiling to chunk around, unlike a single giant statement).
37
+ // Spike-confirmed: batch() joins an already-open exec-level BEGIN rather than erroring, and a
38
+ // literal nested BEGIN hard-errors ("cannot start a transaction within a transaction"), so
39
+ // the shared withTransaction join-not-savepoint helper is what makes this safe to call both
40
+ // standalone and inside reconcile's own transaction.
41
+ async runBatch (sql, paramRows) {
42
+ if (paramRows.length === 0) return;
43
+ await withTransaction(conn, async ()=>{
44
+ await db.batch(paramRows.map((args)=>({
45
+ sql,
46
+ args
47
+ })));
48
+ });
49
+ }
50
+ };
51
+ return conn;
52
+ }
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database'\n// (it lives on @tursodatabase/database-common, a transitive dependency), so its type is derived\n// structurally from Database.prepare()'s return type rather than imported by name.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this\n// wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.\nclass TursoStatementWrapper implements Statement {\n private stmt: TursoStatement;\n\n constructor(stmt: TursoStatement) {\n this.stmt = stmt;\n }\n\n async run(...params: unknown[]): Promise<RunResult> {\n return this.stmt.run(...params);\n }\n\n async get(...params: unknown[]): Promise<unknown> {\n return this.stmt.get(...params);\n }\n\n async all(...params: unknown[]): Promise<unknown[]> {\n return this.stmt.all(...params);\n }\n\n async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n yield* this.stmt.iterate(...params);\n }\n\n columns(): Array<{ name: string }> {\n return this.stmt.columns();\n }\n\n setReadBigInts(enabled: boolean): void {\n this.stmt.safeIntegers(enabled);\n }\n}\n\nexport function createConnection(db: Database): Connection {\n const conn: Connection = {\n async exec(sql: string): Promise<void> {\n await db.exec(sql);\n },\n async prepare(sql: string): Promise<Statement> {\n return new TursoStatementWrapper(await db.prepare(sql));\n },\n // db.batch() is the engine's own bulk idiom: one crossing, N rows, each row its own\n // statement (no 32k bind-variable ceiling to chunk around, unlike a single giant statement).\n // Spike-confirmed: batch() joins an already-open exec-level BEGIN rather than erroring, and a\n // literal nested BEGIN hard-errors (\"cannot start a transaction within a transaction\"), so\n // the shared withTransaction join-not-savepoint helper is what makes this safe to call both\n // standalone and inside reconcile's own transaction.\n async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n if (paramRows.length === 0) return;\n await withTransaction(conn, async () => {\n await db.batch(paramRows.map((args) => ({ sql, args })));\n });\n },\n };\n return conn;\n}\n"],"names":["withTransaction","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","batch","map","args"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAQpD,+FAA+F;AAC/F,gGAAgG;AAChG,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASQ,iBAAiBC,EAAY;IAC3C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpB,MAAMH,GAAGE,IAAI,CAACC;QAChB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAIf,sBAAsB,MAAMY,GAAGI,OAAO,CAACD;QACpD;QACA,oFAAoF;QACpF,6FAA6F;QAC7F,8FAA8F;QAC9F,2FAA2F;QAC3F,4FAA4F;QAC5F,qDAAqD;QACrD,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMpB,gBAAgBc,MAAM;gBAC1B,MAAMD,GAAGQ,KAAK,CAACF,UAAUG,GAAG,CAAC,CAACC,OAAU,CAAA;wBAAEP;wBAAKO;oBAAK,CAAA;YACtD;QACF;IACF;IACA,OAAOT;AACT"}
@@ -0,0 +1,2 @@
1
+ import type { Connection, FieldStat } from '../types.js';
2
+ export declare function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]>;
@@ -0,0 +1,24 @@
1
+ import { quoteIdent } from '../shared.js';
2
+ // Fork of sqlite/fieldStats.ts (house precedent: each store owns its engine-specific SQL, see
3
+ // turso/reconcile.ts). Turso is SQLite dialect (spike-verified: typeof(), GROUP_CONCAT, and
4
+ // FILTER (WHERE ...) all behave identically to node:sqlite for the bigint/number/string/null
5
+ // shapes mapValue() produces), so the SQL text is identical to sqlite's -- only the Connection
6
+ // it runs against differs.
7
+ export async function fieldStats(conn, columns, scopeWhere) {
8
+ const exprs = columns.map((name, i)=>{
9
+ const quoted = quoteIdent(name);
10
+ return `COUNT(${quoted}) AS n${i}, GROUP_CONCAT(DISTINCT typeof(${quoted})) FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;
11
+ });
12
+ const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);
13
+ const row = await stmt.get();
14
+ return columns.map((name, i)=>{
15
+ var _row_, _row_1;
16
+ return {
17
+ field: name,
18
+ coverage: Number((_row_ = row[`n${i}`]) !== null && _row_ !== void 0 ? _row_ : 0),
19
+ type: [
20
+ ...new Set(((_row_1 = row[`t${i}`]) !== null && _row_1 !== void 0 ? _row_1 : '').split(',').filter(Boolean))
21
+ ].sort().join(',')
22
+ };
23
+ });
24
+ }
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// Fork of sqlite/fieldStats.ts (house precedent: each store owns its engine-specific SQL, see\n// turso/reconcile.ts). Turso is SQLite dialect (spike-verified: typeof(), GROUP_CONCAT, and\n// FILTER (WHERE ...) all behave identically to node:sqlite for the bigint/number/string/null\n// shapes mapValue() produces), so the SQL text is identical to sqlite's -- only the Connection\n// it runs against differs.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n const exprs = columns.map((name, i) => {\n const quoted = quoteIdent(name);\n return `COUNT(${quoted}) AS n${i}, GROUP_CONCAT(DISTINCT typeof(${quoted})) FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n });\n const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n const row = (await stmt.get()) as Record<string, number | string | null>;\n return columns.map((name, i) => ({\n field: name,\n coverage: Number(row[`n${i}`] ?? 0),\n type: [...new Set(((row[`t${i}`] as string) ?? '').split(',').filter(Boolean))].sort().join(','),\n }));\n}\n"],"names":["quoteIdent","fieldStats","conn","columns","scopeWhere","exprs","map","name","i","quoted","stmt","prepare","join","row","get","field","coverage","Number","type","Set","split","filter","Boolean","sort"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,8FAA8F;AAC9F,4FAA4F;AAC5F,6FAA6F;AAC7F,+FAA+F;AAC/F,2BAA2B;AAC3B,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAQF,QAAQG,GAAG,CAAC,CAACC,MAAMC;QAC/B,MAAMC,SAAST,WAAWO;QAC1B,OAAO,CAAC,MAAM,EAAEE,OAAO,MAAM,EAAED,EAAE,+BAA+B,EAAEC,OAAO,iBAAiB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;IAC5H;IACA,MAAME,OAAO,MAAMR,KAAKS,OAAO,CAAC,CAAC,OAAO,EAAEN,MAAMO,IAAI,CAAC,MAAM,kBAAkB,EAAER,YAAY;IAC3F,MAAMS,MAAO,MAAMH,KAAKI,GAAG;IAC3B,OAAOX,QAAQG,GAAG,CAAC,CAACC,MAAMC;YAEPK,OACGA;eAHW;YAC/BE,OAAOR;YACPS,UAAUC,QAAOJ,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAAgB;YACjCK,MAAM;mBAAI,IAAIC,IAAI,EAAEN,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAA2B,IAAIO,KAAK,CAAC,KAAKC,MAAM,CAACC;aAAU,CAACC,IAAI,GAAGX,IAAI,CAAC;QAC9F;;AACF"}
@@ -0,0 +1,2 @@
1
+ export declare const TURSO_PACKAGE = "@tursodatabase/database";
2
+ export declare function tursoApi(): Promise<typeof import('@tursodatabase/database')>;
@@ -0,0 +1,19 @@
1
+ import { loadOrInstall, packageNodeModules } from '../native.js';
2
+ export const TURSO_PACKAGE = '@tursodatabase/database';
3
+ const TURSO = {
4
+ store: 'turso',
5
+ pkg: TURSO_PACKAGE,
6
+ sizeHint: '~16MB'
7
+ };
8
+ let tursoApiPromise;
9
+ // The one accessor every consumer of @tursodatabase/database (open.ts) must go through, rather
10
+ // than each doing its own `import('@tursodatabase/database')`: once a fresh install has happened
11
+ // in this process, Node keeps returning that pre-install "not found" miss to any NEW
12
+ // bare-specifier lookup of the same path, so a second independent import elsewhere in the
13
+ // process fails even though the package is now genuinely on disk (the same stale-cache problem
14
+ // loadOrInstall works around for its own retry). Caching the resolved module here means only the
15
+ // first caller ever resolves the bare specifier at all.
16
+ export function tursoApi() {
17
+ if (!tursoApiPromise) tursoApiPromise = loadOrInstall(TURSO, packageNodeModules());
18
+ return tursoApiPromise;
19
+ }
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const TURSO_PACKAGE = '@tursodatabase/database';\n\nconst TURSO: NativeDescriptor = { store: 'turso', pkg: TURSO_PACKAGE, sizeHint: '~16MB' };\n\nlet tursoApiPromise: Promise<typeof import('@tursodatabase/database')> | undefined;\n\n// The one accessor every consumer of @tursodatabase/database (open.ts) must go through, rather\n// than each doing its own `import('@tursodatabase/database')`: once a fresh install has happened\n// in this process, Node keeps returning that pre-install \"not found\" miss to any NEW\n// bare-specifier lookup of the same path, so a second independent import elsewhere in the\n// process fails even though the package is now genuinely on disk (the same stale-cache problem\n// loadOrInstall works around for its own retry). Caching the resolved module here means only the\n// first caller ever resolves the bare specifier at all.\nexport function tursoApi(): Promise<typeof import('@tursodatabase/database')> {\n if (!tursoApiPromise) tursoApiPromise = loadOrInstall<typeof import('@tursodatabase/database')>(TURSO, packageNodeModules());\n return tursoApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","TURSO_PACKAGE","TURSO","store","pkg","sizeHint","tursoApiPromise","tursoApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,gBAAgB,0BAA0B;AAEvD,MAAMC,QAA0B;IAAEC,OAAO;IAASC,KAAKH;IAAeI,UAAU;AAAQ;AAExF,IAAIC;AAEJ,+FAA+F;AAC/F,iGAAiG;AACjG,qFAAqF;AACrF,0FAA0F;AAC1F,+FAA+F;AAC/F,iGAAiG;AACjG,wDAAwD;AACxD,OAAO,SAASC;IACd,IAAI,CAACD,iBAAiBA,kBAAkBP,cAAwDG,OAAOF;IACvG,OAAOM;AACT"}
@@ -0,0 +1,12 @@
1
+ import type { ResolvedConfig } from '../../config/index.js';
2
+ import type { Store } from '../types.js';
3
+ export declare const DB_FILENAME = "cache.turso.db";
4
+ export declare const SCHEMA_VERSION = "1";
5
+ export interface OpenResult {
6
+ store: Store;
7
+ cfg: ResolvedConfig;
8
+ dbPath: string;
9
+ parsed: number;
10
+ warnings: string[];
11
+ }
12
+ export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;
@@ -0,0 +1,113 @@
1
+ import { mkdirSync } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import { featureSignature, STATE_DIR } from '../../config/index.js';
4
+ import { SenseError } from '../../errors.js';
5
+ import { activeFeatures, FEATURES } from '../../features/index.js';
6
+ import { getMeta, setMeta } from '../shared.js';
7
+ import { clearCache } from '../sqlite/open.js';
8
+ import { createConnection } from './connection.js';
9
+ import { TURSO_PACKAGE, tursoApi } from './native.js';
10
+ import { reconcile } from './reconcile.js';
11
+ import { createStore } from './store.js';
12
+ export const DB_FILENAME = 'cache.turso.db';
13
+ // Independent of sqlite's SCHEMA_VERSION and duckdb's -- each store's cache shape evolves
14
+ // separately, and the store name already joins the feature signature (see featureSignature), so
15
+ // a config's `store` key switching cleanly rebuilds rather than reusing another engine's cache.
16
+ export const SCHEMA_VERSION = '1';
17
+ // content is a plain table (not FTS-virtual): phase 1 creates no FTS index at all -- that is
18
+ // phase 2's job (see the plan). Untyped ALTER TABLE ADD COLUMN, like sqlite -- turso's dialect
19
+ // accepts it with no declared type (spike-confirmed), unlike duckdb which requires VARIANT.
20
+ async function ensureSchema(conn, cfg) {
21
+ await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_ctime" REAL, "_size" INTEGER, "_parse_error" TEXT)`);
22
+ await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);
23
+ await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
24
+ await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
25
+ for (const feature of activeFeatures(cfg))await feature.schema(conn);
26
+ if (await getMeta(conn, 'schema_version') === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);
27
+ if (await getMeta(conn, 'features') === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));
28
+ }
29
+ async function connect(cfg) {
30
+ const stateDir = join(cfg.baseDir, STATE_DIR);
31
+ mkdirSync(stateDir, {
32
+ recursive: true
33
+ });
34
+ const dbPath = join(stateDir, DB_FILENAME);
35
+ // Dynamic, not a top-level import: a sqlite or duckdb tree must never even attempt to resolve
36
+ // this optional dependency, so nothing in this store's module graph imports it as a value until
37
+ // a turso tree is actually opened (type-only imports elsewhere are erased and cost nothing
38
+ // either way). Installed on first use if missing (see native.ts's tursoApi).
39
+ let turso;
40
+ try {
41
+ turso = await tursoApi();
42
+ } catch (err) {
43
+ if (err instanceof SenseError) throw err;
44
+ throw new SenseError('STORE_DEPENDENCY_MISSING', `store "turso" needs the ${TURSO_PACKAGE} package (${err.message})`);
45
+ }
46
+ let db;
47
+ try {
48
+ // Floored at the same 30s sqlite opens with. `timeout` is a connect-time option in this
49
+ // client (no synchronous PRAGMA-at-open-time escape hatch), so the real derived value --
50
+ // which depends on this cache's own recorded reconcile_max_ms, readable only once connected
51
+ // -- is installed via a runtime PRAGMA below, right before the open-time reconcile, the same
52
+ // lever sqlite's open() uses (spike-confirmed: raw PRAGMA busy_timeout works as a setter too).
53
+ db = await turso.connect(dbPath, {
54
+ timeout: 30000
55
+ });
56
+ } catch (err) {
57
+ throw new SenseError('STORE_DEPENDENCY_MISSING', `store "turso" failed to open ${dbPath}: ${err.message}`);
58
+ }
59
+ // A throw below (schema, reconcile's COLUMN_LIMIT) must release this handle, or the leaked WAL
60
+ // makes the db file undeletable on Windows. closed marks the rebuild branch that closes and
61
+ // recurses, so the catch never double-closes.
62
+ let closed = false;
63
+ try {
64
+ var _ref;
65
+ const conn = createConnection(db);
66
+ await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
67
+ const version = await getMeta(conn, 'schema_version');
68
+ const features = await getMeta(conn, 'features');
69
+ const wantFeatures = featureSignature(cfg, FEATURES);
70
+ if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
71
+ // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --
72
+ // rebuild instead, same as sqlite's and duckdb's open() (see their comments for why this is
73
+ // announced). No tokenize-only or embed-identity-adopted partial path (sqlite's grace for
74
+ // those): turso has no content.tokenize knob to special-case, and the grace is an
75
+ // optimization, not a correctness requirement -- any mismatch rebuilds, duckdb's shape.
76
+ const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
77
+ console.error(`sense: ${reason}; rebuilding the index`);
78
+ closed = true;
79
+ await db.close();
80
+ clearCache(cfg);
81
+ return connect(cfg);
82
+ }
83
+ await ensureSchema(conn, cfg);
84
+ // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.
85
+ // Installed before reconcile() -- that call is the one that races a watcher's transaction.
86
+ const recordedMaxMs = Number((_ref = await getMeta(conn, 'reconcile_max_ms')) !== null && _ref !== void 0 ? _ref : '0');
87
+ await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600000)}`);
88
+ const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);
89
+ return {
90
+ db,
91
+ conn,
92
+ cfg,
93
+ dbPath,
94
+ parsed,
95
+ warnings
96
+ };
97
+ } catch (err) {
98
+ if (!closed) await db.close();
99
+ throw err;
100
+ }
101
+ }
102
+ // The turso store's open: connects (fully async, see connect() above), then wraps the resulting
103
+ // connection in the Store interface.
104
+ export async function openTurso(cfg) {
105
+ const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);
106
+ return {
107
+ store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir),
108
+ cfg: resolvedCfg,
109
+ dbPath,
110
+ parsed,
111
+ warnings
112
+ };
113
+ }
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { clearCache } from '../sqlite/open.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's SCHEMA_VERSION and duckdb's -- each store's cache shape evolves\n// separately, and the store name already joins the feature signature (see featureSignature), so\n// a config's `store` key switching cleanly rebuilds rather than reusing another engine's cache.\nexport const SCHEMA_VERSION = '1';\n\nexport interface OpenResult {\n store: Store;\n cfg: ResolvedConfig;\n dbPath: string;\n parsed: number;\n warnings: string[];\n}\n\n// content is a plain table (not FTS-virtual): phase 1 creates no FTS index at all -- that is\n// phase 2's job (see the plan). Untyped ALTER TABLE ADD COLUMN, like sqlite -- turso's dialect\n// accepts it with no declared type (spike-confirmed), unlike duckdb which requires VARIANT.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n db: Database;\n conn: Connection;\n cfg: ResolvedConfig;\n dbPath: string;\n parsed: number;\n warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n const stateDir = join(cfg.baseDir, STATE_DIR);\n mkdirSync(stateDir, { recursive: true });\n const dbPath = join(stateDir, DB_FILENAME);\n\n // Dynamic, not a top-level import: a sqlite or duckdb tree must never even attempt to resolve\n // this optional dependency, so nothing in this store's module graph imports it as a value until\n // a turso tree is actually opened (type-only imports elsewhere are erased and cost nothing\n // either way). Installed on first use if missing (see native.ts's tursoApi).\n let turso: Awaited<ReturnType<typeof tursoApi>>;\n try {\n turso = await tursoApi();\n } catch (err) {\n if (err instanceof SenseError) throw err;\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n }\n\n let db: Database;\n try {\n // Floored at the same 30s sqlite opens with. `timeout` is a connect-time option in this\n // client (no synchronous PRAGMA-at-open-time escape hatch), so the real derived value --\n // which depends on this cache's own recorded reconcile_max_ms, readable only once connected\n // -- is installed via a runtime PRAGMA below, right before the open-time reconcile, the same\n // lever sqlite's open() uses (spike-confirmed: raw PRAGMA busy_timeout works as a setter too).\n db = await turso.connect(dbPath, { timeout: 30_000 });\n } catch (err) {\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n }\n // A throw below (schema, reconcile's COLUMN_LIMIT) must release this handle, or the leaked WAL\n // makes the db file undeletable on Windows. closed marks the rebuild branch that closes and\n // recurses, so the catch never double-closes.\n let closed = false;\n try {\n const conn = createConnection(db);\n\n await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n const version = await getMeta(conn, 'schema_version');\n const features = await getMeta(conn, 'features');\n const wantFeatures = featureSignature(cfg, FEATURES);\n if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n // rebuild instead, same as sqlite's and duckdb's open() (see their comments for why this is\n // announced). No tokenize-only or embed-identity-adopted partial path (sqlite's grace for\n // those): turso has no content.tokenize knob to special-case, and the grace is an\n // optimization, not a correctness requirement -- any mismatch rebuilds, duckdb's shape.\n const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n console.error(`sense: ${reason}; rebuilding the index`);\n closed = true;\n await db.close();\n clearCache(cfg);\n return connect(cfg);\n }\n\n await ensureSchema(conn, cfg);\n\n // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n return { db, conn, cfg, dbPath, parsed, warnings };\n } catch (err) {\n if (!closed) await db.close();\n throw err;\n }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n return { store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir), cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","SenseError","activeFeatures","FEATURES","getMeta","setMeta","clearCache","createConnection","TURSO_PACKAGE","tursoApi","reconcile","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","conn","cfg","exec","feature","schema","connect","stateDir","baseDir","recursive","dbPath","turso","err","message","db","timeout","closed","version","features","wantFeatures","reason","console","error","close","recordedMaxMs","Number","Math","min","max","parsed","warnings","openTurso","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,0FAA0F;AAC1F,gGAAgG;AAChG,gGAAgG;AAChG,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,6FAA6F;AAC7F,+FAA+F;AAC/F,4FAA4F;AAC5F,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWhB,eAAec,KAAM,MAAME,QAAQC,MAAM,CAACJ;IAChE,IAAI,AAAC,MAAMX,QAAQW,MAAM,sBAAuB,MAAM,MAAMV,QAAQU,MAAM,kBAAkBF;IAC5F,IAAI,AAAC,MAAMT,QAAQW,MAAM,gBAAiB,MAAM,MAAMV,QAAQU,MAAM,YAAYhB,iBAAiBiB,KAAKb;AACxG;AAWA,eAAeiB,QAAQJ,GAAmB;IACxC,MAAMK,WAAWvB,KAAKkB,IAAIM,OAAO,EAAEtB;IACnCH,UAAUwB,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS1B,KAAKuB,UAAUT;IAE9B,8FAA8F;IAC9F,gGAAgG;IAChG,2FAA2F;IAC3F,6EAA6E;IAC7E,IAAIa;IACJ,IAAI;QACFA,QAAQ,MAAMhB;IAChB,EAAE,OAAOiB,KAAK;QACZ,IAAIA,eAAezB,YAAY,MAAMyB;QACrC,MAAM,IAAIzB,WAAW,4BAA4B,CAAC,wBAAwB,EAAEO,cAAc,UAAU,EAAE,AAACkB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,wFAAwF;QACxF,yFAAyF;QACzF,4FAA4F;QAC5F,6FAA6F;QAC7F,+FAA+F;QAC/FA,KAAK,MAAMH,MAAML,OAAO,CAACI,QAAQ;YAAEK,SAAS;QAAO;IACrD,EAAE,OAAOH,KAAK;QACZ,MAAM,IAAIzB,WAAW,4BAA4B,CAAC,6BAA6B,EAAEuB,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACtH;IACA,+FAA+F;IAC/F,4FAA4F;IAC5F,8CAA8C;IAC9C,IAAIG,SAAS;IACb,IAAI;YA0B4B;QAzB9B,MAAMf,OAAOR,iBAAiBqB;QAE9B,MAAMb,KAAKE,IAAI,CAAC;QAEhB,MAAMc,UAAU,MAAM3B,QAAQW,MAAM;QACpC,MAAMiB,WAAW,MAAM5B,QAAQW,MAAM;QACrC,MAAMkB,eAAelC,iBAAiBiB,KAAKb;QAC3C,IAAI,AAAC4B,YAAY,QAAQA,YAAYlB,kBAAoBmB,aAAa,QAAQA,aAAaC,cAAe;YACxG,4FAA4F;YAC5F,4FAA4F;YAC5F,0FAA0F;YAC1F,kFAAkF;YAClF,wFAAwF;YACxF,MAAMC,SAASH,YAAY,QAAQA,YAAYlB,iBAAiB,mDAAmD;YACnHsB,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtDJ,SAAS;YACT,MAAMF,GAAGS,KAAK;YACd/B,WAAWU;YACX,OAAOI,QAAQJ;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMsB,gBAAgBC,QAAQ,OAAA,MAAMnC,QAAQW,MAAM,iCAApB,kBAAA,OAA4C;QAC1E,MAAMA,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAEuB,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAEhG,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMlC,UAAUK,MAAMC,KAAKA,IAAIM,OAAO;QAEnE,OAAO;YAAEM;YAAIb;YAAMC;YAAKQ;YAAQmB;YAAQC;QAAS;IACnD,EAAE,OAAOlB,KAAK;QACZ,IAAI,CAACI,QAAQ,MAAMF,GAAGS,KAAK;QAC3B,MAAMX;IACR;AACF;AAEA,gGAAgG;AAChG,qCAAqC;AACrC,OAAO,eAAemB,UAAU7B,GAAmB;IACjD,MAAM,EAAEY,EAAE,EAAEb,IAAI,EAAEC,KAAK8B,WAAW,EAAEtB,MAAM,EAAEmB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMxB,QAAQJ;IAC/E,OAAO;QAAE+B,OAAOpC,YAAYiB,IAAIb,MAAM+B,aAAaA,YAAYxB,OAAO;QAAGN,KAAK8B;QAAatB;QAAQmB;QAAQC;IAAS;AACtH"}
@@ -0,0 +1,6 @@
1
+ import type { Config } from '../../config/index.js';
2
+ import type { Connection } from '../types.js';
3
+ export declare function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{
4
+ parsed: number;
5
+ warnings: string[];
6
+ }>;
@@ -0,0 +1,158 @@
1
+ import { SenseError } from '../../errors.js';
2
+ import { activeFeatures } from '../../features/index.js';
3
+ import { progress } from '../../output/progress.js';
4
+ import { listFiles, RESERVED_COLUMNS } from '../../scan/index.js';
5
+ import { reparseFiles } from '../../scan/reparse.js';
6
+ import { getColumns, getMeta, quoteIdent, setMeta } from '../shared.js';
7
+ import { withTransaction } from '../transaction.js';
8
+ // Fork of sqlite/reconcile.ts (house precedent: each store owns its engine-specific SQL), not
9
+ // duckdb's, even though duckdb's content handling is the structurally closer of the two. duckdb
10
+ // dropped sqlite's reconcile_max_ms bookkeeping entirely, and this store's busy timeout (open.ts)
11
+ // is derived from exactly that recorded value -- forking duckdb's file would lose it silently.
12
+ // content itself follows duckdb's shape instead: a plain table, no rowid coupling, no FTS index
13
+ // (phase 2's job, not reconcile's), and no `_seg` sidecars (turso's unspaced-script path is not
14
+ // FTS5-shaped).
15
+ const CORE_FRONTMATTER_COLUMNS = new Set([
16
+ 'path',
17
+ '_mtime',
18
+ '_ctime',
19
+ '_size',
20
+ '_parse_error'
21
+ ]);
22
+ // Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with no
23
+ // error). The real fence is a SELECT projecting more than this many result columns, which fails
24
+ // to prepare (spike-measured, both `SELECT *` and an explicit list), so a wide frontmatter table
25
+ // would pass every write silently and only break the first time something selects across it.
26
+ // Enforced here, before ALTER, the same shape as sqlite's compile-time fence.
27
+ const MAX_FRONTMATTER_COLUMNS = 2000;
28
+ // No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.
29
+ const INSERT_CONTENT_SQL = `INSERT INTO content ("path", title, summary, text) VALUES (?, ?, ?, ?)`;
30
+ function contentRow(doc) {
31
+ return [
32
+ doc.relPath,
33
+ doc.search.title,
34
+ doc.search.summary,
35
+ doc.search.text
36
+ ];
37
+ }
38
+ export async function reconcile(conn, cfg, baseDir) {
39
+ const files = listFiles(cfg, baseDir);
40
+ const currentSet = new Set(files.map((f)=>f.relPath));
41
+ const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
42
+ const existingRows = await existingStmt.all();
43
+ const existing = new Map(existingRows.map((r)=>[
44
+ r.path,
45
+ r
46
+ ]));
47
+ const vanished = existingRows.filter((r)=>!currentSet.has(r.path)).map((r)=>r.path);
48
+ const toReparse = files.filter((f)=>{
49
+ const row = existing.get(f.relPath);
50
+ return !row || row._mtime !== f.mtimeMs || row._size !== f.size;
51
+ });
52
+ if (vanished.length === 0 && toReparse.length === 0) return {
53
+ parsed: 0,
54
+ warnings: []
55
+ };
56
+ const features = activeFeatures(cfg);
57
+ const seenColumns = await getColumns(conn);
58
+ // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
59
+ // reconciles stay silent (progress() has a threshold).
60
+ const report = progress('reparsing files', toReparse.length);
61
+ const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
62
+ report.finish();
63
+ for (const col of newColumns)seenColumns.add(col);
64
+ const allColumns = [
65
+ ...seenColumns
66
+ ];
67
+ if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {
68
+ throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);
69
+ }
70
+ // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a
71
+ // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).
72
+ const writableColumns = allColumns.filter((c)=>CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));
73
+ // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's identity stable across reparses.
74
+ const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(()=>'?').join(', ')}) ON CONFLICT("path") DO UPDATE SET ${writableColumns.filter((c)=>c !== 'path').map((c)=>`${quoteIdent(c)} = excluded.${quoteIdent(c)}`).join(', ')}`;
75
+ const added = toReparse.filter((f)=>!existing.has(f.relPath)).map((f)=>f.relPath);
76
+ const delta = {
77
+ files,
78
+ reparsed: parsedDocs.map((d)=>d.relPath),
79
+ added,
80
+ vanished
81
+ };
82
+ const addedSet = new Set(added);
83
+ const reparsedExisting = parsedDocs.map((d)=>d.relPath).filter((p)=>!addedSet.has(p));
84
+ const txStart = Date.now();
85
+ await withTransaction(conn, async ()=>{
86
+ var _feature_remove, _feature_afterReconcile;
87
+ for (const col of newColumns)await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);
88
+ if (parsedDocs.length > 0) {
89
+ const rows = parsedDocs.map((doc)=>writableColumns.map((col)=>{
90
+ var _doc_data_col;
91
+ if (col === 'path') return doc.relPath;
92
+ if (col === '_mtime') return doc.mtimeMs;
93
+ if (col === '_ctime') return doc.ctimeMs;
94
+ if (col === '_size') return doc.size;
95
+ // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.
96
+ if (col === '_parse_error') return doc.parseError;
97
+ return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
98
+ }));
99
+ await conn.runBatch(insertSql, rows);
100
+ }
101
+ // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5
102
+ // content), so vanished and reparsed docs delete in one pass.
103
+ const contentTouched = [
104
+ ...vanished,
105
+ ...reparsedExisting
106
+ ];
107
+ if (contentTouched.length > 0) await conn.runBatch('DELETE FROM content WHERE "path" = ?', contentTouched.map((p)=>[
108
+ p
109
+ ]));
110
+ if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));
111
+ if (vanished.length > 0) await conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[
112
+ p
113
+ ]));
114
+ // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;
115
+ // new docs have nothing to clear, which keeps cold builds linear.
116
+ const presetTouched = [
117
+ ...vanished,
118
+ ...reparsedExisting
119
+ ];
120
+ if (presetTouched.length > 0) await conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', presetTouched.map((p)=>[
121
+ p
122
+ ]));
123
+ const presetRows = [];
124
+ for (const doc of parsedDocs)for (const presetName of doc.presets)presetRows.push([
125
+ doc.relPath,
126
+ presetName
127
+ ]);
128
+ if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?)', presetRows);
129
+ const removedPaths = [
130
+ ...vanished,
131
+ ...reparsedExisting
132
+ ];
133
+ if (removedPaths.length > 0) for (const feature of features)await ((_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, removedPaths, delta));
134
+ for (const feature of features){
135
+ var _feature_store;
136
+ const docsForFeature = parsedDocs.map((doc)=>({
137
+ path: doc.relPath,
138
+ extracted: doc.extracted[feature.name]
139
+ }));
140
+ await ((_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta));
141
+ }
142
+ for (const feature of features)await ((_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta));
143
+ });
144
+ // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the
145
+ // observed max so a big watcher reconcile's lock hold is what the next open bounds its wait
146
+ // against (the same mechanism sqlite's reconcile records, and the one this store's busy
147
+ // timeout is derived from).
148
+ const durationMs = Date.now() - txStart;
149
+ const prevRaw = await getMeta(conn, 'reconcile_max_ms');
150
+ // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)
151
+ // still gets recorded instead of losing to the "nothing recorded yet" default.
152
+ const prevMax = prevRaw === null ? -1 : Number(prevRaw);
153
+ if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));
154
+ return {
155
+ parsed: parsedDocs.length,
156
+ warnings
157
+ };
158
+ }
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Fork of sqlite/reconcile.ts (house precedent: each store owns its engine-specific SQL), not\n// duckdb's, even though duckdb's content handling is the structurally closer of the two. duckdb\n// dropped sqlite's reconcile_max_ms bookkeeping entirely, and this store's busy timeout (open.ts)\n// is derived from exactly that recorded value -- forking duckdb's file would lose it silently.\n// content itself follows duckdb's shape instead: a plain table, no rowid coupling, no FTS index\n// (phase 2's job, not reconcile's), and no `_seg` sidecars (turso's unspaced-script path is not\n// FTS5-shaped).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with no\n// error). The real fence is a SELECT projecting more than this many result columns, which fails\n// to prepare (spike-measured, both `SELECT *` and an explicit list), so a wide frontmatter table\n// would pass every write silently and only break the first time something selects across it.\n// Enforced here, before ALTER, the same shape as sqlite's compile-time fence.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n const files = listFiles(cfg, baseDir);\n const currentSet = new Set(files.map((f) => f.relPath));\n\n const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n const existing = new Map(existingRows.map((r) => [r.path, r]));\n const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n const toReparse = files.filter((f) => {\n const row = existing.get(f.relPath);\n return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n });\n\n if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n const features = activeFeatures(cfg);\n const seenColumns = await getColumns(conn);\n\n // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n // reconciles stay silent (progress() has a threshold).\n const report = progress('reparsing files', toReparse.length);\n const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n report.finish();\n for (const col of newColumns) seenColumns.add(col);\n\n const allColumns = [...seenColumns];\n if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n throw new SenseError(\n 'COLUMN_LIMIT',\n `frontmatter would need ${allColumns.length} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n );\n }\n // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's identity stable across reparses.\n const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n .filter((c) => c !== 'path')\n .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n .join(', ')}`;\n\n const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n const addedSet = new Set(added);\n const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n const txStart = Date.now();\n await withTransaction(conn, async () => {\n for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n if (parsedDocs.length > 0) {\n const rows = parsedDocs.map((doc) =>\n writableColumns.map((col) => {\n if (col === 'path') return doc.relPath;\n if (col === '_mtime') return doc.mtimeMs;\n if (col === '_ctime') return doc.ctimeMs;\n if (col === '_size') return doc.size;\n // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n if (col === '_parse_error') return doc.parseError;\n return doc.data[col] ?? null;\n })\n );\n await conn.runBatch(insertSql, rows);\n }\n\n // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n // content), so vanished and reparsed docs delete in one pass.\n const contentTouched = [...vanished, ...reparsedExisting];\n if (contentTouched.length > 0)\n await conn.runBatch(\n 'DELETE FROM content WHERE \"path\" = ?',\n contentTouched.map((p) => [p])\n );\n if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n if (vanished.length > 0)\n await conn.runBatch(\n 'DELETE FROM frontmatter WHERE \"path\" = ?',\n vanished.map((p) => [p])\n );\n\n // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n // new docs have nothing to clear, which keeps cold builds linear.\n const presetTouched = [...vanished, ...reparsedExisting];\n if (presetTouched.length > 0)\n await conn.runBatch(\n 'DELETE FROM preset_files WHERE \"path\" = ?',\n presetTouched.map((p) => [p])\n );\n const presetRows: unknown[][] = [];\n for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n const removedPaths = [...vanished, ...reparsedExisting];\n if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n for (const feature of features) {\n const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n await feature.store?.(conn, docsForFeature, delta);\n }\n for (const feature of features) await feature.afterReconcile?.(conn, delta);\n });\n\n // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n // observed max so a big watcher reconcile's lock hold is what the next open bounds its wait\n // against (the same mechanism sqlite's reconcile records, and the one this store's busy\n // timeout is derived from).\n const durationMs = Date.now() - txStart;\n const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["SenseError","activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","getMeta","quoteIdent","setMeta","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","txStart","Date","now","feature","exec","rows","ctimeMs","parseError","data","runBatch","contentTouched","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile","durationMs","prevRaw","prevMax","Number","String"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAClE,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,eAAe;AACxE,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,8FAA8F;AAC9F,gGAAgG;AAChG,kGAAkG;AAClG,+FAA+F;AAC/F,gGAAgG;AAChG,gGAAgG;AAChG,gBAAgB;AAChB,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,iGAAiG;AACjG,gGAAgG;AAChG,iGAAiG;AACjG,6FAA6F;AAC7F,8EAA8E;AAC9E,MAAMC,0BAA0B;AAEhC,yFAAyF;AACzF,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQvB,UAAUqB,KAAKC;IAC7B,MAAME,aAAa,IAAIf,IAAIc,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEZ,OAAO;QAClC,OAAO,CAACyB,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWlD,eAAeuB;IAChC,MAAM4B,cAAc,MAAM9C,WAAWiB;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8B,SAASnD,SAAS,mBAAmBuC,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMnD,aAAaoC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,IAAIS,WAAWb,MAAM,GAAGnC,yBAAyB;QAC/C,MAAM,IAAIb,WACR,gBACA,CAAC,uBAAuB,EAAE6D,WAAWb,MAAM,CAAC,2DAA2D,EAAEnC,wBAAwB,gPAAgP,CAAC;IAEtX;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMiD,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMpD,yBAAyB6B,GAAG,CAACuB,MAAM,CAAC3D,iBAAiBoC,GAAG,CAACuB;IAC1G,uFAAuF;IACvF,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACpB,YAAYyD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGvD,WAAWuD,GAAG,YAAY,EAAEvD,WAAWuD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAMkD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO;QAAGiD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI1D,IAAIsD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO,EAAEsB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMjE,gBAAgBa,MAAM;YA+CyCqD,iBAK7BA;QAnDtC,KAAK,MAAMjB,OAAOH,WAAY,MAAMjC,KAAKsD,IAAI,CAAC,CAAC,mCAAmC,EAAErE,WAAWmD,MAAM;QAErG,IAAIJ,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM8B,OAAOvB,WAAW3B,GAAG,CAAC,CAACZ,MAC3B8C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAOZ3C;oBANP,IAAI2C,QAAQ,QAAQ,OAAO3C,IAAIC,OAAO;oBACtC,IAAI0C,QAAQ,UAAU,OAAO3C,IAAI6B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO3C,IAAI+D,OAAO;oBACxC,IAAIpB,QAAQ,SAAS,OAAO3C,IAAI+B,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO3C,IAAIgE,UAAU;oBACjD,QAAOhE,gBAAAA,IAAIiE,IAAI,CAACtB,IAAI,cAAb3C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMO,KAAK2D,QAAQ,CAAClB,WAAWc;QACjC;QAEA,0FAA0F;QAC1F,8DAA8D;QAC9D,MAAMK,iBAAiB;eAAI7C;eAAaiC;SAAiB;QACzD,IAAIY,eAAenC,MAAM,GAAG,GAC1B,MAAMzB,KAAK2D,QAAQ,CACjB,wCACAC,eAAevD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEjC,IAAIjB,WAAWP,MAAM,GAAG,GAAG,MAAMzB,KAAK2D,QAAQ,CAACpE,oBAAoByC,WAAW3B,GAAG,CAACb;QAElF,IAAIuB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAK2D,QAAQ,CACjB,4CACA5C,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMY,gBAAgB;eAAI9C;eAAaiC;SAAiB;QACxD,IAAIa,cAAcpC,MAAM,GAAG,GACzB,MAAMzB,KAAK2D,QAAQ,CACjB,6CACAE,cAAcxD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMa,aAA0B,EAAE;QAClC,KAAK,MAAMrE,OAAOuC,WAAY,KAAK,MAAM+B,cAActE,IAAIuE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACxE,IAAIC,OAAO;YAAEqE;SAAW;QAC7G,IAAID,WAAWrC,MAAM,GAAG,GAAG,MAAMzB,KAAK2D,QAAQ,CAAC,2DAA2DG;QAE1G,MAAMI,eAAe;eAAInD;eAAaiC;SAAiB;QACvD,IAAIkB,aAAazC,MAAM,GAAG,GAAG,KAAK,MAAM4B,WAAWzB,SAAU,QAAMyB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBrD,MAAMkE,cAActB;QACxG,KAAK,MAAMS,WAAWzB,SAAU;gBAExByB;YADN,MAAMe,iBAAiCpC,WAAW3B,GAAG,CAAC,CAACZ,MAAS,CAAA;oBAAEqB,MAAMrB,IAAIC,OAAO;oBAAE2E,WAAW5E,IAAI4E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBrD,MAAMoE,gBAAgBxB;QAC9C;QACA,KAAK,MAAMS,WAAWzB,SAAU,QAAMyB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBrD,MAAM4C;IACvE;IAEA,0FAA0F;IAC1F,4FAA4F;IAC5F,wFAAwF;IACxF,4BAA4B;IAC5B,MAAM6B,aAAatB,KAAKC,GAAG,KAAKF;IAChC,MAAMwB,UAAU,MAAM1F,QAAQgB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM2E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS,MAAMzF,QAAQc,MAAM,oBAAoB6E,OAAOJ;IAEzE,OAAO;QAAE/C,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}
@@ -0,0 +1,5 @@
1
+ import type { Database } from '@tursodatabase/database';
2
+ import type { Config } from '../../config/index.js';
3
+ import type { Capability, Connection, Store } from '../types.js';
4
+ export declare const CAPABILITIES: ReadonlySet<Capability>;
5
+ export declare function createStore(db: Database, conn: Connection, cfg: Config, baseDir: string): Store;