sensemaking 0.19.2 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/README.md +0 -1
  2. package/dist/cjs/chunk/extract.d.cts +1 -0
  3. package/dist/cjs/chunk/extract.d.ts +1 -0
  4. package/dist/cjs/chunk/extract.js +23 -5
  5. package/dist/cjs/chunk/extract.js.map +1 -1
  6. package/dist/cjs/chunk/index.d.cts +1 -1
  7. package/dist/cjs/chunk/index.d.ts +1 -1
  8. package/dist/cjs/chunk/index.js +3 -0
  9. package/dist/cjs/chunk/index.js.map +1 -1
  10. package/dist/cjs/config/access.d.cts +0 -1
  11. package/dist/cjs/config/access.d.ts +0 -1
  12. package/dist/cjs/config/access.js +10 -17
  13. package/dist/cjs/config/access.js.map +1 -1
  14. package/dist/cjs/config/index.d.cts +1 -1
  15. package/dist/cjs/config/index.d.ts +1 -1
  16. package/dist/cjs/config/index.js +0 -3
  17. package/dist/cjs/config/index.js.map +1 -1
  18. package/dist/cjs/config/types.d.cts +0 -3
  19. package/dist/cjs/config/types.d.ts +0 -3
  20. package/dist/cjs/config/types.js.map +1 -1
  21. package/dist/cjs/config/validate.js +6 -24
  22. package/dist/cjs/config/validate.js.map +1 -1
  23. package/dist/cjs/errors.d.cts +1 -1
  24. package/dist/cjs/errors.d.ts +1 -1
  25. package/dist/cjs/errors.js.map +1 -1
  26. package/dist/cjs/features/links.js +4 -5
  27. package/dist/cjs/features/links.js.map +1 -1
  28. package/dist/cjs/features/sections.js +4 -1
  29. package/dist/cjs/features/sections.js.map +1 -1
  30. package/dist/cjs/scan/pool.d.cts +15 -0
  31. package/dist/cjs/scan/pool.d.ts +15 -0
  32. package/dist/cjs/scan/pool.js +263 -0
  33. package/dist/cjs/scan/pool.js.map +1 -0
  34. package/dist/cjs/scan/reparse.d.cts +2 -0
  35. package/dist/cjs/scan/reparse.d.ts +2 -0
  36. package/dist/cjs/scan/reparse.js +15 -76
  37. package/dist/cjs/scan/reparse.js.map +1 -1
  38. package/dist/cjs/store/builder.d.cts +19 -0
  39. package/dist/cjs/store/builder.d.ts +19 -0
  40. package/dist/cjs/store/builder.js +618 -0
  41. package/dist/cjs/store/builder.js.map +1 -0
  42. package/dist/cjs/store/duckdb/lexical.d.cts +1 -0
  43. package/dist/cjs/store/duckdb/lexical.d.ts +1 -0
  44. package/dist/cjs/store/duckdb/lexical.js +21 -5
  45. package/dist/cjs/store/duckdb/lexical.js.map +1 -1
  46. package/dist/cjs/store/duckdb/open.d.cts +9 -8
  47. package/dist/cjs/store/duckdb/open.d.ts +9 -8
  48. package/dist/cjs/store/duckdb/open.js +77 -123
  49. package/dist/cjs/store/duckdb/open.js.map +1 -1
  50. package/dist/cjs/store/duckdb/reconcile.js +34 -5
  51. package/dist/cjs/store/duckdb/reconcile.js.map +1 -1
  52. package/dist/cjs/store/duckdb/store.d.cts +1 -2
  53. package/dist/cjs/store/duckdb/store.d.ts +1 -2
  54. package/dist/cjs/store/duckdb/store.js +1 -26
  55. package/dist/cjs/store/duckdb/store.js.map +1 -1
  56. package/dist/cjs/store/embed-scope.d.cts +2 -0
  57. package/dist/cjs/store/embed-scope.d.ts +2 -0
  58. package/dist/cjs/store/embed-scope.js +85 -0
  59. package/dist/cjs/store/embed-scope.js.map +1 -0
  60. package/dist/cjs/store/feature-scope.d.cts +8 -0
  61. package/dist/cjs/store/feature-scope.d.ts +8 -0
  62. package/dist/cjs/store/feature-scope.js +107 -0
  63. package/dist/cjs/store/feature-scope.js.map +1 -0
  64. package/dist/cjs/store/index.d.cts +8 -3
  65. package/dist/cjs/store/index.d.ts +8 -3
  66. package/dist/cjs/store/index.js +36 -7
  67. package/dist/cjs/store/index.js.map +1 -1
  68. package/dist/cjs/store/open.d.cts +15 -0
  69. package/dist/cjs/store/open.d.ts +15 -0
  70. package/dist/cjs/store/open.js +608 -0
  71. package/dist/cjs/store/open.js.map +1 -0
  72. package/dist/cjs/store/preset-scope.d.cts +3 -0
  73. package/dist/cjs/store/preset-scope.d.ts +3 -0
  74. package/dist/cjs/store/preset-scope.js +293 -0
  75. package/dist/cjs/store/preset-scope.js.map +1 -0
  76. package/dist/cjs/store/reconcile.d.cts +2 -1
  77. package/dist/cjs/store/reconcile.d.ts +2 -1
  78. package/dist/cjs/store/reconcile.js +175 -179
  79. package/dist/cjs/store/reconcile.js.map +1 -1
  80. package/dist/cjs/store/{sqlite/signature.d.cts → signature.d.cts} +0 -3
  81. package/dist/cjs/store/{sqlite/signature.d.ts → signature.d.ts} +0 -3
  82. package/dist/cjs/store/signature.js +164 -0
  83. package/dist/cjs/store/signature.js.map +1 -0
  84. package/dist/cjs/store/sqlite/lexical.d.cts +1 -1
  85. package/dist/cjs/store/sqlite/lexical.d.ts +1 -1
  86. package/dist/cjs/store/sqlite/lexical.js +2 -2
  87. package/dist/cjs/store/sqlite/lexical.js.map +1 -1
  88. package/dist/cjs/store/sqlite/open.d.cts +8 -9
  89. package/dist/cjs/store/sqlite/open.d.ts +8 -9
  90. package/dist/cjs/store/sqlite/open.js +82 -311
  91. package/dist/cjs/store/sqlite/open.js.map +1 -1
  92. package/dist/cjs/store/sqlite/reconcile.d.cts +1 -1
  93. package/dist/cjs/store/sqlite/reconcile.d.ts +1 -1
  94. package/dist/cjs/store/sqlite/reconcile.js +78 -16
  95. package/dist/cjs/store/sqlite/reconcile.js.map +1 -1
  96. package/dist/cjs/store/sqlite/sql-functions.d.cts +1 -1
  97. package/dist/cjs/store/sqlite/sql-functions.d.ts +1 -1
  98. package/dist/cjs/store/sqlite/sql-functions.js +3 -3
  99. package/dist/cjs/store/sqlite/sql-functions.js.map +1 -1
  100. package/dist/cjs/store/sqlite/store.d.cts +1 -2
  101. package/dist/cjs/store/sqlite/store.d.ts +1 -2
  102. package/dist/cjs/store/sqlite/store.js +2 -15
  103. package/dist/cjs/store/sqlite/store.js.map +1 -1
  104. package/dist/cjs/store/turso/open.d.cts +6 -11
  105. package/dist/cjs/store/turso/open.d.ts +6 -11
  106. package/dist/cjs/store/turso/open.js +85 -171
  107. package/dist/cjs/store/turso/open.js.map +1 -1
  108. package/dist/cjs/store/turso/reconcile.d.cts +2 -0
  109. package/dist/cjs/store/turso/reconcile.d.ts +2 -0
  110. package/dist/cjs/store/turso/reconcile.js +88 -8
  111. package/dist/cjs/store/turso/reconcile.js.map +1 -1
  112. package/dist/cjs/store/turso/store.d.cts +1 -2
  113. package/dist/cjs/store/turso/store.d.ts +1 -2
  114. package/dist/cjs/store/turso/store.js +1 -13
  115. package/dist/cjs/store/turso/store.js.map +1 -1
  116. package/dist/cjs/store/types.d.cts +16 -6
  117. package/dist/cjs/store/types.d.ts +16 -6
  118. package/dist/cjs/store/types.js.map +1 -1
  119. package/dist/cjs/text/strip.js +3 -6
  120. package/dist/cjs/text/strip.js.map +1 -1
  121. package/dist/cjs/watch.js +28 -16
  122. package/dist/cjs/watch.js.map +1 -1
  123. package/dist/esm/chunk/extract.d.ts +1 -0
  124. package/dist/esm/chunk/extract.js +10 -2
  125. package/dist/esm/chunk/extract.js.map +1 -1
  126. package/dist/esm/chunk/index.d.ts +1 -1
  127. package/dist/esm/chunk/index.js +1 -1
  128. package/dist/esm/chunk/index.js.map +1 -1
  129. package/dist/esm/config/access.d.ts +0 -1
  130. package/dist/esm/config/access.js +12 -17
  131. package/dist/esm/config/access.js.map +1 -1
  132. package/dist/esm/config/index.d.ts +1 -1
  133. package/dist/esm/config/index.js +1 -1
  134. package/dist/esm/config/index.js.map +1 -1
  135. package/dist/esm/config/types.d.ts +0 -3
  136. package/dist/esm/config/types.js.map +1 -1
  137. package/dist/esm/config/validate.js +6 -22
  138. package/dist/esm/config/validate.js.map +1 -1
  139. package/dist/esm/errors.d.ts +1 -1
  140. package/dist/esm/errors.js.map +1 -1
  141. package/dist/esm/features/links.js +3 -4
  142. package/dist/esm/features/links.js.map +1 -1
  143. package/dist/esm/features/sections.js +4 -1
  144. package/dist/esm/features/sections.js.map +1 -1
  145. package/dist/esm/scan/pool.d.ts +15 -0
  146. package/dist/esm/scan/pool.js +77 -0
  147. package/dist/esm/scan/pool.js.map +1 -0
  148. package/dist/esm/scan/reparse.d.ts +2 -0
  149. package/dist/esm/scan/reparse.js +11 -58
  150. package/dist/esm/scan/reparse.js.map +1 -1
  151. package/dist/esm/store/builder.d.ts +19 -0
  152. package/dist/esm/store/builder.js +163 -0
  153. package/dist/esm/store/builder.js.map +1 -0
  154. package/dist/esm/store/duckdb/lexical.d.ts +1 -0
  155. package/dist/esm/store/duckdb/lexical.js +22 -9
  156. package/dist/esm/store/duckdb/lexical.js.map +1 -1
  157. package/dist/esm/store/duckdb/open.d.ts +9 -8
  158. package/dist/esm/store/duckdb/open.js +36 -58
  159. package/dist/esm/store/duckdb/open.js.map +1 -1
  160. package/dist/esm/store/duckdb/reconcile.js +15 -3
  161. package/dist/esm/store/duckdb/reconcile.js.map +1 -1
  162. package/dist/esm/store/duckdb/store.d.ts +1 -2
  163. package/dist/esm/store/duckdb/store.js +2 -11
  164. package/dist/esm/store/duckdb/store.js.map +1 -1
  165. package/dist/esm/store/embed-scope.d.ts +2 -0
  166. package/dist/esm/store/embed-scope.js +28 -0
  167. package/dist/esm/store/embed-scope.js.map +1 -0
  168. package/dist/esm/store/feature-scope.d.ts +8 -0
  169. package/dist/esm/store/feature-scope.js +49 -0
  170. package/dist/esm/store/feature-scope.js.map +1 -0
  171. package/dist/esm/store/index.d.ts +8 -3
  172. package/dist/esm/store/index.js +22 -7
  173. package/dist/esm/store/index.js.map +1 -1
  174. package/dist/esm/store/open.d.ts +15 -0
  175. package/dist/esm/store/open.js +187 -0
  176. package/dist/esm/store/open.js.map +1 -0
  177. package/dist/esm/store/preset-scope.d.ts +3 -0
  178. package/dist/esm/store/preset-scope.js +77 -0
  179. package/dist/esm/store/preset-scope.js.map +1 -0
  180. package/dist/esm/store/reconcile.d.ts +2 -1
  181. package/dist/esm/store/reconcile.js +33 -6
  182. package/dist/esm/store/reconcile.js.map +1 -1
  183. package/dist/esm/store/{sqlite/signature.d.ts → signature.d.ts} +0 -3
  184. package/dist/esm/store/signature.js +37 -0
  185. package/dist/esm/store/signature.js.map +1 -0
  186. package/dist/esm/store/sqlite/lexical.d.ts +1 -1
  187. package/dist/esm/store/sqlite/lexical.js +4 -4
  188. package/dist/esm/store/sqlite/lexical.js.map +1 -1
  189. package/dist/esm/store/sqlite/open.d.ts +8 -9
  190. package/dist/esm/store/sqlite/open.js +41 -147
  191. package/dist/esm/store/sqlite/open.js.map +1 -1
  192. package/dist/esm/store/sqlite/reconcile.d.ts +1 -1
  193. package/dist/esm/store/sqlite/reconcile.js +14 -14
  194. package/dist/esm/store/sqlite/reconcile.js.map +1 -1
  195. package/dist/esm/store/sqlite/sql-functions.d.ts +1 -1
  196. package/dist/esm/store/sqlite/sql-functions.js +3 -3
  197. package/dist/esm/store/sqlite/sql-functions.js.map +1 -1
  198. package/dist/esm/store/sqlite/store.d.ts +1 -2
  199. package/dist/esm/store/sqlite/store.js +3 -9
  200. package/dist/esm/store/sqlite/store.js.map +1 -1
  201. package/dist/esm/store/turso/open.d.ts +6 -11
  202. package/dist/esm/store/turso/open.js +34 -78
  203. package/dist/esm/store/turso/open.js.map +1 -1
  204. package/dist/esm/store/turso/reconcile.d.ts +2 -0
  205. package/dist/esm/store/turso/reconcile.js +18 -3
  206. package/dist/esm/store/turso/reconcile.js.map +1 -1
  207. package/dist/esm/store/turso/store.d.ts +1 -2
  208. package/dist/esm/store/turso/store.js +2 -7
  209. package/dist/esm/store/turso/store.js.map +1 -1
  210. package/dist/esm/store/types.d.ts +16 -6
  211. package/dist/esm/store/types.js.map +1 -1
  212. package/dist/esm/text/strip.js +3 -4
  213. package/dist/esm/text/strip.js.map +1 -1
  214. package/dist/esm/watch.js +5 -3
  215. package/dist/esm/watch.js.map +1 -1
  216. package/package.json +4 -3
  217. package/schema.json +0 -11
  218. package/dist/cjs/store/sqlite/signature.js +0 -380
  219. package/dist/cjs/store/sqlite/signature.js.map +0 -1
  220. package/dist/esm/store/sqlite/signature.js +0 -58
  221. package/dist/esm/store/sqlite/signature.js.map +0 -1
@@ -14,6 +14,9 @@ _export(exports, {
14
14
  get createLexicalIndex () {
15
15
  return createLexicalIndex;
16
16
  },
17
+ get markContentStale () {
18
+ return markContentStale;
19
+ },
17
20
  get queryLexical () {
18
21
  return queryLexical;
19
22
  }
@@ -237,6 +240,22 @@ function unsupportedOperator(terms) {
237
240
  }
238
241
  return null;
239
242
  }
243
+ // One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it
244
+ // stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.
245
+ var ftsState = new WeakMap();
246
+ function stateFor(conn) {
247
+ var state = ftsState.get(conn);
248
+ if (!state) {
249
+ state = {
250
+ stale: true
251
+ };
252
+ ftsState.set(conn, state);
253
+ }
254
+ return state;
255
+ }
256
+ function markContentStale(conn) {
257
+ stateFor(conn).stale = true;
258
+ }
240
259
  // A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token
241
260
  // (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.
242
261
  function splitTerms(terms) {
@@ -480,15 +499,12 @@ function queryLexical(conn, terms, opts, state) {
480
499
  })();
481
500
  }
482
501
  function createLexicalIndex(conn) {
483
- var state = {
484
- stale: true
485
- };
486
502
  return {
487
503
  query: function query(terms, opts) {
488
- return queryLexical(conn, terms, opts, state);
504
+ return queryLexical(conn, terms, opts, stateFor(conn));
489
505
  },
490
506
  markStale: function markStale() {
491
- state.stale = true;
507
+ return markContentStale(conn);
492
508
  }
493
509
  };
494
510
  }
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n for (const { label, re } of FTS5_OPERATORS) {\n const m = withoutPhrases.match(re);\n if (m) return { label, token: m[0].trim() };\n }\n return null;\n}\n\ninterface FtsIndexState {\n stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n const words: string[] = [];\n const substrings: string[] = [];\n const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n const phrase = inner.trim();\n if (phrase.length === 0) return ' ';\n substrings.push(phrase);\n if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n return ' ';\n });\n for (const tok of withoutPhrases.split(/\\s+/)) {\n if (tok.length === 0) continue;\n if (hasUnspacedRun(tok)) substrings.push(tok);\n else words.push(tok);\n }\n return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n if (!state.stale) return;\n await conn.exec('INSTALL fts; LOAD fts;');\n await withTransaction(conn, async () => {\n // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n });\n state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n const scoreParts: string[] = [];\n const gateParts: string[] = [];\n const scoreParams: unknown[] = [];\n const gateParams: unknown[] = [];\n\n if (words.length > 0) {\n const wordQuery = words.join(' ');\n for (const field of FIELDS) {\n scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n scoreParams.push(wordQuery);\n }\n gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n gateParams.push(wordQuery);\n }\n\n for (const raw of substrings) {\n const needle = raw.toLowerCase();\n for (const field of FIELDS) {\n scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n scoreParams.push(needle);\n }\n gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n gateParams.push(needle);\n }\n\n return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n const unsupported = unsupportedOperator(terms);\n if (unsupported !== null) {\n throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n }\n const { whereJoin, whereCond, scopeCond, limit } = opts;\n const { words, substrings } = splitTerms(terms);\n if (words.length === 0 && substrings.length === 0) return [];\n\n if (words.length > 0) await ensureFtsFresh(conn, state);\n\n const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n const sql = `SELECT path, NULL AS hit FROM (\n SELECT content.\"path\" AS path, (${scoreSql}) AS score\n FROM content\n ${whereJoin}\n WHERE ${gateSql}\n ${whereCond} ${scopeCond}\n ) sq ORDER BY score DESC, path LIMIT ?`;\n const stmt = await conn.prepare(sql);\n return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches `content`),\n// and store.ts's reconcile wrapper flips it back to true whenever content changes, so the next query rebuilds first.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n const state: FtsIndexState = { stale: true };\n return {\n query: (terms, opts) => queryLexical(conn, terms, opts, state),\n markStale: () => {\n state.stale = true;\n },\n };\n}\n"],"names":["createLexicalIndex","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","hasUnspacedRun","split","w","tok","ensureFtsFresh","conn","state","stale","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","sql","SenseError","prepare","all","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QA8H9CA;eAAAA;;QAzBMC;eAAAA;;;wBApGK;yBACI;6BACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAMA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASC,WAAWP,KAAa;IAC/B,IAAMQ,QAAkB,EAAE;IAC1B,IAAMC,aAAuB,EAAE;IAC/B,IAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACQ,IAAIC;QACtD,IAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAACG,IAAAA,yBAAc,EAACH,SAAS;gBAAK,kCAAA,2BAAA;;gBAAL,QAAK,YAAWA,OAAOI,KAAK,CAAC,2BAAxB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,IAAN;oBAAgC,IAAIA,EAAEJ,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACG;;;gBAA7D;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAA+D;QACjG,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAahB,eAAee,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAME,MAAN;YACH,IAAIA,IAAIL,MAAM,KAAK,GAAG;YACtB,IAAIE,IAAAA,yBAAc,EAACG,MAAMT,WAAWK,IAAI,CAACI;iBACpCV,MAAMM,IAAI,CAACI;QAClB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAO;QAAEV,OAAAA;QAAOC,YAAAA;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,SAAeU,eAAeC,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;;;oBAClB;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAACJ,MAAM;;;;;4CAC1B,6HAA6H;4CAC7H,0JAA0J;4CAC1J;;gDAAMA,KAAKG,IAAI,CAAC;;;4CAAhB;;;;;;4BACF;;;;oBAJA;oBAKAF,MAAMC,KAAK,GAAG;;;;;;IAChB;;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASG,kBAAkBjB,KAAe,EAAEC,UAAoB;IAC9D,IAAMiB,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAIrB,MAAMK,MAAM,GAAG,GAAG;QACpB,IAAMiB,YAAYtB,MAAMuB,IAAI,CAAC;YACxB,kCAAA,2BAAA;;YAAL,QAAK,YAAexC,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAMyC,QAAN;gBACHN,WAAWZ,IAAI,CAAC,AAAC,GAAgGkB,OAA9FxC,YAAY,CAACwC,MAAM,EAAC,6EAAgF,OAANA,OAAM;gBACvHJ,YAAYd,IAAI,CAACgB;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAILH,UAAUb,IAAI,CAAC;QACfe,WAAWf,IAAI,CAACgB;IAClB;QAEK,mCAAA,4BAAA;;QAAL,QAAK,aAAarB,+BAAb,UAAA,8BAAA,SAAA,0BAAA,kCAAyB;YAAzB,IAAMwB,MAAN;YACH,IAAMC,SAASD,IAAIE,WAAW;gBACzB,mCAAA,4BAAA;;gBAAL,QAAK,aAAe5C,2BAAf,UAAA,8BAAA,SAAA,0BAAA,kCAAuB;oBAAvB,IAAMyC,SAAN;oBACHN,WAAWZ,IAAI,CAAC,AAAC,GAAyDkB,OAAvDxC,YAAY,CAACwC,OAAM,EAAC,qCAAyC,OAANA,QAAM;oBAChFJ,YAAYd,IAAI,CAACoB;gBACnB;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILP,UAAUb,IAAI,CAAC;YACfe,WAAWf,IAAI,CAACoB;QAClB;;QARK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ,AAAC,qBAAGV,oBAAa,qBAAGC;IAAY;AACvH;AAIO,SAAevC,aAAa8B,IAAgB,EAAEpB,KAAa,EAAEuC,IAAyB,EAAElB,KAAoB;;YAoBnGmB,OAnBRC,aAIEC,WAAWC,WAAWC,WAAWC,OACXtC,aAAtBC,OAAOC,YAKuBgB,oBAA9BW,UAAUC,SAASC,QACrBQ,KAOAN;;;;oBAlBAC,cAAc1C,oBAAoBC;oBACxC,IAAIyC,gBAAgB,MAAM;wBACxB,MAAM,IAAIM,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEN,OAAvBA,YAAY5C,KAAK,EAAC,OAAqDG,OAAhDyC,YAAYpC,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQoC,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACXtC,cAAAA,WAAWP,QAAjCQ,QAAsBD,YAAtBC,OAAOC,aAAeF,YAAfE;oBACf,IAAID,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG;;;;yBAE/CL,CAAAA,MAAMK,MAAM,GAAG,CAAA,GAAfL;;;;oBAAkB;;wBAAMW,eAAeC,MAAMC;;;oBAA3B;;;oBAEgBI,qBAAAA,kBAAkBjB,OAAOC,aAAvD2B,WAA8BX,mBAA9BW,UAAUC,UAAoBZ,mBAApBY,SAASC,SAAWb,mBAAXa;oBACrBQ,MAAM,AAAC,wEAGTJ,OAFgCN,UAAS,sCAGnCC,OADNK,WAAU,gBAEVC,OADMN,SAAQ,UACDO,OAAbD,WAAU,KAAa,OAAVC,WAAU;oBAEd;;wBAAMxB,KAAK4B,OAAO,CAACF;;;oBAA1BN,OAAO;oBACL;;wBAAMA,CAAAA,QAAAA,MAAKS,GAAG,OAART,OAAAA,AAAS,qBAAGF;4BAAQO;;;;oBAAlC;;wBAAQ;;;;IACV;;AAIO,SAASxD,mBAAmB+B,IAAgB;IACjD,IAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL4B,OAAO,SAAPA,MAAQlD,OAAOuC;mBAASjD,aAAa8B,MAAMpB,OAAOuC,MAAMlB;;QACxD8B,WAAW,SAAXA;YACE9B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n for (const { label, re } of FTS5_OPERATORS) {\n const m = withoutPhrases.match(re);\n if (m) return { label, token: m[0].trim() };\n }\n return null;\n}\n\ninterface FtsIndexState {\n stale: boolean;\n}\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nfunction stateFor(conn: Connection): FtsIndexState {\n let state = ftsState.get(conn);\n if (!state) {\n state = { stale: true };\n ftsState.set(conn, state);\n }\n return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`, so the next lexical query\n// rebuilds instead of searching stale content.\nexport function markContentStale(conn: Connection): void {\n stateFor(conn).stale = true;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n const words: string[] = [];\n const substrings: string[] = [];\n const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n const phrase = inner.trim();\n if (phrase.length === 0) return ' ';\n substrings.push(phrase);\n if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n return ' ';\n });\n for (const tok of withoutPhrases.split(/\\s+/)) {\n if (tok.length === 0) continue;\n if (hasUnspacedRun(tok)) substrings.push(tok);\n else words.push(tok);\n }\n return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n if (!state.stale) return;\n await conn.exec('INSTALL fts; LOAD fts;');\n await withTransaction(conn, async () => {\n // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n });\n state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n const scoreParts: string[] = [];\n const gateParts: string[] = [];\n const scoreParams: unknown[] = [];\n const gateParams: unknown[] = [];\n\n if (words.length > 0) {\n const wordQuery = words.join(' ');\n for (const field of FIELDS) {\n scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n scoreParams.push(wordQuery);\n }\n gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n gateParams.push(wordQuery);\n }\n\n for (const raw of substrings) {\n const needle = raw.toLowerCase();\n for (const field of FIELDS) {\n scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n scoreParams.push(needle);\n }\n gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n gateParams.push(needle);\n }\n\n return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n const unsupported = unsupportedOperator(terms);\n if (unsupported !== null) {\n throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n }\n const { whereJoin, whereCond, scopeCond, limit } = opts;\n const { words, substrings } = splitTerms(terms);\n if (words.length === 0 && substrings.length === 0) return [];\n\n if (words.length > 0) await ensureFtsFresh(conn, state);\n\n const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n const sql = `SELECT path, NULL AS hit FROM (\n SELECT content.\"path\" AS path, (${scoreSql}) AS score\n FROM content\n ${whereJoin}\n WHERE ${gateSql}\n ${whereCond} ${scopeCond}\n ) sq ORDER BY score DESC, path LIMIT ?`;\n const stmt = await conn.prepare(sql);\n return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches\n// `content`), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n return {\n query: (terms, opts) => queryLexical(conn, terms, opts, stateFor(conn)),\n markStale: () => markContentStale(conn),\n };\n}\n"],"names":["createLexicalIndex","markContentStale","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","ftsState","WeakMap","stateFor","conn","state","get","stale","set","splitTerms","words","substrings","_m","inner","phrase","length","push","hasUnspacedRun","split","w","tok","ensureFtsFresh","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","sql","SenseError","prepare","all","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QAgJ9CA;eAAAA;;QA/FAC;eAAAA;;QAsEMC;eAAAA;;;wBAtHK;yBACI;6BACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAMA,6FAA6F;AAC7F,0GAA0G;AAC1G,IAAMC,WAAW,IAAIC;AACrB,SAASC,SAASC,IAAgB;IAChC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACVA,QAAQ;YAAEE,OAAO;QAAK;QACtBN,SAASO,GAAG,CAACJ,MAAMC;IACrB;IACA,OAAOA;AACT;AAIO,SAAStB,iBAAiBqB,IAAgB;IAC/CD,SAASC,MAAMG,KAAK,GAAG;AACzB;AAEA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASE,WAAWf,KAAa;IAC/B,IAAMgB,QAAkB,EAAE;IAC1B,IAAMC,aAAuB,EAAE;IAC/B,IAAMhB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACgB,IAAIC;QACtD,IAAMC,SAASD,MAAMb,IAAI;QACzB,IAAIc,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAACG,IAAAA,yBAAc,EAACH,SAAS;gBAAK,kCAAA,2BAAA;;gBAAL,QAAK,YAAWA,OAAOI,KAAK,CAAC,2BAAxB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,IAAN;oBAAgC,IAAIA,EAAEJ,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACG;;;gBAA7D;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAA+D;QACjG,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAaxB,eAAeuB,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAME,MAAN;YACH,IAAIA,IAAIL,MAAM,KAAK,GAAG;YACtB,IAAIE,IAAAA,yBAAc,EAACG,MAAMT,WAAWK,IAAI,CAACI;iBACpCV,MAAMM,IAAI,CAACI;QAClB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAO;QAAEV,OAAAA;QAAOC,YAAAA;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,SAAeU,eAAejB,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAME,KAAK,EAAE;;;oBAClB;;wBAAMH,KAAKkB,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAACnB,MAAM;;;;;4CAC1B,6HAA6H;4CAC7H,0JAA0J;4CAC1J;;gDAAMA,KAAKkB,IAAI,CAAC;;;4CAAhB;;;;;;4BACF;;;;oBAJA;oBAKAjB,MAAME,KAAK,GAAG;;;;;;IAChB;;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASiB,kBAAkBd,KAAe,EAAEC,UAAoB;IAC9D,IAAMc,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAIlB,MAAMK,MAAM,GAAG,GAAG;QACpB,IAAMc,YAAYnB,MAAMoB,IAAI,CAAC;YACxB,kCAAA,2BAAA;;YAAL,QAAK,YAAe7C,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAM8C,QAAN;gBACHN,WAAWT,IAAI,CAAC,AAAC,GAAgGe,OAA9F7C,YAAY,CAAC6C,MAAM,EAAC,6EAAgF,OAANA,OAAM;gBACvHJ,YAAYX,IAAI,CAACa;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAILH,UAAUV,IAAI,CAAC;QACfY,WAAWZ,IAAI,CAACa;IAClB;QAEK,mCAAA,4BAAA;;QAAL,QAAK,aAAalB,+BAAb,UAAA,8BAAA,SAAA,0BAAA,kCAAyB;YAAzB,IAAMqB,MAAN;YACH,IAAMC,SAASD,IAAIE,WAAW;gBACzB,mCAAA,4BAAA;;gBAAL,QAAK,aAAejD,2BAAf,UAAA,8BAAA,SAAA,0BAAA,kCAAuB;oBAAvB,IAAM8C,SAAN;oBACHN,WAAWT,IAAI,CAAC,AAAC,GAAyDe,OAAvD7C,YAAY,CAAC6C,OAAM,EAAC,qCAAyC,OAANA,QAAM;oBAChFJ,YAAYX,IAAI,CAACiB;gBACnB;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILP,UAAUV,IAAI,CAAC;YACfY,WAAWZ,IAAI,CAACiB;QAClB;;QARK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ,AAAC,qBAAGV,oBAAa,qBAAGC;IAAY;AACvH;AAIO,SAAe5C,aAAaoB,IAAgB,EAAEV,KAAa,EAAE4C,IAAyB,EAAEjC,KAAoB;;YAoBnGkC,OAnBRC,aAIEC,WAAWC,WAAWC,WAAWC,OACXnC,aAAtBC,OAAOC,YAKuBa,oBAA9BW,UAAUC,SAASC,QACrBQ,KAOAN;;;;oBAlBAC,cAAc/C,oBAAoBC;oBACxC,IAAI8C,gBAAgB,MAAM;wBACxB,MAAM,IAAIM,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEN,OAAvBA,YAAYjD,KAAK,EAAC,OAAqDG,OAAhD8C,YAAYzC,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQyC,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACXnC,cAAAA,WAAWf,QAAjCgB,QAAsBD,YAAtBC,OAAOC,aAAeF,YAAfE;oBACf,IAAID,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG;;;;yBAE/CL,CAAAA,MAAMK,MAAM,GAAG,CAAA,GAAfL;;;;oBAAkB;;wBAAMW,eAAejB,MAAMC;;;oBAA3B;;;oBAEgBmB,qBAAAA,kBAAkBd,OAAOC,aAAvDwB,WAA8BX,mBAA9BW,UAAUC,UAAoBZ,mBAApBY,SAASC,SAAWb,mBAAXa;oBACrBQ,MAAM,AAAC,wEAGTJ,OAFgCN,UAAS,sCAGnCC,OADNK,WAAU,gBAEVC,OADMN,SAAQ,UACDO,OAAbD,WAAU,KAAa,OAAVC,WAAU;oBAEd;;wBAAMvC,KAAK2C,OAAO,CAACF;;;oBAA1BN,OAAO;oBACL;;wBAAMA,CAAAA,QAAAA,MAAKS,GAAG,OAART,OAAAA,AAAS,qBAAGF;4BAAQO;;;;oBAAlC;;wBAAQ;;;;IACV;;AAIO,SAAS9D,mBAAmBsB,IAAgB;IACjD,OAAO;QACL6C,OAAO,SAAPA,MAAQvD,OAAO4C;mBAAStD,aAAaoB,MAAMV,OAAO4C,MAAMnC,SAASC;;QACjE8C,WAAW,SAAXA;mBAAiBnE,iBAAiBqB;;IACpC;AACF"}
@@ -1,12 +1,13 @@
1
+ import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
1
2
  import type { ResolvedConfig } from '../../config/index.js';
2
- import type { Store } from '../types.js';
3
+ import type { OpenResult } from '../open.js';
4
+ import type { OpenDialect } from '../types.js';
3
5
  export declare const DB_FILENAME = "cache.duckdb";
4
- export declare const SCHEMA_VERSION = "2";
5
- export interface OpenResult {
6
- store: Store;
7
- cfg: ResolvedConfig;
8
- dbPath: string;
9
- parsed: number;
10
- warnings: string[];
6
+ export declare const SCHEMA_VERSION = "3";
7
+ export type { OpenResult };
8
+ interface DuckdbHandle {
9
+ instance: DuckDBInstance;
10
+ duckdb: DuckDBConnection;
11
11
  }
12
+ export declare const duckdbOpenDialect: OpenDialect<DuckdbHandle>;
12
13
  export declare function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult>;
@@ -1,12 +1,13 @@
1
+ import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
1
2
  import type { ResolvedConfig } from '../../config/index.js';
2
- import type { Store } from '../types.js';
3
+ import type { OpenResult } from '../open.js';
4
+ import type { OpenDialect } from '../types.js';
3
5
  export declare const DB_FILENAME = "cache.duckdb";
4
- export declare const SCHEMA_VERSION = "2";
5
- export interface OpenResult {
6
- store: Store;
7
- cfg: ResolvedConfig;
8
- dbPath: string;
9
- parsed: number;
10
- warnings: string[];
6
+ export declare const SCHEMA_VERSION = "3";
7
+ export type { OpenResult };
8
+ interface DuckdbHandle {
9
+ instance: DuckDBInstance;
10
+ duckdb: DuckDBConnection;
11
11
  }
12
+ export declare const duckdbOpenDialect: OpenDialect<DuckdbHandle>;
12
13
  export declare function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult>;
@@ -15,23 +15,22 @@ _export(exports, {
15
15
  get SCHEMA_VERSION () {
16
16
  return SCHEMA_VERSION;
17
17
  },
18
+ get duckdbOpenDialect () {
19
+ return duckdbOpenDialect;
20
+ },
18
21
  get openDuckdb () {
19
22
  return openDuckdb;
20
23
  }
21
24
  });
22
- var _nodefs = require("node:fs");
23
- var _nodepath = require("node:path");
24
25
  var _indexts = require("../../config/index.js");
25
- var _handoffts = require("../../embed/handoff.js");
26
26
  var _typests = require("../../embed/types.js");
27
27
  var _errorsts = require("../../errors.js");
28
28
  var _indexts1 = require("../../features/index.js");
29
- var _cachets = require("../cache.js");
30
- var _reconcilets = require("../reconcile.js");
29
+ var _opents = require("../open.js");
31
30
  var _sharedts = require("../shared.js");
32
31
  var _connectionts = require("./connection.js");
33
32
  var _nativets = require("./native.js");
34
- var _reconcilets1 = require("./reconcile.js");
33
+ var _reconcilets = require("./reconcile.js");
35
34
  var _sqlfunctionsts = require("./sql-functions.js");
36
35
  var _storets = require("./store.js");
37
36
  function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
@@ -166,10 +165,10 @@ function _ts_generator(thisArg, body) {
166
165
  }
167
166
  }
168
167
  var DB_FILENAME = 'cache.duckdb';
169
- var SCHEMA_VERSION = '2';
168
+ var SCHEMA_VERSION = '3';
170
169
  // `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
171
- // No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.
172
- function ensureSchema(conn, cfg) {
170
+ // No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
171
+ function ensureSchema(_handle, conn, cfg) {
173
172
  return _async_to_generator(function() {
174
173
  var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, feature, err;
175
174
  return _ts_generator(this, function(_state) {
@@ -312,45 +311,50 @@ function ensureSchema(conn, cfg) {
312
311
  });
313
312
  })();
314
313
  }
315
- function connect(cfg) {
314
+ // Order matters: the connection must be gone before the instance closes the WAL.
315
+ function close(handle) {
316
+ return _async_to_generator(function() {
317
+ return _ts_generator(this, function(_state) {
318
+ handle.duckdb.disconnectSync();
319
+ handle.instance.closeSync();
320
+ return [
321
+ 2
322
+ ];
323
+ });
324
+ })();
325
+ }
326
+ function connect(dbPath, _cfg) {
316
327
  return _async_to_generator(function() {
317
- var stateDir, dbPath, DuckDBInstance, err, duckdb, instance, err1, closed, conn, version, features, wantFeatures, reason, _ref, parsed, warnings, err2;
328
+ var DuckDBInstance, err, duckdb, instance, err1, conn, err2;
318
329
  return _ts_generator(this, function(_state) {
319
330
  switch(_state.label){
320
331
  case 0:
321
- stateDir = (0, _nodepath.join)(cfg.baseDir, _indexts.STATE_DIR);
322
- (0, _nodefs.mkdirSync)(stateDir, {
323
- recursive: true
324
- });
325
- dbPath = (0, _nodepath.join)(stateDir, DB_FILENAME);
326
- _state.label = 1;
327
- case 1:
328
332
  _state.trys.push([
329
- 1,
330
- 3,
333
+ 0,
334
+ 2,
331
335
  ,
332
- 4
336
+ 3
333
337
  ]);
334
338
  return [
335
339
  4,
336
340
  (0, _nativets.duckdbApi)()
337
341
  ];
338
- case 2:
342
+ case 1:
339
343
  DuckDBInstance = _state.sent().DuckDBInstance;
340
344
  return [
341
345
  3,
342
- 4
346
+ 3
343
347
  ];
344
- case 3:
348
+ case 2:
345
349
  err = _state.sent();
346
350
  if (_instanceof(err, _errorsts.SenseError)) throw err;
347
351
  throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "duckdb" needs the '.concat(_nativets.DUCKDB_PACKAGE, " package (").concat(err.message, ")"));
348
- case 4:
352
+ case 3:
349
353
  _state.trys.push([
350
- 4,
351
- 7,
354
+ 3,
355
+ 6,
352
356
  ,
353
- 8
357
+ 7
354
358
  ]);
355
359
  return [
356
360
  4,
@@ -358,7 +362,7 @@ function connect(cfg) {
358
362
  storage_compatibility_version: 'v1.5.0'
359
363
  })
360
364
  ];
361
- case 5:
365
+ case 4:
362
366
  // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns ("VARIANT
363
367
  // columns are not supported in storage versions prior to v1.5.0"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.
364
368
  instance = _state.sent();
@@ -366,102 +370,54 @@ function connect(cfg) {
366
370
  4,
367
371
  instance.connect()
368
372
  ];
369
- case 6:
373
+ case 5:
370
374
  duckdb = _state.sent();
371
375
  return [
372
376
  3,
373
- 8
377
+ 7
374
378
  ];
375
- case 7:
379
+ case 6:
376
380
  err1 = _state.sent();
377
381
  // create() may have succeeded before connect() failed: close it, or its WAL stays open.
378
382
  instance === null || instance === void 0 ? void 0 : instance.closeSync();
379
383
  throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "duckdb" failed to open '.concat(dbPath, ": ").concat(err1.message));
380
- case 8:
381
- // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL
382
- // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.
383
- closed = false;
384
- _state.label = 9;
385
- case 9:
384
+ case 7:
386
385
  _state.trys.push([
386
+ 7,
387
387
  9,
388
- 16,
389
388
  ,
390
- 17
389
+ 11
391
390
  ]);
392
391
  return [
393
392
  4,
394
393
  (0, _sqlfunctionsts.registerFunctions)(duckdb)
395
394
  ];
396
- case 10:
395
+ case 8:
397
396
  _state.sent();
398
397
  conn = (0, _connectionts.createConnection)(duckdb);
399
- return [
400
- 4,
401
- conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)')
402
- ];
403
- case 11:
404
- _state.sent();
405
- return [
406
- 4,
407
- (0, _sharedts.getMeta)(conn, 'schema_version')
408
- ];
409
- case 12:
410
- version = _state.sent();
411
- return [
412
- 4,
413
- (0, _sharedts.getMeta)(conn, 'features')
414
- ];
415
- case 13:
416
- features = _state.sent();
417
- wantFeatures = (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES);
418
- if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
419
- // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --
420
- // rebuild instead, same as sqlite's open() (see its comment for why this is announced).
421
- reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
422
- console.error("sense: ".concat(reason, "; rebuilding the index"));
423
- // Close before clearCache deletes the files underneath the still-open instance.
424
- closed = true;
425
- duckdb.disconnectSync();
426
- instance.closeSync();
427
- (0, _cachets.clearCache)(cfg);
428
- return [
429
- 2,
430
- connect(cfg)
431
- ];
432
- }
433
- return [
434
- 4,
435
- ensureSchema(conn, cfg)
436
- ];
437
- case 14:
438
- _state.sent();
439
- return [
440
- 4,
441
- (0, _reconcilets.reconcile)(conn, cfg, cfg.baseDir, _reconcilets1.duckdbDialect)
442
- ];
443
- case 15:
444
- _ref = _state.sent(), parsed = _ref.parsed, warnings = _ref.warnings;
445
398
  return [
446
399
  2,
447
400
  {
448
- instance: instance,
449
- duckdb: duckdb,
450
- conn: conn,
451
- cfg: cfg,
452
- dbPath: dbPath,
453
- parsed: parsed,
454
- warnings: warnings
401
+ handle: {
402
+ instance: instance,
403
+ duckdb: duckdb
404
+ },
405
+ conn: conn
455
406
  }
456
407
  ];
457
- case 16:
408
+ case 9:
458
409
  err2 = _state.sent();
459
- if (!closed) {
460
- duckdb.disconnectSync();
461
- instance === null || instance === void 0 ? void 0 : instance.closeSync();
462
- }
410
+ return [
411
+ 4,
412
+ close({
413
+ instance: instance,
414
+ duckdb: duckdb
415
+ })
416
+ ];
417
+ case 10:
418
+ _state.sent();
463
419
  throw err2;
464
- case 17:
420
+ case 11:
465
421
  return [
466
422
  2
467
423
  ];
@@ -469,32 +425,30 @@ function connect(cfg) {
469
425
  });
470
426
  })();
471
427
  }
428
+ var duckdbOpenDialect = {
429
+ filename: DB_FILENAME,
430
+ schemaVersion: SCHEMA_VERSION,
431
+ reconcileDialect: _reconcilets.duckdbDialect,
432
+ connect: connect,
433
+ close: close,
434
+ // duckdb's file lock spans the connection's life, and each platform words the refusal its own
435
+ // way: posix "Could not set lock on file ... Conflicting lock is held in <exe> (PID n)", Windows
436
+ // "Cannot open file ... being used by another process". Both are the same condition.
437
+ isLocked: function isLocked(err) {
438
+ return /Could not set lock on file|being used by another process/.test(err.message);
439
+ },
440
+ ensureSchema: ensureSchema,
441
+ createStore: function createStore(handle, conn) {
442
+ return (0, _storets.createStore)(handle.instance, handle.duckdb, conn);
443
+ }
444
+ };
472
445
  function openDuckdb(cfg) {
473
446
  return _async_to_generator(function() {
474
- var _ref, instance, duckdb, conn, resolvedCfg, dbPath, parsed, warnings, store;
475
447
  return _ts_generator(this, function(_state) {
476
- switch(_state.label){
477
- case 0:
478
- return [
479
- 4,
480
- connect(cfg)
481
- ];
482
- case 1:
483
- _ref = _state.sent(), instance = _ref.instance, duckdb = _ref.duckdb, conn = _ref.conn, resolvedCfg = _ref.cfg, dbPath = _ref.dbPath, parsed = _ref.parsed, warnings = _ref.warnings;
484
- store = (0, _storets.createStore)(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);
485
- // reconcile ran before this object existed, so its chunk text is keyed by the connection.
486
- (0, _handoffts.rekeyChunkText)(conn, store);
487
- return [
488
- 2,
489
- {
490
- store: store,
491
- cfg: resolvedCfg,
492
- dbPath: dbPath,
493
- parsed: parsed,
494
- warnings: warnings
495
- }
496
- ];
497
- }
448
+ return [
449
+ 2,
450
+ (0, _opents.openWithDialect)(cfg, duckdbOpenDialect)
451
+ ];
498
452
  });
499
453
  })();
500
454
  }
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n store: Store;\n cfg: ResolvedConfig;\n dbPath: string;\n parsed: number;\n warnings: string[];\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n for (const feature of activeFeatures(cfg)) {\n // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n if (feature.name === 'embed') {\n await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n continue;\n }\n await feature.schema(conn);\n }\n if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<{ instance: DuckDBInstance; duckdb: DuckDBConnection; conn: Connection; cfg: ResolvedConfig; dbPath: string; parsed: number; warnings: string[] }> {\n const stateDir = join(cfg.baseDir, STATE_DIR);\n mkdirSync(stateDir, { recursive: true });\n const dbPath = join(stateDir, DB_FILENAME);\n\n // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n try {\n ({ DuckDBInstance } = await duckdbApi());\n } catch (err) {\n if (err instanceof SenseError) throw err;\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n }\n\n let duckdb: DuckDBConnection;\n let instance: DuckDBInstance | undefined;\n try {\n // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n duckdb = await instance.connect();\n } catch (err) {\n // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n instance?.closeSync();\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n }\n // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL\n // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.\n let closed = false;\n try {\n await registerFunctions(duckdb);\n const conn = createConnection(duckdb);\n\n await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n const version = await getMeta(conn, 'schema_version');\n const features = await getMeta(conn, 'features');\n const wantFeatures = featureSignature(cfg, FEATURES);\n if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n // rebuild instead, same as sqlite's open() (see its comment for why this is announced).\n const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n console.error(`sense: ${reason}; rebuilding the index`);\n // Close before clearCache deletes the files underneath the still-open instance.\n closed = true;\n duckdb.disconnectSync();\n instance.closeSync();\n clearCache(cfg);\n return connect(cfg);\n }\n\n await ensureSchema(conn, cfg);\n\n const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, duckdbDialect);\n return { instance, duckdb, conn, cfg, dbPath, parsed, warnings };\n } catch (err) {\n if (!closed) {\n duckdb.disconnectSync();\n instance?.closeSync();\n }\n throw err;\n }\n}\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n const store = createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);\n // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n rekeyChunkText(conn, store);\n return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openDuckdb","ensureSchema","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","DuckDBInstance","err","duckdb","instance","closed","version","features","wantFeatures","reason","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","closeSync","registerFunctions","createConnection","console","error","disconnectSync","clearCache","reconcile","duckdbDialect","resolvedCfg","store","createStore","rekeyChunkText"],"mappings":";;;;;;;;;;;QAmBaA;eAAAA;;QAGAC;eAAAA;;QA+FSC;eAAAA;;;sBArHI;wBACL;uBAGuB;yBACb;uBACJ;wBACA;wBACc;uBACd;2BACD;wBACO;4BAEA;wBACS;4BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMF,cAAc;AAGpB,IAAMC,iBAAiB;AAU9B,8LAA8L;AAC9L,2JAA2J;AAC3J,SAAeE,aAAaC,IAAgB,EAAEC,GAAW;;YAKlD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBH;;;oBAAtC;;;oBACjD;;wBAAMW,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,QAAQX,GAAmB;;YAClCY,UAEAC,QAIFC,gBAGKC,KAKLC,QACAC,UAMKF,MAOLG,QAGInB,MAIAoB,SACAC,UACAC,cAIEC,QAYqB,MAArBC,QAAQC,UAETT;;;;oBAvDHH,WAAWa,IAAAA,cAAI,EAACzB,IAAI0B,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAAChB,UAAU;wBAAEiB,WAAW;oBAAK;oBAChChB,SAASY,IAAAA,cAAI,EAACb,UAAUjB;;;;;;;;;oBAMN;;wBAAMmC,IAAAA,mBAAS;;;oBAAlChB,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAegB,oBAAU,GAAE,MAAMhB;oBACrC,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACjB,IAAckB,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAMnB,eAAeoB,MAAM,CAACrB,QAAQ;4BAAEsB,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KlB,WAAW;oBACF;;wBAAMA,SAASN,OAAO;;;oBAA/BK,SAAS;;;;;;oBACFD;oBACP,wFAAwF;oBACxFE,qBAAAA,+BAAAA,SAAUmB,SAAS;oBACnB,MAAM,IAAIL,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXlB,QAAO,MAA2B,OAAvB,AAACE,KAAckB,OAAO;;oBAErH,2FAA2F;oBAC3F,uFAAuF;oBACnFf,SAAS;;;;;;;;;oBAEX;;wBAAMmB,IAAAA,iCAAiB,EAACrB;;;oBAAxB;oBACMjB,OAAOuC,IAAAA,8BAAgB,EAACtB;oBAE9B;;wBAAMjB,KAAKG,IAAI,CAAC;;;oBAAhB;oBAEgB;;wBAAMK,IAAAA,iBAAO,EAACR,MAAM;;;oBAA9BoB,UAAU;oBACC;;wBAAMZ,IAAAA,iBAAO,EAACR,MAAM;;;oBAA/BqB,WAAW;oBACXC,eAAeZ,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;oBACnD,IAAI,AAACS,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,cAAe;wBACxG,4FAA4F;wBAC5F,wFAAwF;wBAClFC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;wBACnH2C,QAAQC,KAAK,CAAC,AAAC,UAAgB,OAAPlB,QAAO;wBAC/B,gFAAgF;wBAChFJ,SAAS;wBACTF,OAAOyB,cAAc;wBACrBxB,SAASmB,SAAS;wBAClBM,IAAAA,mBAAU,EAAC1C;wBACX;;4BAAOW,QAAQX;;oBACjB;oBAEA;;wBAAMF,aAAaC,MAAMC;;;oBAAzB;oBAE6B;;wBAAM2C,IAAAA,sBAAS,EAAC5C,MAAMC,KAAKA,IAAI0B,OAAO,EAAEkB,2BAAa;;;oBAArD,OAAA,eAArBrB,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAChB;;wBAAO;4BAAEP,UAAAA;4BAAUD,QAAAA;4BAAQjB,MAAAA;4BAAMC,KAAAA;4BAAKa,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;oBACxDT;oBACP,IAAI,CAACG,QAAQ;wBACXF,OAAOyB,cAAc;wBACrBxB,qBAAAA,+BAAAA,SAAUmB,SAAS;oBACrB;oBACA,MAAMrB;;;;;;;IAEV;;AAEO,SAAelB,WAAWG,GAAmB;;YAC6B,MAAvEiB,UAAUD,QAAQjB,MAAW8C,aAAahC,QAAQU,QAAQC,UAC5DsB;;;;oBADyE;;wBAAMnC,QAAQX;;;oBAAd,OAAA,eAAvEiB,WAAuE,KAAvEA,UAAUD,SAA6D,KAA7DA,QAAQjB,OAAqD,KAArDA,MAAW8C,cAA0C,KAA/C7C,KAAkBa,SAA6B,KAA7BA,QAAQU,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAC5DsB,QAAQC,IAAAA,oBAAW,EAAC9B,UAAUD,QAAQjB,MAAM8C,aAAaA,YAAYnB,OAAO;oBAClF,0FAA0F;oBAC1FsB,IAAAA,yBAAc,EAACjD,MAAM+C;oBACrB;;wBAAO;4BAAEA,OAAOA;4BAAO9C,KAAK6C;4BAAahC,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACpE"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '3';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n instance: DuckDBInstance;\n duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n for (const feature of activeFeatures(cfg)) {\n // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n if (feature.name === 'embed') {\n await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n continue;\n }\n await feature.schema(conn);\n }\n if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n handle.duckdb.disconnectSync();\n handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n try {\n ({ DuckDBInstance } = await duckdbApi());\n } catch (err) {\n if (err instanceof SenseError) throw err;\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n }\n\n let duckdb: DuckDBConnection;\n let instance: DuckDBInstance | undefined;\n try {\n // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n duckdb = await instance.connect();\n } catch (err) {\n // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n instance?.closeSync();\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n }\n // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n try {\n await registerFunctions(duckdb);\n const conn = createConnection(duckdb);\n return { handle: { instance, duckdb }, conn };\n } catch (err) {\n await close({ instance, duckdb });\n throw err;\n }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n filename: DB_FILENAME,\n schemaVersion: SCHEMA_VERSION,\n reconcileDialect: duckdbDialect,\n connect,\n close,\n // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n // \"Cannot open file ... being used by another process\". Both are the same condition.\n isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n ensureSchema,\n createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","_handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAGAC;eAAAA;;QAwEAC;eAAAA;;QAcSC;eAAAA;;;uBAvGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,gGAAgG;AAChG,SAAeG,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;;YAKzE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMa,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMC,MAAoB;;;YACvCA,OAAOC,MAAM,CAACC,cAAc;YAC5BF,OAAOG,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLR,QACAE,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BJ,SAAS;;;;;;oBACFQ;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACf;;;oBAAxB;oBACMd,OAAO8B,IAAAA,8BAAgB,EAAChB;oBAC9B;;wBAAO;4BAAED,QAAQ;gCAAEG,UAAAA;gCAAUF,QAAAA;4BAAO;4BAAGd,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMV,MAAM;4BAAEI,UAAAA;4BAAUF,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMQ;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAN,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFuB,UAAU,SAAVA,SAAWb;eAAQ,2DAA2Dc,IAAI,CAACd,IAAII,OAAO;;IAC9F5B,cAAAA;IACAuC,aAAa,SAAbA,YAAcxB,QAAQb;eAASqC,IAAAA,oBAAW,EAACxB,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAEd;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKL;;;IAC9B"}
@@ -9,6 +9,8 @@ Object.defineProperty(exports, "duckdbDialect", {
9
9
  }
10
10
  });
11
11
  var _errorsts = require("../../errors.js");
12
+ var _sharedts = require("../shared.js");
13
+ var _lexicalts = require("./lexical.js");
12
14
  function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
13
15
  try {
14
16
  var info = gen[key](arg);
@@ -185,6 +187,37 @@ function reconcileContent(conn, touched, docs, _delta) {
185
187
  _state.sent();
186
188
  _state.label = 4;
187
189
  case 4:
190
+ // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,
191
+ // not here (lexical.ts's FtsIndexState).
192
+ (0, _lexicalts.markContentStale)(conn);
193
+ return [
194
+ 2
195
+ ];
196
+ }
197
+ });
198
+ })();
199
+ }
200
+ // DuckDB rejects more than one ALTER command per statement ("Parser Error: Only one ALTER
201
+ // command per statement is supported", measured), so every name's clause joins into one string
202
+ // and runs as a single exec() -- one column-add "leg" through the driver instead of `names.length`.
203
+ // VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()
204
+ // produces for one key across files.
205
+ function addColumns(conn, names) {
206
+ return _async_to_generator(function() {
207
+ return _ts_generator(this, function(_state) {
208
+ switch(_state.label){
209
+ case 0:
210
+ if (names.length === 0) return [
211
+ 2
212
+ ];
213
+ return [
214
+ 4,
215
+ conn.exec(names.map(function(name) {
216
+ return "ALTER TABLE frontmatter ADD COLUMN ".concat((0, _sharedts.quoteIdent)(name), " VARIANT");
217
+ }).join('; '))
218
+ ];
219
+ case 1:
220
+ _state.sent();
188
221
  return [
189
222
  2
190
223
  ];
@@ -201,11 +234,7 @@ var duckdbDialect = {
201
234
  throw new _errorsts.SenseError('COLUMN_LIMIT', "frontmatter would need ".concat(count, " columns, crossing this store's sanity limit (").concat(MAX_FRONTMATTER_COLUMNS, "). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys."));
202
235
  }
203
236
  },
204
- // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold
205
- // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.
206
- columnTypeSuffix: function columnTypeSuffix() {
207
- return ' VARIANT';
208
- },
237
+ addColumns: addColumns,
209
238
  reconcileContent: reconcileContent
210
239
  };
211
240
  /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n if (touched.length > 0)\n await conn.runBatch(\n 'DELETE FROM content WHERE \"path\" = ?',\n touched.map((p) => [p])\n );\n if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n beginMode: () => 'BEGIN',\n checkColumnLimit(count) {\n if (count > MAX_FRONTMATTER_COLUMNS) {\n throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n }\n },\n // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n columnTypeSuffix: () => ' VARIANT',\n reconcileContent,\n};\n"],"names":["duckdbDialect","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","beginMode","checkColumnLimit","count","SenseError","columnTypeSuffix"],"mappings":";;;;+BA+BaA;;;eAAAA;;;wBA/Bc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAK3B,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEL,CAAAA,KAAKE,MAAM,GAAG,CAAA,GAAdF;;;;oBAAiB;;wBAAMF,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;;;oBAAjD;;;;;;;;IACvB;;AAEO,IAAMF,gBAAkC;IAC7CmB,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQZ,yBAAyB;YACnC,MAAM,IAAIa,oBAAU,CAAC,gBAAgB,AAAC,0BAA+Eb,OAAtDY,OAAM,kDAAwE,OAAxBZ,yBAAwB;QAC/I;IACF;IACA,yFAAyF;IACzF,2FAA2F;IAC3Fc,kBAAkB,SAAlBA;eAAwB;;IACxBb,kBAAAA;AACF"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n if (touched.length > 0)\n await conn.runBatch(\n 'DELETE FROM content WHERE \"path\" = ?',\n touched.map((p) => [p])\n );\n if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n // not here (lexical.ts's FtsIndexState).\n markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n if (names.length === 0) return;\n await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n beginMode: () => 'BEGIN',\n checkColumnLimit(count) {\n if (count > MAX_FRONTMATTER_COLUMNS) {\n throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n }\n },\n addColumns,\n reconcileContent,\n};\n"],"names":["duckdbDialect","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","markContentStale","addColumns","names","exec","name","quoteIdent","join","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BA8CaA;;;eAAAA;;;wBA9Cc;wBAGA;yBAEM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEjC,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEL,CAAAA,KAAKE,MAAM,GAAG,CAAA,GAAdF;;;;oBAAiB;;wBAAMF,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;;;oBAAjD;;;oBACrB,6FAA6F;oBAC7F,yCAAyC;oBACzCiB,IAAAA,2BAAgB,EAACR;;;;;;IACnB;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAeS,WAAWT,IAAgB,EAAEU,KAAe;;;;;oBACzD,IAAIA,MAAMN,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMJ,KAAKW,IAAI,CAACD,MAAMJ,GAAG,CAAC,SAACM;mCAAS,AAAC,sCAAsD,OAAjBC,IAAAA,oBAAU,EAACD,OAAM;2BAAWE,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEO,IAAMzB,gBAAkC;IAC7C0B,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQnB,yBAAyB;YACnC,MAAM,IAAIoB,oBAAU,CAAC,gBAAgB,AAAC,0BAA+EpB,OAAtDmB,OAAM,kDAAwE,OAAxBnB,yBAAwB;QAC/I;IACF;IACAW,YAAAA;IACAV,kBAAAA;AACF"}
@@ -1,5 +1,4 @@
1
1
  import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
2
- import type { Config } from '../../config/index.js';
3
2
  import type { Capability, Connection, Store } from '../types.js';
4
3
  export declare const CAPABILITIES: ReadonlySet<Capability>;
5
- export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store;
4
+ export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store;
@@ -1,5 +1,4 @@
1
1
  import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
2
- import type { Config } from '../../config/index.js';
3
2
  import type { Capability, Connection, Store } from '../types.js';
4
3
  export declare const CAPABILITIES: ReadonlySet<Capability>;
5
- export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store;
4
+ export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store;