sensemaking 0.19.2 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -1
- package/dist/cjs/chunk/extract.d.cts +1 -0
- package/dist/cjs/chunk/extract.d.ts +1 -0
- package/dist/cjs/chunk/extract.js +23 -5
- package/dist/cjs/chunk/extract.js.map +1 -1
- package/dist/cjs/chunk/index.d.cts +1 -1
- package/dist/cjs/chunk/index.d.ts +1 -1
- package/dist/cjs/chunk/index.js +3 -0
- package/dist/cjs/chunk/index.js.map +1 -1
- package/dist/cjs/config/access.d.cts +0 -1
- package/dist/cjs/config/access.d.ts +0 -1
- package/dist/cjs/config/access.js +10 -17
- package/dist/cjs/config/access.js.map +1 -1
- package/dist/cjs/config/index.d.cts +1 -1
- package/dist/cjs/config/index.d.ts +1 -1
- package/dist/cjs/config/index.js +0 -3
- package/dist/cjs/config/index.js.map +1 -1
- package/dist/cjs/config/types.d.cts +0 -3
- package/dist/cjs/config/types.d.ts +0 -3
- package/dist/cjs/config/types.js.map +1 -1
- package/dist/cjs/config/validate.js +6 -24
- package/dist/cjs/config/validate.js.map +1 -1
- package/dist/cjs/errors.d.cts +1 -1
- package/dist/cjs/errors.d.ts +1 -1
- package/dist/cjs/errors.js.map +1 -1
- package/dist/cjs/features/links.js +4 -5
- package/dist/cjs/features/links.js.map +1 -1
- package/dist/cjs/features/sections.js +4 -1
- package/dist/cjs/features/sections.js.map +1 -1
- package/dist/cjs/scan/pool.d.cts +15 -0
- package/dist/cjs/scan/pool.d.ts +15 -0
- package/dist/cjs/scan/pool.js +263 -0
- package/dist/cjs/scan/pool.js.map +1 -0
- package/dist/cjs/scan/reparse.d.cts +2 -0
- package/dist/cjs/scan/reparse.d.ts +2 -0
- package/dist/cjs/scan/reparse.js +15 -76
- package/dist/cjs/scan/reparse.js.map +1 -1
- package/dist/cjs/store/builder.d.cts +19 -0
- package/dist/cjs/store/builder.d.ts +19 -0
- package/dist/cjs/store/builder.js +618 -0
- package/dist/cjs/store/builder.js.map +1 -0
- package/dist/cjs/store/duckdb/lexical.d.cts +1 -0
- package/dist/cjs/store/duckdb/lexical.d.ts +1 -0
- package/dist/cjs/store/duckdb/lexical.js +21 -5
- package/dist/cjs/store/duckdb/lexical.js.map +1 -1
- package/dist/cjs/store/duckdb/open.d.cts +9 -8
- package/dist/cjs/store/duckdb/open.d.ts +9 -8
- package/dist/cjs/store/duckdb/open.js +77 -123
- package/dist/cjs/store/duckdb/open.js.map +1 -1
- package/dist/cjs/store/duckdb/reconcile.js +34 -5
- package/dist/cjs/store/duckdb/reconcile.js.map +1 -1
- package/dist/cjs/store/duckdb/store.d.cts +1 -2
- package/dist/cjs/store/duckdb/store.d.ts +1 -2
- package/dist/cjs/store/duckdb/store.js +1 -26
- package/dist/cjs/store/duckdb/store.js.map +1 -1
- package/dist/cjs/store/embed-scope.d.cts +2 -0
- package/dist/cjs/store/embed-scope.d.ts +2 -0
- package/dist/cjs/store/embed-scope.js +85 -0
- package/dist/cjs/store/embed-scope.js.map +1 -0
- package/dist/cjs/store/feature-scope.d.cts +8 -0
- package/dist/cjs/store/feature-scope.d.ts +8 -0
- package/dist/cjs/store/feature-scope.js +107 -0
- package/dist/cjs/store/feature-scope.js.map +1 -0
- package/dist/cjs/store/index.d.cts +8 -3
- package/dist/cjs/store/index.d.ts +8 -3
- package/dist/cjs/store/index.js +36 -7
- package/dist/cjs/store/index.js.map +1 -1
- package/dist/cjs/store/open.d.cts +15 -0
- package/dist/cjs/store/open.d.ts +15 -0
- package/dist/cjs/store/open.js +608 -0
- package/dist/cjs/store/open.js.map +1 -0
- package/dist/cjs/store/preset-scope.d.cts +3 -0
- package/dist/cjs/store/preset-scope.d.ts +3 -0
- package/dist/cjs/store/preset-scope.js +293 -0
- package/dist/cjs/store/preset-scope.js.map +1 -0
- package/dist/cjs/store/reconcile.d.cts +2 -1
- package/dist/cjs/store/reconcile.d.ts +2 -1
- package/dist/cjs/store/reconcile.js +175 -179
- package/dist/cjs/store/reconcile.js.map +1 -1
- package/dist/cjs/store/{sqlite/signature.d.cts → signature.d.cts} +0 -3
- package/dist/cjs/store/{sqlite/signature.d.ts → signature.d.ts} +0 -3
- package/dist/cjs/store/signature.js +164 -0
- package/dist/cjs/store/signature.js.map +1 -0
- package/dist/cjs/store/sqlite/lexical.d.cts +1 -1
- package/dist/cjs/store/sqlite/lexical.d.ts +1 -1
- package/dist/cjs/store/sqlite/lexical.js +2 -2
- package/dist/cjs/store/sqlite/lexical.js.map +1 -1
- package/dist/cjs/store/sqlite/open.d.cts +8 -9
- package/dist/cjs/store/sqlite/open.d.ts +8 -9
- package/dist/cjs/store/sqlite/open.js +82 -311
- package/dist/cjs/store/sqlite/open.js.map +1 -1
- package/dist/cjs/store/sqlite/reconcile.d.cts +1 -1
- package/dist/cjs/store/sqlite/reconcile.d.ts +1 -1
- package/dist/cjs/store/sqlite/reconcile.js +78 -16
- package/dist/cjs/store/sqlite/reconcile.js.map +1 -1
- package/dist/cjs/store/sqlite/sql-functions.d.cts +1 -1
- package/dist/cjs/store/sqlite/sql-functions.d.ts +1 -1
- package/dist/cjs/store/sqlite/sql-functions.js +3 -3
- package/dist/cjs/store/sqlite/sql-functions.js.map +1 -1
- package/dist/cjs/store/sqlite/store.d.cts +1 -2
- package/dist/cjs/store/sqlite/store.d.ts +1 -2
- package/dist/cjs/store/sqlite/store.js +2 -15
- package/dist/cjs/store/sqlite/store.js.map +1 -1
- package/dist/cjs/store/turso/open.d.cts +6 -11
- package/dist/cjs/store/turso/open.d.ts +6 -11
- package/dist/cjs/store/turso/open.js +85 -171
- package/dist/cjs/store/turso/open.js.map +1 -1
- package/dist/cjs/store/turso/reconcile.d.cts +2 -0
- package/dist/cjs/store/turso/reconcile.d.ts +2 -0
- package/dist/cjs/store/turso/reconcile.js +88 -8
- package/dist/cjs/store/turso/reconcile.js.map +1 -1
- package/dist/cjs/store/turso/store.d.cts +1 -2
- package/dist/cjs/store/turso/store.d.ts +1 -2
- package/dist/cjs/store/turso/store.js +1 -13
- package/dist/cjs/store/turso/store.js.map +1 -1
- package/dist/cjs/store/types.d.cts +16 -6
- package/dist/cjs/store/types.d.ts +16 -6
- package/dist/cjs/store/types.js.map +1 -1
- package/dist/cjs/text/strip.js +3 -6
- package/dist/cjs/text/strip.js.map +1 -1
- package/dist/cjs/watch.js +28 -16
- package/dist/cjs/watch.js.map +1 -1
- package/dist/esm/chunk/extract.d.ts +1 -0
- package/dist/esm/chunk/extract.js +10 -2
- package/dist/esm/chunk/extract.js.map +1 -1
- package/dist/esm/chunk/index.d.ts +1 -1
- package/dist/esm/chunk/index.js +1 -1
- package/dist/esm/chunk/index.js.map +1 -1
- package/dist/esm/config/access.d.ts +0 -1
- package/dist/esm/config/access.js +12 -17
- package/dist/esm/config/access.js.map +1 -1
- package/dist/esm/config/index.d.ts +1 -1
- package/dist/esm/config/index.js +1 -1
- package/dist/esm/config/index.js.map +1 -1
- package/dist/esm/config/types.d.ts +0 -3
- package/dist/esm/config/types.js.map +1 -1
- package/dist/esm/config/validate.js +6 -22
- package/dist/esm/config/validate.js.map +1 -1
- package/dist/esm/errors.d.ts +1 -1
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/features/links.js +3 -4
- package/dist/esm/features/links.js.map +1 -1
- package/dist/esm/features/sections.js +4 -1
- package/dist/esm/features/sections.js.map +1 -1
- package/dist/esm/scan/pool.d.ts +15 -0
- package/dist/esm/scan/pool.js +77 -0
- package/dist/esm/scan/pool.js.map +1 -0
- package/dist/esm/scan/reparse.d.ts +2 -0
- package/dist/esm/scan/reparse.js +11 -58
- package/dist/esm/scan/reparse.js.map +1 -1
- package/dist/esm/store/builder.d.ts +19 -0
- package/dist/esm/store/builder.js +163 -0
- package/dist/esm/store/builder.js.map +1 -0
- package/dist/esm/store/duckdb/lexical.d.ts +1 -0
- package/dist/esm/store/duckdb/lexical.js +22 -9
- package/dist/esm/store/duckdb/lexical.js.map +1 -1
- package/dist/esm/store/duckdb/open.d.ts +9 -8
- package/dist/esm/store/duckdb/open.js +36 -58
- package/dist/esm/store/duckdb/open.js.map +1 -1
- package/dist/esm/store/duckdb/reconcile.js +15 -3
- package/dist/esm/store/duckdb/reconcile.js.map +1 -1
- package/dist/esm/store/duckdb/store.d.ts +1 -2
- package/dist/esm/store/duckdb/store.js +2 -11
- package/dist/esm/store/duckdb/store.js.map +1 -1
- package/dist/esm/store/embed-scope.d.ts +2 -0
- package/dist/esm/store/embed-scope.js +28 -0
- package/dist/esm/store/embed-scope.js.map +1 -0
- package/dist/esm/store/feature-scope.d.ts +8 -0
- package/dist/esm/store/feature-scope.js +49 -0
- package/dist/esm/store/feature-scope.js.map +1 -0
- package/dist/esm/store/index.d.ts +8 -3
- package/dist/esm/store/index.js +22 -7
- package/dist/esm/store/index.js.map +1 -1
- package/dist/esm/store/open.d.ts +15 -0
- package/dist/esm/store/open.js +187 -0
- package/dist/esm/store/open.js.map +1 -0
- package/dist/esm/store/preset-scope.d.ts +3 -0
- package/dist/esm/store/preset-scope.js +77 -0
- package/dist/esm/store/preset-scope.js.map +1 -0
- package/dist/esm/store/reconcile.d.ts +2 -1
- package/dist/esm/store/reconcile.js +33 -6
- package/dist/esm/store/reconcile.js.map +1 -1
- package/dist/esm/store/{sqlite/signature.d.ts → signature.d.ts} +0 -3
- package/dist/esm/store/signature.js +37 -0
- package/dist/esm/store/signature.js.map +1 -0
- package/dist/esm/store/sqlite/lexical.d.ts +1 -1
- package/dist/esm/store/sqlite/lexical.js +4 -4
- package/dist/esm/store/sqlite/lexical.js.map +1 -1
- package/dist/esm/store/sqlite/open.d.ts +8 -9
- package/dist/esm/store/sqlite/open.js +41 -147
- package/dist/esm/store/sqlite/open.js.map +1 -1
- package/dist/esm/store/sqlite/reconcile.d.ts +1 -1
- package/dist/esm/store/sqlite/reconcile.js +14 -14
- package/dist/esm/store/sqlite/reconcile.js.map +1 -1
- package/dist/esm/store/sqlite/sql-functions.d.ts +1 -1
- package/dist/esm/store/sqlite/sql-functions.js +3 -3
- package/dist/esm/store/sqlite/sql-functions.js.map +1 -1
- package/dist/esm/store/sqlite/store.d.ts +1 -2
- package/dist/esm/store/sqlite/store.js +3 -9
- package/dist/esm/store/sqlite/store.js.map +1 -1
- package/dist/esm/store/turso/open.d.ts +6 -11
- package/dist/esm/store/turso/open.js +34 -78
- package/dist/esm/store/turso/open.js.map +1 -1
- package/dist/esm/store/turso/reconcile.d.ts +2 -0
- package/dist/esm/store/turso/reconcile.js +18 -3
- package/dist/esm/store/turso/reconcile.js.map +1 -1
- package/dist/esm/store/turso/store.d.ts +1 -2
- package/dist/esm/store/turso/store.js +2 -7
- package/dist/esm/store/turso/store.js.map +1 -1
- package/dist/esm/store/types.d.ts +16 -6
- package/dist/esm/store/types.js.map +1 -1
- package/dist/esm/text/strip.js +3 -4
- package/dist/esm/text/strip.js.map +1 -1
- package/dist/esm/watch.js +5 -3
- package/dist/esm/watch.js.map +1 -1
- package/package.json +4 -3
- package/schema.json +0 -11
- package/dist/cjs/store/sqlite/signature.js +0 -380
- package/dist/cjs/store/sqlite/signature.js.map +0 -1
- package/dist/esm/store/sqlite/signature.js +0 -58
- package/dist/esm/store/sqlite/signature.js.map +0 -1
|
@@ -14,6 +14,9 @@ _export(exports, {
|
|
|
14
14
|
get createLexicalIndex () {
|
|
15
15
|
return createLexicalIndex;
|
|
16
16
|
},
|
|
17
|
+
get markContentStale () {
|
|
18
|
+
return markContentStale;
|
|
19
|
+
},
|
|
17
20
|
get queryLexical () {
|
|
18
21
|
return queryLexical;
|
|
19
22
|
}
|
|
@@ -237,6 +240,22 @@ function unsupportedOperator(terms) {
|
|
|
237
240
|
}
|
|
238
241
|
return null;
|
|
239
242
|
}
|
|
243
|
+
// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it
|
|
244
|
+
// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.
|
|
245
|
+
var ftsState = new WeakMap();
|
|
246
|
+
function stateFor(conn) {
|
|
247
|
+
var state = ftsState.get(conn);
|
|
248
|
+
if (!state) {
|
|
249
|
+
state = {
|
|
250
|
+
stale: true
|
|
251
|
+
};
|
|
252
|
+
ftsState.set(conn, state);
|
|
253
|
+
}
|
|
254
|
+
return state;
|
|
255
|
+
}
|
|
256
|
+
function markContentStale(conn) {
|
|
257
|
+
stateFor(conn).stale = true;
|
|
258
|
+
}
|
|
240
259
|
// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token
|
|
241
260
|
// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.
|
|
242
261
|
function splitTerms(terms) {
|
|
@@ -480,15 +499,12 @@ function queryLexical(conn, terms, opts, state) {
|
|
|
480
499
|
})();
|
|
481
500
|
}
|
|
482
501
|
function createLexicalIndex(conn) {
|
|
483
|
-
var state = {
|
|
484
|
-
stale: true
|
|
485
|
-
};
|
|
486
502
|
return {
|
|
487
503
|
query: function query(terms, opts) {
|
|
488
|
-
return queryLexical(conn, terms, opts,
|
|
504
|
+
return queryLexical(conn, terms, opts, stateFor(conn));
|
|
489
505
|
},
|
|
490
506
|
markStale: function markStale() {
|
|
491
|
-
|
|
507
|
+
return markContentStale(conn);
|
|
492
508
|
}
|
|
493
509
|
};
|
|
494
510
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n for (const { label, re } of FTS5_OPERATORS) {\n const m = withoutPhrases.match(re);\n if (m) return { label, token: m[0].trim() };\n }\n return null;\n}\n\ninterface FtsIndexState {\n stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n const words: string[] = [];\n const substrings: string[] = [];\n const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n const phrase = inner.trim();\n if (phrase.length === 0) return ' ';\n substrings.push(phrase);\n if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n return ' ';\n });\n for (const tok of withoutPhrases.split(/\\s+/)) {\n if (tok.length === 0) continue;\n if (hasUnspacedRun(tok)) substrings.push(tok);\n else words.push(tok);\n }\n return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n if (!state.stale) return;\n await conn.exec('INSTALL fts; LOAD fts;');\n await withTransaction(conn, async () => {\n // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n });\n state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n const scoreParts: string[] = [];\n const gateParts: string[] = [];\n const scoreParams: unknown[] = [];\n const gateParams: unknown[] = [];\n\n if (words.length > 0) {\n const wordQuery = words.join(' ');\n for (const field of FIELDS) {\n scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n scoreParams.push(wordQuery);\n }\n gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n gateParams.push(wordQuery);\n }\n\n for (const raw of substrings) {\n const needle = raw.toLowerCase();\n for (const field of FIELDS) {\n scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n scoreParams.push(needle);\n }\n gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n gateParams.push(needle);\n }\n\n return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n const unsupported = unsupportedOperator(terms);\n if (unsupported !== null) {\n throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n }\n const { whereJoin, whereCond, scopeCond, limit } = opts;\n const { words, substrings } = splitTerms(terms);\n if (words.length === 0 && substrings.length === 0) return [];\n\n if (words.length > 0) await ensureFtsFresh(conn, state);\n\n const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n const sql = `SELECT path, NULL AS hit FROM (\n SELECT content.\"path\" AS path, (${scoreSql}) AS score\n FROM content\n ${whereJoin}\n WHERE ${gateSql}\n ${whereCond} ${scopeCond}\n ) sq ORDER BY score DESC, path LIMIT ?`;\n const stmt = await conn.prepare(sql);\n return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches `content`),\n// and store.ts's reconcile wrapper flips it back to true whenever content changes, so the next query rebuilds first.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n const state: FtsIndexState = { stale: true };\n return {\n query: (terms, opts) => queryLexical(conn, terms, opts, state),\n markStale: () => {\n state.stale = true;\n },\n };\n}\n"],"names":["createLexicalIndex","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","hasUnspacedRun","split","w","tok","ensureFtsFresh","conn","state","stale","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","sql","SenseError","prepare","all","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QA8H9CA;eAAAA;;QAzBMC;eAAAA;;;wBApGK;yBACI;6BACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAMA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASC,WAAWP,KAAa;IAC/B,IAAMQ,QAAkB,EAAE;IAC1B,IAAMC,aAAuB,EAAE;IAC/B,IAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACQ,IAAIC;QACtD,IAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAACG,IAAAA,yBAAc,EAACH,SAAS;gBAAK,kCAAA,2BAAA;;gBAAL,QAAK,YAAWA,OAAOI,KAAK,CAAC,2BAAxB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,IAAN;oBAAgC,IAAIA,EAAEJ,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACG;;;gBAA7D;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAA+D;QACjG,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAahB,eAAee,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAME,MAAN;YACH,IAAIA,IAAIL,MAAM,KAAK,GAAG;YACtB,IAAIE,IAAAA,yBAAc,EAACG,MAAMT,WAAWK,IAAI,CAACI;iBACpCV,MAAMM,IAAI,CAACI;QAClB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAO;QAAEV,OAAAA;QAAOC,YAAAA;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,SAAeU,eAAeC,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;;;oBAClB;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAACJ,MAAM;;;;;4CAC1B,6HAA6H;4CAC7H,0JAA0J;4CAC1J;;gDAAMA,KAAKG,IAAI,CAAC;;;4CAAhB;;;;;;4BACF;;;;oBAJA;oBAKAF,MAAMC,KAAK,GAAG;;;;;;IAChB;;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASG,kBAAkBjB,KAAe,EAAEC,UAAoB;IAC9D,IAAMiB,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAIrB,MAAMK,MAAM,GAAG,GAAG;QACpB,IAAMiB,YAAYtB,MAAMuB,IAAI,CAAC;YACxB,kCAAA,2BAAA;;YAAL,QAAK,YAAexC,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAMyC,QAAN;gBACHN,WAAWZ,IAAI,CAAC,AAAC,GAAgGkB,OAA9FxC,YAAY,CAACwC,MAAM,EAAC,6EAAgF,OAANA,OAAM;gBACvHJ,YAAYd,IAAI,CAACgB;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAILH,UAAUb,IAAI,CAAC;QACfe,WAAWf,IAAI,CAACgB;IAClB;QAEK,mCAAA,4BAAA;;QAAL,QAAK,aAAarB,+BAAb,UAAA,8BAAA,SAAA,0BAAA,kCAAyB;YAAzB,IAAMwB,MAAN;YACH,IAAMC,SAASD,IAAIE,WAAW;gBACzB,mCAAA,4BAAA;;gBAAL,QAAK,aAAe5C,2BAAf,UAAA,8BAAA,SAAA,0BAAA,kCAAuB;oBAAvB,IAAMyC,SAAN;oBACHN,WAAWZ,IAAI,CAAC,AAAC,GAAyDkB,OAAvDxC,YAAY,CAACwC,OAAM,EAAC,qCAAyC,OAANA,QAAM;oBAChFJ,YAAYd,IAAI,CAACoB;gBACnB;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILP,UAAUb,IAAI,CAAC;YACfe,WAAWf,IAAI,CAACoB;QAClB;;QARK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ,AAAC,qBAAGV,oBAAa,qBAAGC;IAAY;AACvH;AAIO,SAAevC,aAAa8B,IAAgB,EAAEpB,KAAa,EAAEuC,IAAyB,EAAElB,KAAoB;;YAoBnGmB,OAnBRC,aAIEC,WAAWC,WAAWC,WAAWC,OACXtC,aAAtBC,OAAOC,YAKuBgB,oBAA9BW,UAAUC,SAASC,QACrBQ,KAOAN;;;;oBAlBAC,cAAc1C,oBAAoBC;oBACxC,IAAIyC,gBAAgB,MAAM;wBACxB,MAAM,IAAIM,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEN,OAAvBA,YAAY5C,KAAK,EAAC,OAAqDG,OAAhDyC,YAAYpC,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQoC,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACXtC,cAAAA,WAAWP,QAAjCQ,QAAsBD,YAAtBC,OAAOC,aAAeF,YAAfE;oBACf,IAAID,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG;;;;yBAE/CL,CAAAA,MAAMK,MAAM,GAAG,CAAA,GAAfL;;;;oBAAkB;;wBAAMW,eAAeC,MAAMC;;;oBAA3B;;;oBAEgBI,qBAAAA,kBAAkBjB,OAAOC,aAAvD2B,WAA8BX,mBAA9BW,UAAUC,UAAoBZ,mBAApBY,SAASC,SAAWb,mBAAXa;oBACrBQ,MAAM,AAAC,wEAGTJ,OAFgCN,UAAS,sCAGnCC,OADNK,WAAU,gBAEVC,OADMN,SAAQ,UACDO,OAAbD,WAAU,KAAa,OAAVC,WAAU;oBAEd;;wBAAMxB,KAAK4B,OAAO,CAACF;;;oBAA1BN,OAAO;oBACL;;wBAAMA,CAAAA,QAAAA,MAAKS,GAAG,OAART,OAAAA,AAAS,qBAAGF;4BAAQO;;;;oBAAlC;;wBAAQ;;;;IACV;;AAIO,SAASxD,mBAAmB+B,IAAgB;IACjD,IAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL4B,OAAO,SAAPA,MAAQlD,OAAOuC;mBAASjD,aAAa8B,MAAMpB,OAAOuC,MAAMlB;;QACxD8B,WAAW,SAAXA;YACE9B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n for (const { label, re } of FTS5_OPERATORS) {\n const m = withoutPhrases.match(re);\n if (m) return { label, token: m[0].trim() };\n }\n return null;\n}\n\ninterface FtsIndexState {\n stale: boolean;\n}\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nfunction stateFor(conn: Connection): FtsIndexState {\n let state = ftsState.get(conn);\n if (!state) {\n state = { stale: true };\n ftsState.set(conn, state);\n }\n return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`, so the next lexical query\n// rebuilds instead of searching stale content.\nexport function markContentStale(conn: Connection): void {\n stateFor(conn).stale = true;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n const words: string[] = [];\n const substrings: string[] = [];\n const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n const phrase = inner.trim();\n if (phrase.length === 0) return ' ';\n substrings.push(phrase);\n if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n return ' ';\n });\n for (const tok of withoutPhrases.split(/\\s+/)) {\n if (tok.length === 0) continue;\n if (hasUnspacedRun(tok)) substrings.push(tok);\n else words.push(tok);\n }\n return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n if (!state.stale) return;\n await conn.exec('INSTALL fts; LOAD fts;');\n await withTransaction(conn, async () => {\n // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n });\n state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n const scoreParts: string[] = [];\n const gateParts: string[] = [];\n const scoreParams: unknown[] = [];\n const gateParams: unknown[] = [];\n\n if (words.length > 0) {\n const wordQuery = words.join(' ');\n for (const field of FIELDS) {\n scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n scoreParams.push(wordQuery);\n }\n gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n gateParams.push(wordQuery);\n }\n\n for (const raw of substrings) {\n const needle = raw.toLowerCase();\n for (const field of FIELDS) {\n scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n scoreParams.push(needle);\n }\n gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n gateParams.push(needle);\n }\n\n return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n const unsupported = unsupportedOperator(terms);\n if (unsupported !== null) {\n throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n }\n const { whereJoin, whereCond, scopeCond, limit } = opts;\n const { words, substrings } = splitTerms(terms);\n if (words.length === 0 && substrings.length === 0) return [];\n\n if (words.length > 0) await ensureFtsFresh(conn, state);\n\n const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n const sql = `SELECT path, NULL AS hit FROM (\n SELECT content.\"path\" AS path, (${scoreSql}) AS score\n FROM content\n ${whereJoin}\n WHERE ${gateSql}\n ${whereCond} ${scopeCond}\n ) sq ORDER BY score DESC, path LIMIT ?`;\n const stmt = await conn.prepare(sql);\n return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches\n// `content`), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n return {\n query: (terms, opts) => queryLexical(conn, terms, opts, stateFor(conn)),\n markStale: () => markContentStale(conn),\n };\n}\n"],"names":["createLexicalIndex","markContentStale","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","ftsState","WeakMap","stateFor","conn","state","get","stale","set","splitTerms","words","substrings","_m","inner","phrase","length","push","hasUnspacedRun","split","w","tok","ensureFtsFresh","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","sql","SenseError","prepare","all","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QAgJ9CA;eAAAA;;QA/FAC;eAAAA;;QAsEMC;eAAAA;;;wBAtHK;yBACI;6BACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAMA,6FAA6F;AAC7F,0GAA0G;AAC1G,IAAMC,WAAW,IAAIC;AACrB,SAASC,SAASC,IAAgB;IAChC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACVA,QAAQ;YAAEE,OAAO;QAAK;QACtBN,SAASO,GAAG,CAACJ,MAAMC;IACrB;IACA,OAAOA;AACT;AAIO,SAAStB,iBAAiBqB,IAAgB;IAC/CD,SAASC,MAAMG,KAAK,GAAG;AACzB;AAEA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASE,WAAWf,KAAa;IAC/B,IAAMgB,QAAkB,EAAE;IAC1B,IAAMC,aAAuB,EAAE;IAC/B,IAAMhB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACgB,IAAIC;QACtD,IAAMC,SAASD,MAAMb,IAAI;QACzB,IAAIc,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAACG,IAAAA,yBAAc,EAACH,SAAS;gBAAK,kCAAA,2BAAA;;gBAAL,QAAK,YAAWA,OAAOI,KAAK,CAAC,2BAAxB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,IAAN;oBAAgC,IAAIA,EAAEJ,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACG;;;gBAA7D;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAA+D;QACjG,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAaxB,eAAeuB,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAME,MAAN;YACH,IAAIA,IAAIL,MAAM,KAAK,GAAG;YACtB,IAAIE,IAAAA,yBAAc,EAACG,MAAMT,WAAWK,IAAI,CAACI;iBACpCV,MAAMM,IAAI,CAACI;QAClB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAO;QAAEV,OAAAA;QAAOC,YAAAA;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,SAAeU,eAAejB,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAME,KAAK,EAAE;;;oBAClB;;wBAAMH,KAAKkB,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAACnB,MAAM;;;;;4CAC1B,6HAA6H;4CAC7H,0JAA0J;4CAC1J;;gDAAMA,KAAKkB,IAAI,CAAC;;;4CAAhB;;;;;;4BACF;;;;oBAJA;oBAKAjB,MAAME,KAAK,GAAG;;;;;;IAChB;;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASiB,kBAAkBd,KAAe,EAAEC,UAAoB;IAC9D,IAAMc,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAIlB,MAAMK,MAAM,GAAG,GAAG;QACpB,IAAMc,YAAYnB,MAAMoB,IAAI,CAAC;YACxB,kCAAA,2BAAA;;YAAL,QAAK,YAAe7C,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAM8C,QAAN;gBACHN,WAAWT,IAAI,CAAC,AAAC,GAAgGe,OAA9F7C,YAAY,CAAC6C,MAAM,EAAC,6EAAgF,OAANA,OAAM;gBACvHJ,YAAYX,IAAI,CAACa;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAILH,UAAUV,IAAI,CAAC;QACfY,WAAWZ,IAAI,CAACa;IAClB;QAEK,mCAAA,4BAAA;;QAAL,QAAK,aAAalB,+BAAb,UAAA,8BAAA,SAAA,0BAAA,kCAAyB;YAAzB,IAAMqB,MAAN;YACH,IAAMC,SAASD,IAAIE,WAAW;gBACzB,mCAAA,4BAAA;;gBAAL,QAAK,aAAejD,2BAAf,UAAA,8BAAA,SAAA,0BAAA,kCAAuB;oBAAvB,IAAM8C,SAAN;oBACHN,WAAWT,IAAI,CAAC,AAAC,GAAyDe,OAAvD7C,YAAY,CAAC6C,OAAM,EAAC,qCAAyC,OAANA,QAAM;oBAChFJ,YAAYX,IAAI,CAACiB;gBACnB;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILP,UAAUV,IAAI,CAAC;YACfY,WAAWZ,IAAI,CAACiB;QAClB;;QARK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ,AAAC,qBAAGV,oBAAa,qBAAGC;IAAY;AACvH;AAIO,SAAe5C,aAAaoB,IAAgB,EAAEV,KAAa,EAAE4C,IAAyB,EAAEjC,KAAoB;;YAoBnGkC,OAnBRC,aAIEC,WAAWC,WAAWC,WAAWC,OACXnC,aAAtBC,OAAOC,YAKuBa,oBAA9BW,UAAUC,SAASC,QACrBQ,KAOAN;;;;oBAlBAC,cAAc/C,oBAAoBC;oBACxC,IAAI8C,gBAAgB,MAAM;wBACxB,MAAM,IAAIM,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEN,OAAvBA,YAAYjD,KAAK,EAAC,OAAqDG,OAAhD8C,YAAYzC,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQyC,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACXnC,cAAAA,WAAWf,QAAjCgB,QAAsBD,YAAtBC,OAAOC,aAAeF,YAAfE;oBACf,IAAID,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG;;;;yBAE/CL,CAAAA,MAAMK,MAAM,GAAG,CAAA,GAAfL;;;;oBAAkB;;wBAAMW,eAAejB,MAAMC;;;oBAA3B;;;oBAEgBmB,qBAAAA,kBAAkBd,OAAOC,aAAvDwB,WAA8BX,mBAA9BW,UAAUC,UAAoBZ,mBAApBY,SAASC,SAAWb,mBAAXa;oBACrBQ,MAAM,AAAC,wEAGTJ,OAFgCN,UAAS,sCAGnCC,OADNK,WAAU,gBAEVC,OADMN,SAAQ,UACDO,OAAbD,WAAU,KAAa,OAAVC,WAAU;oBAEd;;wBAAMvC,KAAK2C,OAAO,CAACF;;;oBAA1BN,OAAO;oBACL;;wBAAMA,CAAAA,QAAAA,MAAKS,GAAG,OAART,OAAAA,AAAS,qBAAGF;4BAAQO;;;;oBAAlC;;wBAAQ;;;;IACV;;AAIO,SAAS9D,mBAAmBsB,IAAgB;IACjD,OAAO;QACL6C,OAAO,SAAPA,MAAQvD,OAAO4C;mBAAStD,aAAaoB,MAAMV,OAAO4C,MAAMnC,SAASC;;QACjE8C,WAAW,SAAXA;mBAAiBnE,iBAAiBqB;;IACpC;AACF"}
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
+
import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
|
|
1
2
|
import type { ResolvedConfig } from '../../config/index.js';
|
|
2
|
-
import type {
|
|
3
|
+
import type { OpenResult } from '../open.js';
|
|
4
|
+
import type { OpenDialect } from '../types.js';
|
|
3
5
|
export declare const DB_FILENAME = "cache.duckdb";
|
|
4
|
-
export declare const SCHEMA_VERSION = "
|
|
5
|
-
export
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
parsed: number;
|
|
10
|
-
warnings: string[];
|
|
6
|
+
export declare const SCHEMA_VERSION = "3";
|
|
7
|
+
export type { OpenResult };
|
|
8
|
+
interface DuckdbHandle {
|
|
9
|
+
instance: DuckDBInstance;
|
|
10
|
+
duckdb: DuckDBConnection;
|
|
11
11
|
}
|
|
12
|
+
export declare const duckdbOpenDialect: OpenDialect<DuckdbHandle>;
|
|
12
13
|
export declare function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult>;
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
+
import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
|
|
1
2
|
import type { ResolvedConfig } from '../../config/index.js';
|
|
2
|
-
import type {
|
|
3
|
+
import type { OpenResult } from '../open.js';
|
|
4
|
+
import type { OpenDialect } from '../types.js';
|
|
3
5
|
export declare const DB_FILENAME = "cache.duckdb";
|
|
4
|
-
export declare const SCHEMA_VERSION = "
|
|
5
|
-
export
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
parsed: number;
|
|
10
|
-
warnings: string[];
|
|
6
|
+
export declare const SCHEMA_VERSION = "3";
|
|
7
|
+
export type { OpenResult };
|
|
8
|
+
interface DuckdbHandle {
|
|
9
|
+
instance: DuckDBInstance;
|
|
10
|
+
duckdb: DuckDBConnection;
|
|
11
11
|
}
|
|
12
|
+
export declare const duckdbOpenDialect: OpenDialect<DuckdbHandle>;
|
|
12
13
|
export declare function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult>;
|
|
@@ -15,23 +15,22 @@ _export(exports, {
|
|
|
15
15
|
get SCHEMA_VERSION () {
|
|
16
16
|
return SCHEMA_VERSION;
|
|
17
17
|
},
|
|
18
|
+
get duckdbOpenDialect () {
|
|
19
|
+
return duckdbOpenDialect;
|
|
20
|
+
},
|
|
18
21
|
get openDuckdb () {
|
|
19
22
|
return openDuckdb;
|
|
20
23
|
}
|
|
21
24
|
});
|
|
22
|
-
var _nodefs = require("node:fs");
|
|
23
|
-
var _nodepath = require("node:path");
|
|
24
25
|
var _indexts = require("../../config/index.js");
|
|
25
|
-
var _handoffts = require("../../embed/handoff.js");
|
|
26
26
|
var _typests = require("../../embed/types.js");
|
|
27
27
|
var _errorsts = require("../../errors.js");
|
|
28
28
|
var _indexts1 = require("../../features/index.js");
|
|
29
|
-
var
|
|
30
|
-
var _reconcilets = require("../reconcile.js");
|
|
29
|
+
var _opents = require("../open.js");
|
|
31
30
|
var _sharedts = require("../shared.js");
|
|
32
31
|
var _connectionts = require("./connection.js");
|
|
33
32
|
var _nativets = require("./native.js");
|
|
34
|
-
var
|
|
33
|
+
var _reconcilets = require("./reconcile.js");
|
|
35
34
|
var _sqlfunctionsts = require("./sql-functions.js");
|
|
36
35
|
var _storets = require("./store.js");
|
|
37
36
|
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
|
|
@@ -166,10 +165,10 @@ function _ts_generator(thisArg, body) {
|
|
|
166
165
|
}
|
|
167
166
|
}
|
|
168
167
|
var DB_FILENAME = 'cache.duckdb';
|
|
169
|
-
var SCHEMA_VERSION = '
|
|
168
|
+
var SCHEMA_VERSION = '3';
|
|
170
169
|
// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
|
|
171
|
-
// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer
|
|
172
|
-
function ensureSchema(conn, cfg) {
|
|
170
|
+
// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
|
|
171
|
+
function ensureSchema(_handle, conn, cfg) {
|
|
173
172
|
return _async_to_generator(function() {
|
|
174
173
|
var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, feature, err;
|
|
175
174
|
return _ts_generator(this, function(_state) {
|
|
@@ -312,45 +311,50 @@ function ensureSchema(conn, cfg) {
|
|
|
312
311
|
});
|
|
313
312
|
})();
|
|
314
313
|
}
|
|
315
|
-
|
|
314
|
+
// Order matters: the connection must be gone before the instance closes the WAL.
|
|
315
|
+
function close(handle) {
|
|
316
|
+
return _async_to_generator(function() {
|
|
317
|
+
return _ts_generator(this, function(_state) {
|
|
318
|
+
handle.duckdb.disconnectSync();
|
|
319
|
+
handle.instance.closeSync();
|
|
320
|
+
return [
|
|
321
|
+
2
|
|
322
|
+
];
|
|
323
|
+
});
|
|
324
|
+
})();
|
|
325
|
+
}
|
|
326
|
+
function connect(dbPath, _cfg) {
|
|
316
327
|
return _async_to_generator(function() {
|
|
317
|
-
var
|
|
328
|
+
var DuckDBInstance, err, duckdb, instance, err1, conn, err2;
|
|
318
329
|
return _ts_generator(this, function(_state) {
|
|
319
330
|
switch(_state.label){
|
|
320
331
|
case 0:
|
|
321
|
-
stateDir = (0, _nodepath.join)(cfg.baseDir, _indexts.STATE_DIR);
|
|
322
|
-
(0, _nodefs.mkdirSync)(stateDir, {
|
|
323
|
-
recursive: true
|
|
324
|
-
});
|
|
325
|
-
dbPath = (0, _nodepath.join)(stateDir, DB_FILENAME);
|
|
326
|
-
_state.label = 1;
|
|
327
|
-
case 1:
|
|
328
332
|
_state.trys.push([
|
|
329
|
-
|
|
330
|
-
|
|
333
|
+
0,
|
|
334
|
+
2,
|
|
331
335
|
,
|
|
332
|
-
|
|
336
|
+
3
|
|
333
337
|
]);
|
|
334
338
|
return [
|
|
335
339
|
4,
|
|
336
340
|
(0, _nativets.duckdbApi)()
|
|
337
341
|
];
|
|
338
|
-
case
|
|
342
|
+
case 1:
|
|
339
343
|
DuckDBInstance = _state.sent().DuckDBInstance;
|
|
340
344
|
return [
|
|
341
345
|
3,
|
|
342
|
-
|
|
346
|
+
3
|
|
343
347
|
];
|
|
344
|
-
case
|
|
348
|
+
case 2:
|
|
345
349
|
err = _state.sent();
|
|
346
350
|
if (_instanceof(err, _errorsts.SenseError)) throw err;
|
|
347
351
|
throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "duckdb" needs the '.concat(_nativets.DUCKDB_PACKAGE, " package (").concat(err.message, ")"));
|
|
348
|
-
case
|
|
352
|
+
case 3:
|
|
349
353
|
_state.trys.push([
|
|
350
|
-
|
|
351
|
-
|
|
354
|
+
3,
|
|
355
|
+
6,
|
|
352
356
|
,
|
|
353
|
-
|
|
357
|
+
7
|
|
354
358
|
]);
|
|
355
359
|
return [
|
|
356
360
|
4,
|
|
@@ -358,7 +362,7 @@ function connect(cfg) {
|
|
|
358
362
|
storage_compatibility_version: 'v1.5.0'
|
|
359
363
|
})
|
|
360
364
|
];
|
|
361
|
-
case
|
|
365
|
+
case 4:
|
|
362
366
|
// On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns ("VARIANT
|
|
363
367
|
// columns are not supported in storage versions prior to v1.5.0"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.
|
|
364
368
|
instance = _state.sent();
|
|
@@ -366,102 +370,54 @@ function connect(cfg) {
|
|
|
366
370
|
4,
|
|
367
371
|
instance.connect()
|
|
368
372
|
];
|
|
369
|
-
case
|
|
373
|
+
case 5:
|
|
370
374
|
duckdb = _state.sent();
|
|
371
375
|
return [
|
|
372
376
|
3,
|
|
373
|
-
|
|
377
|
+
7
|
|
374
378
|
];
|
|
375
|
-
case
|
|
379
|
+
case 6:
|
|
376
380
|
err1 = _state.sent();
|
|
377
381
|
// create() may have succeeded before connect() failed: close it, or its WAL stays open.
|
|
378
382
|
instance === null || instance === void 0 ? void 0 : instance.closeSync();
|
|
379
383
|
throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "duckdb" failed to open '.concat(dbPath, ": ").concat(err1.message));
|
|
380
|
-
case
|
|
381
|
-
// A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL
|
|
382
|
-
// then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.
|
|
383
|
-
closed = false;
|
|
384
|
-
_state.label = 9;
|
|
385
|
-
case 9:
|
|
384
|
+
case 7:
|
|
386
385
|
_state.trys.push([
|
|
386
|
+
7,
|
|
387
387
|
9,
|
|
388
|
-
16,
|
|
389
388
|
,
|
|
390
|
-
|
|
389
|
+
11
|
|
391
390
|
]);
|
|
392
391
|
return [
|
|
393
392
|
4,
|
|
394
393
|
(0, _sqlfunctionsts.registerFunctions)(duckdb)
|
|
395
394
|
];
|
|
396
|
-
case
|
|
395
|
+
case 8:
|
|
397
396
|
_state.sent();
|
|
398
397
|
conn = (0, _connectionts.createConnection)(duckdb);
|
|
399
|
-
return [
|
|
400
|
-
4,
|
|
401
|
-
conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)')
|
|
402
|
-
];
|
|
403
|
-
case 11:
|
|
404
|
-
_state.sent();
|
|
405
|
-
return [
|
|
406
|
-
4,
|
|
407
|
-
(0, _sharedts.getMeta)(conn, 'schema_version')
|
|
408
|
-
];
|
|
409
|
-
case 12:
|
|
410
|
-
version = _state.sent();
|
|
411
|
-
return [
|
|
412
|
-
4,
|
|
413
|
-
(0, _sharedts.getMeta)(conn, 'features')
|
|
414
|
-
];
|
|
415
|
-
case 13:
|
|
416
|
-
features = _state.sent();
|
|
417
|
-
wantFeatures = (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES);
|
|
418
|
-
if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
|
|
419
|
-
// Reconcile only reparses changed files, so a stale cache can't be patched incrementally --
|
|
420
|
-
// rebuild instead, same as sqlite's open() (see its comment for why this is announced).
|
|
421
|
-
reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
|
|
422
|
-
console.error("sense: ".concat(reason, "; rebuilding the index"));
|
|
423
|
-
// Close before clearCache deletes the files underneath the still-open instance.
|
|
424
|
-
closed = true;
|
|
425
|
-
duckdb.disconnectSync();
|
|
426
|
-
instance.closeSync();
|
|
427
|
-
(0, _cachets.clearCache)(cfg);
|
|
428
|
-
return [
|
|
429
|
-
2,
|
|
430
|
-
connect(cfg)
|
|
431
|
-
];
|
|
432
|
-
}
|
|
433
|
-
return [
|
|
434
|
-
4,
|
|
435
|
-
ensureSchema(conn, cfg)
|
|
436
|
-
];
|
|
437
|
-
case 14:
|
|
438
|
-
_state.sent();
|
|
439
|
-
return [
|
|
440
|
-
4,
|
|
441
|
-
(0, _reconcilets.reconcile)(conn, cfg, cfg.baseDir, _reconcilets1.duckdbDialect)
|
|
442
|
-
];
|
|
443
|
-
case 15:
|
|
444
|
-
_ref = _state.sent(), parsed = _ref.parsed, warnings = _ref.warnings;
|
|
445
398
|
return [
|
|
446
399
|
2,
|
|
447
400
|
{
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
parsed: parsed,
|
|
454
|
-
warnings: warnings
|
|
401
|
+
handle: {
|
|
402
|
+
instance: instance,
|
|
403
|
+
duckdb: duckdb
|
|
404
|
+
},
|
|
405
|
+
conn: conn
|
|
455
406
|
}
|
|
456
407
|
];
|
|
457
|
-
case
|
|
408
|
+
case 9:
|
|
458
409
|
err2 = _state.sent();
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
410
|
+
return [
|
|
411
|
+
4,
|
|
412
|
+
close({
|
|
413
|
+
instance: instance,
|
|
414
|
+
duckdb: duckdb
|
|
415
|
+
})
|
|
416
|
+
];
|
|
417
|
+
case 10:
|
|
418
|
+
_state.sent();
|
|
463
419
|
throw err2;
|
|
464
|
-
case
|
|
420
|
+
case 11:
|
|
465
421
|
return [
|
|
466
422
|
2
|
|
467
423
|
];
|
|
@@ -469,32 +425,30 @@ function connect(cfg) {
|
|
|
469
425
|
});
|
|
470
426
|
})();
|
|
471
427
|
}
|
|
428
|
+
var duckdbOpenDialect = {
|
|
429
|
+
filename: DB_FILENAME,
|
|
430
|
+
schemaVersion: SCHEMA_VERSION,
|
|
431
|
+
reconcileDialect: _reconcilets.duckdbDialect,
|
|
432
|
+
connect: connect,
|
|
433
|
+
close: close,
|
|
434
|
+
// duckdb's file lock spans the connection's life, and each platform words the refusal its own
|
|
435
|
+
// way: posix "Could not set lock on file ... Conflicting lock is held in <exe> (PID n)", Windows
|
|
436
|
+
// "Cannot open file ... being used by another process". Both are the same condition.
|
|
437
|
+
isLocked: function isLocked(err) {
|
|
438
|
+
return /Could not set lock on file|being used by another process/.test(err.message);
|
|
439
|
+
},
|
|
440
|
+
ensureSchema: ensureSchema,
|
|
441
|
+
createStore: function createStore(handle, conn) {
|
|
442
|
+
return (0, _storets.createStore)(handle.instance, handle.duckdb, conn);
|
|
443
|
+
}
|
|
444
|
+
};
|
|
472
445
|
function openDuckdb(cfg) {
|
|
473
446
|
return _async_to_generator(function() {
|
|
474
|
-
var _ref, instance, duckdb, conn, resolvedCfg, dbPath, parsed, warnings, store;
|
|
475
447
|
return _ts_generator(this, function(_state) {
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
connect(cfg)
|
|
481
|
-
];
|
|
482
|
-
case 1:
|
|
483
|
-
_ref = _state.sent(), instance = _ref.instance, duckdb = _ref.duckdb, conn = _ref.conn, resolvedCfg = _ref.cfg, dbPath = _ref.dbPath, parsed = _ref.parsed, warnings = _ref.warnings;
|
|
484
|
-
store = (0, _storets.createStore)(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);
|
|
485
|
-
// reconcile ran before this object existed, so its chunk text is keyed by the connection.
|
|
486
|
-
(0, _handoffts.rekeyChunkText)(conn, store);
|
|
487
|
-
return [
|
|
488
|
-
2,
|
|
489
|
-
{
|
|
490
|
-
store: store,
|
|
491
|
-
cfg: resolvedCfg,
|
|
492
|
-
dbPath: dbPath,
|
|
493
|
-
parsed: parsed,
|
|
494
|
-
warnings: warnings
|
|
495
|
-
}
|
|
496
|
-
];
|
|
497
|
-
}
|
|
448
|
+
return [
|
|
449
|
+
2,
|
|
450
|
+
(0, _opents.openWithDialect)(cfg, duckdbOpenDialect)
|
|
451
|
+
];
|
|
498
452
|
});
|
|
499
453
|
})();
|
|
500
454
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n store: Store;\n cfg: ResolvedConfig;\n dbPath: string;\n parsed: number;\n warnings: string[];\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n for (const feature of activeFeatures(cfg)) {\n // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n if (feature.name === 'embed') {\n await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n continue;\n }\n await feature.schema(conn);\n }\n if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<{ instance: DuckDBInstance; duckdb: DuckDBConnection; conn: Connection; cfg: ResolvedConfig; dbPath: string; parsed: number; warnings: string[] }> {\n const stateDir = join(cfg.baseDir, STATE_DIR);\n mkdirSync(stateDir, { recursive: true });\n const dbPath = join(stateDir, DB_FILENAME);\n\n // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n try {\n ({ DuckDBInstance } = await duckdbApi());\n } catch (err) {\n if (err instanceof SenseError) throw err;\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n }\n\n let duckdb: DuckDBConnection;\n let instance: DuckDBInstance | undefined;\n try {\n // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n duckdb = await instance.connect();\n } catch (err) {\n // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n instance?.closeSync();\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n }\n // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL\n // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.\n let closed = false;\n try {\n await registerFunctions(duckdb);\n const conn = createConnection(duckdb);\n\n await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n const version = await getMeta(conn, 'schema_version');\n const features = await getMeta(conn, 'features');\n const wantFeatures = featureSignature(cfg, FEATURES);\n if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n // rebuild instead, same as sqlite's open() (see its comment for why this is announced).\n const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n console.error(`sense: ${reason}; rebuilding the index`);\n // Close before clearCache deletes the files underneath the still-open instance.\n closed = true;\n duckdb.disconnectSync();\n instance.closeSync();\n clearCache(cfg);\n return connect(cfg);\n }\n\n await ensureSchema(conn, cfg);\n\n const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, duckdbDialect);\n return { instance, duckdb, conn, cfg, dbPath, parsed, warnings };\n } catch (err) {\n if (!closed) {\n duckdb.disconnectSync();\n instance?.closeSync();\n }\n throw err;\n }\n}\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n const store = createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);\n // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n rekeyChunkText(conn, store);\n return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openDuckdb","ensureSchema","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","DuckDBInstance","err","duckdb","instance","closed","version","features","wantFeatures","reason","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","closeSync","registerFunctions","createConnection","console","error","disconnectSync","clearCache","reconcile","duckdbDialect","resolvedCfg","store","createStore","rekeyChunkText"],"mappings":";;;;;;;;;;;QAmBaA;eAAAA;;QAGAC;eAAAA;;QA+FSC;eAAAA;;;sBArHI;wBACL;uBAGuB;yBACb;uBACJ;wBACA;wBACc;uBACd;2BACD;wBACO;4BAEA;wBACS;4BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMF,cAAc;AAGpB,IAAMC,iBAAiB;AAU9B,8LAA8L;AAC9L,2JAA2J;AAC3J,SAAeE,aAAaC,IAAgB,EAAEC,GAAW;;YAKlD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBH;;;oBAAtC;;;oBACjD;;wBAAMW,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,QAAQX,GAAmB;;YAClCY,UAEAC,QAIFC,gBAGKC,KAKLC,QACAC,UAMKF,MAOLG,QAGInB,MAIAoB,SACAC,UACAC,cAIEC,QAYqB,MAArBC,QAAQC,UAETT;;;;oBAvDHH,WAAWa,IAAAA,cAAI,EAACzB,IAAI0B,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAAChB,UAAU;wBAAEiB,WAAW;oBAAK;oBAChChB,SAASY,IAAAA,cAAI,EAACb,UAAUjB;;;;;;;;;oBAMN;;wBAAMmC,IAAAA,mBAAS;;;oBAAlChB,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAegB,oBAAU,GAAE,MAAMhB;oBACrC,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACjB,IAAckB,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAMnB,eAAeoB,MAAM,CAACrB,QAAQ;4BAAEsB,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KlB,WAAW;oBACF;;wBAAMA,SAASN,OAAO;;;oBAA/BK,SAAS;;;;;;oBACFD;oBACP,wFAAwF;oBACxFE,qBAAAA,+BAAAA,SAAUmB,SAAS;oBACnB,MAAM,IAAIL,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXlB,QAAO,MAA2B,OAAvB,AAACE,KAAckB,OAAO;;oBAErH,2FAA2F;oBAC3F,uFAAuF;oBACnFf,SAAS;;;;;;;;;oBAEX;;wBAAMmB,IAAAA,iCAAiB,EAACrB;;;oBAAxB;oBACMjB,OAAOuC,IAAAA,8BAAgB,EAACtB;oBAE9B;;wBAAMjB,KAAKG,IAAI,CAAC;;;oBAAhB;oBAEgB;;wBAAMK,IAAAA,iBAAO,EAACR,MAAM;;;oBAA9BoB,UAAU;oBACC;;wBAAMZ,IAAAA,iBAAO,EAACR,MAAM;;;oBAA/BqB,WAAW;oBACXC,eAAeZ,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;oBACnD,IAAI,AAACS,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,cAAe;wBACxG,4FAA4F;wBAC5F,wFAAwF;wBAClFC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;wBACnH2C,QAAQC,KAAK,CAAC,AAAC,UAAgB,OAAPlB,QAAO;wBAC/B,gFAAgF;wBAChFJ,SAAS;wBACTF,OAAOyB,cAAc;wBACrBxB,SAASmB,SAAS;wBAClBM,IAAAA,mBAAU,EAAC1C;wBACX;;4BAAOW,QAAQX;;oBACjB;oBAEA;;wBAAMF,aAAaC,MAAMC;;;oBAAzB;oBAE6B;;wBAAM2C,IAAAA,sBAAS,EAAC5C,MAAMC,KAAKA,IAAI0B,OAAO,EAAEkB,2BAAa;;;oBAArD,OAAA,eAArBrB,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAChB;;wBAAO;4BAAEP,UAAAA;4BAAUD,QAAAA;4BAAQjB,MAAAA;4BAAMC,KAAAA;4BAAKa,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;oBACxDT;oBACP,IAAI,CAACG,QAAQ;wBACXF,OAAOyB,cAAc;wBACrBxB,qBAAAA,+BAAAA,SAAUmB,SAAS;oBACrB;oBACA,MAAMrB;;;;;;;IAEV;;AAEO,SAAelB,WAAWG,GAAmB;;YAC6B,MAAvEiB,UAAUD,QAAQjB,MAAW8C,aAAahC,QAAQU,QAAQC,UAC5DsB;;;;oBADyE;;wBAAMnC,QAAQX;;;oBAAd,OAAA,eAAvEiB,WAAuE,KAAvEA,UAAUD,SAA6D,KAA7DA,QAAQjB,OAAqD,KAArDA,MAAW8C,cAA0C,KAA/C7C,KAAkBa,SAA6B,KAA7BA,QAAQU,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAC5DsB,QAAQC,IAAAA,oBAAW,EAAC9B,UAAUD,QAAQjB,MAAM8C,aAAaA,YAAYnB,OAAO;oBAClF,0FAA0F;oBAC1FsB,IAAAA,yBAAc,EAACjD,MAAM+C;oBACrB;;wBAAO;4BAAEA,OAAOA;4BAAO9C,KAAK6C;4BAAahC,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACpE"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '3';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n instance: DuckDBInstance;\n duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n for (const feature of activeFeatures(cfg)) {\n // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n if (feature.name === 'embed') {\n await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n continue;\n }\n await feature.schema(conn);\n }\n if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n handle.duckdb.disconnectSync();\n handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n try {\n ({ DuckDBInstance } = await duckdbApi());\n } catch (err) {\n if (err instanceof SenseError) throw err;\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n }\n\n let duckdb: DuckDBConnection;\n let instance: DuckDBInstance | undefined;\n try {\n // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n duckdb = await instance.connect();\n } catch (err) {\n // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n instance?.closeSync();\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n }\n // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n try {\n await registerFunctions(duckdb);\n const conn = createConnection(duckdb);\n return { handle: { instance, duckdb }, conn };\n } catch (err) {\n await close({ instance, duckdb });\n throw err;\n }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n filename: DB_FILENAME,\n schemaVersion: SCHEMA_VERSION,\n reconcileDialect: duckdbDialect,\n connect,\n close,\n // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n // \"Cannot open file ... being used by another process\". Both are the same condition.\n isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n ensureSchema,\n createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","_handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAGAC;eAAAA;;QAwEAC;eAAAA;;QAcSC;eAAAA;;;uBAvGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,gGAAgG;AAChG,SAAeG,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;;YAKzE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMa,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMC,MAAoB;;;YACvCA,OAAOC,MAAM,CAACC,cAAc;YAC5BF,OAAOG,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLR,QACAE,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BJ,SAAS;;;;;;oBACFQ;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACf;;;oBAAxB;oBACMd,OAAO8B,IAAAA,8BAAgB,EAAChB;oBAC9B;;wBAAO;4BAAED,QAAQ;gCAAEG,UAAAA;gCAAUF,QAAAA;4BAAO;4BAAGd,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMV,MAAM;4BAAEI,UAAAA;4BAAUF,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMQ;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAN,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFuB,UAAU,SAAVA,SAAWb;eAAQ,2DAA2Dc,IAAI,CAACd,IAAII,OAAO;;IAC9F5B,cAAAA;IACAuC,aAAa,SAAbA,YAAcxB,QAAQb;eAASqC,IAAAA,oBAAW,EAACxB,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAEd;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKL;;;IAC9B"}
|
|
@@ -9,6 +9,8 @@ Object.defineProperty(exports, "duckdbDialect", {
|
|
|
9
9
|
}
|
|
10
10
|
});
|
|
11
11
|
var _errorsts = require("../../errors.js");
|
|
12
|
+
var _sharedts = require("../shared.js");
|
|
13
|
+
var _lexicalts = require("./lexical.js");
|
|
12
14
|
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
|
|
13
15
|
try {
|
|
14
16
|
var info = gen[key](arg);
|
|
@@ -185,6 +187,37 @@ function reconcileContent(conn, touched, docs, _delta) {
|
|
|
185
187
|
_state.sent();
|
|
186
188
|
_state.label = 4;
|
|
187
189
|
case 4:
|
|
190
|
+
// content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,
|
|
191
|
+
// not here (lexical.ts's FtsIndexState).
|
|
192
|
+
(0, _lexicalts.markContentStale)(conn);
|
|
193
|
+
return [
|
|
194
|
+
2
|
|
195
|
+
];
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
})();
|
|
199
|
+
}
|
|
200
|
+
// DuckDB rejects more than one ALTER command per statement ("Parser Error: Only one ALTER
|
|
201
|
+
// command per statement is supported", measured), so every name's clause joins into one string
|
|
202
|
+
// and runs as a single exec() -- one column-add "leg" through the driver instead of `names.length`.
|
|
203
|
+
// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()
|
|
204
|
+
// produces for one key across files.
|
|
205
|
+
function addColumns(conn, names) {
|
|
206
|
+
return _async_to_generator(function() {
|
|
207
|
+
return _ts_generator(this, function(_state) {
|
|
208
|
+
switch(_state.label){
|
|
209
|
+
case 0:
|
|
210
|
+
if (names.length === 0) return [
|
|
211
|
+
2
|
|
212
|
+
];
|
|
213
|
+
return [
|
|
214
|
+
4,
|
|
215
|
+
conn.exec(names.map(function(name) {
|
|
216
|
+
return "ALTER TABLE frontmatter ADD COLUMN ".concat((0, _sharedts.quoteIdent)(name), " VARIANT");
|
|
217
|
+
}).join('; '))
|
|
218
|
+
];
|
|
219
|
+
case 1:
|
|
220
|
+
_state.sent();
|
|
188
221
|
return [
|
|
189
222
|
2
|
|
190
223
|
];
|
|
@@ -201,11 +234,7 @@ var duckdbDialect = {
|
|
|
201
234
|
throw new _errorsts.SenseError('COLUMN_LIMIT', "frontmatter would need ".concat(count, " columns, crossing this store's sanity limit (").concat(MAX_FRONTMATTER_COLUMNS, "). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys."));
|
|
202
235
|
}
|
|
203
236
|
},
|
|
204
|
-
|
|
205
|
-
// the mixed bigint/number/string/null shapes mapValue() produces for one key across files.
|
|
206
|
-
columnTypeSuffix: function columnTypeSuffix() {
|
|
207
|
-
return ' VARIANT';
|
|
208
|
-
},
|
|
237
|
+
addColumns: addColumns,
|
|
209
238
|
reconcileContent: reconcileContent
|
|
210
239
|
};
|
|
211
240
|
/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n if (touched.length > 0)\n await conn.runBatch(\n 'DELETE FROM content WHERE \"path\" = ?',\n touched.map((p) => [p])\n );\n if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n beginMode: () => 'BEGIN',\n checkColumnLimit(count) {\n if (count > MAX_FRONTMATTER_COLUMNS) {\n throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n }\n },\n
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n if (touched.length > 0)\n await conn.runBatch(\n 'DELETE FROM content WHERE \"path\" = ?',\n touched.map((p) => [p])\n );\n if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n // not here (lexical.ts's FtsIndexState).\n markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n if (names.length === 0) return;\n await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n beginMode: () => 'BEGIN',\n checkColumnLimit(count) {\n if (count > MAX_FRONTMATTER_COLUMNS) {\n throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n }\n },\n addColumns,\n reconcileContent,\n};\n"],"names":["duckdbDialect","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","markContentStale","addColumns","names","exec","name","quoteIdent","join","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BA8CaA;;;eAAAA;;;wBA9Cc;wBAGA;yBAEM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEjC,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEL,CAAAA,KAAKE,MAAM,GAAG,CAAA,GAAdF;;;;oBAAiB;;wBAAMF,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;;;oBAAjD;;;oBACrB,6FAA6F;oBAC7F,yCAAyC;oBACzCiB,IAAAA,2BAAgB,EAACR;;;;;;IACnB;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAeS,WAAWT,IAAgB,EAAEU,KAAe;;;;;oBACzD,IAAIA,MAAMN,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMJ,KAAKW,IAAI,CAACD,MAAMJ,GAAG,CAAC,SAACM;mCAAS,AAAC,sCAAsD,OAAjBC,IAAAA,oBAAU,EAACD,OAAM;2BAAWE,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEO,IAAMzB,gBAAkC;IAC7C0B,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQnB,yBAAyB;YACnC,MAAM,IAAIoB,oBAAU,CAAC,gBAAgB,AAAC,0BAA+EpB,OAAtDmB,OAAM,kDAAwE,OAAxBnB,yBAAwB;QAC/I;IACF;IACAW,YAAAA;IACAV,kBAAAA;AACF"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
|
|
2
|
-
import type { Config } from '../../config/index.js';
|
|
3
2
|
import type { Capability, Connection, Store } from '../types.js';
|
|
4
3
|
export declare const CAPABILITIES: ReadonlySet<Capability>;
|
|
5
|
-
export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection
|
|
4
|
+
export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
|
|
2
|
-
import type { Config } from '../../config/index.js';
|
|
3
2
|
import type { Capability, Connection, Store } from '../types.js';
|
|
4
3
|
export declare const CAPABILITIES: ReadonlySet<Capability>;
|
|
5
|
-
export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection
|
|
4
|
+
export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store;
|