sensemaking 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -10
- package/dist/cjs/cli/index.d.cts +1 -1
- package/dist/cjs/cli/index.d.ts +1 -1
- package/dist/cjs/cli/index.js +1 -1
- package/dist/cjs/cli/index.js.map +1 -1
- package/dist/cjs/cli/path.js +5 -29
- package/dist/cjs/cli/path.js.map +1 -1
- package/dist/cjs/cli/related.js +4 -3
- package/dist/cjs/cli/related.js.map +1 -1
- package/dist/cjs/commands/index.d.cts +6 -3
- package/dist/cjs/commands/index.d.ts +6 -3
- package/dist/cjs/commands/index.js +4 -0
- package/dist/cjs/commands/index.js.map +1 -1
- package/dist/cjs/commands/map.d.cts +14 -5
- package/dist/cjs/commands/map.d.ts +14 -5
- package/dist/cjs/commands/map.js +42 -35
- package/dist/cjs/commands/map.js.map +1 -1
- package/dist/cjs/commands/path.d.cts +14 -0
- package/dist/cjs/commands/path.d.ts +14 -0
- package/dist/cjs/commands/path.js +209 -0
- package/dist/cjs/commands/path.js.map +1 -0
- package/dist/cjs/commands/peek.d.cts +16 -1
- package/dist/cjs/commands/peek.d.ts +16 -1
- package/dist/cjs/commands/peek.js +71 -33
- package/dist/cjs/commands/peek.js.map +1 -1
- package/dist/cjs/commands/related.d.cts +16 -3
- package/dist/cjs/commands/related.d.ts +16 -3
- package/dist/cjs/commands/related.js +131 -98
- package/dist/cjs/commands/related.js.map +1 -1
- package/dist/cjs/commands/scope.d.cts +1 -0
- package/dist/cjs/commands/scope.d.ts +1 -0
- package/dist/cjs/commands/scope.js +149 -0
- package/dist/cjs/commands/scope.js.map +1 -1
- package/dist/cjs/commands/search.d.cts +26 -3
- package/dist/cjs/commands/search.d.ts +26 -3
- package/dist/cjs/commands/search.js +211 -71
- package/dist/cjs/commands/search.js.map +1 -1
- package/dist/cjs/commands/signals.d.cts +4 -3
- package/dist/cjs/commands/signals.d.ts +4 -3
- package/dist/cjs/commands/signals.js +13 -4
- package/dist/cjs/commands/signals.js.map +1 -1
- package/dist/cjs/commands/snapshot.d.cts +3 -0
- package/dist/cjs/commands/snapshot.d.ts +3 -0
- package/dist/cjs/commands/snapshot.js +180 -0
- package/dist/cjs/commands/snapshot.js.map +1 -0
- package/dist/cjs/embed/cohere.js +41 -16
- package/dist/cjs/embed/cohere.js.map +1 -1
- package/dist/cjs/embed/endpoint.d.cts +2 -0
- package/dist/cjs/embed/endpoint.d.ts +2 -0
- package/dist/cjs/embed/endpoint.js +30 -0
- package/dist/cjs/embed/endpoint.js.map +1 -0
- package/dist/cjs/embed/handoff.d.cts +7 -2
- package/dist/cjs/embed/handoff.d.ts +7 -2
- package/dist/cjs/embed/handoff.js +10 -0
- package/dist/cjs/embed/handoff.js.map +1 -1
- package/dist/cjs/embed/http.d.cts +18 -1
- package/dist/cjs/embed/http.d.ts +18 -1
- package/dist/cjs/embed/http.js +239 -40
- package/dist/cjs/embed/http.js.map +1 -1
- package/dist/cjs/embed/openai.js +39 -17
- package/dist/cjs/embed/openai.js.map +1 -1
- package/dist/cjs/embed/query.d.cts +9 -3
- package/dist/cjs/embed/query.d.ts +9 -3
- package/dist/cjs/embed/query.js +257 -90
- package/dist/cjs/embed/query.js.map +1 -1
- package/dist/cjs/embed/registry.d.cts +2 -2
- package/dist/cjs/embed/registry.d.ts +2 -2
- package/dist/cjs/embed/registry.js +36 -4
- package/dist/cjs/embed/registry.js.map +1 -1
- package/dist/cjs/embed/static.js +6 -2
- package/dist/cjs/embed/static.js.map +1 -1
- package/dist/cjs/embed/store.js +42 -77
- package/dist/cjs/embed/store.js.map +1 -1
- package/dist/cjs/embed/types.d.cts +5 -2
- package/dist/cjs/embed/types.d.ts +5 -2
- package/dist/cjs/embed/types.js.map +1 -1
- package/dist/cjs/features/embed.js +290 -43
- package/dist/cjs/features/embed.js.map +1 -1
- package/dist/cjs/graph/traverse.d.cts +2 -2
- package/dist/cjs/graph/traverse.d.ts +2 -2
- package/dist/cjs/graph/traverse.js +15 -5
- package/dist/cjs/graph/traverse.js.map +1 -1
- package/dist/cjs/index.d.cts +2 -2
- package/dist/cjs/index.d.ts +2 -2
- package/dist/cjs/index.js +6 -0
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/output/output.d.cts +4 -36
- package/dist/cjs/output/output.d.ts +4 -36
- package/dist/cjs/output/output.js +1 -2
- package/dist/cjs/output/output.js.map +1 -1
- package/dist/cjs/scan/pool.d.cts +2 -1
- package/dist/cjs/scan/pool.d.ts +2 -1
- package/dist/cjs/scan/pool.js +116 -27
- package/dist/cjs/scan/pool.js.map +1 -1
- package/dist/cjs/scan/reparse.d.cts +1 -0
- package/dist/cjs/scan/reparse.d.ts +1 -0
- package/dist/cjs/scan/reparse.js +75 -21
- package/dist/cjs/scan/reparse.js.map +1 -1
- package/dist/cjs/store/builder.d.cts +11 -21
- package/dist/cjs/store/builder.d.ts +11 -21
- package/dist/cjs/store/builder.js +1077 -438
- package/dist/cjs/store/builder.js.map +1 -1
- package/dist/cjs/store/duckdb/connection.js +16 -2
- package/dist/cjs/store/duckdb/connection.js.map +1 -1
- package/dist/cjs/store/duckdb/open.d.cts +1 -1
- package/dist/cjs/store/duckdb/open.d.ts +1 -1
- package/dist/cjs/store/duckdb/open.js +73 -65
- package/dist/cjs/store/duckdb/open.js.map +1 -1
- package/dist/cjs/store/duckdb/reconcile.js +533 -15
- package/dist/cjs/store/duckdb/reconcile.js.map +1 -1
- package/dist/cjs/store/embed-scope.js +19 -9
- package/dist/cjs/store/embed-scope.js.map +1 -1
- package/dist/cjs/store/index.d.cts +6 -0
- package/dist/cjs/store/index.d.ts +6 -0
- package/dist/cjs/store/index.js +8 -4
- package/dist/cjs/store/index.js.map +1 -1
- package/dist/cjs/store/open.d.cts +6 -1
- package/dist/cjs/store/open.d.ts +6 -1
- package/dist/cjs/store/open.js +230 -307
- package/dist/cjs/store/open.js.map +1 -1
- package/dist/cjs/store/readiness.d.cts +5 -0
- package/dist/cjs/store/readiness.d.ts +5 -0
- package/dist/cjs/store/readiness.js +39 -0
- package/dist/cjs/store/readiness.js.map +1 -0
- package/dist/cjs/store/reconcile.d.cts +26 -1
- package/dist/cjs/store/reconcile.d.ts +26 -1
- package/dist/cjs/store/reconcile.js +670 -576
- package/dist/cjs/store/reconcile.js.map +1 -1
- package/dist/cjs/store/signature.d.cts +1 -0
- package/dist/cjs/store/signature.d.ts +1 -0
- package/dist/cjs/store/signature.js +9 -0
- package/dist/cjs/store/signature.js.map +1 -1
- package/dist/cjs/store/sqlite/open.d.cts +1 -1
- package/dist/cjs/store/sqlite/open.d.ts +1 -1
- package/dist/cjs/store/sqlite/open.js +84 -90
- package/dist/cjs/store/sqlite/open.js.map +1 -1
- package/dist/cjs/store/sqlite/reconcile.js.map +1 -1
- package/dist/cjs/store/sqlite/store.js +63 -60
- package/dist/cjs/store/sqlite/store.js.map +1 -1
- package/dist/cjs/store/sqlite/vectors.d.cts +2 -2
- package/dist/cjs/store/sqlite/vectors.d.ts +2 -2
- package/dist/cjs/store/sqlite/vectors.js +8 -4
- package/dist/cjs/store/sqlite/vectors.js.map +1 -1
- package/dist/cjs/store/stages.js.map +1 -1
- package/dist/cjs/store/turso/connection.d.cts +1 -0
- package/dist/cjs/store/turso/connection.d.ts +1 -0
- package/dist/cjs/store/turso/connection.js +319 -94
- package/dist/cjs/store/turso/connection.js.map +1 -1
- package/dist/cjs/store/turso/open.d.cts +1 -1
- package/dist/cjs/store/turso/open.d.ts +1 -1
- package/dist/cjs/store/turso/open.js +70 -74
- package/dist/cjs/store/turso/open.js.map +1 -1
- package/dist/cjs/store/turso/store.js +25 -23
- package/dist/cjs/store/turso/store.js.map +1 -1
- package/dist/cjs/store/turso/vectors.d.cts +2 -2
- package/dist/cjs/store/turso/vectors.d.ts +2 -2
- package/dist/cjs/store/turso/vectors.js +26 -16
- package/dist/cjs/store/turso/vectors.js.map +1 -1
- package/dist/cjs/store/types.d.cts +4 -1
- package/dist/cjs/store/types.d.ts +4 -1
- package/dist/cjs/store/types.js.map +1 -1
- package/dist/cjs/store/vectors.d.cts +14 -1
- package/dist/cjs/store/vectors.d.ts +14 -1
- package/dist/cjs/store/vectors.js +61 -0
- package/dist/cjs/store/vectors.js.map +1 -1
- package/dist/cjs/watch.d.cts +6 -0
- package/dist/cjs/watch.d.ts +6 -0
- package/dist/cjs/watch.js +42 -10
- package/dist/cjs/watch.js.map +1 -1
- package/dist/cjs/workers/parse.d.cts +4 -3
- package/dist/cjs/workers/parse.d.ts +4 -3
- package/dist/cjs/workers/parse.js +19 -17
- package/dist/cjs/workers/parse.js.map +1 -1
- package/dist/esm/cli/index.d.ts +1 -1
- package/dist/esm/cli/index.js +1 -1
- package/dist/esm/cli/index.js.map +1 -1
- package/dist/esm/cli/path.js +4 -10
- package/dist/esm/cli/path.js.map +1 -1
- package/dist/esm/cli/related.js +5 -3
- package/dist/esm/cli/related.js.map +1 -1
- package/dist/esm/commands/index.d.ts +6 -3
- package/dist/esm/commands/index.js +1 -0
- package/dist/esm/commands/index.js.map +1 -1
- package/dist/esm/commands/map.d.ts +14 -5
- package/dist/esm/commands/map.js +2 -0
- package/dist/esm/commands/map.js.map +1 -1
- package/dist/esm/commands/path.d.ts +14 -0
- package/dist/esm/commands/path.js +33 -0
- package/dist/esm/commands/path.js.map +1 -0
- package/dist/esm/commands/peek.d.ts +16 -1
- package/dist/esm/commands/peek.js +35 -9
- package/dist/esm/commands/peek.js.map +1 -1
- package/dist/esm/commands/related.d.ts +16 -3
- package/dist/esm/commands/related.js +45 -28
- package/dist/esm/commands/related.js.map +1 -1
- package/dist/esm/commands/scope.d.ts +1 -0
- package/dist/esm/commands/scope.js +58 -0
- package/dist/esm/commands/scope.js.map +1 -1
- package/dist/esm/commands/search.d.ts +26 -3
- package/dist/esm/commands/search.js +85 -18
- package/dist/esm/commands/search.js.map +1 -1
- package/dist/esm/commands/signals.d.ts +4 -3
- package/dist/esm/commands/signals.js +11 -4
- package/dist/esm/commands/signals.js.map +1 -1
- package/dist/esm/commands/snapshot.d.ts +3 -0
- package/dist/esm/commands/snapshot.js +25 -0
- package/dist/esm/commands/snapshot.js.map +1 -0
- package/dist/esm/embed/cohere.js +35 -10
- package/dist/esm/embed/cohere.js.map +1 -1
- package/dist/esm/embed/endpoint.d.ts +2 -0
- package/dist/esm/embed/endpoint.js +11 -0
- package/dist/esm/embed/endpoint.js.map +1 -0
- package/dist/esm/embed/handoff.d.ts +7 -2
- package/dist/esm/embed/handoff.js +7 -0
- package/dist/esm/embed/handoff.js.map +1 -1
- package/dist/esm/embed/http.d.ts +18 -1
- package/dist/esm/embed/http.js +137 -17
- package/dist/esm/embed/http.js.map +1 -1
- package/dist/esm/embed/openai.js +34 -12
- package/dist/esm/embed/openai.js.map +1 -1
- package/dist/esm/embed/query.d.ts +9 -3
- package/dist/esm/embed/query.js +98 -26
- package/dist/esm/embed/query.js.map +1 -1
- package/dist/esm/embed/registry.d.ts +2 -2
- package/dist/esm/embed/registry.js +34 -5
- package/dist/esm/embed/registry.js.map +1 -1
- package/dist/esm/embed/static.js +10 -2
- package/dist/esm/embed/static.js.map +1 -1
- package/dist/esm/embed/store.js +30 -21
- package/dist/esm/embed/store.js.map +1 -1
- package/dist/esm/embed/types.d.ts +5 -2
- package/dist/esm/embed/types.js.map +1 -1
- package/dist/esm/features/embed.js +82 -21
- package/dist/esm/features/embed.js.map +1 -1
- package/dist/esm/graph/traverse.d.ts +2 -2
- package/dist/esm/graph/traverse.js +4 -2
- package/dist/esm/graph/traverse.js.map +1 -1
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/output/output.d.ts +4 -36
- package/dist/esm/output/output.js +1 -2
- package/dist/esm/output/output.js.map +1 -1
- package/dist/esm/scan/pool.d.ts +2 -1
- package/dist/esm/scan/pool.js +80 -23
- package/dist/esm/scan/pool.js.map +1 -1
- package/dist/esm/scan/reparse.d.ts +1 -0
- package/dist/esm/scan/reparse.js +40 -12
- package/dist/esm/scan/reparse.js.map +1 -1
- package/dist/esm/store/builder.d.ts +11 -21
- package/dist/esm/store/builder.js +337 -133
- package/dist/esm/store/builder.js.map +1 -1
- package/dist/esm/store/duckdb/connection.js +9 -1
- package/dist/esm/store/duckdb/connection.js.map +1 -1
- package/dist/esm/store/duckdb/open.d.ts +1 -1
- package/dist/esm/store/duckdb/open.js +13 -9
- package/dist/esm/store/duckdb/open.js.map +1 -1
- package/dist/esm/store/duckdb/reconcile.js +145 -10
- package/dist/esm/store/duckdb/reconcile.js.map +1 -1
- package/dist/esm/store/embed-scope.js +16 -6
- package/dist/esm/store/embed-scope.js.map +1 -1
- package/dist/esm/store/index.d.ts +6 -0
- package/dist/esm/store/index.js +10 -4
- package/dist/esm/store/index.js.map +1 -1
- package/dist/esm/store/open.d.ts +6 -1
- package/dist/esm/store/open.js +77 -87
- package/dist/esm/store/open.js.map +1 -1
- package/dist/esm/store/readiness.d.ts +5 -0
- package/dist/esm/store/readiness.js +11 -0
- package/dist/esm/store/readiness.js.map +1 -0
- package/dist/esm/store/reconcile.d.ts +26 -1
- package/dist/esm/store/reconcile.js +142 -120
- package/dist/esm/store/reconcile.js.map +1 -1
- package/dist/esm/store/signature.d.ts +1 -0
- package/dist/esm/store/signature.js +8 -0
- package/dist/esm/store/signature.js.map +1 -1
- package/dist/esm/store/sqlite/open.d.ts +1 -1
- package/dist/esm/store/sqlite/open.js +12 -12
- package/dist/esm/store/sqlite/open.js.map +1 -1
- package/dist/esm/store/sqlite/reconcile.js +2 -2
- package/dist/esm/store/sqlite/reconcile.js.map +1 -1
- package/dist/esm/store/sqlite/store.js +21 -18
- package/dist/esm/store/sqlite/store.js.map +1 -1
- package/dist/esm/store/sqlite/vectors.d.ts +2 -2
- package/dist/esm/store/sqlite/vectors.js +7 -5
- package/dist/esm/store/sqlite/vectors.js.map +1 -1
- package/dist/esm/store/stages.js.map +1 -1
- package/dist/esm/store/turso/connection.d.ts +1 -0
- package/dist/esm/store/turso/connection.js +97 -28
- package/dist/esm/store/turso/connection.js.map +1 -1
- package/dist/esm/store/turso/open.d.ts +1 -1
- package/dist/esm/store/turso/open.js +9 -8
- package/dist/esm/store/turso/open.js.map +1 -1
- package/dist/esm/store/turso/store.js +16 -14
- package/dist/esm/store/turso/store.js.map +1 -1
- package/dist/esm/store/turso/vectors.d.ts +2 -2
- package/dist/esm/store/turso/vectors.js +27 -17
- package/dist/esm/store/turso/vectors.js.map +1 -1
- package/dist/esm/store/types.d.ts +4 -1
- package/dist/esm/store/types.js.map +1 -1
- package/dist/esm/store/vectors.d.ts +14 -1
- package/dist/esm/store/vectors.js +55 -0
- package/dist/esm/store/vectors.js.map +1 -1
- package/dist/esm/watch.d.ts +6 -0
- package/dist/esm/watch.js +40 -14
- package/dist/esm/watch.js.map +1 -1
- package/dist/esm/workers/parse.d.ts +4 -3
- package/dist/esm/workers/parse.js +19 -17
- package/dist/esm/workers/parse.js.map +1 -1
- package/package.json +1 -1
- package/skills/sense/SKILL.md +4 -2
- package/skills/sense/references/stores/duckdb.md +1 -1
- package/skills/sense-setup/SKILL.md +2 -2
- package/skills/sense-setup/references/embeddings.md +1 -1
- package/skills/sense-setup/references/store-benchmarks.md +5 -5
- package/skills/sense-setup/references/stores/duckdb.md +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n if (touched.length > 0)\n await conn.runBatch(\n 'DELETE FROM content WHERE \"path\" = ?',\n touched.map((p) => [p])\n );\n // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n // connection, so no second writer can have landed one of these rows: nothing can conflict.\n await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n // Build prepares the FTS index after reconcile. This persisted mark commits with the content\n // change, so an interrupted build cannot expose stale lexical results.\n await markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n if (names.length === 0) return;\n await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n beginMode: () => 'BEGIN',\n checkColumnLimit(count) {\n if (count > MAX_FRONTMATTER_COLUMNS) {\n throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n }\n },\n addColumns,\n reconcileContent,\n};\n"],"names":["duckdbDialect","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","quoteIdent","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","appendRows","markContentStale","addColumns","names","exec","name","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BAiDaA;;;eAAAA;;;wBAjDc;wBAGY;yBAEN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEjC,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,IAAMC,qBAAqB,AAAC,wBAAkE,OAA3CD,gBAAgBE,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC,OAAM;AAE9F,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQb,GAAG,CAAC,SAACkB;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;oBAIF,6FAA6F;oBAC7F,2FAA2F;oBAC3F;;wBAAMC,IAAAA,oBAAU,EAACP,MAAM,WAAWd,iBAAiBC,oBAAoBe,KAAKd,GAAG,CAACG;;;oBAAhF;oBACA,6FAA6F;oBAC7F,uEAAuE;oBACvE;;wBAAMiB,IAAAA,2BAAgB,EAACR;;;oBAAvB;;;;;;IACF;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAeS,WAAWT,IAAgB,EAAEU,KAAe;;;;;oBACzD,IAAIA,MAAMN,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMJ,KAAKW,IAAI,CAACD,MAAMtB,GAAG,CAAC,SAACwB;mCAAS,AAAC,sCAAsD,OAAjBvB,IAAAA,oBAAU,EAACuB,OAAM;2BAAWtB,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEO,IAAML,gBAAkC;IAC7C4B,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQjB,yBAAyB;YACnC,MAAM,IAAIkB,oBAAU,CAAC,gBAAgB,AAAC,0BAA+ElB,OAAtDiB,OAAM,kDAAwE,OAAxBjB,yBAAwB;QAC/I;IACF;IACAW,YAAAA;IACAV,kBAAAA;AACF"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport type { DuckdbConnection } from './connection.ts';\nimport { markContentStale } from './lexical.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so changed rows are maintained\n// here; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n// Bound the extra persisted text held for exact comparison, even during a full reparse.\nconst CONTENT_LOOKUP_CHUNK = 128;\n\ninterface ContentRow {\n path: string;\n title: string | null;\n summary: string | null;\n text: string | null;\n}\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\nconst FRONTMATTER_UPDATE_CHUNK = 128;\n\nasync function withNativeStatement<T>(native: DuckDBConnection, sql: string, fn: (stmt: DuckDBPreparedStatement) => Promise<T>): Promise<T> {\n const stmt = await native.prepare(sql);\n const errors: unknown[] = [];\n let result!: T;\n try {\n result = await fn(stmt);\n } catch (err) {\n errors.push(err);\n }\n try {\n stmt.destroySync();\n } catch (err) {\n errors.push(err);\n }\n if (errors.length === 1) throw errors[0];\n if (errors.length > 1) throw new AggregateError(errors, 'frontmatter statement and cleanup failed');\n return result;\n}\n\n// Wide writes amplify native commit allocations even for one changed stamp.\n// Skip only known, exactly equal values; uncertain representations still write.\nasync function updateFrontmatter(conn: Connection, writableColumns: string[], rows: unknown[][]): Promise<void> {\n if (rows.length === 0) return;\n const native = (conn as DuckdbConnection).duckdb;\n const { DuckDBVariantValue, DuckDBTypeId } = await duckdbApi();\n const physical = await withNativeStatement(native, 'PRAGMA table_info(frontmatter)', async (stmt) => {\n const reader = await stmt.runAndReadAll();\n return reader.getRowObjectsJS() as Array<{ name: string; type: string }>;\n });\n const writable = new Map(writableColumns.map((name, i) => [name, i]));\n const pathIndex = writable.get('path');\n if (pathIndex === undefined) throw new Error('frontmatter update requires a writable path column');\n const columnTypes = new Map(physical.map((column) => [column.name, column.type]));\n const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')})`;\n const sameValue = (old: DuckDBValue, value: unknown, column: string): boolean => {\n if (value === null || value === undefined) return old === null;\n if (columnTypes.get(column) !== 'VARIANT') return Object.is(old, value);\n if (!(old instanceof DuckDBVariantValue)) return false;\n const typeId = typeof value === 'string' ? DuckDBTypeId.VARCHAR : typeof value === 'bigint' ? DuckDBTypeId.HUGEINT : typeof value === 'number' && !Number.isInteger(value) ? DuckDBTypeId.DOUBLE : undefined;\n return typeId !== undefined && old.type?.typeId === typeId && Object.is(old.value, value);\n };\n for (let i = 0; i < rows.length; i += FRONTMATTER_UPDATE_CHUNK) {\n const chunk = rows.slice(i, i + FRONTMATTER_UPDATE_CHUNK);\n const paths = chunk.map((row) => row[pathIndex] as string);\n const placeholders = paths.map(() => '?').join(', ');\n const selected = writableColumns.map(quoteIdent).join(', ');\n const existing = await withNativeStatement(native, `SELECT ${selected} FROM frontmatter WHERE \"path\" IN (${placeholders})`, async (stmt) => {\n stmt.bind(paths);\n const reader = await stmt.runAndReadAll();\n // Native wrappers distinguish numeric tags and SQL NULL from VARIANT null.\n return new Map(reader.getRowObjects().map((row) => [row.path as string, row]));\n });\n const plans = chunk.map((row) => {\n const old = existing.get(row[pathIndex] as string);\n const changed = old ? writableColumns.flatMap((column, index) => (sameValue(old[column], row[index], column) ? [] : [index])) : [];\n return { row, missing: old === undefined, changed, signature: changed.join(',') };\n });\n for (let position = 0; position < plans.length; ) {\n const plan = plans[position];\n if (!plan.missing && plan.changed.length === 0) {\n ++position;\n continue;\n }\n let end = position + 1;\n while (end < plans.length && plans[end].missing === plan.missing && plans[end].signature === plan.signature) ++end;\n if (plan.missing) {\n await appendRows(\n conn,\n 'frontmatter',\n writableColumns,\n insertSql,\n plans.slice(position, end).map((repair) => repair.row)\n );\n } else {\n const assignments = plan.changed.map((index) => `${quoteIdent(writableColumns[index])} = ?`).join(', ');\n await withNativeStatement(native, `UPDATE frontmatter SET ${assignments} WHERE \"path\" = ?`, async (stmt) => {\n for (let rowIndex = position; rowIndex < end; ++rowIndex) {\n const row = plans[rowIndex].row;\n stmt.bind([...plan.changed.map((index) => row[index] ?? null), row[pathIndex]] as DuckDBValue[]);\n await stmt.run();\n }\n });\n }\n position = end;\n }\n }\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n const added = new Set(delta.added);\n const reparsed = docs.filter((doc) => !added.has(doc.relPath));\n const changed = new Set(delta.added);\n const removed: string[] = [];\n for (let i = 0; i < reparsed.length; i += CONTENT_LOOKUP_CHUNK) {\n const chunk = reparsed.slice(i, i + CONTENT_LOOKUP_CHUNK);\n const stmt = await conn.prepare(`SELECT \"path\", title, summary, text FROM content WHERE \"path\" IN (${chunk.map(() => '?').join(', ')})`);\n const rows = (await stmt.all(...chunk.map((doc) => doc.relPath))) as ContentRow[];\n const existing = new Map(rows.map((row) => [row.path, row]));\n for (const doc of chunk) {\n const row = existing.get(doc.relPath);\n if (row && row.title === doc.search.title && row.summary === doc.search.summary && row.text === doc.search.text) continue;\n changed.add(doc.relPath);\n if (row) removed.push(doc.relPath);\n }\n }\n\n const docPaths = new Set(docs.map((doc) => doc.relPath));\n const vanished = touched.filter((path) => !docPaths.has(path));\n for (let i = 0; i < vanished.length; i += CONTENT_LOOKUP_CHUNK) {\n const chunk = vanished.slice(i, i + CONTENT_LOOKUP_CHUNK);\n const stmt = await conn.prepare(`SELECT \"path\" FROM content WHERE \"path\" IN (${chunk.map(() => '?').join(', ')})`);\n const rows = (await stmt.all(...chunk)) as Array<{ path: string }>;\n removed.push(...rows.map((row) => row.path));\n }\n\n if (removed.length > 0)\n await conn.runBatch(\n 'DELETE FROM content WHERE \"path\" = ?',\n removed.map((p) => [p])\n );\n // Keep document order and the existing insert/error semantics for added paths.\n const changedDocs = docs.filter((doc) => changed.has(doc.relPath));\n await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, changedDocs.map(contentRow));\n // Commit the stale mark with actual content changes. Identical rows leave prior staleness\n // untouched, and FTS preparation still owns missing-index recovery.\n if (removed.length > 0 || changedDocs.length > 0) await markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n if (names.length === 0) return;\n await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n beginMode: () => 'BEGIN',\n checkColumnLimit(count) {\n if (count > MAX_FRONTMATTER_COLUMNS) {\n throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n }\n },\n addColumns,\n updateFrontmatter,\n reconcileContent,\n};\n"],"names":["duckdbDialect","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","quoteIdent","join","CONTENT_LOOKUP_CHUNK","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","FRONTMATTER_UPDATE_CHUNK","withNativeStatement","native","sql","fn","stmt","errors","result","err","prepare","push","destroySync","length","AggregateError","updateFrontmatter","conn","writableColumns","rows","DuckDBVariantValue","DuckDBTypeId","physical","writable","pathIndex","columnTypes","insertSql","sameValue","i","chunk","paths","placeholders","selected","existing","plans","position","plan","end","assignments","missing","changed","signature","appendRows","slice","repair","row","index","rowIndex","bind","run","reader","runAndReadAll","Map","getRowObjects","path","old","get","flatMap","column","undefined","duckdb","duckdbApi","getRowObjectsJS","name","Error","type","value","Object","is","typeId","VARCHAR","HUGEINT","Number","isInteger","DOUBLE","reconcileContent","touched","docs","delta","added","reparsed","removed","docPaths","vanished","changedDocs","Set","filter","has","all","add","runBatch","p","markContentStale","addColumns","names","exec","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BAgLaA;;;eAAAA;;;wBA/Kc;wBAGY;yBAGN;wBACP;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,qFAAqF;AACrF,kGAAkG;AAClG,iFAAiF;AACjF,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,IAAMC,qBAAqB,AAAC,wBAAkE,OAA3CD,gBAAgBE,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC,OAAM;AAC9F,wFAAwF;AACxF,IAAMC,uBAAuB;AAS7B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAChC,IAAMC,2BAA2B;AAEjC,SAAeC,oBAAuBC,MAAwB,EAAEC,GAAW,EAAEC,EAAiD;;YACtHC,MACAC,QACFC,QAGKC;;;;oBALI;;wBAAMN,OAAOO,OAAO,CAACN;;;oBAA5BE,OAAO;oBACPC;;;;;;;;;oBAGK;;wBAAMF,GAAGC;;;oBAAlBE,SAAS;;;;;;oBACFC;oBACPF,OAAOI,IAAI,CAACF;;;;;;oBAEd,IAAI;wBACFH,KAAKM,WAAW;oBAClB,EAAE,OAAOH,KAAK;wBACZF,OAAOI,IAAI,CAACF;oBACd;oBACA,IAAIF,OAAOM,MAAM,KAAK,GAAG,MAAMN,MAAM,CAAC,EAAE;oBACxC,IAAIA,OAAOM,MAAM,GAAG,GAAG,MAAM,IAAIC,eAAeP,QAAQ;oBACxD;;wBAAOC;;;;IACT;;AAEA,4EAA4E;AAC5E,gFAAgF;AAChF,SAAeO,kBAAkBC,IAAgB,EAAEC,eAAyB,EAAEC,IAAiB;;mBAEvFf,QACuC,MAArCgB,oBAAoBC,cACtBC,UAIAC,UACAC,WAEAC,aACAC,WACAC,WAOGC;;;;;mCACDC,OACAC,OACAC,cACAC,UACAC,UAMAC,OAKGC;;;;;4CACDC,MAKFC,KAWIC;;;;oDAhBFF,OAAOF,KAAK,CAACC,UAAS;oDAC5B,IAAI,CAACC,KAAKG,OAAO,IAAIH,KAAKI,OAAO,CAAC1B,MAAM,KAAK,GAAG;wDAC9C,EAAEqB;wDACF;;mFAAA;;oDACF;oDACIE,MAAMF,YAAW;oDACrB,MAAOE,MAAMH,MAAMpB,MAAM,IAAIoB,KAAK,CAACG,IAAI,CAACE,OAAO,KAAKH,KAAKG,OAAO,IAAIL,KAAK,CAACG,IAAI,CAACI,SAAS,KAAKL,KAAKK,SAAS,CAAE,EAAEJ;yDAC3GD,KAAKG,OAAO,EAAZH;;;;oDACF;;wDAAMM,IAAAA,oBAAU,EACdzB,MACA,eACAC,iBACAQ,WACAQ,MAAMS,KAAK,CAACR,WAAUE,KAAK/C,GAAG,CAAC,SAACsD;mEAAWA,OAAOC,GAAG;;;;oDALvD;;;;;;oDAQMP,cAAcF,KAAKI,OAAO,CAAClD,GAAG,CAAC,SAACwD;+DAAU,AAAC,GAAqC,OAAnCvD,IAAAA,oBAAU,EAAC2B,eAAe,CAAC4B,MAAM,GAAE;uDAAOtD,IAAI,CAAC;oDAClG;;wDAAMW,oBAAoBC,QAAQ,AAAC,0BAAqC,OAAZkC,aAAY,sBAAoB,SAAO/B;;2EACxFwC;;;;;oFACDF;;;;4FAAAA,MAAMX,KAAK,CAACa,SAAS,CAACF,GAAG;4FAC/BtC,KAAKyC,IAAI,CAAC,AAAC,qBAAGZ,KAAKI,OAAO,CAAClD,GAAG,CAAC,SAACwD;oGAAUD;wGAAAA,aAAAA,GAAG,CAACC,MAAM,cAAVD,wBAAAA,aAAc;;gGAAOA,GAAG,CAACrB,UAAU;;4FAC7E;;gGAAMjB,KAAK0C,GAAG;;;4FAAd;;;;;;4EACF;4EAJSF,WAAWZ;;;iFAAUY,CAAAA,WAAWV,GAAE;;;;;;;;;;;;4EAAG,EAAEU;;;;;;;;;;;4DAKlD;;;;oDANA;;;oDAQFZ,YAAWE;;;;;;;oCACb;oCA1CMR,QAAQV,KAAKwB,KAAK,CAACf,GAAGA,IAAI1B;oCAC1B4B,QAAQD,MAAMvC,GAAG,CAAC,SAACuD;+CAAQA,GAAG,CAACrB,UAAU;;oCACzCO,eAAeD,MAAMxC,GAAG,CAAC;+CAAM;uCAAKE,IAAI,CAAC;oCACzCwC,WAAWd,gBAAgB5B,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC;oCACrC;;wCAAMW,oBAAoBC,QAAQ,AAAC,UAAuD2B,OAA9CC,UAAS,uCAAkD,OAAbD,cAAa,MAAI,SAAOxB;;oDAE3H2C;;;;4DADN3C,KAAKyC,IAAI,CAAClB;4DACK;;gEAAMvB,KAAK4C,aAAa;;;4DAAjCD,SAAS;4DACf,2EAA2E;4DAC3E;;gEAAO,IAAIE,IAAIF,OAAOG,aAAa,GAAG/D,GAAG,CAAC,SAACuD;2EAAQ;wEAACA,IAAIS,IAAI;wEAAYT;qEAAI;;;;;4CAC9E;;;;oCALMZ,WAAW;oCAMXC,QAAQL,MAAMvC,GAAG,CAAC,SAACuD;wCACvB,IAAMU,MAAMtB,SAASuB,GAAG,CAACX,GAAG,CAACrB,UAAU;wCACvC,IAAMgB,UAAUe,MAAMrC,gBAAgBuC,OAAO,CAAC,SAACC,QAAQZ;mDAAWnB,UAAU4B,GAAG,CAACG,OAAO,EAAEb,GAAG,CAACC,MAAM,EAAEY,UAAU,EAAE,GAAG;gDAACZ;6CAAM;6CAAK,EAAE;wCAClI,OAAO;4CAAED,KAAAA;4CAAKN,SAASgB,QAAQI;4CAAWnB,SAAAA;4CAASC,WAAWD,QAAQhD,IAAI,CAAC;wCAAK;oCAClF;oCACS2C,WAAW;;;yCAAGA,CAAAA,WAAWD,MAAMpB,MAAM,AAAD;;;;;;;;;;;;;;;;;;;;;;oBA4B/C;oBA/DA,IAAIK,KAAKL,MAAM,KAAK,GAAG;;;oBACjBV,SAAS,AAACa,KAA0B2C,MAAM;oBACH;;wBAAMC,IAAAA,mBAAS;;;oBAAf,OAAA,eAArCzC,qBAAqC,KAArCA,oBAAoBC,eAAiB,KAAjBA;oBACX;;wBAAMlB,oBAAoBC,QAAQ,kCAAkC,SAAOG;;oCACpF2C;;;;4CAAS;;gDAAM3C,KAAK4C,aAAa;;;4CAAjCD,SAAS;4CACf;;gDAAOA,OAAOY,eAAe;;;;4BAC/B;;;;oBAHMxC,WAAW;oBAIXC,WAAW,IAAI6B,IAAIlC,gBAAgB5B,GAAG,CAAC,SAACyE,MAAMnC;+BAAM;4BAACmC;4BAAMnC;yBAAE;;oBAC7DJ,YAAYD,SAASiC,GAAG,CAAC;oBAC/B,IAAIhC,cAAcmC,WAAW,MAAM,IAAIK,MAAM;oBACvCvC,cAAc,IAAI2B,IAAI9B,SAAShC,GAAG,CAAC,SAACoE;+BAAW;4BAACA,OAAOK,IAAI;4BAAEL,OAAOO,IAAI;yBAAC;;oBACzEvC,YAAY,AAAC,4BAAkFR,OAAvDA,gBAAgB5B,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAAsD,OAA1C0B,gBAAgB5B,GAAG,CAAC;+BAAM;uBAAKE,IAAI,CAAC,OAAM;oBACzImC,YAAY,mBAAC4B,KAAkBW,OAAgBR;4BAKpBH;wBAJ/B,IAAIW,UAAU,QAAQA,UAAUP,WAAW,OAAOJ,QAAQ;wBAC1D,IAAI9B,YAAY+B,GAAG,CAACE,YAAY,WAAW,OAAOS,OAAOC,EAAE,CAACb,KAAKW;wBACjE,IAAI,CAAEX,AAAG,YAAHA,KAAenC,qBAAqB,OAAO;wBACjD,IAAMiD,SAAS,OAAOH,UAAU,WAAW7C,aAAaiD,OAAO,GAAG,CAAA,OAAOJ,sCAAP,SAAOA,MAAI,MAAM,WAAW7C,aAAakD,OAAO,GAAG,OAAOL,UAAU,YAAY,CAACM,OAAOC,SAAS,CAACP,SAAS7C,aAAaqD,MAAM,GAAGf;wBACnM,OAAOU,WAAWV,aAAaJ,EAAAA,YAAAA,IAAIU,IAAI,cAARV,gCAAAA,UAAUc,MAAM,MAAKA,UAAUF,OAAOC,EAAE,CAACb,IAAIW,KAAK,EAAEA;oBACrF;oBACStC,IAAI;;;yBAAGA,CAAAA,IAAIT,KAAKL,MAAM,AAAD;;;;;;;;;;;;oBAAGc,KAAK1B;;;;;;;;;;;IA6CxC;;AAEA,SAAeyE,iBAAiB1D,IAAgB,EAAE2D,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;;YACrGC,OACAC,UACAxC,SACAyC,SACGrD,GAGarB,OAFdsB,OACAtB,MACAY,MACAc,UACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMtC,KACHkD,KAOJqC,UACAC,UACGvD,IAGarB,QACpB0E,UAHMpD,QACAtB,OACAY,OAUFiE;;;;oBAhCAL,QAAQ,IAAIM,IAAIP,MAAMC,KAAK;oBAC3BC,WAAWH,KAAKS,MAAM,CAAC,SAAC3F;+BAAQ,CAACoF,MAAMQ,GAAG,CAAC5F,IAAIC,OAAO;;oBACtD4C,UAAU,IAAI6C,IAAIP,MAAMC,KAAK;oBAC7BE;oBACGrD,IAAI;;;yBAAGA,CAAAA,IAAIoD,SAASlE,MAAM,AAAD;;;;oBAC1Be,QAAQmD,SAASrC,KAAK,CAACf,GAAGA,IAAInC;oBACvB;;wBAAMwB,KAAKN,OAAO,CAAC,AAAC,qEAAoG,OAAhCkB,MAAMvC,GAAG,CAAC;mCAAM;2BAAKE,IAAI,CAAC,OAAM;;;oBAA/He,OAAO;oBACC;;wBAAMA,CAAAA,QAAAA,MAAKiF,GAAG,OAARjF,OAAS,qBAAGsB,MAAMvC,GAAG,CAAC,SAACK;mCAAQA,IAAIC,OAAO;;;;oBAAxDuB,OAAQ;oBACRc,WAAW,IAAImB,IAAIjC,KAAK7B,GAAG,CAAC,SAACuD;+BAAQ;4BAACA,IAAIS,IAAI;4BAAET;yBAAI;;oBACrD,kCAAA,2BAAA;;wBAAL,IAAK,YAAahB,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdlC,MAAN;4BACGkD,MAAMZ,SAASuB,GAAG,CAAC7D,IAAIC,OAAO;4BACpC,IAAIiD,OAAOA,IAAI/C,KAAK,KAAKH,IAAIE,MAAM,CAACC,KAAK,IAAI+C,IAAI9C,OAAO,KAAKJ,IAAIE,MAAM,CAACE,OAAO,IAAI8C,IAAI7C,IAAI,KAAKL,IAAIE,MAAM,CAACG,IAAI,EAAE;4BACjHwC,QAAQiD,GAAG,CAAC9F,IAAIC,OAAO;4BACvB,IAAIiD,KAAKoC,QAAQrE,IAAI,CAACjB,IAAIC,OAAO;wBACnC;;wBALK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBAL8BgC,KAAKnC;;;;;;oBAapCyF,WAAW,IAAIG,IAAIR,KAAKvF,GAAG,CAAC,SAACK;+BAAQA,IAAIC,OAAO;;oBAChDuF,WAAWP,QAAQU,MAAM,CAAC,SAAChC;+BAAS,CAAC4B,SAASK,GAAG,CAACjC;;oBAC/C1B,KAAI;;;yBAAGA,CAAAA,KAAIuD,SAASrE,MAAM,AAAD;;;;oBAC1Be,SAAQsD,SAASxC,KAAK,CAACf,IAAGA,KAAInC;oBACvB;;wBAAMwB,KAAKN,OAAO,CAAC,AAAC,+CAA8E,OAAhCkB,OAAMvC,GAAG,CAAC;mCAAM;2BAAKE,IAAI,CAAC,OAAM;;;oBAAzGe,QAAO;oBACC;;wBAAMA,CAAAA,SAAAA,OAAKiF,GAAG,OAARjF,QAAS,qBAAGsB;;;oBAA1BV,QAAQ;oBACd8D,CAAAA,WAAAA,SAAQrE,IAAI,OAAZqE,UAAa,qBAAG9D,MAAK7B,GAAG,CAAC,SAACuD;+BAAQA,IAAIS,IAAI;;;;oBAJP1B,MAAKnC;;;;;;yBAOtCwF,CAAAA,QAAQnE,MAAM,GAAG,CAAA,GAAjBmE;;;;oBACF;;wBAAMhE,KAAKyE,QAAQ,CACjB,wCACAT,QAAQ3F,GAAG,CAAC,SAACqG;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;oBAIF,+EAA+E;oBACzEP,cAAcP,KAAKS,MAAM,CAAC,SAAC3F;+BAAQ6C,QAAQ+C,GAAG,CAAC5F,IAAIC,OAAO;;oBAChE;;wBAAM8C,IAAAA,oBAAU,EAACzB,MAAM,WAAW7B,iBAAiBC,oBAAoB+F,YAAY9F,GAAG,CAACI;;;oBAAvF;yBAGIuF,CAAAA,QAAQnE,MAAM,GAAG,KAAKsE,YAAYtE,MAAM,GAAG,CAAA,GAA3CmE;;;;oBAA8C;;wBAAMW,IAAAA,2BAAgB,EAAC3E;;;oBAAvB;;;;;;;;IACpD;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAe4E,WAAW5E,IAAgB,EAAE6E,KAAe;;;;;oBACzD,IAAIA,MAAMhF,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMG,KAAK8E,IAAI,CAACD,MAAMxG,GAAG,CAAC,SAACyE;mCAAS,AAAC,sCAAsD,OAAjBxE,IAAAA,oBAAU,EAACwE,OAAM;2BAAWvE,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEO,IAAML,gBAAkC;IAC7C6G,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQjG,yBAAyB;YACnC,MAAM,IAAIkG,oBAAU,CAAC,gBAAgB,AAAC,0BAA+ElG,OAAtDiG,OAAM,kDAAwE,OAAxBjG,yBAAwB;QAC/I;IACF;IACA4F,YAAAA;IACA7E,mBAAAA;IACA2D,kBAAAA;AACF"}
|
|
@@ -56,8 +56,8 @@ function _unsupported_iterable_to_array(o, minLen) {
|
|
|
56
56
|
if (n === "Map" || n === "Set") return Array.from(n);
|
|
57
57
|
if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
|
|
58
58
|
}
|
|
59
|
-
// One embed segment's provider/model/chunkVersion, ignoring any trailing `@identity`.
|
|
60
|
-
//
|
|
59
|
+
// One embed segment's provider/model/endpoint/chunkVersion, ignoring any trailing `@identity`.
|
|
60
|
+
// The endpoint-less HTTP shape remains readable so upgrading invalidates vectors, not core data.
|
|
61
61
|
function parseEmbedSegment(sig) {
|
|
62
62
|
var part = sig.split('|').find(function(p) {
|
|
63
63
|
return p.startsWith('embed:');
|
|
@@ -65,13 +65,23 @@ function parseEmbedSegment(sig) {
|
|
|
65
65
|
if (part === undefined || part === 'embed:off') return null;
|
|
66
66
|
var at = part.indexOf('@');
|
|
67
67
|
var body = at === -1 ? part : part.slice(0, at);
|
|
68
|
-
var
|
|
69
|
-
if (
|
|
70
|
-
|
|
68
|
+
var withEndpoint = /^embed:(openai|cohere):(.+):endpoint=([0-9a-f]{64}):(chunk:v\d+(?::\d+)?)$/.exec(body);
|
|
69
|
+
if (withEndpoint) {
|
|
70
|
+
var _withEndpoint = _sliced_to_array(withEndpoint, 5), provider = _withEndpoint[1], model = _withEndpoint[2], endpoint = _withEndpoint[3], chunkVersion = _withEndpoint[4];
|
|
71
|
+
return {
|
|
72
|
+
provider: provider,
|
|
73
|
+
model: model,
|
|
74
|
+
endpoint: endpoint,
|
|
75
|
+
chunkVersion: chunkVersion
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
var legacy = /^embed:(static|openai|cohere):(.+):(chunk:v\d+(?::\d+)?)$/.exec(body);
|
|
79
|
+
if (!legacy) return null;
|
|
80
|
+
var _legacy = _sliced_to_array(legacy, 4), provider1 = _legacy[1], model1 = _legacy[2], chunkVersion1 = _legacy[3];
|
|
71
81
|
return {
|
|
72
|
-
provider:
|
|
73
|
-
model:
|
|
74
|
-
chunkVersion:
|
|
82
|
+
provider: provider1,
|
|
83
|
+
model: model1,
|
|
84
|
+
chunkVersion: chunkVersion1
|
|
75
85
|
};
|
|
76
86
|
}
|
|
77
87
|
function classifyEmbedChange(before, after) {
|
|
@@ -79,7 +89,7 @@ function classifyEmbedChange(before, after) {
|
|
|
79
89
|
var a = parseEmbedSegment(after);
|
|
80
90
|
if (b === null || a === null) return null;
|
|
81
91
|
if (b.chunkVersion !== a.chunkVersion) return 'chunk';
|
|
82
|
-
if (b.provider !== a.provider || b.model !== a.model) return 'model';
|
|
92
|
+
if (b.provider !== a.provider || b.model !== a.model || b.endpoint !== a.endpoint) return 'model';
|
|
83
93
|
return null;
|
|
84
94
|
}
|
|
85
95
|
/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/embed-scope.ts"],"sourcesContent":["// Decomposition of the `embed:` featureSignature segment (config/access.ts's featureSignature,\n// features/embed.ts's signature()), for open()'s narrow embed-invalidation routing.\n\nexport type EmbedChangeKind = 'model' | 'chunk';\n\ninterface EmbedSegment {\n provider: string;\n model: string;\n chunkVersion: string;\n}\n\n// One embed segment's provider/model/chunkVersion, ignoring any trailing `@identity
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/embed-scope.ts"],"sourcesContent":["// Decomposition of the `embed:` featureSignature segment (config/access.ts's featureSignature,\n// features/embed.ts's signature()), for open()'s narrow embed-invalidation routing.\n\nexport type EmbedChangeKind = 'model' | 'chunk';\n\ninterface EmbedSegment {\n provider: string;\n model: string;\n endpoint?: string;\n chunkVersion: string;\n}\n\n// One embed segment's provider/model/endpoint/chunkVersion, ignoring any trailing `@identity`.\n// The endpoint-less HTTP shape remains readable so upgrading invalidates vectors, not core data.\nfunction parseEmbedSegment(sig: string): EmbedSegment | null {\n const part = sig.split('|').find((p) => p.startsWith('embed:'));\n if (part === undefined || part === 'embed:off') return null;\n const at = part.indexOf('@');\n const body = at === -1 ? part : part.slice(0, at);\n const withEndpoint = /^embed:(openai|cohere):(.+):endpoint=([0-9a-f]{64}):(chunk:v\\d+(?::\\d+)?)$/.exec(body);\n if (withEndpoint) {\n const [, provider, model, endpoint, chunkVersion] = withEndpoint;\n return { provider, model, endpoint, chunkVersion };\n }\n const legacy = /^embed:(static|openai|cohere):(.+):(chunk:v\\d+(?::\\d+)?)$/.exec(body);\n if (!legacy) return null;\n const [, provider, model, chunkVersion] = legacy;\n return { provider, model, chunkVersion };\n}\n\n// 'chunk' when chunkTokens or the chunk version moved (boundaries changed, embeddings rebuild);\n// 'model' when only the provider or model moved (rows unchanged, values stale); null otherwise.\nexport function classifyEmbedChange(before: string, after: string): EmbedChangeKind | null {\n const b = parseEmbedSegment(before);\n const a = parseEmbedSegment(after);\n if (b === null || a === null) return null;\n if (b.chunkVersion !== a.chunkVersion) return 'chunk';\n if (b.provider !== a.provider || b.model !== a.model || b.endpoint !== a.endpoint) return 'model';\n return null;\n}\n"],"names":["classifyEmbedChange","parseEmbedSegment","sig","part","split","find","p","startsWith","undefined","at","indexOf","body","slice","withEndpoint","exec","provider","model","endpoint","chunkVersion","legacy","before","after","b","a"],"mappings":"AAAA,+FAA+F;AAC/F,oFAAoF;;;;;+BA+BpEA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AApBhB,+FAA+F;AAC/F,iGAAiG;AACjG,SAASC,kBAAkBC,GAAW;IACpC,IAAMC,OAAOD,IAAIE,KAAK,CAAC,KAAKC,IAAI,CAAC,SAACC;eAAMA,EAAEC,UAAU,CAAC;;IACrD,IAAIJ,SAASK,aAAaL,SAAS,aAAa,OAAO;IACvD,IAAMM,KAAKN,KAAKO,OAAO,CAAC;IACxB,IAAMC,OAAOF,OAAO,CAAC,IAAIN,OAAOA,KAAKS,KAAK,CAAC,GAAGH;IAC9C,IAAMI,eAAe,6EAA6EC,IAAI,CAACH;IACvG,IAAIE,cAAc;QAChB,IAAoDA,iCAAAA,kBAA3CE,WAA2CF,kBAAjCG,QAAiCH,kBAA1BI,WAA0BJ,kBAAhBK,eAAgBL;QACpD,OAAO;YAAEE,UAAAA;YAAUC,OAAAA;YAAOC,UAAAA;YAAUC,cAAAA;QAAa;IACnD;IACA,IAAMC,SAAS,4DAA4DL,IAAI,CAACH;IAChF,IAAI,CAACQ,QAAQ,OAAO;IACpB,IAA0CA,2BAAAA,YAAjCJ,YAAiCI,YAAvBH,SAAuBG,YAAhBD,gBAAgBC;IAC1C,OAAO;QAAEJ,UAAAA;QAAUC,OAAAA;QAAOE,cAAAA;IAAa;AACzC;AAIO,SAASlB,oBAAoBoB,MAAc,EAAEC,KAAa;IAC/D,IAAMC,IAAIrB,kBAAkBmB;IAC5B,IAAMG,IAAItB,kBAAkBoB;IAC5B,IAAIC,MAAM,QAAQC,MAAM,MAAM,OAAO;IACrC,IAAID,EAAEJ,YAAY,KAAKK,EAAEL,YAAY,EAAE,OAAO;IAC9C,IAAII,EAAEP,QAAQ,KAAKQ,EAAER,QAAQ,IAAIO,EAAEN,KAAK,KAAKO,EAAEP,KAAK,IAAIM,EAAEL,QAAQ,KAAKM,EAAEN,QAAQ,EAAE,OAAO;IAC1F,OAAO;AACT"}
|
|
@@ -3,13 +3,18 @@ import { clearCache } from './cache.js';
|
|
|
3
3
|
import type { InternalOpenOptions, OpenOptions, OpenResult } from './open.js';
|
|
4
4
|
import type { Stages } from './stages.js';
|
|
5
5
|
export declare function openStoreFor(cfg: ResolvedConfig, options: InternalOpenOptions): Promise<OpenResult>;
|
|
6
|
+
/** Opens a store through the root `open` API, preparing capabilities unless `build` is `false`. */
|
|
6
7
|
export declare function openStore(cfg: ResolvedConfig, options?: OpenOptions): Promise<OpenResult>;
|
|
7
8
|
export declare function lexicalReadiness(cfg: ResolvedConfig): Promise<{
|
|
8
9
|
ready: boolean;
|
|
9
10
|
error: string | null;
|
|
10
11
|
}>;
|
|
12
|
+
/** Options for the root `build(config, options)` API. */
|
|
11
13
|
export interface BuildOptions {
|
|
14
|
+
/** Recreates the derived cache before building. */
|
|
12
15
|
force?: boolean;
|
|
16
|
+
/** Cancels supported provider I/O and stops between phases after active native work finishes. */
|
|
17
|
+
signal?: AbortSignal;
|
|
13
18
|
}
|
|
14
19
|
export interface BuildResult {
|
|
15
20
|
cfg: ResolvedConfig;
|
|
@@ -18,6 +23,7 @@ export interface BuildResult {
|
|
|
18
23
|
warnings: string[];
|
|
19
24
|
stages: Stages;
|
|
20
25
|
}
|
|
26
|
+
/** Builds every configured capability, waiting for active native work and cleanup before settling. */
|
|
21
27
|
export declare function buildIndex(cfg: ResolvedConfig, options?: BuildOptions): Promise<BuildResult>;
|
|
22
28
|
export declare function storeFilename(cfg: ResolvedConfig): string;
|
|
23
29
|
export type { OpenOptions, OpenResult } from './open.js';
|
|
@@ -3,13 +3,18 @@ import { clearCache } from './cache.js';
|
|
|
3
3
|
import type { InternalOpenOptions, OpenOptions, OpenResult } from './open.js';
|
|
4
4
|
import type { Stages } from './stages.js';
|
|
5
5
|
export declare function openStoreFor(cfg: ResolvedConfig, options: InternalOpenOptions): Promise<OpenResult>;
|
|
6
|
+
/** Opens a store through the root `open` API, preparing capabilities unless `build` is `false`. */
|
|
6
7
|
export declare function openStore(cfg: ResolvedConfig, options?: OpenOptions): Promise<OpenResult>;
|
|
7
8
|
export declare function lexicalReadiness(cfg: ResolvedConfig): Promise<{
|
|
8
9
|
ready: boolean;
|
|
9
10
|
error: string | null;
|
|
10
11
|
}>;
|
|
12
|
+
/** Options for the root `build(config, options)` API. */
|
|
11
13
|
export interface BuildOptions {
|
|
14
|
+
/** Recreates the derived cache before building. */
|
|
12
15
|
force?: boolean;
|
|
16
|
+
/** Cancels supported provider I/O and stops between phases after active native work finishes. */
|
|
17
|
+
signal?: AbortSignal;
|
|
13
18
|
}
|
|
14
19
|
export interface BuildResult {
|
|
15
20
|
cfg: ResolvedConfig;
|
|
@@ -18,6 +23,7 @@ export interface BuildResult {
|
|
|
18
23
|
warnings: string[];
|
|
19
24
|
stages: Stages;
|
|
20
25
|
}
|
|
26
|
+
/** Builds every configured capability, waiting for active native work and cleanup before settling. */
|
|
21
27
|
export declare function buildIndex(cfg: ResolvedConfig, options?: BuildOptions): Promise<BuildResult>;
|
|
22
28
|
export declare function storeFilename(cfg: ResolvedConfig): string;
|
|
23
29
|
export type { OpenOptions, OpenResult } from './open.js';
|
package/dist/cjs/store/index.js
CHANGED
|
@@ -265,8 +265,9 @@ function validateRequirements(cfg, entry, requirements) {
|
|
|
265
265
|
}
|
|
266
266
|
function openStoreFor(cfg, options) {
|
|
267
267
|
return _async_to_generator(function() {
|
|
268
|
-
var _options_requirements, entry;
|
|
268
|
+
var _options_requirements, _options_signal, entry;
|
|
269
269
|
return _ts_generator(this, function(_state) {
|
|
270
|
+
(_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
|
|
270
271
|
entry = entryFor((0, _indexts.storeName)(cfg));
|
|
271
272
|
validateRequirements(cfg, entry, (_options_requirements = options.requirements) !== null && _options_requirements !== void 0 ? _options_requirements : new Set([
|
|
272
273
|
'core'
|
|
@@ -291,7 +292,8 @@ function openStore(_0) {
|
|
|
291
292
|
build: buildEnabled,
|
|
292
293
|
requirements: buildEnabled ? requirementsFor(cfg) : new Set([
|
|
293
294
|
'core'
|
|
294
|
-
])
|
|
295
|
+
]),
|
|
296
|
+
signal: options.signal
|
|
295
297
|
})
|
|
296
298
|
];
|
|
297
299
|
});
|
|
@@ -369,18 +371,20 @@ function lexicalReadiness(cfg) {
|
|
|
369
371
|
}
|
|
370
372
|
function buildIndex(_0) {
|
|
371
373
|
return _async_to_generator(function(cfg) {
|
|
372
|
-
var options, opened, _store, result;
|
|
374
|
+
var options, _options_signal, opened, _store, result;
|
|
373
375
|
var _arguments = arguments;
|
|
374
376
|
return _ts_generator(this, function(_state) {
|
|
375
377
|
switch(_state.label){
|
|
376
378
|
case 0:
|
|
377
379
|
options = _arguments.length > 1 && _arguments[1] !== void 0 ? _arguments[1] : {};
|
|
380
|
+
(_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
|
|
378
381
|
if (options.force) (0, _cachets.clearCache)(cfg);
|
|
379
382
|
return [
|
|
380
383
|
4,
|
|
381
384
|
openStoreFor(cfg, {
|
|
382
385
|
build: true,
|
|
383
|
-
requirements: requirementsFor(cfg)
|
|
386
|
+
requirements: requirementsFor(cfg),
|
|
387
|
+
signal: options.signal
|
|
384
388
|
})
|
|
385
389
|
];
|
|
386
390
|
case 1:
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + the public open/build boundary. Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { clearCache } from './cache.ts';\nimport { DB_FILENAME as DUCKDB_FILENAME, openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { BuildRequirement, InternalOpenOptions, OpenOptions, OpenResult } from './open.ts';\nimport { openSqlite, DB_FILENAME as SQLITE_FILENAME } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport type { Stages } from './stages.ts';\nimport { openTurso, DB_FILENAME as TURSO_FILENAME } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n capabilities: ReadonlySet<Capability>;\n filename: string;\n open: (cfg: ResolvedConfig, options?: InternalOpenOptions) => Promise<OpenResult>;\n}\n\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n sqlite: { capabilities: SQLITE_CAPABILITIES, filename: SQLITE_FILENAME, open: openSqlite },\n duckdb: { capabilities: DUCKDB_CAPABILITIES, filename: DUCKDB_FILENAME, open: openDuckdb },\n turso: { capabilities: TURSO_CAPABILITIES, filename: TURSO_FILENAME, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n const entry = REGISTRY[name];\n if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n return entry;\n}\n\nfunction requirementsFor(cfg: ResolvedConfig): Set<BuildRequirement> {\n const requirements = new Set<BuildRequirement>(['core', 'lexical']);\n if (anyPresetEmbeds(cfg)) requirements.add('vectors');\n return requirements;\n}\n\nfunction validateRequirements(cfg: ResolvedConfig, entry: StoreEntry, requirements: ReadonlySet<BuildRequirement>): void {\n if (requirements.has('vectors') && anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n const name = storeName(cfg);\n throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n }\n}\n\nexport async function openStoreFor(cfg: ResolvedConfig, options: InternalOpenOptions): Promise<OpenResult> {\n const entry = entryFor(storeName(cfg));\n validateRequirements(cfg, entry, options.requirements ?? new Set<BuildRequirement>(['core']));\n return entry.open(cfg, options);\n}\n\nexport async function openStore(cfg: ResolvedConfig, options: OpenOptions = {}): Promise<OpenResult> {\n const buildEnabled = options.build !== false;\n return openStoreFor(cfg, {\n build: buildEnabled,\n requirements: buildEnabled ? requirementsFor(cfg) : new Set<BuildRequirement>(['core']),\n });\n}\n\nexport async function lexicalReadiness(cfg: ResolvedConfig): Promise<{ ready: boolean; error: string | null }> {\n let opened: OpenResult;\n try {\n opened = await openStoreFor(cfg, { build: false, requirements: new Set<BuildRequirement>(['core', 'lexical']) });\n } catch (err) {\n if (err instanceof SenseError && err.code === 'INDEX_NOT_READY') return { ready: false, error: err.message };\n throw err;\n }\n try {\n return { ready: true, error: null };\n } finally {\n await opened.store.close();\n }\n}\n\nexport interface BuildOptions {\n force?: boolean;\n}\n\nexport interface BuildResult {\n cfg: ResolvedConfig;\n dbPath: string;\n parsed: number;\n warnings: string[];\n stages: Stages;\n}\n\nexport async function buildIndex(cfg: ResolvedConfig, options: BuildOptions = {}): Promise<BuildResult> {\n if (options.force) clearCache(cfg);\n const opened = await openStoreFor(cfg, { build: true, requirements: requirementsFor(cfg) });\n try {\n const { store: _store, ...result } = opened;\n return result;\n } finally {\n await opened.store.close();\n }\n}\n\nexport function storeFilename(cfg: ResolvedConfig): string {\n return entryFor(storeName(cfg)).filename;\n}\n\nexport type { OpenOptions, OpenResult } from './open.ts';\nexport { docCount } from './open.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Stages } from './stages.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\nexport { clearCache };\n"],"names":["CAPABILITY_NAMES","DB_FILENAME","SCHEMA_VERSION","buildIndex","clearCache","docCount","getMeta","lexicalReadiness","openStore","openStoreFor","setMeta","storeFilename","REGISTRY","sqlite","capabilities","SQLITE_CAPABILITIES","filename","SQLITE_FILENAME","open","openSqlite","duckdb","DUCKDB_CAPABILITIES","DUCKDB_FILENAME","openDuckdb","turso","TURSO_CAPABILITIES","TURSO_FILENAME","openTurso","entryFor","name","entry","SenseError","Object","keys","join","requirementsFor","cfg","requirements","Set","anyPresetEmbeds","add","validateRequirements","has","storeName","options","buildEnabled","build","opened","err","code","ready","error","message","store","close","_store","result","force"],"mappings":"AAAA,gGAAgG;AAChG,kEAAkE;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + the public open/build boundary. Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { clearCache } from './cache.ts';\nimport { DB_FILENAME as DUCKDB_FILENAME, openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { BuildRequirement, InternalOpenOptions, OpenOptions, OpenResult } from './open.ts';\nimport { openSqlite, DB_FILENAME as SQLITE_FILENAME } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport type { Stages } from './stages.ts';\nimport { openTurso, DB_FILENAME as TURSO_FILENAME } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n capabilities: ReadonlySet<Capability>;\n filename: string;\n open: (cfg: ResolvedConfig, options?: InternalOpenOptions) => Promise<OpenResult>;\n}\n\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n sqlite: { capabilities: SQLITE_CAPABILITIES, filename: SQLITE_FILENAME, open: openSqlite },\n duckdb: { capabilities: DUCKDB_CAPABILITIES, filename: DUCKDB_FILENAME, open: openDuckdb },\n turso: { capabilities: TURSO_CAPABILITIES, filename: TURSO_FILENAME, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n const entry = REGISTRY[name];\n if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n return entry;\n}\n\nfunction requirementsFor(cfg: ResolvedConfig): Set<BuildRequirement> {\n const requirements = new Set<BuildRequirement>(['core', 'lexical']);\n if (anyPresetEmbeds(cfg)) requirements.add('vectors');\n return requirements;\n}\n\nfunction validateRequirements(cfg: ResolvedConfig, entry: StoreEntry, requirements: ReadonlySet<BuildRequirement>): void {\n if (requirements.has('vectors') && anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n const name = storeName(cfg);\n throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n }\n}\n\nexport async function openStoreFor(cfg: ResolvedConfig, options: InternalOpenOptions): Promise<OpenResult> {\n options.signal?.throwIfAborted();\n const entry = entryFor(storeName(cfg));\n validateRequirements(cfg, entry, options.requirements ?? new Set<BuildRequirement>(['core']));\n return entry.open(cfg, options);\n}\n\n/** Opens a store through the root `open` API, preparing capabilities unless `build` is `false`. */\nexport async function openStore(cfg: ResolvedConfig, options: OpenOptions = {}): Promise<OpenResult> {\n const buildEnabled = options.build !== false;\n return openStoreFor(cfg, {\n build: buildEnabled,\n requirements: buildEnabled ? requirementsFor(cfg) : new Set<BuildRequirement>(['core']),\n signal: options.signal,\n });\n}\n\nexport async function lexicalReadiness(cfg: ResolvedConfig): Promise<{ ready: boolean; error: string | null }> {\n let opened: OpenResult;\n try {\n opened = await openStoreFor(cfg, { build: false, requirements: new Set<BuildRequirement>(['core', 'lexical']) });\n } catch (err) {\n if (err instanceof SenseError && err.code === 'INDEX_NOT_READY') return { ready: false, error: err.message };\n throw err;\n }\n try {\n return { ready: true, error: null };\n } finally {\n await opened.store.close();\n }\n}\n\n/** Options for the root `build(config, options)` API. */\nexport interface BuildOptions {\n /** Recreates the derived cache before building. */\n force?: boolean;\n /** Cancels supported provider I/O and stops between phases after active native work finishes. */\n signal?: AbortSignal;\n}\n\nexport interface BuildResult {\n cfg: ResolvedConfig;\n dbPath: string;\n parsed: number;\n warnings: string[];\n stages: Stages;\n}\n\n/** Builds every configured capability, waiting for active native work and cleanup before settling. */\nexport async function buildIndex(cfg: ResolvedConfig, options: BuildOptions = {}): Promise<BuildResult> {\n options.signal?.throwIfAborted();\n if (options.force) clearCache(cfg);\n const opened = await openStoreFor(cfg, { build: true, requirements: requirementsFor(cfg), signal: options.signal });\n try {\n const { store: _store, ...result } = opened;\n return result;\n } finally {\n await opened.store.close();\n }\n}\n\nexport function storeFilename(cfg: ResolvedConfig): string {\n return entryFor(storeName(cfg)).filename;\n}\n\nexport type { OpenOptions, OpenResult } from './open.ts';\nexport { docCount } from './open.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Stages } from './stages.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\nexport { clearCache };\n"],"names":["CAPABILITY_NAMES","DB_FILENAME","SCHEMA_VERSION","buildIndex","clearCache","docCount","getMeta","lexicalReadiness","openStore","openStoreFor","setMeta","storeFilename","REGISTRY","sqlite","capabilities","SQLITE_CAPABILITIES","filename","SQLITE_FILENAME","open","openSqlite","duckdb","DUCKDB_CAPABILITIES","DUCKDB_FILENAME","openDuckdb","turso","TURSO_CAPABILITIES","TURSO_FILENAME","openTurso","entryFor","name","entry","SenseError","Object","keys","join","requirementsFor","cfg","requirements","Set","anyPresetEmbeds","add","validateRequirements","has","storeName","options","signal","throwIfAborted","buildEnabled","build","opened","err","code","ready","error","message","store","close","_store","result","force"],"mappings":"AAAA,gGAAgG;AAChG,kEAAkE;;;;;;;;;;;;QAsHzDA;eAAAA,yBAAgB;;QAHhBC;eAAAA,oBAAW;;QAAEC;eAAAA,uBAAc;;QAnBdC;eAAAA;;QAuBbC;eAAAA,mBAAU;;QANVC;eAAAA,iBAAQ;;QACRC;eAAAA,iBAAO;;QAlDMC;eAAAA;;QATAC;eAAAA;;QARAC;eAAAA;;QAmEJC;eAAAA,iBAAO;;QANTC;eAAAA;;;uBAzG2B;wBAChB;uBACA;sBACgC;uBACP;uBAEO;wBACP;uBAEK;wBACN;uBAoG1B;wBACQ;uBAIA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAhGjC,IAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcC,sBAAmB;QAAEC,UAAUC,oBAAe;QAAEC,MAAMC,mBAAU;IAAC;IACzFC,QAAQ;QAAEN,cAAcO,qBAAmB;QAAEL,UAAUM,mBAAe;QAAEJ,MAAMK,kBAAU;IAAC;IACzFC,OAAO;QAAEV,cAAcW,sBAAkB;QAAET,UAAUU,oBAAc;QAAER,MAAMS,kBAAS;IAAC;AACvF;AAEA,SAASC,SAASC,IAAe;IAC/B,IAAMC,QAAQlB,QAAQ,CAACiB,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIC,oBAAU,CAAC,iBAAiB,AAAC,0BAA8CC,OAArBH,MAAK,kBAAiD,OAAjCG,OAAOC,IAAI,CAACrB,UAAUsB,IAAI,CAAC;IAC5H,OAAOJ;AACT;AAEA,SAASK,gBAAgBC,GAAmB;IAC1C,IAAMC,eAAe,IAAIC,IAAsB;QAAC;QAAQ;KAAU;IAClE,IAAIC,IAAAA,wBAAe,EAACH,MAAMC,aAAaG,GAAG,CAAC;IAC3C,OAAOH;AACT;AAEA,SAASI,qBAAqBL,GAAmB,EAAEN,KAAiB,EAAEO,YAA2C;IAC/G,IAAIA,aAAaK,GAAG,CAAC,cAAcH,IAAAA,wBAAe,EAACH,QAAQ,CAACN,MAAMhB,YAAY,CAAC4B,GAAG,CAAC,YAAY;QAC7F,IAAMb,OAAOc,IAAAA,kBAAS,EAACP;QACvB,MAAM,IAAIL,oBAAU,CAAC,4BAA4B,AAAC,UAAc,OAALF,MAAK;IAClE;AACF;AAEO,SAAepB,aAAa2B,GAAmB,EAAEQ,OAA4B;;YAGjDA,uBAFjCA,iBACMd;;aADNc,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;YACxBhB,QAAQF,SAASe,IAAAA,kBAAS,EAACP;YACjCK,qBAAqBL,KAAKN,QAAOc,wBAAAA,QAAQP,YAAY,cAApBO,mCAAAA,wBAAwB,IAAIN;gBAAuB;;YACpF;;gBAAOR,MAAMZ,IAAI,CAACkB,KAAKQ;;;IACzB;;AAGO,SAAepC;wCAAU4B,GAAmB;YAAEQ,SAC7CG;;;YAD6CH,UAAAA,oEAAuB,CAAC;YACrEG,eAAeH,QAAQI,KAAK,KAAK;YACvC;;gBAAOvC,aAAa2B,KAAK;oBACvBY,OAAOD;oBACPV,cAAcU,eAAeZ,gBAAgBC,OAAO,IAAIE;wBAAuB;;oBAC/EO,QAAQD,QAAQC,MAAM;gBACxB;;;IACF;;AAEO,SAAetC,iBAAiB6B,GAAmB;;YACpDa,QAGKC;;;;;;;;;;oBADE;;wBAAMzC,aAAa2B,KAAK;4BAAEY,OAAO;4BAAOX,cAAc,IAAIC;gCAAuB;gCAAQ;;wBAAY;;;oBAA9GW,SAAS;;;;;;oBACFC;oBACP,IAAIA,AAAG,YAAHA,KAAenB,oBAAU,KAAImB,IAAIC,IAAI,KAAK,mBAAmB;;wBAAO;4BAAEC,OAAO;4BAAOC,OAAOH,IAAII,OAAO;wBAAC;;oBAC3G,MAAMJ;;;;;;;;oBAGN;;wBAAO;4BAAEE,OAAO;4BAAMC,OAAO;wBAAK;;;oBAElC;;wBAAMJ,OAAOM,KAAK,CAACC,KAAK;;;oBAAxB;;;;;;;;;;IAEJ;;AAmBO,SAAerD;wCAAWiC,GAAmB;YAAEQ,SACpDA,iBAEMK,QAEWQ,QAAWC;;;;;oBALwBd,UAAAA,oEAAwB,CAAC;qBAC7EA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;oBAC9B,IAAIF,QAAQe,KAAK,EAAEvD,IAAAA,mBAAU,EAACgC;oBACf;;wBAAM3B,aAAa2B,KAAK;4BAAEY,OAAO;4BAAMX,cAAcF,gBAAgBC;4BAAMS,QAAQD,QAAQC,MAAM;wBAAC;;;oBAA3GI,SAAS;;;;;;;;;oBAEEQ,SAAsBR,OAA7BM,OAAkBG,oCAAWT;;;oBACrC;;wBAAOS;;;oBAEP;;wBAAMT,OAAOM,KAAK,CAACC,KAAK;;;oBAAxB;;;;;;;;;;IAEJ;;AAEO,SAAS7C,cAAcyB,GAAmB;IAC/C,OAAOR,SAASe,IAAAA,kBAAS,EAACP,MAAMpB,QAAQ;AAC1C"}
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
import type { ResolvedConfig } from '../config/index.js';
|
|
2
|
+
import type { EmbedCallOptions } from '../embed/types.js';
|
|
2
3
|
import type { Stages } from './stages.js';
|
|
3
4
|
import type { OpenDialect, Store } from './types.js';
|
|
4
5
|
export type BuildRequirement = 'core' | 'lexical' | 'vectors';
|
|
6
|
+
/** Options for the root `open(config, options)` API. */
|
|
5
7
|
export interface OpenOptions {
|
|
8
|
+
/** Prepares configured capabilities by default; `false` opens compatible published state without repair. */
|
|
6
9
|
build?: boolean;
|
|
10
|
+
/** Cancels supported provider I/O and stops between phases after active native work finishes. */
|
|
11
|
+
signal?: AbortSignal;
|
|
7
12
|
}
|
|
8
13
|
export interface InternalOpenOptions extends OpenOptions {
|
|
9
14
|
requirements?: ReadonlySet<BuildRequirement>;
|
|
@@ -16,6 +21,6 @@ export interface OpenResult {
|
|
|
16
21
|
warnings: string[];
|
|
17
22
|
stages: Stages;
|
|
18
23
|
}
|
|
19
|
-
export declare function prepareDocumentEmbeddings(store: Store, cfg: ResolvedConfig, allowed?: ReadonlySet<string
|
|
24
|
+
export declare function prepareDocumentEmbeddings(store: Store, cfg: ResolvedConfig, allowed?: ReadonlySet<string>, options?: EmbedCallOptions): Promise<void>;
|
|
20
25
|
export declare function openWithDialect<Handle>(input: ResolvedConfig, dialect: OpenDialect<Handle>, options?: InternalOpenOptions): Promise<OpenResult>;
|
|
21
26
|
export declare function docCount(store: Store): Promise<number>;
|
package/dist/cjs/store/open.d.ts
CHANGED
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
import type { ResolvedConfig } from '../config/index.js';
|
|
2
|
+
import type { EmbedCallOptions } from '../embed/types.js';
|
|
2
3
|
import type { Stages } from './stages.js';
|
|
3
4
|
import type { OpenDialect, Store } from './types.js';
|
|
4
5
|
export type BuildRequirement = 'core' | 'lexical' | 'vectors';
|
|
6
|
+
/** Options for the root `open(config, options)` API. */
|
|
5
7
|
export interface OpenOptions {
|
|
8
|
+
/** Prepares configured capabilities by default; `false` opens compatible published state without repair. */
|
|
6
9
|
build?: boolean;
|
|
10
|
+
/** Cancels supported provider I/O and stops between phases after active native work finishes. */
|
|
11
|
+
signal?: AbortSignal;
|
|
7
12
|
}
|
|
8
13
|
export interface InternalOpenOptions extends OpenOptions {
|
|
9
14
|
requirements?: ReadonlySet<BuildRequirement>;
|
|
@@ -16,6 +21,6 @@ export interface OpenResult {
|
|
|
16
21
|
warnings: string[];
|
|
17
22
|
stages: Stages;
|
|
18
23
|
}
|
|
19
|
-
export declare function prepareDocumentEmbeddings(store: Store, cfg: ResolvedConfig, allowed?: ReadonlySet<string
|
|
24
|
+
export declare function prepareDocumentEmbeddings(store: Store, cfg: ResolvedConfig, allowed?: ReadonlySet<string>, options?: EmbedCallOptions): Promise<void>;
|
|
20
25
|
export declare function openWithDialect<Handle>(input: ResolvedConfig, dialect: OpenDialect<Handle>, options?: InternalOpenOptions): Promise<OpenResult>;
|
|
21
26
|
export declare function docCount(store: Store): Promise<number>;
|