pi-vault-mind 0.14.2 → 0.16.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +293 -0
  2. package/README.md +5 -5
  3. package/agents/broadcaster.agent.md +2 -2
  4. package/agents/heavy-lifter.agent.md +1 -1
  5. package/agents/main.agent.md +29 -0
  6. package/agents/manager.agent.md +1 -1
  7. package/agents/miner.agent.md +1 -1
  8. package/dist/packages/obsidian/src/client.js +331 -0
  9. package/dist/packages/obsidian/src/types.js +1 -0
  10. package/dist/src/activity.d.ts +49 -0
  11. package/dist/src/activity.js +175 -0
  12. package/dist/src/agent-bus/Agent.d.ts +9 -1
  13. package/dist/src/agent-bus/Agent.js +18 -2
  14. package/dist/src/agent-bus/index.d.ts +5 -5
  15. package/dist/src/agent-bus/index.js +1 -1
  16. package/dist/src/agent-queue.js +12 -3
  17. package/dist/src/agents/BroadcasterAgent.d.ts +1 -1
  18. package/dist/src/agents/BroadcasterAgent.js +2 -2
  19. package/dist/src/agents/HeavyLifterAgent.d.ts +1 -1
  20. package/dist/src/agents/HeavyLifterAgent.js +2 -2
  21. package/dist/src/agents/ManagerAgent.js +1 -1
  22. package/dist/src/agents/MinerAgent.d.ts +1 -1
  23. package/dist/src/agents/MinerAgent.js +2 -2
  24. package/dist/src/agents/index.d.ts +2 -2
  25. package/dist/src/agents/index.js +2 -2
  26. package/dist/src/auth.d.ts +17 -15
  27. package/dist/src/auth.js +26 -37
  28. package/dist/src/autosync.js +2 -2
  29. package/dist/src/commands.d.ts +2 -19
  30. package/dist/src/commands.js +301 -231
  31. package/dist/src/config-keys.d.ts +16 -0
  32. package/dist/src/config-keys.js +45 -0
  33. package/dist/src/discover-schema.js +4 -4
  34. package/dist/src/edits.d.ts +21 -0
  35. package/dist/src/edits.js +75 -0
  36. package/dist/src/embedding-probe.d.ts +34 -0
  37. package/dist/src/embedding-probe.js +142 -0
  38. package/dist/src/embedding-providers.d.ts +133 -0
  39. package/dist/src/embedding-providers.js +173 -0
  40. package/dist/src/embedding-secrets.d.ts +38 -0
  41. package/dist/src/embedding-secrets.js +291 -0
  42. package/dist/src/engine.d.ts +6 -0
  43. package/dist/src/engine.js +78 -6
  44. package/dist/src/events.js +7 -10
  45. package/dist/src/extension-packages.d.ts +9 -0
  46. package/dist/src/extension-packages.js +14 -0
  47. package/dist/src/git.d.ts +19 -0
  48. package/dist/src/git.js +78 -0
  49. package/dist/src/graph.d.ts +2 -1
  50. package/dist/src/graph.js +47 -13
  51. package/dist/src/identities-config.d.ts +6 -0
  52. package/dist/src/identities-config.js +39 -0
  53. package/dist/src/identity-injector.js +1 -1
  54. package/dist/src/index.js +9 -26
  55. package/dist/src/lance.d.ts +9 -0
  56. package/dist/src/lance.js +143 -203
  57. package/dist/src/modal-client.d.ts +16 -0
  58. package/dist/src/modal-client.js +16 -0
  59. package/dist/src/modal-config.d.ts +37 -14
  60. package/dist/src/modal-config.js +59 -32
  61. package/dist/src/model-router.d.ts +27 -0
  62. package/dist/src/model-router.js +67 -0
  63. package/dist/src/models.d.ts +42 -0
  64. package/dist/src/models.js +78 -0
  65. package/dist/src/personalize.js +17 -3
  66. package/dist/src/scaffold.d.ts +48 -0
  67. package/dist/src/scaffold.js +246 -0
  68. package/dist/src/server.d.ts +47 -4
  69. package/dist/src/server.js +988 -221
  70. package/dist/src/session-search.d.ts +63 -0
  71. package/dist/src/session-search.js +379 -0
  72. package/dist/src/settings-ui.d.ts +2 -2
  73. package/dist/src/settings-ui.js +76 -135
  74. package/dist/src/sync.d.ts +3 -3
  75. package/dist/src/sync.js +19 -15
  76. package/dist/src/tool-catalog.d.ts +18 -0
  77. package/dist/src/tool-catalog.js +232 -0
  78. package/dist/src/tools/marksman.js +1 -1
  79. package/dist/src/tools.js +111 -65
  80. package/dist/src/types.d.ts +127 -71
  81. package/dist/src/utils.d.ts +57 -6
  82. package/dist/src/utils.js +114 -28
  83. package/dist/src/vault-tools.d.ts +34 -0
  84. package/dist/src/vault-tools.js +99 -0
  85. package/dist/src/vault-writer.js +4 -1
  86. package/dist/src/vm-handlers.js +140 -43
  87. package/dist/src/watcher.js +9 -9
  88. package/dist/test/activity-tool.test.js +54 -0
  89. package/dist/test/agent-bus/Agent.test.js +174 -0
  90. package/dist/test/agent-bus/AgentLoader.test.js +180 -0
  91. package/dist/test/agent-bus/AgentRegistry.test.js +86 -0
  92. package/dist/test/agent-bus/LLMProvider.test.js +162 -0
  93. package/dist/test/agent-bus/TaskQueue.test.js +256 -0
  94. package/dist/test/agent-dispatch-context.test.js +73 -0
  95. package/dist/test/agent-queue.test.js +1 -1
  96. package/dist/test/agents/Agents.test.js +282 -0
  97. package/dist/test/auth.test.js +57 -56
  98. package/dist/test/bridge-identity.test.js +34 -3
  99. package/dist/test/classify-search-mode.test.js +51 -0
  100. package/dist/test/commands.test.js +410 -0
  101. package/dist/test/config-merge.test.js +98 -0
  102. package/dist/test/context-capture.test.js +56 -0
  103. package/dist/test/discover-schema.test.js +8 -5
  104. package/dist/test/dispatch.test.js +45 -91
  105. package/dist/test/edits.test.js +144 -0
  106. package/dist/test/embedding-probe.test.js +424 -0
  107. package/dist/test/embedding-providers.test.js +157 -0
  108. package/dist/test/embedding-secrets.test.js +210 -0
  109. package/dist/test/engine-identity.test.js +1 -1
  110. package/dist/test/events.test.js +675 -0
  111. package/dist/test/git-bridge.test.js +160 -0
  112. package/dist/test/graph.test.js +464 -0
  113. package/dist/test/helpers/server.js +81 -0
  114. package/dist/test/identity-injector.test.js +197 -0
  115. package/dist/test/index.test.js +5 -7
  116. package/dist/test/lance-modal.test.js +67 -2
  117. package/dist/test/lance.test.js +429 -0
  118. package/dist/test/modal-client.test.js +1 -1
  119. package/dist/test/modal-config.test.js +92 -79
  120. package/dist/test/model-readiness.test.js +122 -0
  121. package/dist/test/model-router.test.js +99 -0
  122. package/dist/test/obsidian-client.test.js +42 -0
  123. package/dist/test/personalize.test.js +1 -1
  124. package/dist/test/rest-models.test.js +153 -0
  125. package/dist/test/rest-queue.test.js +135 -87
  126. package/dist/test/rest-setup.test.js +625 -115
  127. package/dist/test/rest-vm.test.js +355 -94
  128. package/dist/test/server-cors.test.js +48 -0
  129. package/dist/test/server-ws.test.js +61 -34
  130. package/dist/test/session-crud.test.js +323 -0
  131. package/dist/test/session-search.test.js +139 -0
  132. package/dist/test/settings-ui.test.js +113 -4
  133. package/dist/test/sync.test.js +12 -12
  134. package/dist/test/tombstone.test.js +8 -10
  135. package/dist/test/tool-catalog.test.js +86 -0
  136. package/dist/test/utils.test.js +588 -0
  137. package/dist/test/vault-tools.test.js +103 -12
  138. package/dist/test/vault-writer.test.js +371 -0
  139. package/dist/test/vm-handlers.test.js +620 -0
  140. package/extension-packages.json +9 -0
  141. package/package.json +107 -90
  142. package/pi-vault-mind.config.example.json +24 -2
  143. package/scripts/check-deps.sh +35 -32
  144. package/scripts/cli-e2e.mjs +215 -0
  145. package/scripts/configuration-e2e.mjs +783 -0
  146. package/scripts/e2e-commands.mjs +391 -0
  147. package/scripts/e2e-ws.mjs +3 -2
  148. package/scripts/fetch-modal-token.sh +4 -3
  149. package/scripts/generate-config-keys-json.mjs +45 -0
  150. package/scripts/generate-extension-packages-json.mjs +55 -0
  151. package/scripts/pi-session-query +0 -0
  152. package/scripts/publish-obsidian-plugin.sh +0 -0
  153. package/scripts/reset-test-vault.sh +2 -2
  154. package/scripts/setup-test-config.sh +0 -0
  155. package/scripts/setup-vault-pi.sh +48 -9
  156. package/scripts/test-slash-commands.mjs +2 -2
  157. package/scripts/vault-pi.sh +12 -3
  158. package/skills/github-release-notes-file/SKILL.md +42 -0
  159. package/skills/obsidian-guided-test-session/SKILL.md +110 -0
  160. package/skills/obsidian-interactive-test/SKILL.md +88 -0
  161. package/skills/obsidian-plugin-publish/SKILL.md +58 -0
  162. package/skills/pi-vault-mind-release/SKILL.md +141 -0
  163. package/skills/pre-test-release/SKILL.md +116 -0
  164. package/dist/test/auto-indexer.test.js +0 -2
package/dist/src/lance.js CHANGED
@@ -4,7 +4,8 @@ import * as lancedb from "@lancedb/lancedb";
4
4
  import { LanceSchema, TextEmbeddingFunction } from "@lancedb/lancedb/embedding";
5
5
  import * as arrow from "apache-arrow";
6
6
  import { EmbeddingCoalescer } from "./embed-queue.js";
7
- import { createModalClient, namespacedTableName, resolveDim, resolveModel, } from "./modal-config.js";
7
+ import { createProvider } from "./embedding-providers.js";
8
+ import { namespacedTableName, resolveDim, resolveModel } from "./modal-config.js";
8
9
  let db = null;
9
10
  // ── Connection ──────────────────────────────────────────────────────────────
10
11
  export const connect = async (dataDir) => {
@@ -22,151 +23,46 @@ export const resetConnection = () => {
22
23
  modalTableCache.clear();
23
24
  modalTableInFlight.clear();
24
25
  nativeDimCache.clear();
26
+ providerCache.clear();
25
27
  void disposeCoalescers();
26
28
  };
27
- // ── Embedding Functions ─────────────────────────────────────────────────────
29
+ // ── Lance Adapter ───────────────────────────────────────────────────────────
30
+ // `TextEmbeddingFunction` is LanceDB's store-specific contract. The
31
+ // store-agnostic `EmbeddingProvider` (see src/embedding-providers.ts) is
32
+ // the source of truth; this adapter is the only place that knows both
33
+ // interfaces. Adding a new vector store (Qdrant, Pinecone, pgvector, …)
34
+ // means writing a new adapter here; the provider code does not change.
28
35
  /**
29
- * OpenAI-compatible embedding function that POSTs to /v1/embeddings.
30
- * Works with any OpenAI-compatible endpoint (Ollama, Modal, cloud providers).
36
+ * Wraps a store-agnostic `EmbeddingProvider` into LanceDB's
37
+ * `TextEmbeddingFunction`. The provider supplies `embed()`; the adapter
38
+ * supplies the schema helpers (`sourceField`, `vectorField`,
39
+ * `embeddingDataType`) and the task-routed convenience methods
40
+ * (`computeSourceEmbeddings` for the storage path, `computeQueryEmbeddings`
41
+ * for the search path).
31
42
  */
32
- class OpenAICompatibleEmbeddingFunction extends TextEmbeddingFunction {
33
- url;
34
- model;
35
- apiKey;
36
- ndimsValue;
37
- constructor(options) {
43
+ class LanceEmbeddingAdapter extends TextEmbeddingFunction {
44
+ provider;
45
+ constructor(provider) {
38
46
  super();
39
- this.url = options.url;
40
- this.model = options.model;
41
- this.apiKey = options.apiKey;
42
- this.ndimsValue = options.dims;
47
+ this.provider = provider;
43
48
  }
44
49
  ndims() {
45
- return this.ndimsValue;
50
+ return this.provider.dim();
46
51
  }
47
52
  embeddingDataType() {
48
53
  return new arrow.Float32();
49
54
  }
50
- async init() {
51
- // Best-effort connectivity check
52
- try {
53
- const baseUrl = this.url.replace(/\/v1\/embeddings\/?$/, "").replace(/\/+$/, "");
54
- const headers = {};
55
- if (this.apiKey)
56
- headers.Authorization = `Bearer ${this.apiKey}`;
57
- await fetch(`${baseUrl}/v1/models`, { headers });
58
- }
59
- catch (err) {
60
- console.warn(`[pi-vault-mind] Cannot reach embedding endpoint at ${this.url}: ${err}`);
61
- }
62
- }
63
- async generateEmbeddings(texts) {
64
- const endpoint = this.url.endsWith("/v1/embeddings")
65
- ? this.url
66
- : `${this.url.replace(/\/+$/, "")}/v1/embeddings`;
67
- const headers = { "Content-Type": "application/json" };
68
- if (this.apiKey)
69
- headers.Authorization = `Bearer ${this.apiKey}`;
70
- const resp = await fetch(endpoint, {
71
- method: "POST",
72
- headers,
73
- body: JSON.stringify({ model: this.model, input: texts }),
74
- });
75
- if (!resp.ok) {
76
- const body = await resp.text().catch(() => "");
77
- throw new Error(`Embed failed: ${resp.status} ${body}`);
78
- }
79
- const data = (await resp.json());
80
- if (!data.data || !Array.isArray(data.data)) {
81
- throw new Error("Invalid embedding response");
82
- }
83
- return data.data.map((d) => d.embedding);
84
- }
85
- }
86
- /**
87
- * Transformers.js fallback — fully offline, works on any Node.js.
88
- */
89
- class TransformersEmbeddingFunction extends TextEmbeddingFunction {
90
- extractor = null;
91
- initPromise = null;
92
- ndims() {
93
- return 384;
94
- }
95
- embeddingDataType() {
96
- return new arrow.Float32();
97
- }
98
- async init() {
99
- if (!this.initPromise) {
100
- this.initPromise = (async () => {
101
- const { pipeline } = await import("@xenova/transformers");
102
- this.extractor = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2");
103
- })();
104
- }
105
- return this.initPromise;
106
- }
107
- async generateEmbeddings(texts) {
108
- await this.init();
109
- const embeddings = [];
110
- for (const text of texts) {
111
- const result = await this.extractor(text, { pooling: "mean", normalize: true });
112
- embeddings.push(Array.from(result.data));
113
- }
114
- return embeddings;
115
- }
116
- }
117
- // ── Modal Provider ──────────────────────────────────────────────────────────
118
- /**
119
- * Embedding function backed by the Modal `/embed` endpoint (see
120
- * `src/modal-client.ts`). Used two ways:
121
- *
122
- * 1. As a `TextEmbeddingFunction` for graph tables that still auto-embed via
123
- * `sourceField` (document task for storage, query task for search).
124
- * 2. As the `embedFn` wrapped by the `EmbeddingCoalescer` for the main
125
- * collection path, where append/ingest is debounced + batched and search
126
- * bypasses the debounce via `embedImmediate`.
127
- *
128
- * The main collection path does NOT auto-embed — it inserts precomputed
129
- * vectors into namespaced `col_{collection}__{model}__{dim}` tables (see
130
- * `upsertEntry`/`searchHybrid` below and ADR-3). One canonical model owns the
131
- * vector space; a single query never mixes `model__dim` spaces.
132
- */
133
- class ModalEmbeddingFunction extends TextEmbeddingFunction {
134
- cfg;
135
- collection;
136
- ndimsValue;
137
- constructor(cfg, collection, ndimsValue) {
138
- super();
139
- this.cfg = cfg;
140
- this.collection = collection;
141
- this.ndimsValue = ndimsValue;
142
- }
143
- ndims() {
144
- return this.ndimsValue;
145
- }
146
- embeddingDataType() {
147
- return new arrow.Float32();
148
- }
149
- /** Batched embed against `/embed`. `task` selects query vs document. */
55
+ /** Batched embed. `task` selects query vs document. */
150
56
  async generateEmbeddings(texts, task = "document") {
151
- const client = createModalClient(this.cfg);
152
- if (!client)
153
- throw new Error("Modal provider not configured (missing baseUrl or token).");
154
- const res = await client.embed(texts, {
155
- model: resolveModel(this.cfg, this.collection),
156
- dim: resolveDim(this.cfg, this.collection),
157
- task,
158
- });
159
- // Cache the model's effective output dim for table resolution.
160
- rememberNativeDim(resolveModel(this.cfg, this.collection), res.dim);
161
- return res.vectors;
162
- }
163
- // Storage path (LanceDB auto-embed via sourceField) → document task.
57
+ return this.provider.embed(texts, task);
58
+ }
59
+ /** Storage path (LanceDB auto-embed via `sourceField`) → document task. */
164
60
  async computeSourceEmbeddings(texts) {
165
- return this.generateEmbeddings(texts, "document");
61
+ return this.provider.embed(texts, "document");
166
62
  }
167
- // Query path (LanceDB nearestTo via embedding function) → query task.
63
+ /** Query path (LanceDB `nearestTo`) → query task. */
168
64
  async computeQueryEmbeddings(data) {
169
- const vecs = await this.generateEmbeddings([data], "query");
65
+ const vecs = await this.provider.embed([data], "query");
170
66
  return vecs[0];
171
67
  }
172
68
  }
@@ -196,14 +92,31 @@ const effectiveDim = (cfg, collection) => {
196
92
  * `cfg.embedding.coalesce` (Q6).
197
93
  */
198
94
  const coalescerCache = new Map();
95
+ /** Provider promise cache — `createProvider` is async; we memoize by key. */
96
+ const providerCache = new Map();
199
97
  const coalesceKey = (cfg, collection) => `modal:${resolveModel(cfg, collection)}:${resolveDim(cfg, collection) ?? "native"}`;
98
+ const getProvider = (cfg, collection) => {
99
+ const key = coalesceKey(cfg, collection);
100
+ let p = providerCache.get(key);
101
+ if (!p) {
102
+ p = createProvider(cfg, { collection });
103
+ providerCache.set(key, p);
104
+ }
105
+ return p;
106
+ };
200
107
  const getCoalescer = (cfg, collection) => {
201
108
  const key = coalesceKey(cfg, collection);
202
109
  const existing = coalescerCache.get(key);
203
110
  if (existing)
204
111
  return existing;
205
- const fn = new ModalEmbeddingFunction(cfg, collection);
206
- const embedFn = (texts, task) => fn.generateEmbeddings(texts, task);
112
+ // createProvider is async; the coalescer's embedFn closure fires after
113
+ // the provider is ready. If the provider rejects at init(), the first
114
+ // embed call surfaces that error to the caller.
115
+ let provider;
116
+ const providerPromise = getProvider(cfg, collection).then((p) => {
117
+ provider = p;
118
+ });
119
+ const embedFn = (texts, task) => providerPromise.then(() => provider.embed(texts, task));
207
120
  const co = cfg.embedding.coalesce ?? {};
208
121
  const coalescer = new EmbeddingCoalescer({
209
122
  embedFn,
@@ -341,10 +254,7 @@ const modalFtsFallback = async (dataDir, collection, model, query, limit, identi
341
254
  * null when no same-space fallback is available, so the caller degrades to FTS.
342
255
  */
343
256
  const fallbackQueryEmbed = async (cfg, collection, text) => {
344
- const fb = cfg.embedding.modal?.fallback;
345
- if (fb?.enabled === false)
346
- return null;
347
- if (fb?.provider !== "openai-compatible")
257
+ if (cfg.embedding.fallback?.enabled === false)
348
258
  return null;
349
259
  if (!cfg.embedding.localUrl)
350
260
  return null;
@@ -352,13 +262,8 @@ const fallbackQueryEmbed = async (cfg, collection, text) => {
352
262
  if ((cfg.embedding.model || "embeddinggemma") !== canonical)
353
263
  return null;
354
264
  try {
355
- const fn = new OpenAICompatibleEmbeddingFunction({
356
- url: cfg.embedding.localUrl,
357
- model: cfg.embedding.model || "embeddinggemma",
358
- apiKey: cfg.embedding.apiKey,
359
- });
360
- await fn.init();
361
- const vecs = await fn.generateEmbeddings([text]);
265
+ const fn = await createProvider(cfg, { collection });
266
+ const vecs = await fn.embed([text], "query");
362
267
  return vecs[0];
363
268
  }
364
269
  catch (err) {
@@ -381,42 +286,11 @@ const disposeCoalescers = async () => {
381
286
  };
382
287
  // ── Embedding Provider Factory ──────────────────────────────────────────────
383
288
  const getEmbeddingFunction = async (cfg) => {
384
- if (cfg.embedding.remoteUrl) {
385
- const dim = effectiveDim(cfg);
386
- return new ModalEmbeddingFunction(cfg, undefined, dim);
387
- }
388
- if (cfg.embedding.localUrl) {
389
- const fn = new OpenAICompatibleEmbeddingFunction({
390
- url: cfg.embedding.localUrl,
391
- model: cfg.embedding.model || "embeddinggemma",
392
- apiKey: cfg.embedding.apiKey,
393
- });
394
- await fn.init();
395
- return fn;
396
- }
397
- if (cfg.embedding.useTransformers !== false) {
398
- const fn = new TransformersEmbeddingFunction();
399
- await fn.init();
400
- return fn;
401
- }
402
- throw new Error("No embedding provider configured. Set remoteUrl, localUrl, or useTransformers: true.");
289
+ const dims = effectiveDim(cfg);
290
+ const provider = await createProvider(cfg, { dims });
291
+ return new LanceEmbeddingAdapter(provider);
403
292
  };
404
293
  // ── Schema Builders ─────────────────────────────────────────────────────────
405
- function makeCollectionSchema(embeddingFn) {
406
- return LanceSchema({
407
- id: new arrow.Utf8(),
408
- domain: new arrow.Utf8(),
409
- source: new arrow.Utf8(),
410
- fact: embeddingFn.sourceField(),
411
- tag: new arrow.Utf8(),
412
- artifact: new arrow.Utf8(),
413
- created_at: new arrow.Utf8(),
414
- origin_identity: new arrow.Utf8(),
415
- visibility: new arrow.Utf8(),
416
- identity: new arrow.Utf8(),
417
- vector: embeddingFn.vectorField(),
418
- });
419
- }
420
294
  /**
421
295
  * Backwards-compat: ensure the `identity` column exists on tables created before
422
296
  * this field was added. The column is nullable so old rows remain readable.
@@ -440,6 +314,12 @@ function makeEntitySchema(embeddingFn) {
440
314
  vector: embeddingFn.vectorField(),
441
315
  });
442
316
  }
317
+ async function assertPlainCollectionSchema(table, tableName) {
318
+ const schema = await table.schema();
319
+ if (schema.metadata?.has("embedding_functions")) {
320
+ throw new Error(`Old-format LanceDB table "${tableName}" uses embedded LanceSchema metadata. Run /vm index --reembed to migrate it to the explicit-vector schema.`);
321
+ }
322
+ }
443
323
  function makeRelationSchema(embeddingFn) {
444
324
  return LanceSchema({
445
325
  id: new arrow.Utf8(),
@@ -456,20 +336,23 @@ function makeRelationSchema(embeddingFn) {
456
336
  });
457
337
  }
458
338
  let tables = {};
459
- const getCollectionTable = async (dataDir, collectionName, cfg) => {
339
+ const getCollectionTable = async (dataDir, collectionName, cfg, vectorDim) => {
460
340
  const conn = await connect(dataDir);
461
341
  const tableName = `collection_${collectionName}`;
462
342
  if (tables[tableName])
463
343
  return tables[tableName];
464
- const embeddingFn = await getEmbeddingFunction(cfg);
465
- const schema = makeCollectionSchema(embeddingFn);
466
344
  const existing = await conn.tableNames();
467
345
  if (existing.includes(tableName)) {
468
346
  tables[tableName] = await conn.openTable(tableName);
347
+ await assertPlainCollectionSchema(tables[tableName], tableName);
469
348
  await ensureIdentityColumn(tables[tableName]);
470
349
  return tables[tableName];
471
350
  }
472
- tables[tableName] = await conn.createEmptyTable(tableName, schema);
351
+ const dim = vectorDim ?? effectiveDim(cfg, collectionName);
352
+ if (dim == null) {
353
+ throw new Error(`Collection "${collectionName}" needs a known embedding dimension. Set vaultMind.embedding.dim, or embed a row first.`);
354
+ }
355
+ tables[tableName] = await conn.createEmptyTable(tableName, makeModalCollectionSchema(dim));
473
356
  if (cfg.ftsEnabled !== false) {
474
357
  try {
475
358
  await tables[tableName].createIndex("fact", { config: lancedb.Index.fts() });
@@ -492,7 +375,7 @@ const assertGraphDim = (cfg, kind) => {
492
375
  throw new Error(`Graph ${kind} table needs a known embedding dimension. Set vaultMind.embedding.dim, or run an embed/sync first so the native dim is cached.`);
493
376
  }
494
377
  };
495
- const getEntityTable = async (dataDir, cfg) => {
378
+ const _getEntityTable = async (dataDir, cfg) => {
496
379
  const conn = await connect(dataDir);
497
380
  const tableName = "entities";
498
381
  if (tables[tableName])
@@ -508,7 +391,7 @@ const getEntityTable = async (dataDir, cfg) => {
508
391
  tables[tableName] = await conn.createEmptyTable(tableName, schema);
509
392
  return tables[tableName];
510
393
  };
511
- const getRelationTable = async (dataDir, cfg) => {
394
+ const _getRelationTable = async (dataDir, cfg) => {
512
395
  const conn = await connect(dataDir);
513
396
  const tableName = "relations";
514
397
  if (tables[tableName])
@@ -525,11 +408,14 @@ const getRelationTable = async (dataDir, cfg) => {
525
408
  return tables[tableName];
526
409
  };
527
410
  // ── Public API ──────────────────────────────────────────────────────────────
528
- const isModal = (cfg) => !!cfg.embedding.remoteUrl && !!cfg.embedding.modal;
411
+ const isModal = (cfg) => !!cfg.embedding.remoteUrl;
529
412
  export const upsertEntry = async (dataDir, collectionName, entry, cfg, identity) => {
530
413
  if (isModal(cfg))
531
414
  return upsertEntryModal(dataDir, collectionName, entry, cfg, identity);
532
- const table = await getCollectionTable(dataDir, collectionName, cfg);
415
+ const embeddingFn = await getEmbeddingFunction(cfg);
416
+ const text = entry.fact || "";
417
+ const vectors = await embeddingFn.computeSourceEmbeddings([text]);
418
+ const table = await getCollectionTable(dataDir, collectionName, cfg, vectors[0]?.length);
533
419
  const row = {
534
420
  id: entry.id || crypto.randomUUID(),
535
421
  domain: entry.domain || "",
@@ -541,6 +427,7 @@ export const upsertEntry = async (dataDir, collectionName, entry, cfg, identity)
541
427
  origin_identity: identity?.id || "",
542
428
  visibility: identity?.visibility || "vault",
543
429
  identity: entry.identity || identity?.id || "",
430
+ vector: vectors[0],
544
431
  };
545
432
  await table.add([row]);
546
433
  };
@@ -590,10 +477,7 @@ const upsertEntryModal = async (dataDir, collectionName, entry, cfg, identity) =
590
477
  };
591
478
  /** Offline fallback embed for a document (storage). Same-space local only. */
592
479
  const fallbackDocumentEmbed = async (cfg, collection, text) => {
593
- const fb = cfg.embedding.modal?.fallback;
594
- if (fb?.enabled === false)
595
- return null;
596
- if (fb?.provider !== "openai-compatible")
480
+ if (cfg.embedding.fallback?.enabled === false)
597
481
  return null;
598
482
  if (!cfg.embedding.localUrl)
599
483
  return null;
@@ -601,13 +485,8 @@ const fallbackDocumentEmbed = async (cfg, collection, text) => {
601
485
  if ((cfg.embedding.model || "embeddinggemma") !== canonical)
602
486
  return null;
603
487
  try {
604
- const fn = new OpenAICompatibleEmbeddingFunction({
605
- url: cfg.embedding.localUrl,
606
- model: cfg.embedding.model || "embeddinggemma",
607
- apiKey: cfg.embedding.apiKey,
608
- });
609
- await fn.init();
610
- const vecs = await fn.generateEmbeddings([text]);
488
+ const fn = await createProvider(cfg, { collection });
489
+ const vecs = await fn.embed([text], "document");
611
490
  return vecs[0];
612
491
  }
613
492
  catch (err) {
@@ -646,13 +525,22 @@ export const upsertPrecomputed = async (dataDir, collectionName, model, dim, row
646
525
  });
647
526
  await table.mergeInsert(["id"]).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(mapped);
648
527
  };
649
- /** Filter results by visibility: show public entries + entries owned by the reader. No identity = no filter (backward compat). */
650
- function filterByVisibility(results, identity) {
528
+ function isVisibilityRecord(value) {
529
+ return value !== null && typeof value === "object";
530
+ }
531
+ /** Filter results by visibility: show public entries + entries owned by the reader. No identity = no filter (backward compat).
532
+ * Records without visibility/origin_identity fields (e.g. graph entities/relations) are treated as public. */
533
+ export function filterByVisibility(results, identity) {
651
534
  if (!identity)
652
535
  return results;
653
536
  return results.filter((r) => {
537
+ if (!isVisibilityRecord(r))
538
+ return true;
654
539
  const vis = r.visibility;
655
540
  const origin = r.origin_identity;
541
+ // Graph entities/relations have no visibility fields; they are already gated at upsert time.
542
+ if (vis === undefined && origin === undefined)
543
+ return true;
656
544
  return vis === "public" || origin === identity.id;
657
545
  });
658
546
  }
@@ -700,10 +588,19 @@ export const queryCollection = async (collectionPath, schema, query, filters, li
700
588
  export const searchHybrid = async (dataDir, collectionName, query, limit, cfg, identity) => {
701
589
  if (isModal(cfg))
702
590
  return searchHybridModal(dataDir, collectionName, query, limit, cfg, identity);
591
+ const tableName = `collection_${collectionName}`;
592
+ const conn = await connect(dataDir);
593
+ if (!tables[tableName] && !(await conn.tableNames()).includes(tableName))
594
+ return [];
703
595
  const table = await getCollectionTable(dataDir, collectionName, cfg);
704
596
  const embeddingFn = await getEmbeddingFunction(cfg);
705
597
  const queryVector = await embeddingFn.generateEmbeddings([query]);
706
- const results = await table.query().nearestTo(queryVector[0]).limit(limit).toArray();
598
+ const results = await table
599
+ .query()
600
+ .nearestTo(queryVector[0])
601
+ .column("vector")
602
+ .limit(limit)
603
+ .toArray();
707
604
  return filterByVisibility(results, identity);
708
605
  };
709
606
  /**
@@ -736,7 +633,12 @@ const searchHybridModal = async (dataDir, collectionName, query, limit, cfg, ide
736
633
  return modalFtsFallback(dataDir, collectionName, model, query, limit, identity);
737
634
  }
738
635
  try {
739
- const results = await found.table.query().nearestTo(queryVector).limit(limit).toArray();
636
+ const results = await found.table
637
+ .query()
638
+ .nearestTo(queryVector)
639
+ .column("vector")
640
+ .limit(limit)
641
+ .toArray();
740
642
  return filterByVisibility(results, identity);
741
643
  }
742
644
  catch (err) {
@@ -752,17 +654,55 @@ export const searchFts = async (dataDir, collectionName, query, limit, cfg, iden
752
654
  if (!found)
753
655
  return [];
754
656
  try {
755
- const results = await found.table.search(query).limit(limit).toArray();
657
+ const results = await found.table
658
+ .query()
659
+ .nearestToText(query, ["fact"])
660
+ .limit(limit)
661
+ .toArray();
756
662
  return filterByVisibility(results, identity);
757
663
  }
758
664
  catch {
759
665
  return [];
760
666
  }
761
667
  }
668
+ const tableName = `collection_${collectionName}`;
669
+ const conn = await connect(dataDir);
670
+ if (!tables[tableName] && !(await conn.tableNames()).includes(tableName))
671
+ return [];
762
672
  const table = await getCollectionTable(dataDir, collectionName, cfg);
763
- const results = await table.search(query).limit(limit).toArray();
673
+ const results = await table.query().nearestToText(query, ["fact"]).limit(limit).toArray();
764
674
  return filterByVisibility(results, identity);
765
675
  };
676
+ /**
677
+ * Rank-merge vector and FTS results: vector hits score higher, deduplicated by
678
+ * id or fact. Each result is tagged with `_source: "vector" | "fts"`. Falls
679
+ * back gracefully — if either path fails, the other still contributes.
680
+ */
681
+ export const searchHybridRanked = async (dataDir, collectionName, query, limit, cfg, identity) => {
682
+ const [vectorResults, ftsResults] = await Promise.all([
683
+ searchHybrid(dataDir, collectionName, query, limit, cfg, identity).catch(() => []),
684
+ searchFts(dataDir, collectionName, query, limit, cfg, identity).catch(() => []),
685
+ ]);
686
+ const seen = new Set();
687
+ const merged = [];
688
+ for (const r of vectorResults) {
689
+ const row = r;
690
+ const key = (row.id ?? row.fact ?? JSON.stringify(row));
691
+ if (!seen.has(key)) {
692
+ seen.add(key);
693
+ merged.push({ ...row, _source: "vector" });
694
+ }
695
+ }
696
+ for (const r of ftsResults) {
697
+ const row = r;
698
+ const key = (row.id ?? row.fact ?? JSON.stringify(row));
699
+ if (!seen.has(key)) {
700
+ seen.add(key);
701
+ merged.push({ ...row, _source: "fts" });
702
+ }
703
+ }
704
+ return merged.slice(0, limit);
705
+ };
766
706
  export const getStatus = async (dataDir) => {
767
707
  const conn = await connect(dataDir);
768
708
  const names = await conn.tableNames();
@@ -183,4 +183,20 @@ export declare class ModalEmbeddingClient {
183
183
  since?: number;
184
184
  limit?: number;
185
185
  }): Promise<number>;
186
+ /**
187
+ * Submit bulk embedding jobs for multiple collections, wait for all to
188
+ * complete, and return per-collection results. Used by the /vm/reindex
189
+ * endpoint to trigger a full remote reindex from the status tab.
190
+ */
191
+ reindexAll(jobs: Array<{
192
+ collection: string;
193
+ records: JobRecord[];
194
+ }>, opts?: {
195
+ model?: string;
196
+ dim?: number;
197
+ }): Promise<Array<{
198
+ collection: string;
199
+ job: JobSubmitResult;
200
+ status: JobStatus;
201
+ }>>;
186
202
  }
@@ -187,4 +187,20 @@ export class ModalEmbeddingClient {
187
187
  return watermark;
188
188
  }
189
189
  }
190
+ /**
191
+ * Submit bulk embedding jobs for multiple collections, wait for all to
192
+ * complete, and return per-collection results. Used by the /vm/reindex
193
+ * endpoint to trigger a full remote reindex from the status tab.
194
+ */
195
+ async reindexAll(jobs, opts = {}) {
196
+ const results = [];
197
+ for (const { collection, records } of jobs) {
198
+ if (records.length === 0)
199
+ continue;
200
+ const submitted = await this.submitJob(collection, records, opts);
201
+ const status = await this.waitForJob(submitted.job_id);
202
+ results.push({ collection, job: submitted, status });
203
+ }
204
+ return results;
205
+ }
190
206
  }
@@ -10,23 +10,29 @@
10
10
  */
11
11
  import { ModalEmbeddingClient } from "./modal-client.js";
12
12
  import type { VaultMindConfig } from "./types.js";
13
- /** Env var name for the Modal bearer token (preferred over config). */
14
- export declare const MODAL_TOKEN_ENV = "PVM_API_TOKEN";
13
+ /** Env var name for the unscoped remote embedding API key (preferred over config). */
14
+ export declare const MODAL_TOKEN_ENV = "PVM_REMOTE_EMBEDDING_API_KEY";
15
15
  /** Env var name for the read-scoped Modal token (search/export). */
16
16
  export declare const MODAL_READ_TOKEN_ENV = "PVM_MODAL_READ_TOKEN";
17
17
  /** Env var name for the write-scoped Modal token (embed/reindex). */
18
18
  export declare const MODAL_WRITE_TOKEN_ENV = "PVM_MODAL_WRITE_TOKEN";
19
- /** Dotenv-style fallback path for the Modal token. */
20
- export declare const MODAL_TOKEN_ENV_PATH: string;
19
+ /**
20
+ * Dotenv fallback path for the Modal token, resolved from the vault cwd:
21
+ * `<vault>/.vault-mind/vault-mind.env`. Lazy so tests can override cwd.
22
+ */
23
+ export declare const modalTokenEnvPath: (cwd?: string) => string;
21
24
  /** Canonical deployed Modal app name. */
22
25
  export declare const MODAL_APP_NAME = "pi-vault-mind-embed";
23
26
  /** Derive the Modal service URL from a workspace slug and canonical app name. */
24
27
  export declare const modalUrl: (workspace: string) => string;
25
28
  /**
26
- * Resolve the Modal API token. Resolution order:
27
- * 1. `PVM_API_TOKEN` env var
28
- * 2. `~/.pi/agent/vault-mind.env`
29
- * 3. `vaultMind.embedding.modal.apiToken` in config
29
+ * Resolve the unscoped remote embedding API token. Resolution order:
30
+ * 1. `PVM_REMOTE_EMBEDDING_API_KEY` env var
31
+ * 2. `<vault>/.vault-mind/vault-mind.env`
32
+ * 3. `vaultMind.embedding.remoteApiKey` in config
33
+ *
34
+ * `PVM_API_TOKEN` is intentionally not consulted — it is reserved for the
35
+ * extension's bridge authentication.
30
36
  *
31
37
  * Never log the resolved token.
32
38
  */
@@ -35,14 +41,17 @@ export declare const resolveModalToken: (cfg: VaultMindConfig) => string | undef
35
41
  * Resolve a scoped Modal token for the requested operation.
36
42
  * Resolution order:
37
43
  * 1. `PVM_MODAL_{READ|WRITE}_TOKEN` env var
38
- * 2. `~/.pi/agent/vault-mind.env`
39
- * 3. `vaultMind.embedding.modal.{readToken|writeToken}` in config
40
- * 4. Fallback to the unscoped `apiToken` (env, dotenv, then config)
44
+ * 2. `<vault>/.vault-mind/vault-mind.env`
45
+ * 3. `vaultMind.embedding.remoteReadApiKey` / `remoteWriteApiKey` in config
46
+ * 4. Fallback to the unscoped `PVM_REMOTE_EMBEDDING_API_KEY` (env, dotenv, then config)
47
+ *
48
+ * `PVM_API_TOKEN` is intentionally not consulted — it is reserved for the
49
+ * extension's bridge authentication.
41
50
  *
42
51
  * Never log the resolved token.
43
52
  */
44
53
  export declare const getModalToken: (cfg: VaultMindConfig, operation: "read" | "write") => string | undefined;
45
- /** Resolve the explicit baseUrl or derive it from workspace. */
54
+ /** Derive the remote URL from the top-level `remoteUrl` or the `workspace` shortcut. */
46
55
  export declare const resolveBaseUrl: (cfg: VaultMindConfig) => string | undefined;
47
56
  /** Resolve the configured workspace slug, if any. */
48
57
  export declare const resolveWorkspace: (cfg: VaultMindConfig) => string | undefined;
@@ -62,9 +71,23 @@ export declare const createModalClient: (cfg: VaultMindConfig) => ModalEmbedding
62
71
  export declare const resolveModel: (cfg: VaultMindConfig, collection?: string) => string;
63
72
  /**
64
73
  * The configured output dimension for a collection, if known. Per-collection
65
- * overrides win; otherwise the global Modal config `dim`. Returns undefined
66
- * when unset — callers resolve the native dim lazily via `/models`.
74
+ * overrides win; otherwise the top-level `dim` on `EmbeddingConfig`.
75
+ * Returns undefined when unset — callers resolve the native dim lazily via `/models`.
67
76
  */
68
77
  export declare const resolveDim: (cfg: VaultMindConfig, collection?: string) => number | undefined;
69
78
  /** Namespaced local table name mirroring the server's ADR-3 scheme. */
70
79
  export declare const namespacedTableName: (collection: string, model: string, dim: number) => string;
80
+ /**
81
+ * Resolve the server-side (Modal) collection table prefix for cross-vault
82
+ * isolation on a shared Modal deployment. Unset or `"auto"` derives it from
83
+ * the vault slug. The result is normalized (no `__`, no leading/trailing `_`)
84
+ * so it is safe for the server's `col_(.+)__(.+)__(\d+)$` parse.
85
+ *
86
+ * Applies ONLY to remote table names — local LanceDB tables stay unprefixed.
87
+ */
88
+ export declare const resolveCollectionPrefix: (cfg: VaultMindConfig, vaultPath: string) => string | undefined;
89
+ /**
90
+ * Prepend the collection prefix to a collection name for server-side table
91
+ * identity. When `prefix` is undefined, returns the collection name unchanged.
92
+ */
93
+ export declare const serverCollectionName: (prefix: string | undefined, collection: string) => string;