pi-vault-mind 0.14.2 → 0.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +293 -0
- package/README.md +5 -5
- package/agents/broadcaster.agent.md +2 -2
- package/agents/heavy-lifter.agent.md +1 -1
- package/agents/main.agent.md +29 -0
- package/agents/manager.agent.md +1 -1
- package/agents/miner.agent.md +1 -1
- package/dist/packages/obsidian/src/client.js +331 -0
- package/dist/packages/obsidian/src/types.js +1 -0
- package/dist/src/activity.d.ts +49 -0
- package/dist/src/activity.js +175 -0
- package/dist/src/agent-bus/Agent.d.ts +9 -1
- package/dist/src/agent-bus/Agent.js +18 -2
- package/dist/src/agent-bus/index.d.ts +5 -5
- package/dist/src/agent-bus/index.js +1 -1
- package/dist/src/agent-queue.js +12 -3
- package/dist/src/agents/BroadcasterAgent.d.ts +1 -1
- package/dist/src/agents/BroadcasterAgent.js +2 -2
- package/dist/src/agents/HeavyLifterAgent.d.ts +1 -1
- package/dist/src/agents/HeavyLifterAgent.js +2 -2
- package/dist/src/agents/ManagerAgent.js +1 -1
- package/dist/src/agents/MinerAgent.d.ts +1 -1
- package/dist/src/agents/MinerAgent.js +2 -2
- package/dist/src/agents/index.d.ts +2 -2
- package/dist/src/agents/index.js +2 -2
- package/dist/src/auth.d.ts +17 -15
- package/dist/src/auth.js +26 -37
- package/dist/src/autosync.js +2 -2
- package/dist/src/commands.d.ts +2 -19
- package/dist/src/commands.js +301 -231
- package/dist/src/config-keys.d.ts +16 -0
- package/dist/src/config-keys.js +45 -0
- package/dist/src/discover-schema.js +4 -4
- package/dist/src/edits.d.ts +21 -0
- package/dist/src/edits.js +75 -0
- package/dist/src/embedding-probe.d.ts +34 -0
- package/dist/src/embedding-probe.js +142 -0
- package/dist/src/embedding-providers.d.ts +133 -0
- package/dist/src/embedding-providers.js +173 -0
- package/dist/src/embedding-secrets.d.ts +38 -0
- package/dist/src/embedding-secrets.js +291 -0
- package/dist/src/engine.d.ts +6 -0
- package/dist/src/engine.js +78 -6
- package/dist/src/events.js +7 -10
- package/dist/src/extension-packages.d.ts +9 -0
- package/dist/src/extension-packages.js +14 -0
- package/dist/src/git.d.ts +19 -0
- package/dist/src/git.js +78 -0
- package/dist/src/graph.d.ts +2 -1
- package/dist/src/graph.js +47 -13
- package/dist/src/identities-config.d.ts +6 -0
- package/dist/src/identities-config.js +39 -0
- package/dist/src/identity-injector.js +1 -1
- package/dist/src/index.js +9 -26
- package/dist/src/lance.d.ts +9 -0
- package/dist/src/lance.js +143 -203
- package/dist/src/modal-client.d.ts +16 -0
- package/dist/src/modal-client.js +16 -0
- package/dist/src/modal-config.d.ts +37 -14
- package/dist/src/modal-config.js +59 -32
- package/dist/src/model-router.d.ts +27 -0
- package/dist/src/model-router.js +67 -0
- package/dist/src/models.d.ts +42 -0
- package/dist/src/models.js +78 -0
- package/dist/src/personalize.js +17 -3
- package/dist/src/scaffold.d.ts +48 -0
- package/dist/src/scaffold.js +246 -0
- package/dist/src/server.d.ts +47 -4
- package/dist/src/server.js +988 -221
- package/dist/src/session-search.d.ts +63 -0
- package/dist/src/session-search.js +379 -0
- package/dist/src/settings-ui.d.ts +2 -2
- package/dist/src/settings-ui.js +76 -135
- package/dist/src/sync.d.ts +3 -3
- package/dist/src/sync.js +19 -15
- package/dist/src/tool-catalog.d.ts +18 -0
- package/dist/src/tool-catalog.js +232 -0
- package/dist/src/tools/marksman.js +1 -1
- package/dist/src/tools.js +111 -65
- package/dist/src/types.d.ts +127 -71
- package/dist/src/utils.d.ts +57 -6
- package/dist/src/utils.js +114 -28
- package/dist/src/vault-tools.d.ts +34 -0
- package/dist/src/vault-tools.js +99 -0
- package/dist/src/vault-writer.js +4 -1
- package/dist/src/vm-handlers.js +140 -43
- package/dist/src/watcher.js +9 -9
- package/dist/test/activity-tool.test.js +54 -0
- package/dist/test/agent-bus/Agent.test.js +174 -0
- package/dist/test/agent-bus/AgentLoader.test.js +180 -0
- package/dist/test/agent-bus/AgentRegistry.test.js +86 -0
- package/dist/test/agent-bus/LLMProvider.test.js +162 -0
- package/dist/test/agent-bus/TaskQueue.test.js +256 -0
- package/dist/test/agent-dispatch-context.test.js +73 -0
- package/dist/test/agent-queue.test.js +1 -1
- package/dist/test/agents/Agents.test.js +282 -0
- package/dist/test/auth.test.js +57 -56
- package/dist/test/bridge-identity.test.js +34 -3
- package/dist/test/classify-search-mode.test.js +51 -0
- package/dist/test/commands.test.js +410 -0
- package/dist/test/config-merge.test.js +98 -0
- package/dist/test/context-capture.test.js +56 -0
- package/dist/test/discover-schema.test.js +8 -5
- package/dist/test/dispatch.test.js +45 -91
- package/dist/test/edits.test.js +144 -0
- package/dist/test/embedding-probe.test.js +424 -0
- package/dist/test/embedding-providers.test.js +157 -0
- package/dist/test/embedding-secrets.test.js +210 -0
- package/dist/test/engine-identity.test.js +1 -1
- package/dist/test/events.test.js +675 -0
- package/dist/test/git-bridge.test.js +160 -0
- package/dist/test/graph.test.js +464 -0
- package/dist/test/helpers/server.js +81 -0
- package/dist/test/identity-injector.test.js +197 -0
- package/dist/test/index.test.js +5 -7
- package/dist/test/lance-modal.test.js +67 -2
- package/dist/test/lance.test.js +429 -0
- package/dist/test/modal-client.test.js +1 -1
- package/dist/test/modal-config.test.js +92 -79
- package/dist/test/model-readiness.test.js +122 -0
- package/dist/test/model-router.test.js +99 -0
- package/dist/test/obsidian-client.test.js +42 -0
- package/dist/test/personalize.test.js +1 -1
- package/dist/test/rest-models.test.js +153 -0
- package/dist/test/rest-queue.test.js +135 -87
- package/dist/test/rest-setup.test.js +625 -115
- package/dist/test/rest-vm.test.js +355 -94
- package/dist/test/server-cors.test.js +48 -0
- package/dist/test/server-ws.test.js +61 -34
- package/dist/test/session-crud.test.js +323 -0
- package/dist/test/session-search.test.js +139 -0
- package/dist/test/settings-ui.test.js +113 -4
- package/dist/test/sync.test.js +12 -12
- package/dist/test/tombstone.test.js +8 -10
- package/dist/test/tool-catalog.test.js +86 -0
- package/dist/test/utils.test.js +588 -0
- package/dist/test/vault-tools.test.js +103 -12
- package/dist/test/vault-writer.test.js +371 -0
- package/dist/test/vm-handlers.test.js +620 -0
- package/extension-packages.json +9 -0
- package/package.json +107 -90
- package/pi-vault-mind.config.example.json +24 -2
- package/scripts/check-deps.sh +35 -32
- package/scripts/cli-e2e.mjs +215 -0
- package/scripts/configuration-e2e.mjs +783 -0
- package/scripts/e2e-commands.mjs +391 -0
- package/scripts/e2e-ws.mjs +3 -2
- package/scripts/fetch-modal-token.sh +4 -3
- package/scripts/generate-config-keys-json.mjs +45 -0
- package/scripts/generate-extension-packages-json.mjs +55 -0
- package/scripts/pi-session-query +0 -0
- package/scripts/publish-obsidian-plugin.sh +0 -0
- package/scripts/reset-test-vault.sh +2 -2
- package/scripts/setup-test-config.sh +0 -0
- package/scripts/setup-vault-pi.sh +48 -9
- package/scripts/test-slash-commands.mjs +2 -2
- package/scripts/vault-pi.sh +12 -3
- package/skills/github-release-notes-file/SKILL.md +42 -0
- package/skills/obsidian-guided-test-session/SKILL.md +110 -0
- package/skills/obsidian-interactive-test/SKILL.md +88 -0
- package/skills/obsidian-plugin-publish/SKILL.md +58 -0
- package/skills/pi-vault-mind-release/SKILL.md +141 -0
- package/skills/pre-test-release/SKILL.md +116 -0
- package/dist/test/auto-indexer.test.js +0 -2
package/dist/src/lance.js
CHANGED
|
@@ -4,7 +4,8 @@ import * as lancedb from "@lancedb/lancedb";
|
|
|
4
4
|
import { LanceSchema, TextEmbeddingFunction } from "@lancedb/lancedb/embedding";
|
|
5
5
|
import * as arrow from "apache-arrow";
|
|
6
6
|
import { EmbeddingCoalescer } from "./embed-queue.js";
|
|
7
|
-
import {
|
|
7
|
+
import { createProvider } from "./embedding-providers.js";
|
|
8
|
+
import { namespacedTableName, resolveDim, resolveModel } from "./modal-config.js";
|
|
8
9
|
let db = null;
|
|
9
10
|
// ── Connection ──────────────────────────────────────────────────────────────
|
|
10
11
|
export const connect = async (dataDir) => {
|
|
@@ -22,151 +23,46 @@ export const resetConnection = () => {
|
|
|
22
23
|
modalTableCache.clear();
|
|
23
24
|
modalTableInFlight.clear();
|
|
24
25
|
nativeDimCache.clear();
|
|
26
|
+
providerCache.clear();
|
|
25
27
|
void disposeCoalescers();
|
|
26
28
|
};
|
|
27
|
-
// ──
|
|
29
|
+
// ── Lance Adapter ───────────────────────────────────────────────────────────
|
|
30
|
+
// `TextEmbeddingFunction` is LanceDB's store-specific contract. The
|
|
31
|
+
// store-agnostic `EmbeddingProvider` (see src/embedding-providers.ts) is
|
|
32
|
+
// the source of truth; this adapter is the only place that knows both
|
|
33
|
+
// interfaces. Adding a new vector store (Qdrant, Pinecone, pgvector, …)
|
|
34
|
+
// means writing a new adapter here; the provider code does not change.
|
|
28
35
|
/**
|
|
29
|
-
*
|
|
30
|
-
*
|
|
36
|
+
* Wraps a store-agnostic `EmbeddingProvider` into LanceDB's
|
|
37
|
+
* `TextEmbeddingFunction`. The provider supplies `embed()`; the adapter
|
|
38
|
+
* supplies the schema helpers (`sourceField`, `vectorField`,
|
|
39
|
+
* `embeddingDataType`) and the task-routed convenience methods
|
|
40
|
+
* (`computeSourceEmbeddings` for the storage path, `computeQueryEmbeddings`
|
|
41
|
+
* for the search path).
|
|
31
42
|
*/
|
|
32
|
-
class
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
apiKey;
|
|
36
|
-
ndimsValue;
|
|
37
|
-
constructor(options) {
|
|
43
|
+
class LanceEmbeddingAdapter extends TextEmbeddingFunction {
|
|
44
|
+
provider;
|
|
45
|
+
constructor(provider) {
|
|
38
46
|
super();
|
|
39
|
-
this.
|
|
40
|
-
this.model = options.model;
|
|
41
|
-
this.apiKey = options.apiKey;
|
|
42
|
-
this.ndimsValue = options.dims;
|
|
47
|
+
this.provider = provider;
|
|
43
48
|
}
|
|
44
49
|
ndims() {
|
|
45
|
-
return this.
|
|
50
|
+
return this.provider.dim();
|
|
46
51
|
}
|
|
47
52
|
embeddingDataType() {
|
|
48
53
|
return new arrow.Float32();
|
|
49
54
|
}
|
|
50
|
-
|
|
51
|
-
// Best-effort connectivity check
|
|
52
|
-
try {
|
|
53
|
-
const baseUrl = this.url.replace(/\/v1\/embeddings\/?$/, "").replace(/\/+$/, "");
|
|
54
|
-
const headers = {};
|
|
55
|
-
if (this.apiKey)
|
|
56
|
-
headers.Authorization = `Bearer ${this.apiKey}`;
|
|
57
|
-
await fetch(`${baseUrl}/v1/models`, { headers });
|
|
58
|
-
}
|
|
59
|
-
catch (err) {
|
|
60
|
-
console.warn(`[pi-vault-mind] Cannot reach embedding endpoint at ${this.url}: ${err}`);
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
async generateEmbeddings(texts) {
|
|
64
|
-
const endpoint = this.url.endsWith("/v1/embeddings")
|
|
65
|
-
? this.url
|
|
66
|
-
: `${this.url.replace(/\/+$/, "")}/v1/embeddings`;
|
|
67
|
-
const headers = { "Content-Type": "application/json" };
|
|
68
|
-
if (this.apiKey)
|
|
69
|
-
headers.Authorization = `Bearer ${this.apiKey}`;
|
|
70
|
-
const resp = await fetch(endpoint, {
|
|
71
|
-
method: "POST",
|
|
72
|
-
headers,
|
|
73
|
-
body: JSON.stringify({ model: this.model, input: texts }),
|
|
74
|
-
});
|
|
75
|
-
if (!resp.ok) {
|
|
76
|
-
const body = await resp.text().catch(() => "");
|
|
77
|
-
throw new Error(`Embed failed: ${resp.status} ${body}`);
|
|
78
|
-
}
|
|
79
|
-
const data = (await resp.json());
|
|
80
|
-
if (!data.data || !Array.isArray(data.data)) {
|
|
81
|
-
throw new Error("Invalid embedding response");
|
|
82
|
-
}
|
|
83
|
-
return data.data.map((d) => d.embedding);
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
/**
|
|
87
|
-
* Transformers.js fallback — fully offline, works on any Node.js.
|
|
88
|
-
*/
|
|
89
|
-
class TransformersEmbeddingFunction extends TextEmbeddingFunction {
|
|
90
|
-
extractor = null;
|
|
91
|
-
initPromise = null;
|
|
92
|
-
ndims() {
|
|
93
|
-
return 384;
|
|
94
|
-
}
|
|
95
|
-
embeddingDataType() {
|
|
96
|
-
return new arrow.Float32();
|
|
97
|
-
}
|
|
98
|
-
async init() {
|
|
99
|
-
if (!this.initPromise) {
|
|
100
|
-
this.initPromise = (async () => {
|
|
101
|
-
const { pipeline } = await import("@xenova/transformers");
|
|
102
|
-
this.extractor = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2");
|
|
103
|
-
})();
|
|
104
|
-
}
|
|
105
|
-
return this.initPromise;
|
|
106
|
-
}
|
|
107
|
-
async generateEmbeddings(texts) {
|
|
108
|
-
await this.init();
|
|
109
|
-
const embeddings = [];
|
|
110
|
-
for (const text of texts) {
|
|
111
|
-
const result = await this.extractor(text, { pooling: "mean", normalize: true });
|
|
112
|
-
embeddings.push(Array.from(result.data));
|
|
113
|
-
}
|
|
114
|
-
return embeddings;
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
// ── Modal Provider ──────────────────────────────────────────────────────────
|
|
118
|
-
/**
|
|
119
|
-
* Embedding function backed by the Modal `/embed` endpoint (see
|
|
120
|
-
* `src/modal-client.ts`). Used two ways:
|
|
121
|
-
*
|
|
122
|
-
* 1. As a `TextEmbeddingFunction` for graph tables that still auto-embed via
|
|
123
|
-
* `sourceField` (document task for storage, query task for search).
|
|
124
|
-
* 2. As the `embedFn` wrapped by the `EmbeddingCoalescer` for the main
|
|
125
|
-
* collection path, where append/ingest is debounced + batched and search
|
|
126
|
-
* bypasses the debounce via `embedImmediate`.
|
|
127
|
-
*
|
|
128
|
-
* The main collection path does NOT auto-embed — it inserts precomputed
|
|
129
|
-
* vectors into namespaced `col_{collection}__{model}__{dim}` tables (see
|
|
130
|
-
* `upsertEntry`/`searchHybrid` below and ADR-3). One canonical model owns the
|
|
131
|
-
* vector space; a single query never mixes `model__dim` spaces.
|
|
132
|
-
*/
|
|
133
|
-
class ModalEmbeddingFunction extends TextEmbeddingFunction {
|
|
134
|
-
cfg;
|
|
135
|
-
collection;
|
|
136
|
-
ndimsValue;
|
|
137
|
-
constructor(cfg, collection, ndimsValue) {
|
|
138
|
-
super();
|
|
139
|
-
this.cfg = cfg;
|
|
140
|
-
this.collection = collection;
|
|
141
|
-
this.ndimsValue = ndimsValue;
|
|
142
|
-
}
|
|
143
|
-
ndims() {
|
|
144
|
-
return this.ndimsValue;
|
|
145
|
-
}
|
|
146
|
-
embeddingDataType() {
|
|
147
|
-
return new arrow.Float32();
|
|
148
|
-
}
|
|
149
|
-
/** Batched embed against `/embed`. `task` selects query vs document. */
|
|
55
|
+
/** Batched embed. `task` selects query vs document. */
|
|
150
56
|
async generateEmbeddings(texts, task = "document") {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
const res = await client.embed(texts, {
|
|
155
|
-
model: resolveModel(this.cfg, this.collection),
|
|
156
|
-
dim: resolveDim(this.cfg, this.collection),
|
|
157
|
-
task,
|
|
158
|
-
});
|
|
159
|
-
// Cache the model's effective output dim for table resolution.
|
|
160
|
-
rememberNativeDim(resolveModel(this.cfg, this.collection), res.dim);
|
|
161
|
-
return res.vectors;
|
|
162
|
-
}
|
|
163
|
-
// Storage path (LanceDB auto-embed via sourceField) → document task.
|
|
57
|
+
return this.provider.embed(texts, task);
|
|
58
|
+
}
|
|
59
|
+
/** Storage path (LanceDB auto-embed via `sourceField`) → document task. */
|
|
164
60
|
async computeSourceEmbeddings(texts) {
|
|
165
|
-
return this.
|
|
61
|
+
return this.provider.embed(texts, "document");
|
|
166
62
|
}
|
|
167
|
-
|
|
63
|
+
/** Query path (LanceDB `nearestTo`) → query task. */
|
|
168
64
|
async computeQueryEmbeddings(data) {
|
|
169
|
-
const vecs = await this.
|
|
65
|
+
const vecs = await this.provider.embed([data], "query");
|
|
170
66
|
return vecs[0];
|
|
171
67
|
}
|
|
172
68
|
}
|
|
@@ -196,14 +92,31 @@ const effectiveDim = (cfg, collection) => {
|
|
|
196
92
|
* `cfg.embedding.coalesce` (Q6).
|
|
197
93
|
*/
|
|
198
94
|
const coalescerCache = new Map();
|
|
95
|
+
/** Provider promise cache — `createProvider` is async; we memoize by key. */
|
|
96
|
+
const providerCache = new Map();
|
|
199
97
|
const coalesceKey = (cfg, collection) => `modal:${resolveModel(cfg, collection)}:${resolveDim(cfg, collection) ?? "native"}`;
|
|
98
|
+
const getProvider = (cfg, collection) => {
|
|
99
|
+
const key = coalesceKey(cfg, collection);
|
|
100
|
+
let p = providerCache.get(key);
|
|
101
|
+
if (!p) {
|
|
102
|
+
p = createProvider(cfg, { collection });
|
|
103
|
+
providerCache.set(key, p);
|
|
104
|
+
}
|
|
105
|
+
return p;
|
|
106
|
+
};
|
|
200
107
|
const getCoalescer = (cfg, collection) => {
|
|
201
108
|
const key = coalesceKey(cfg, collection);
|
|
202
109
|
const existing = coalescerCache.get(key);
|
|
203
110
|
if (existing)
|
|
204
111
|
return existing;
|
|
205
|
-
|
|
206
|
-
|
|
112
|
+
// createProvider is async; the coalescer's embedFn closure fires after
|
|
113
|
+
// the provider is ready. If the provider rejects at init(), the first
|
|
114
|
+
// embed call surfaces that error to the caller.
|
|
115
|
+
let provider;
|
|
116
|
+
const providerPromise = getProvider(cfg, collection).then((p) => {
|
|
117
|
+
provider = p;
|
|
118
|
+
});
|
|
119
|
+
const embedFn = (texts, task) => providerPromise.then(() => provider.embed(texts, task));
|
|
207
120
|
const co = cfg.embedding.coalesce ?? {};
|
|
208
121
|
const coalescer = new EmbeddingCoalescer({
|
|
209
122
|
embedFn,
|
|
@@ -341,10 +254,7 @@ const modalFtsFallback = async (dataDir, collection, model, query, limit, identi
|
|
|
341
254
|
* null when no same-space fallback is available, so the caller degrades to FTS.
|
|
342
255
|
*/
|
|
343
256
|
const fallbackQueryEmbed = async (cfg, collection, text) => {
|
|
344
|
-
|
|
345
|
-
if (fb?.enabled === false)
|
|
346
|
-
return null;
|
|
347
|
-
if (fb?.provider !== "openai-compatible")
|
|
257
|
+
if (cfg.embedding.fallback?.enabled === false)
|
|
348
258
|
return null;
|
|
349
259
|
if (!cfg.embedding.localUrl)
|
|
350
260
|
return null;
|
|
@@ -352,13 +262,8 @@ const fallbackQueryEmbed = async (cfg, collection, text) => {
|
|
|
352
262
|
if ((cfg.embedding.model || "embeddinggemma") !== canonical)
|
|
353
263
|
return null;
|
|
354
264
|
try {
|
|
355
|
-
const fn =
|
|
356
|
-
|
|
357
|
-
model: cfg.embedding.model || "embeddinggemma",
|
|
358
|
-
apiKey: cfg.embedding.apiKey,
|
|
359
|
-
});
|
|
360
|
-
await fn.init();
|
|
361
|
-
const vecs = await fn.generateEmbeddings([text]);
|
|
265
|
+
const fn = await createProvider(cfg, { collection });
|
|
266
|
+
const vecs = await fn.embed([text], "query");
|
|
362
267
|
return vecs[0];
|
|
363
268
|
}
|
|
364
269
|
catch (err) {
|
|
@@ -381,42 +286,11 @@ const disposeCoalescers = async () => {
|
|
|
381
286
|
};
|
|
382
287
|
// ── Embedding Provider Factory ──────────────────────────────────────────────
|
|
383
288
|
const getEmbeddingFunction = async (cfg) => {
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
}
|
|
388
|
-
if (cfg.embedding.localUrl) {
|
|
389
|
-
const fn = new OpenAICompatibleEmbeddingFunction({
|
|
390
|
-
url: cfg.embedding.localUrl,
|
|
391
|
-
model: cfg.embedding.model || "embeddinggemma",
|
|
392
|
-
apiKey: cfg.embedding.apiKey,
|
|
393
|
-
});
|
|
394
|
-
await fn.init();
|
|
395
|
-
return fn;
|
|
396
|
-
}
|
|
397
|
-
if (cfg.embedding.useTransformers !== false) {
|
|
398
|
-
const fn = new TransformersEmbeddingFunction();
|
|
399
|
-
await fn.init();
|
|
400
|
-
return fn;
|
|
401
|
-
}
|
|
402
|
-
throw new Error("No embedding provider configured. Set remoteUrl, localUrl, or useTransformers: true.");
|
|
289
|
+
const dims = effectiveDim(cfg);
|
|
290
|
+
const provider = await createProvider(cfg, { dims });
|
|
291
|
+
return new LanceEmbeddingAdapter(provider);
|
|
403
292
|
};
|
|
404
293
|
// ── Schema Builders ─────────────────────────────────────────────────────────
|
|
405
|
-
function makeCollectionSchema(embeddingFn) {
|
|
406
|
-
return LanceSchema({
|
|
407
|
-
id: new arrow.Utf8(),
|
|
408
|
-
domain: new arrow.Utf8(),
|
|
409
|
-
source: new arrow.Utf8(),
|
|
410
|
-
fact: embeddingFn.sourceField(),
|
|
411
|
-
tag: new arrow.Utf8(),
|
|
412
|
-
artifact: new arrow.Utf8(),
|
|
413
|
-
created_at: new arrow.Utf8(),
|
|
414
|
-
origin_identity: new arrow.Utf8(),
|
|
415
|
-
visibility: new arrow.Utf8(),
|
|
416
|
-
identity: new arrow.Utf8(),
|
|
417
|
-
vector: embeddingFn.vectorField(),
|
|
418
|
-
});
|
|
419
|
-
}
|
|
420
294
|
/**
|
|
421
295
|
* Backwards-compat: ensure the `identity` column exists on tables created before
|
|
422
296
|
* this field was added. The column is nullable so old rows remain readable.
|
|
@@ -440,6 +314,12 @@ function makeEntitySchema(embeddingFn) {
|
|
|
440
314
|
vector: embeddingFn.vectorField(),
|
|
441
315
|
});
|
|
442
316
|
}
|
|
317
|
+
async function assertPlainCollectionSchema(table, tableName) {
|
|
318
|
+
const schema = await table.schema();
|
|
319
|
+
if (schema.metadata?.has("embedding_functions")) {
|
|
320
|
+
throw new Error(`Old-format LanceDB table "${tableName}" uses embedded LanceSchema metadata. Run /vm index --reembed to migrate it to the explicit-vector schema.`);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
443
323
|
function makeRelationSchema(embeddingFn) {
|
|
444
324
|
return LanceSchema({
|
|
445
325
|
id: new arrow.Utf8(),
|
|
@@ -456,20 +336,23 @@ function makeRelationSchema(embeddingFn) {
|
|
|
456
336
|
});
|
|
457
337
|
}
|
|
458
338
|
let tables = {};
|
|
459
|
-
const getCollectionTable = async (dataDir, collectionName, cfg) => {
|
|
339
|
+
const getCollectionTable = async (dataDir, collectionName, cfg, vectorDim) => {
|
|
460
340
|
const conn = await connect(dataDir);
|
|
461
341
|
const tableName = `collection_${collectionName}`;
|
|
462
342
|
if (tables[tableName])
|
|
463
343
|
return tables[tableName];
|
|
464
|
-
const embeddingFn = await getEmbeddingFunction(cfg);
|
|
465
|
-
const schema = makeCollectionSchema(embeddingFn);
|
|
466
344
|
const existing = await conn.tableNames();
|
|
467
345
|
if (existing.includes(tableName)) {
|
|
468
346
|
tables[tableName] = await conn.openTable(tableName);
|
|
347
|
+
await assertPlainCollectionSchema(tables[tableName], tableName);
|
|
469
348
|
await ensureIdentityColumn(tables[tableName]);
|
|
470
349
|
return tables[tableName];
|
|
471
350
|
}
|
|
472
|
-
|
|
351
|
+
const dim = vectorDim ?? effectiveDim(cfg, collectionName);
|
|
352
|
+
if (dim == null) {
|
|
353
|
+
throw new Error(`Collection "${collectionName}" needs a known embedding dimension. Set vaultMind.embedding.dim, or embed a row first.`);
|
|
354
|
+
}
|
|
355
|
+
tables[tableName] = await conn.createEmptyTable(tableName, makeModalCollectionSchema(dim));
|
|
473
356
|
if (cfg.ftsEnabled !== false) {
|
|
474
357
|
try {
|
|
475
358
|
await tables[tableName].createIndex("fact", { config: lancedb.Index.fts() });
|
|
@@ -492,7 +375,7 @@ const assertGraphDim = (cfg, kind) => {
|
|
|
492
375
|
throw new Error(`Graph ${kind} table needs a known embedding dimension. Set vaultMind.embedding.dim, or run an embed/sync first so the native dim is cached.`);
|
|
493
376
|
}
|
|
494
377
|
};
|
|
495
|
-
const
|
|
378
|
+
const _getEntityTable = async (dataDir, cfg) => {
|
|
496
379
|
const conn = await connect(dataDir);
|
|
497
380
|
const tableName = "entities";
|
|
498
381
|
if (tables[tableName])
|
|
@@ -508,7 +391,7 @@ const getEntityTable = async (dataDir, cfg) => {
|
|
|
508
391
|
tables[tableName] = await conn.createEmptyTable(tableName, schema);
|
|
509
392
|
return tables[tableName];
|
|
510
393
|
};
|
|
511
|
-
const
|
|
394
|
+
const _getRelationTable = async (dataDir, cfg) => {
|
|
512
395
|
const conn = await connect(dataDir);
|
|
513
396
|
const tableName = "relations";
|
|
514
397
|
if (tables[tableName])
|
|
@@ -525,11 +408,14 @@ const getRelationTable = async (dataDir, cfg) => {
|
|
|
525
408
|
return tables[tableName];
|
|
526
409
|
};
|
|
527
410
|
// ── Public API ──────────────────────────────────────────────────────────────
|
|
528
|
-
const isModal = (cfg) => !!cfg.embedding.remoteUrl
|
|
411
|
+
const isModal = (cfg) => !!cfg.embedding.remoteUrl;
|
|
529
412
|
export const upsertEntry = async (dataDir, collectionName, entry, cfg, identity) => {
|
|
530
413
|
if (isModal(cfg))
|
|
531
414
|
return upsertEntryModal(dataDir, collectionName, entry, cfg, identity);
|
|
532
|
-
const
|
|
415
|
+
const embeddingFn = await getEmbeddingFunction(cfg);
|
|
416
|
+
const text = entry.fact || "";
|
|
417
|
+
const vectors = await embeddingFn.computeSourceEmbeddings([text]);
|
|
418
|
+
const table = await getCollectionTable(dataDir, collectionName, cfg, vectors[0]?.length);
|
|
533
419
|
const row = {
|
|
534
420
|
id: entry.id || crypto.randomUUID(),
|
|
535
421
|
domain: entry.domain || "",
|
|
@@ -541,6 +427,7 @@ export const upsertEntry = async (dataDir, collectionName, entry, cfg, identity)
|
|
|
541
427
|
origin_identity: identity?.id || "",
|
|
542
428
|
visibility: identity?.visibility || "vault",
|
|
543
429
|
identity: entry.identity || identity?.id || "",
|
|
430
|
+
vector: vectors[0],
|
|
544
431
|
};
|
|
545
432
|
await table.add([row]);
|
|
546
433
|
};
|
|
@@ -590,10 +477,7 @@ const upsertEntryModal = async (dataDir, collectionName, entry, cfg, identity) =
|
|
|
590
477
|
};
|
|
591
478
|
/** Offline fallback embed for a document (storage). Same-space local only. */
|
|
592
479
|
const fallbackDocumentEmbed = async (cfg, collection, text) => {
|
|
593
|
-
|
|
594
|
-
if (fb?.enabled === false)
|
|
595
|
-
return null;
|
|
596
|
-
if (fb?.provider !== "openai-compatible")
|
|
480
|
+
if (cfg.embedding.fallback?.enabled === false)
|
|
597
481
|
return null;
|
|
598
482
|
if (!cfg.embedding.localUrl)
|
|
599
483
|
return null;
|
|
@@ -601,13 +485,8 @@ const fallbackDocumentEmbed = async (cfg, collection, text) => {
|
|
|
601
485
|
if ((cfg.embedding.model || "embeddinggemma") !== canonical)
|
|
602
486
|
return null;
|
|
603
487
|
try {
|
|
604
|
-
const fn =
|
|
605
|
-
|
|
606
|
-
model: cfg.embedding.model || "embeddinggemma",
|
|
607
|
-
apiKey: cfg.embedding.apiKey,
|
|
608
|
-
});
|
|
609
|
-
await fn.init();
|
|
610
|
-
const vecs = await fn.generateEmbeddings([text]);
|
|
488
|
+
const fn = await createProvider(cfg, { collection });
|
|
489
|
+
const vecs = await fn.embed([text], "document");
|
|
611
490
|
return vecs[0];
|
|
612
491
|
}
|
|
613
492
|
catch (err) {
|
|
@@ -646,13 +525,22 @@ export const upsertPrecomputed = async (dataDir, collectionName, model, dim, row
|
|
|
646
525
|
});
|
|
647
526
|
await table.mergeInsert(["id"]).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(mapped);
|
|
648
527
|
};
|
|
649
|
-
|
|
650
|
-
|
|
528
|
+
function isVisibilityRecord(value) {
|
|
529
|
+
return value !== null && typeof value === "object";
|
|
530
|
+
}
|
|
531
|
+
/** Filter results by visibility: show public entries + entries owned by the reader. No identity = no filter (backward compat).
|
|
532
|
+
* Records without visibility/origin_identity fields (e.g. graph entities/relations) are treated as public. */
|
|
533
|
+
export function filterByVisibility(results, identity) {
|
|
651
534
|
if (!identity)
|
|
652
535
|
return results;
|
|
653
536
|
return results.filter((r) => {
|
|
537
|
+
if (!isVisibilityRecord(r))
|
|
538
|
+
return true;
|
|
654
539
|
const vis = r.visibility;
|
|
655
540
|
const origin = r.origin_identity;
|
|
541
|
+
// Graph entities/relations have no visibility fields; they are already gated at upsert time.
|
|
542
|
+
if (vis === undefined && origin === undefined)
|
|
543
|
+
return true;
|
|
656
544
|
return vis === "public" || origin === identity.id;
|
|
657
545
|
});
|
|
658
546
|
}
|
|
@@ -700,10 +588,19 @@ export const queryCollection = async (collectionPath, schema, query, filters, li
|
|
|
700
588
|
export const searchHybrid = async (dataDir, collectionName, query, limit, cfg, identity) => {
|
|
701
589
|
if (isModal(cfg))
|
|
702
590
|
return searchHybridModal(dataDir, collectionName, query, limit, cfg, identity);
|
|
591
|
+
const tableName = `collection_${collectionName}`;
|
|
592
|
+
const conn = await connect(dataDir);
|
|
593
|
+
if (!tables[tableName] && !(await conn.tableNames()).includes(tableName))
|
|
594
|
+
return [];
|
|
703
595
|
const table = await getCollectionTable(dataDir, collectionName, cfg);
|
|
704
596
|
const embeddingFn = await getEmbeddingFunction(cfg);
|
|
705
597
|
const queryVector = await embeddingFn.generateEmbeddings([query]);
|
|
706
|
-
const results = await table
|
|
598
|
+
const results = await table
|
|
599
|
+
.query()
|
|
600
|
+
.nearestTo(queryVector[0])
|
|
601
|
+
.column("vector")
|
|
602
|
+
.limit(limit)
|
|
603
|
+
.toArray();
|
|
707
604
|
return filterByVisibility(results, identity);
|
|
708
605
|
};
|
|
709
606
|
/**
|
|
@@ -736,7 +633,12 @@ const searchHybridModal = async (dataDir, collectionName, query, limit, cfg, ide
|
|
|
736
633
|
return modalFtsFallback(dataDir, collectionName, model, query, limit, identity);
|
|
737
634
|
}
|
|
738
635
|
try {
|
|
739
|
-
const results = await found.table
|
|
636
|
+
const results = await found.table
|
|
637
|
+
.query()
|
|
638
|
+
.nearestTo(queryVector)
|
|
639
|
+
.column("vector")
|
|
640
|
+
.limit(limit)
|
|
641
|
+
.toArray();
|
|
740
642
|
return filterByVisibility(results, identity);
|
|
741
643
|
}
|
|
742
644
|
catch (err) {
|
|
@@ -752,17 +654,55 @@ export const searchFts = async (dataDir, collectionName, query, limit, cfg, iden
|
|
|
752
654
|
if (!found)
|
|
753
655
|
return [];
|
|
754
656
|
try {
|
|
755
|
-
const results = await found.table
|
|
657
|
+
const results = await found.table
|
|
658
|
+
.query()
|
|
659
|
+
.nearestToText(query, ["fact"])
|
|
660
|
+
.limit(limit)
|
|
661
|
+
.toArray();
|
|
756
662
|
return filterByVisibility(results, identity);
|
|
757
663
|
}
|
|
758
664
|
catch {
|
|
759
665
|
return [];
|
|
760
666
|
}
|
|
761
667
|
}
|
|
668
|
+
const tableName = `collection_${collectionName}`;
|
|
669
|
+
const conn = await connect(dataDir);
|
|
670
|
+
if (!tables[tableName] && !(await conn.tableNames()).includes(tableName))
|
|
671
|
+
return [];
|
|
762
672
|
const table = await getCollectionTable(dataDir, collectionName, cfg);
|
|
763
|
-
const results = await table.
|
|
673
|
+
const results = await table.query().nearestToText(query, ["fact"]).limit(limit).toArray();
|
|
764
674
|
return filterByVisibility(results, identity);
|
|
765
675
|
};
|
|
676
|
+
/**
|
|
677
|
+
* Rank-merge vector and FTS results: vector hits score higher, deduplicated by
|
|
678
|
+
* id or fact. Each result is tagged with `_source: "vector" | "fts"`. Falls
|
|
679
|
+
* back gracefully — if either path fails, the other still contributes.
|
|
680
|
+
*/
|
|
681
|
+
export const searchHybridRanked = async (dataDir, collectionName, query, limit, cfg, identity) => {
|
|
682
|
+
const [vectorResults, ftsResults] = await Promise.all([
|
|
683
|
+
searchHybrid(dataDir, collectionName, query, limit, cfg, identity).catch(() => []),
|
|
684
|
+
searchFts(dataDir, collectionName, query, limit, cfg, identity).catch(() => []),
|
|
685
|
+
]);
|
|
686
|
+
const seen = new Set();
|
|
687
|
+
const merged = [];
|
|
688
|
+
for (const r of vectorResults) {
|
|
689
|
+
const row = r;
|
|
690
|
+
const key = (row.id ?? row.fact ?? JSON.stringify(row));
|
|
691
|
+
if (!seen.has(key)) {
|
|
692
|
+
seen.add(key);
|
|
693
|
+
merged.push({ ...row, _source: "vector" });
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
for (const r of ftsResults) {
|
|
697
|
+
const row = r;
|
|
698
|
+
const key = (row.id ?? row.fact ?? JSON.stringify(row));
|
|
699
|
+
if (!seen.has(key)) {
|
|
700
|
+
seen.add(key);
|
|
701
|
+
merged.push({ ...row, _source: "fts" });
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
return merged.slice(0, limit);
|
|
705
|
+
};
|
|
766
706
|
export const getStatus = async (dataDir) => {
|
|
767
707
|
const conn = await connect(dataDir);
|
|
768
708
|
const names = await conn.tableNames();
|
|
@@ -183,4 +183,20 @@ export declare class ModalEmbeddingClient {
|
|
|
183
183
|
since?: number;
|
|
184
184
|
limit?: number;
|
|
185
185
|
}): Promise<number>;
|
|
186
|
+
/**
|
|
187
|
+
* Submit bulk embedding jobs for multiple collections, wait for all to
|
|
188
|
+
* complete, and return per-collection results. Used by the /vm/reindex
|
|
189
|
+
* endpoint to trigger a full remote reindex from the status tab.
|
|
190
|
+
*/
|
|
191
|
+
reindexAll(jobs: Array<{
|
|
192
|
+
collection: string;
|
|
193
|
+
records: JobRecord[];
|
|
194
|
+
}>, opts?: {
|
|
195
|
+
model?: string;
|
|
196
|
+
dim?: number;
|
|
197
|
+
}): Promise<Array<{
|
|
198
|
+
collection: string;
|
|
199
|
+
job: JobSubmitResult;
|
|
200
|
+
status: JobStatus;
|
|
201
|
+
}>>;
|
|
186
202
|
}
|
package/dist/src/modal-client.js
CHANGED
|
@@ -187,4 +187,20 @@ export class ModalEmbeddingClient {
|
|
|
187
187
|
return watermark;
|
|
188
188
|
}
|
|
189
189
|
}
|
|
190
|
+
/**
|
|
191
|
+
* Submit bulk embedding jobs for multiple collections, wait for all to
|
|
192
|
+
* complete, and return per-collection results. Used by the /vm/reindex
|
|
193
|
+
* endpoint to trigger a full remote reindex from the status tab.
|
|
194
|
+
*/
|
|
195
|
+
async reindexAll(jobs, opts = {}) {
|
|
196
|
+
const results = [];
|
|
197
|
+
for (const { collection, records } of jobs) {
|
|
198
|
+
if (records.length === 0)
|
|
199
|
+
continue;
|
|
200
|
+
const submitted = await this.submitJob(collection, records, opts);
|
|
201
|
+
const status = await this.waitForJob(submitted.job_id);
|
|
202
|
+
results.push({ collection, job: submitted, status });
|
|
203
|
+
}
|
|
204
|
+
return results;
|
|
205
|
+
}
|
|
190
206
|
}
|
|
@@ -10,23 +10,29 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import { ModalEmbeddingClient } from "./modal-client.js";
|
|
12
12
|
import type { VaultMindConfig } from "./types.js";
|
|
13
|
-
/** Env var name for the
|
|
14
|
-
export declare const MODAL_TOKEN_ENV = "
|
|
13
|
+
/** Env var name for the unscoped remote embedding API key (preferred over config). */
|
|
14
|
+
export declare const MODAL_TOKEN_ENV = "PVM_REMOTE_EMBEDDING_API_KEY";
|
|
15
15
|
/** Env var name for the read-scoped Modal token (search/export). */
|
|
16
16
|
export declare const MODAL_READ_TOKEN_ENV = "PVM_MODAL_READ_TOKEN";
|
|
17
17
|
/** Env var name for the write-scoped Modal token (embed/reindex). */
|
|
18
18
|
export declare const MODAL_WRITE_TOKEN_ENV = "PVM_MODAL_WRITE_TOKEN";
|
|
19
|
-
/**
|
|
20
|
-
|
|
19
|
+
/**
|
|
20
|
+
* Dotenv fallback path for the Modal token, resolved from the vault cwd:
|
|
21
|
+
* `<vault>/.vault-mind/vault-mind.env`. Lazy so tests can override cwd.
|
|
22
|
+
*/
|
|
23
|
+
export declare const modalTokenEnvPath: (cwd?: string) => string;
|
|
21
24
|
/** Canonical deployed Modal app name. */
|
|
22
25
|
export declare const MODAL_APP_NAME = "pi-vault-mind-embed";
|
|
23
26
|
/** Derive the Modal service URL from a workspace slug and canonical app name. */
|
|
24
27
|
export declare const modalUrl: (workspace: string) => string;
|
|
25
28
|
/**
|
|
26
|
-
* Resolve the
|
|
27
|
-
* 1. `
|
|
28
|
-
* 2.
|
|
29
|
-
* 3. `vaultMind.embedding.
|
|
29
|
+
* Resolve the unscoped remote embedding API token. Resolution order:
|
|
30
|
+
* 1. `PVM_REMOTE_EMBEDDING_API_KEY` env var
|
|
31
|
+
* 2. `<vault>/.vault-mind/vault-mind.env`
|
|
32
|
+
* 3. `vaultMind.embedding.remoteApiKey` in config
|
|
33
|
+
*
|
|
34
|
+
* `PVM_API_TOKEN` is intentionally not consulted — it is reserved for the
|
|
35
|
+
* extension's bridge authentication.
|
|
30
36
|
*
|
|
31
37
|
* Never log the resolved token.
|
|
32
38
|
*/
|
|
@@ -35,14 +41,17 @@ export declare const resolveModalToken: (cfg: VaultMindConfig) => string | undef
|
|
|
35
41
|
* Resolve a scoped Modal token for the requested operation.
|
|
36
42
|
* Resolution order:
|
|
37
43
|
* 1. `PVM_MODAL_{READ|WRITE}_TOKEN` env var
|
|
38
|
-
* 2.
|
|
39
|
-
* 3. `vaultMind.embedding.
|
|
40
|
-
* 4. Fallback to the unscoped `
|
|
44
|
+
* 2. `<vault>/.vault-mind/vault-mind.env`
|
|
45
|
+
* 3. `vaultMind.embedding.remoteReadApiKey` / `remoteWriteApiKey` in config
|
|
46
|
+
* 4. Fallback to the unscoped `PVM_REMOTE_EMBEDDING_API_KEY` (env, dotenv, then config)
|
|
47
|
+
*
|
|
48
|
+
* `PVM_API_TOKEN` is intentionally not consulted — it is reserved for the
|
|
49
|
+
* extension's bridge authentication.
|
|
41
50
|
*
|
|
42
51
|
* Never log the resolved token.
|
|
43
52
|
*/
|
|
44
53
|
export declare const getModalToken: (cfg: VaultMindConfig, operation: "read" | "write") => string | undefined;
|
|
45
|
-
/**
|
|
54
|
+
/** Derive the remote URL from the top-level `remoteUrl` or the `workspace` shortcut. */
|
|
46
55
|
export declare const resolveBaseUrl: (cfg: VaultMindConfig) => string | undefined;
|
|
47
56
|
/** Resolve the configured workspace slug, if any. */
|
|
48
57
|
export declare const resolveWorkspace: (cfg: VaultMindConfig) => string | undefined;
|
|
@@ -62,9 +71,23 @@ export declare const createModalClient: (cfg: VaultMindConfig) => ModalEmbedding
|
|
|
62
71
|
export declare const resolveModel: (cfg: VaultMindConfig, collection?: string) => string;
|
|
63
72
|
/**
|
|
64
73
|
* The configured output dimension for a collection, if known. Per-collection
|
|
65
|
-
* overrides win; otherwise the
|
|
66
|
-
* when unset — callers resolve the native dim lazily via `/models`.
|
|
74
|
+
* overrides win; otherwise the top-level `dim` on `EmbeddingConfig`.
|
|
75
|
+
* Returns undefined when unset — callers resolve the native dim lazily via `/models`.
|
|
67
76
|
*/
|
|
68
77
|
export declare const resolveDim: (cfg: VaultMindConfig, collection?: string) => number | undefined;
|
|
69
78
|
/** Namespaced local table name mirroring the server's ADR-3 scheme. */
|
|
70
79
|
export declare const namespacedTableName: (collection: string, model: string, dim: number) => string;
|
|
80
|
+
/**
|
|
81
|
+
* Resolve the server-side (Modal) collection table prefix for cross-vault
|
|
82
|
+
* isolation on a shared Modal deployment. Unset or `"auto"` derives it from
|
|
83
|
+
* the vault slug. The result is normalized (no `__`, no leading/trailing `_`)
|
|
84
|
+
* so it is safe for the server's `col_(.+)__(.+)__(\d+)$` parse.
|
|
85
|
+
*
|
|
86
|
+
* Applies ONLY to remote table names — local LanceDB tables stay unprefixed.
|
|
87
|
+
*/
|
|
88
|
+
export declare const resolveCollectionPrefix: (cfg: VaultMindConfig, vaultPath: string) => string | undefined;
|
|
89
|
+
/**
|
|
90
|
+
* Prepend the collection prefix to a collection name for server-side table
|
|
91
|
+
* identity. When `prefix` is undefined, returns the collection name unchanged.
|
|
92
|
+
*/
|
|
93
|
+
export declare const serverCollectionName: (prefix: string | undefined, collection: string) => string;
|