pi-vault-mind 0.14.2 → 0.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +293 -0
- package/README.md +5 -5
- package/agents/broadcaster.agent.md +2 -2
- package/agents/heavy-lifter.agent.md +1 -1
- package/agents/main.agent.md +29 -0
- package/agents/manager.agent.md +1 -1
- package/agents/miner.agent.md +1 -1
- package/dist/packages/obsidian/src/client.js +331 -0
- package/dist/packages/obsidian/src/types.js +1 -0
- package/dist/src/activity.d.ts +49 -0
- package/dist/src/activity.js +175 -0
- package/dist/src/agent-bus/Agent.d.ts +9 -1
- package/dist/src/agent-bus/Agent.js +18 -2
- package/dist/src/agent-bus/index.d.ts +5 -5
- package/dist/src/agent-bus/index.js +1 -1
- package/dist/src/agent-queue.js +12 -3
- package/dist/src/agents/BroadcasterAgent.d.ts +1 -1
- package/dist/src/agents/BroadcasterAgent.js +2 -2
- package/dist/src/agents/HeavyLifterAgent.d.ts +1 -1
- package/dist/src/agents/HeavyLifterAgent.js +2 -2
- package/dist/src/agents/ManagerAgent.js +1 -1
- package/dist/src/agents/MinerAgent.d.ts +1 -1
- package/dist/src/agents/MinerAgent.js +2 -2
- package/dist/src/agents/index.d.ts +2 -2
- package/dist/src/agents/index.js +2 -2
- package/dist/src/auth.d.ts +17 -15
- package/dist/src/auth.js +26 -37
- package/dist/src/autosync.js +2 -2
- package/dist/src/commands.d.ts +2 -19
- package/dist/src/commands.js +301 -231
- package/dist/src/config-keys.d.ts +16 -0
- package/dist/src/config-keys.js +45 -0
- package/dist/src/discover-schema.js +4 -4
- package/dist/src/edits.d.ts +21 -0
- package/dist/src/edits.js +75 -0
- package/dist/src/embedding-probe.d.ts +34 -0
- package/dist/src/embedding-probe.js +142 -0
- package/dist/src/embedding-providers.d.ts +133 -0
- package/dist/src/embedding-providers.js +173 -0
- package/dist/src/embedding-secrets.d.ts +38 -0
- package/dist/src/embedding-secrets.js +291 -0
- package/dist/src/engine.d.ts +6 -0
- package/dist/src/engine.js +78 -6
- package/dist/src/events.js +7 -10
- package/dist/src/extension-packages.d.ts +9 -0
- package/dist/src/extension-packages.js +14 -0
- package/dist/src/git.d.ts +19 -0
- package/dist/src/git.js +78 -0
- package/dist/src/graph.d.ts +2 -1
- package/dist/src/graph.js +47 -13
- package/dist/src/identities-config.d.ts +6 -0
- package/dist/src/identities-config.js +39 -0
- package/dist/src/identity-injector.js +1 -1
- package/dist/src/index.js +9 -26
- package/dist/src/lance.d.ts +9 -0
- package/dist/src/lance.js +143 -203
- package/dist/src/modal-client.d.ts +16 -0
- package/dist/src/modal-client.js +16 -0
- package/dist/src/modal-config.d.ts +37 -14
- package/dist/src/modal-config.js +59 -32
- package/dist/src/model-router.d.ts +27 -0
- package/dist/src/model-router.js +67 -0
- package/dist/src/models.d.ts +42 -0
- package/dist/src/models.js +78 -0
- package/dist/src/personalize.js +17 -3
- package/dist/src/scaffold.d.ts +48 -0
- package/dist/src/scaffold.js +246 -0
- package/dist/src/server.d.ts +47 -4
- package/dist/src/server.js +988 -221
- package/dist/src/session-search.d.ts +63 -0
- package/dist/src/session-search.js +379 -0
- package/dist/src/settings-ui.d.ts +2 -2
- package/dist/src/settings-ui.js +76 -135
- package/dist/src/sync.d.ts +3 -3
- package/dist/src/sync.js +19 -15
- package/dist/src/tool-catalog.d.ts +18 -0
- package/dist/src/tool-catalog.js +232 -0
- package/dist/src/tools/marksman.js +1 -1
- package/dist/src/tools.js +111 -65
- package/dist/src/types.d.ts +127 -71
- package/dist/src/utils.d.ts +57 -6
- package/dist/src/utils.js +114 -28
- package/dist/src/vault-tools.d.ts +34 -0
- package/dist/src/vault-tools.js +99 -0
- package/dist/src/vault-writer.js +4 -1
- package/dist/src/vm-handlers.js +140 -43
- package/dist/src/watcher.js +9 -9
- package/dist/test/activity-tool.test.js +54 -0
- package/dist/test/agent-bus/Agent.test.js +174 -0
- package/dist/test/agent-bus/AgentLoader.test.js +180 -0
- package/dist/test/agent-bus/AgentRegistry.test.js +86 -0
- package/dist/test/agent-bus/LLMProvider.test.js +162 -0
- package/dist/test/agent-bus/TaskQueue.test.js +256 -0
- package/dist/test/agent-dispatch-context.test.js +73 -0
- package/dist/test/agent-queue.test.js +1 -1
- package/dist/test/agents/Agents.test.js +282 -0
- package/dist/test/auth.test.js +57 -56
- package/dist/test/bridge-identity.test.js +34 -3
- package/dist/test/classify-search-mode.test.js +51 -0
- package/dist/test/commands.test.js +410 -0
- package/dist/test/config-merge.test.js +98 -0
- package/dist/test/context-capture.test.js +56 -0
- package/dist/test/discover-schema.test.js +8 -5
- package/dist/test/dispatch.test.js +45 -91
- package/dist/test/edits.test.js +144 -0
- package/dist/test/embedding-probe.test.js +424 -0
- package/dist/test/embedding-providers.test.js +157 -0
- package/dist/test/embedding-secrets.test.js +210 -0
- package/dist/test/engine-identity.test.js +1 -1
- package/dist/test/events.test.js +675 -0
- package/dist/test/git-bridge.test.js +160 -0
- package/dist/test/graph.test.js +464 -0
- package/dist/test/helpers/server.js +81 -0
- package/dist/test/identity-injector.test.js +197 -0
- package/dist/test/index.test.js +5 -7
- package/dist/test/lance-modal.test.js +67 -2
- package/dist/test/lance.test.js +429 -0
- package/dist/test/modal-client.test.js +1 -1
- package/dist/test/modal-config.test.js +92 -79
- package/dist/test/model-readiness.test.js +122 -0
- package/dist/test/model-router.test.js +99 -0
- package/dist/test/obsidian-client.test.js +42 -0
- package/dist/test/personalize.test.js +1 -1
- package/dist/test/rest-models.test.js +153 -0
- package/dist/test/rest-queue.test.js +135 -87
- package/dist/test/rest-setup.test.js +625 -115
- package/dist/test/rest-vm.test.js +355 -94
- package/dist/test/server-cors.test.js +48 -0
- package/dist/test/server-ws.test.js +61 -34
- package/dist/test/session-crud.test.js +323 -0
- package/dist/test/session-search.test.js +139 -0
- package/dist/test/settings-ui.test.js +113 -4
- package/dist/test/sync.test.js +12 -12
- package/dist/test/tombstone.test.js +8 -10
- package/dist/test/tool-catalog.test.js +86 -0
- package/dist/test/utils.test.js +588 -0
- package/dist/test/vault-tools.test.js +103 -12
- package/dist/test/vault-writer.test.js +371 -0
- package/dist/test/vm-handlers.test.js +620 -0
- package/extension-packages.json +9 -0
- package/package.json +107 -90
- package/pi-vault-mind.config.example.json +24 -2
- package/scripts/check-deps.sh +35 -32
- package/scripts/cli-e2e.mjs +215 -0
- package/scripts/configuration-e2e.mjs +783 -0
- package/scripts/e2e-commands.mjs +391 -0
- package/scripts/e2e-ws.mjs +3 -2
- package/scripts/fetch-modal-token.sh +4 -3
- package/scripts/generate-config-keys-json.mjs +45 -0
- package/scripts/generate-extension-packages-json.mjs +55 -0
- package/scripts/pi-session-query +0 -0
- package/scripts/publish-obsidian-plugin.sh +0 -0
- package/scripts/reset-test-vault.sh +2 -2
- package/scripts/setup-test-config.sh +0 -0
- package/scripts/setup-vault-pi.sh +48 -9
- package/scripts/test-slash-commands.mjs +2 -2
- package/scripts/vault-pi.sh +12 -3
- package/skills/github-release-notes-file/SKILL.md +42 -0
- package/skills/obsidian-guided-test-session/SKILL.md +110 -0
- package/skills/obsidian-interactive-test/SKILL.md +88 -0
- package/skills/obsidian-plugin-publish/SKILL.md +58 -0
- package/skills/pi-vault-mind-release/SKILL.md +141 -0
- package/skills/pre-test-release/SKILL.md +116 -0
- package/dist/test/auto-indexer.test.js +0 -2
package/dist/src/settings-ui.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { execFileSync } from "node:child_process";
|
|
2
2
|
import * as fs from "node:fs";
|
|
3
3
|
import * as path from "node:path";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { EMBEDDING_FLAT_KEYS } from "./config-keys.js";
|
|
5
|
+
import { probeEmbeddingEndpoint } from "./embedding-probe.js";
|
|
6
|
+
import { createModalClient, MODAL_TOKEN_ENV, modalUrl, resolveModalToken } from "./modal-config.js";
|
|
7
|
+
import { collectionNames, findConfig, getConfigPath, loadConfig, shrinkHome } from "./utils.js";
|
|
6
8
|
const ONBOARDING_PLUGIN_REQUIREMENTS = [
|
|
7
9
|
{
|
|
8
10
|
id: "obsidian42-brat",
|
|
@@ -192,7 +194,7 @@ export const detectVaultFromCwd = (cwd) => {
|
|
|
192
194
|
export const createCollectionWizard = async (ctx) => {
|
|
193
195
|
const { project: cfgPath } = findConfig(ctx.cwd);
|
|
194
196
|
if (!cfgPath) {
|
|
195
|
-
ctx.ui.notify("No config found. Run /vm
|
|
197
|
+
ctx.ui.notify("No config found. Run /vm setup first.", "error");
|
|
196
198
|
return;
|
|
197
199
|
}
|
|
198
200
|
if (!ctx.hasUI) {
|
|
@@ -235,7 +237,7 @@ export const createCollectionWizard = async (ctx) => {
|
|
|
235
237
|
export const createInjectorWizard = async (ctx) => {
|
|
236
238
|
const { project: cfgPath } = findConfig(ctx.cwd);
|
|
237
239
|
if (!cfgPath) {
|
|
238
|
-
ctx.ui.notify("No config found. Run /vm
|
|
240
|
+
ctx.ui.notify("No config found. Run /vm setup first.", "error");
|
|
239
241
|
return;
|
|
240
242
|
}
|
|
241
243
|
if (!ctx.hasUI) {
|
|
@@ -269,62 +271,8 @@ export const createInjectorWizard = async (ctx) => {
|
|
|
269
271
|
fs.writeFileSync(cfgPath, `${JSON.stringify(existing, null, 2)}\n`, "utf-8");
|
|
270
272
|
ctx.ui.notify(`✅ Injector "${name}" created and configured!`, "info");
|
|
271
273
|
};
|
|
272
|
-
/** Probe Ollama at localhost for available embedding models. */
|
|
273
|
-
const probeOllamaModels = async () => {
|
|
274
|
-
try {
|
|
275
|
-
const res = await fetch("http://127.0.0.1:11434/api/tags", {
|
|
276
|
-
signal: AbortSignal.timeout(3000),
|
|
277
|
-
});
|
|
278
|
-
if (!res.ok)
|
|
279
|
-
return [];
|
|
280
|
-
const data = (await res.json());
|
|
281
|
-
if (!Array.isArray(data.models))
|
|
282
|
-
return [];
|
|
283
|
-
// Filter to embedding-capable models (heuristic: name contains "embed" or known embedding models)
|
|
284
|
-
const embeddingHints = ["embed", "minilm", "nomic", "bge", "gte", "e5", "mxbai"];
|
|
285
|
-
const all = data.models
|
|
286
|
-
.filter((m) => embeddingHints.some((h) => m.name.toLowerCase().includes(h)))
|
|
287
|
-
.map((m) => ({
|
|
288
|
-
name: m.name,
|
|
289
|
-
size: m.size > 1e9 ? `${(m.size / 1e9).toFixed(1)}GB` : `${(m.size / 1e6).toFixed(0)}MB`,
|
|
290
|
-
}));
|
|
291
|
-
// Also include all models as secondary options (user might know what they want)
|
|
292
|
-
const nonEmbedding = data.models
|
|
293
|
-
.filter((m) => !embeddingHints.some((h) => m.name.toLowerCase().includes(h)))
|
|
294
|
-
.map((m) => ({
|
|
295
|
-
name: m.name,
|
|
296
|
-
size: m.size > 1e9 ? `${(m.size / 1e9).toFixed(1)}GB` : `${(m.size / 1e6).toFixed(0)}MB`,
|
|
297
|
-
}));
|
|
298
|
-
return [...all, ...nonEmbedding];
|
|
299
|
-
}
|
|
300
|
-
catch {
|
|
301
|
-
return [];
|
|
302
|
-
}
|
|
303
|
-
};
|
|
304
|
-
/** Probe a Modal deployment for available embedding models via GET /models. */
|
|
305
|
-
const probeModalModels = async (baseUrl, cfg) => {
|
|
306
|
-
try {
|
|
307
|
-
const token = resolveModalToken(cfg);
|
|
308
|
-
const headers = {};
|
|
309
|
-
if (token)
|
|
310
|
-
headers.Authorization = `Bearer ${token}`;
|
|
311
|
-
const res = await fetch(`${baseUrl.replace(/\/$/, "")}/models`, {
|
|
312
|
-
headers,
|
|
313
|
-
signal: AbortSignal.timeout(8000),
|
|
314
|
-
});
|
|
315
|
-
if (!res.ok)
|
|
316
|
-
return [];
|
|
317
|
-
const data = (await res.json());
|
|
318
|
-
if (!Array.isArray(data.models))
|
|
319
|
-
return [];
|
|
320
|
-
return data.models.filter((m) => m.enabled);
|
|
321
|
-
}
|
|
322
|
-
catch {
|
|
323
|
-
return [];
|
|
324
|
-
}
|
|
325
|
-
};
|
|
326
274
|
export const setupWizard = async (ctx, cliArgs) => {
|
|
327
|
-
const
|
|
275
|
+
const existingConfigFile = fs.existsSync(getConfigPath(ctx.cwd));
|
|
328
276
|
// Check if THIS vault has already been configured (not just any global config)
|
|
329
277
|
const { project: existingProject } = findConfig(ctx.cwd);
|
|
330
278
|
const existingVaultCfg = existingProject
|
|
@@ -337,13 +285,13 @@ export const setupWizard = async (ctx, cliArgs) => {
|
|
|
337
285
|
}
|
|
338
286
|
})()
|
|
339
287
|
: {};
|
|
340
|
-
const vaultAlreadyConfigured = !!(existingVaultCfg.vaultMind?.embedding?.
|
|
341
|
-
existingVaultCfg.vaultMind?.embedding?.
|
|
288
|
+
const vaultAlreadyConfigured = !!(existingVaultCfg.vaultMind?.embedding?.[EMBEDDING_FLAT_KEYS[0]] ||
|
|
289
|
+
existingVaultCfg.vaultMind?.embedding?.[EMBEDDING_FLAT_KEYS[1]]);
|
|
342
290
|
const detectedVaultPath = detectVaultFromCwd(ctx.cwd);
|
|
343
291
|
// ── Non-interactive / CLI mode ──────────────────────────────────────────
|
|
344
292
|
if (cliArgs && (cliArgs.vault || cliArgs.remoteUrl || cliArgs.localUrl || cliArgs.workspace)) {
|
|
345
|
-
const config =
|
|
346
|
-
? JSON.parse(fs.readFileSync(
|
|
293
|
+
const config = existingConfigFile
|
|
294
|
+
? JSON.parse(fs.readFileSync(getConfigPath(ctx.cwd), "utf-8"))
|
|
347
295
|
: { vaultMind: { embedding: {}, vaults: {} } };
|
|
348
296
|
config.vaultMind = config.vaultMind || {};
|
|
349
297
|
config.vaultMind.embedding = config.vaultMind.embedding || {};
|
|
@@ -353,33 +301,32 @@ export const setupWizard = async (ctx, cliArgs) => {
|
|
|
353
301
|
config.vaultMind.vaults.default = { path: shrinkHome(effectiveVaultPath), autoSync: true };
|
|
354
302
|
}
|
|
355
303
|
if (cliArgs.remoteUrl) {
|
|
356
|
-
config.vaultMind.embedding
|
|
304
|
+
config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[0]] = cliArgs.remoteUrl;
|
|
357
305
|
}
|
|
358
306
|
if (cliArgs.localUrl) {
|
|
359
|
-
config.vaultMind.embedding
|
|
307
|
+
config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[1]] = cliArgs.localUrl;
|
|
360
308
|
}
|
|
361
309
|
if (cliArgs.model) {
|
|
362
|
-
config.vaultMind.embedding
|
|
310
|
+
config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[2]] = cliArgs.model;
|
|
363
311
|
}
|
|
364
312
|
if (cliArgs.dim) {
|
|
365
|
-
config.vaultMind.embedding
|
|
313
|
+
config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[4]] = cliArgs.dim;
|
|
366
314
|
}
|
|
367
315
|
if (cliArgs.workspace) {
|
|
368
|
-
config.vaultMind.embedding
|
|
369
|
-
config.vaultMind.embedding.modal.workspace = cliArgs.workspace;
|
|
316
|
+
config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[6]] = cliArgs.workspace;
|
|
370
317
|
if (!cliArgs.remoteUrl) {
|
|
371
|
-
config.vaultMind.embedding
|
|
318
|
+
config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[0]] = modalUrl(cliArgs.workspace);
|
|
372
319
|
}
|
|
373
320
|
}
|
|
374
|
-
const dir = path.dirname(
|
|
321
|
+
const dir = path.dirname(getConfigPath(ctx.cwd));
|
|
375
322
|
if (!fs.existsSync(dir))
|
|
376
323
|
fs.mkdirSync(dir, { recursive: true });
|
|
377
|
-
fs.writeFileSync(
|
|
378
|
-
const lines = ["✅
|
|
324
|
+
fs.writeFileSync(getConfigPath(ctx.cwd), `${JSON.stringify(config, null, 2)}\n`, "utf-8");
|
|
325
|
+
const lines = ["✅ Config written:", ` ${getConfigPath(ctx.cwd)}`, ""];
|
|
379
326
|
if (effectiveVaultPath)
|
|
380
327
|
lines.push(` Vault: ${effectiveVaultPath}`);
|
|
381
|
-
if (config.vaultMind.embedding
|
|
382
|
-
lines.push(` Remote URL: ${config.vaultMind.embedding
|
|
328
|
+
if (config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[0]])
|
|
329
|
+
lines.push(` Remote URL: ${config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[0]]}`);
|
|
383
330
|
if (cliArgs.localUrl)
|
|
384
331
|
lines.push(` Local URL: ${cliArgs.localUrl}`);
|
|
385
332
|
if (cliArgs.model)
|
|
@@ -507,7 +454,8 @@ export const setupWizard = async (ctx, cliArgs) => {
|
|
|
507
454
|
let model = "";
|
|
508
455
|
let dim;
|
|
509
456
|
// Probe Ollama for local embedding models
|
|
510
|
-
const
|
|
457
|
+
const ollamaProbe = await probeEmbeddingEndpoint({ target: "local", url: "http://127.0.0.1:11434" }, { config: loadConfig(ctx.cwd).vaultMind });
|
|
458
|
+
const ollamaModels = ollamaProbe.ok ? ollamaProbe.models : [];
|
|
511
459
|
const providerOptions = [
|
|
512
460
|
...(ollamaModels.length > 0 ? ["Local (Ollama — models detected)"] : []),
|
|
513
461
|
"Cloud (Modal workspace)",
|
|
@@ -522,7 +470,7 @@ export const setupWizard = async (ctx, cliArgs) => {
|
|
|
522
470
|
if (provider.startsWith("Local")) {
|
|
523
471
|
localUrl = "http://127.0.0.1:11434/v1";
|
|
524
472
|
// Let user pick from detected Ollama embedding models
|
|
525
|
-
const modelChoice = await ctx.ui.select("Select embedding model:", ollamaModels.map((m) =>
|
|
473
|
+
const modelChoice = await ctx.ui.select("Select embedding model:", ollamaModels.map((m) => m.name));
|
|
526
474
|
if (!modelChoice) {
|
|
527
475
|
ctx.ui.notify("Setup cancelled.", "warning");
|
|
528
476
|
return;
|
|
@@ -538,14 +486,15 @@ export const setupWizard = async (ctx, cliArgs) => {
|
|
|
538
486
|
guidedModalWorkspace = workspaceSlug.trim();
|
|
539
487
|
remoteUrl = modalUrl(guidedModalWorkspace);
|
|
540
488
|
// Probe Modal /models for available embedders
|
|
541
|
-
const
|
|
489
|
+
const modalProbe = await probeEmbeddingEndpoint({ target: "remote", url: remoteUrl }, { config: loadConfig(ctx.cwd).vaultMind });
|
|
490
|
+
const modalModels = modalProbe.ok ? modalProbe.models : [];
|
|
542
491
|
if (modalModels.length > 0) {
|
|
543
|
-
const modalChoice = await ctx.ui.select("Select cloud embedding model:", modalModels.map((m) => `${m.
|
|
492
|
+
const modalChoice = await ctx.ui.select("Select cloud embedding model:", modalModels.map((m) => `${m.id} (dim ${m.dim})`));
|
|
544
493
|
if (modalChoice) {
|
|
545
494
|
model = modalChoice.replace(/\s*\(.*\)$/, "");
|
|
546
|
-
const matched = modalModels.find((m) => m.
|
|
495
|
+
const matched = modalModels.find((m) => m.id === model);
|
|
547
496
|
if (matched)
|
|
548
|
-
dim = matched.
|
|
497
|
+
dim = matched.dim ?? undefined;
|
|
549
498
|
}
|
|
550
499
|
}
|
|
551
500
|
else {
|
|
@@ -593,25 +542,25 @@ export const setupWizard = async (ctx, cliArgs) => {
|
|
|
593
542
|
return;
|
|
594
543
|
}
|
|
595
544
|
// ── Write config ────────────────────────────────────────────────────────
|
|
596
|
-
const
|
|
597
|
-
|
|
545
|
+
const configPath = getConfigPath(ctx.cwd);
|
|
546
|
+
const config = existingConfigFile
|
|
547
|
+
? JSON.parse(fs.readFileSync(configPath, "utf-8"))
|
|
598
548
|
: { vaultMind: { graph: { enabled: true, canvasSync: true }, ftsEnabled: true } };
|
|
599
549
|
config.vaultMind = config.vaultMind || {};
|
|
600
550
|
config.vaultMind.embedding = config.vaultMind.embedding || {};
|
|
601
551
|
config.vaultMind.vaults = config.vaultMind.vaults || {};
|
|
602
552
|
if (remoteUrl)
|
|
603
|
-
config.vaultMind.embedding
|
|
553
|
+
config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[0]] = remoteUrl;
|
|
604
554
|
if (localUrl)
|
|
605
|
-
config.vaultMind.embedding
|
|
555
|
+
config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[1]] = localUrl;
|
|
606
556
|
if (model)
|
|
607
|
-
config.vaultMind.embedding
|
|
557
|
+
config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[2]] = model;
|
|
608
558
|
if (useTransformers)
|
|
609
|
-
config.vaultMind.embedding
|
|
559
|
+
config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[5]] = true;
|
|
610
560
|
if (dim)
|
|
611
|
-
config.vaultMind.embedding
|
|
561
|
+
config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[4]] = dim;
|
|
612
562
|
if (workspace) {
|
|
613
|
-
config.vaultMind.embedding
|
|
614
|
-
config.vaultMind.embedding.modal.workspace = workspace;
|
|
563
|
+
config.vaultMind.embedding[EMBEDDING_FLAT_KEYS[6]] = workspace;
|
|
615
564
|
}
|
|
616
565
|
config.vaultMind.vaults.default = {
|
|
617
566
|
path: shrinkHome(vaultPath),
|
|
@@ -620,30 +569,24 @@ export const setupWizard = async (ctx, cliArgs) => {
|
|
|
620
569
|
};
|
|
621
570
|
config.vaultMind.graph = config.vaultMind.graph || { enabled: true, canvasSync: true };
|
|
622
571
|
config.vaultMind.ftsEnabled = config.vaultMind.ftsEnabled !== false;
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
// Write extension compatibility to the PROJECT config (not global)
|
|
628
|
-
// so pi-context settings are scoped to this vault, not every session.
|
|
629
|
-
const projectCfgPath = path.join(vaultPath, "pi-vault-mind.config.json");
|
|
630
|
-
const projectCfg = fs.existsSync(projectCfgPath)
|
|
631
|
-
? JSON.parse(fs.readFileSync(projectCfgPath, "utf-8"))
|
|
632
|
-
: {};
|
|
633
|
-
projectCfg.extensionCompatibility = projectCfg.extensionCompatibility || {};
|
|
634
|
-
projectCfg.extensionCompatibility["pi-context"] = {
|
|
572
|
+
// Extension-compatibility (pi-context) now lives in the single consolidated
|
|
573
|
+
// config alongside everything else (no separate project file).
|
|
574
|
+
config.extensionCompatibility = config.extensionCompatibility || {};
|
|
575
|
+
config.extensionCompatibility["pi-context"] = {
|
|
635
576
|
tagPatterns: [],
|
|
636
577
|
enhanceInjectors: false,
|
|
637
578
|
autoEnableAcm: true,
|
|
638
579
|
indexContextEvents: true,
|
|
639
|
-
...(
|
|
580
|
+
...(config.extensionCompatibility["pi-context"] || {}),
|
|
640
581
|
enabled: enableContextAutomation,
|
|
641
582
|
};
|
|
642
|
-
|
|
583
|
+
const dir = path.dirname(configPath);
|
|
584
|
+
if (!fs.existsSync(dir))
|
|
585
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
586
|
+
fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf-8");
|
|
643
587
|
ctx.ui.notify([
|
|
644
588
|
"✅ Config written!",
|
|
645
|
-
`
|
|
646
|
-
` Project: ${projectCfgPath}`,
|
|
589
|
+
` Config: ${configPath}`,
|
|
647
590
|
"",
|
|
648
591
|
` Vault: ${vaultPath}`,
|
|
649
592
|
` Remote: ${remoteUrl || "none"}`,
|
|
@@ -658,14 +601,14 @@ export const setupWizard = async (ctx, cliArgs) => {
|
|
|
658
601
|
/**
|
|
659
602
|
* Interactive Modal embedding configuration + "Test connection" action.
|
|
660
603
|
* Walks workspace/base URL, canonical model, dim, offline fallback, and auto-sync
|
|
661
|
-
* (off by default). Token is read from `
|
|
662
|
-
*
|
|
604
|
+
* (off by default). Token is read from `PVM_REMOTE_EMBEDDING_API_KEY` env or
|
|
605
|
+
* `.vault-mind/vault-mind.env` (preferred) — not collected here; only an optional
|
|
663
606
|
* config fallback is offered.
|
|
664
607
|
*/
|
|
665
608
|
export const configureModalWizard = async (ctx) => {
|
|
666
609
|
const { project: cfgPath } = findConfig(ctx.cwd);
|
|
667
610
|
if (!cfgPath) {
|
|
668
|
-
ctx.ui.notify("No config found. Run /vm
|
|
611
|
+
ctx.ui.notify("No config found. Run /vm setup first.", "error");
|
|
669
612
|
return;
|
|
670
613
|
}
|
|
671
614
|
if (!ctx.hasUI) {
|
|
@@ -673,9 +616,9 @@ export const configureModalWizard = async (ctx) => {
|
|
|
673
616
|
return;
|
|
674
617
|
}
|
|
675
618
|
const cfg = loadConfig(ctx.cwd);
|
|
676
|
-
const
|
|
619
|
+
const emb = cfg.vaultMind.embedding;
|
|
677
620
|
// Prefer workspace input; fall back to full URL if user already has one.
|
|
678
|
-
const workspace = await ctx.ui.input("Modal workspace slug (blank if you want to enter the full URL):",
|
|
621
|
+
const workspace = await ctx.ui.input("Modal workspace slug (blank if you want to enter the full URL):", emb.workspace || "");
|
|
679
622
|
if (workspace === undefined)
|
|
680
623
|
return;
|
|
681
624
|
let baseUrl;
|
|
@@ -683,15 +626,15 @@ export const configureModalWizard = async (ctx) => {
|
|
|
683
626
|
baseUrl = modalUrl(workspace);
|
|
684
627
|
}
|
|
685
628
|
else {
|
|
686
|
-
baseUrl = await ctx.ui.input("Modal base URL (deployed ASGI app):",
|
|
629
|
+
baseUrl = await ctx.ui.input("Modal base URL (deployed ASGI app):", emb.remoteUrl ||
|
|
687
630
|
"https://<workspace>--pi-vault-mind-embed-embeddingservice-fastapi-app.modal.run");
|
|
688
631
|
if (!baseUrl)
|
|
689
632
|
return;
|
|
690
633
|
}
|
|
691
|
-
const model = await ctx.ui.input("Canonical embedder model:",
|
|
634
|
+
const model = await ctx.ui.input("Canonical embedder model:", emb.model || "embeddinggemma");
|
|
692
635
|
if (model === undefined)
|
|
693
636
|
return;
|
|
694
|
-
const dimStr = await ctx.ui.input("Output dimension (blank = model native, e.g. 768 for embeddinggemma):",
|
|
637
|
+
const dimStr = await ctx.ui.input("Output dimension (blank = model native, e.g. 768 for embeddinggemma):", emb.dim ? String(emb.dim) : "");
|
|
695
638
|
const dim = dimStr ? Number.parseInt(dimStr, 10) : undefined;
|
|
696
639
|
if (dimStr && !Number.isFinite(dim)) {
|
|
697
640
|
ctx.ui.notify(`Invalid dim: ${dimStr}`, "error");
|
|
@@ -703,7 +646,7 @@ export const configureModalWizard = async (ctx) => {
|
|
|
703
646
|
]);
|
|
704
647
|
let fallbackCfg;
|
|
705
648
|
if (fallback?.startsWith("localUrl"))
|
|
706
|
-
fallbackCfg = { enabled: true
|
|
649
|
+
fallbackCfg = { enabled: true };
|
|
707
650
|
else if (fallback?.startsWith("none"))
|
|
708
651
|
fallbackCfg = { enabled: false };
|
|
709
652
|
const autoSync = await ctx.ui.confirm("Auto-sync", "Enable background vector sync (off by default)? Pulls new server vectors on an interval.");
|
|
@@ -712,28 +655,27 @@ export const configureModalWizard = async (ctx) => {
|
|
|
712
655
|
const intervalStr = await ctx.ui.input("Auto-sync interval (ms):", "300000");
|
|
713
656
|
intervalMs = Number.parseInt(intervalStr || "300000", 10) || 300000;
|
|
714
657
|
}
|
|
715
|
-
// Token guidance (env/dotenv preferred; config
|
|
658
|
+
// Token guidance (env/dotenv preferred; config remoteApiKey is a fallback)
|
|
716
659
|
const tokenEnv = resolveModalToken(loadConfig(ctx.cwd).vaultMind);
|
|
717
660
|
if (!tokenEnv) {
|
|
718
|
-
ctx.ui.notify(`⚠️ No ${MODAL_TOKEN_ENV} env var or
|
|
661
|
+
ctx.ui.notify(`⚠️ No ${MODAL_TOKEN_ENV} env var or .vault-mind/vault-mind.env found. Set it in your shell:\n export ${MODAL_TOKEN_ENV}=<bearer token>\nOr write it to .vault-mind/vault-mind.env. (Env/dotenv preferred; config remoteApiKey is a fallback only.)`, "warning");
|
|
719
662
|
}
|
|
720
|
-
// Persist
|
|
663
|
+
// Persist — flat shape (0.16.2+).
|
|
721
664
|
const existing = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
|
|
722
665
|
existing.vaultMind = existing.vaultMind || {};
|
|
723
666
|
existing.vaultMind.embedding = existing.vaultMind.embedding || {};
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
},
|
|
667
|
+
if (workspace)
|
|
668
|
+
existing.vaultMind.embedding[EMBEDDING_FLAT_KEYS[6]] = workspace;
|
|
669
|
+
existing.vaultMind.embedding[EMBEDDING_FLAT_KEYS[0]] = baseUrl.replace(/\/$/, "");
|
|
670
|
+
existing.vaultMind.embedding[EMBEDDING_FLAT_KEYS[2]] = model || "embeddinggemma";
|
|
671
|
+
if (dim != null)
|
|
672
|
+
existing.vaultMind.embedding[EMBEDDING_FLAT_KEYS[4]] = dim;
|
|
673
|
+
if (fallbackCfg)
|
|
674
|
+
existing.vaultMind.embedding[EMBEDDING_FLAT_KEYS[10]] = fallbackCfg;
|
|
675
|
+
existing.vaultMind.embedding[EMBEDDING_FLAT_KEYS[11]] = {
|
|
676
|
+
...(existing.vaultMind.embedding[EMBEDDING_FLAT_KEYS[11]] || {}),
|
|
677
|
+
autoSync,
|
|
678
|
+
autoSyncIntervalMs: intervalMs,
|
|
737
679
|
};
|
|
738
680
|
fs.writeFileSync(cfgPath, `${JSON.stringify(existing, null, 2)}\n`, "utf-8");
|
|
739
681
|
// Test connection against /health
|
|
@@ -767,7 +709,7 @@ export const openSettingsDashboard = async (ctx) => {
|
|
|
767
709
|
}
|
|
768
710
|
const { project: cfgPath } = findConfig(ctx.cwd);
|
|
769
711
|
if (!cfgPath) {
|
|
770
|
-
ctx.ui.notify("No config found. Run /vm
|
|
712
|
+
ctx.ui.notify("No config found. Run /vm setup first.", "error");
|
|
771
713
|
return;
|
|
772
714
|
}
|
|
773
715
|
let running = true;
|
|
@@ -809,9 +751,8 @@ const showConfigSettings = async (ctx, cfgPath) => {
|
|
|
809
751
|
const merged = loadConfig(ctx.cwd);
|
|
810
752
|
const paths = findConfig(ctx.cwd);
|
|
811
753
|
const lines = [
|
|
812
|
-
"Config
|
|
813
|
-
`-
|
|
814
|
-
`- Project: ${paths.project || "(none)"}`,
|
|
754
|
+
"Config source:",
|
|
755
|
+
`- Config: ${paths.project || "(none)"}`,
|
|
815
756
|
"",
|
|
816
757
|
"Merged runtime config:",
|
|
817
758
|
JSON.stringify(merged.vaultMind, null, 2),
|
package/dist/src/sync.d.ts
CHANGED
|
@@ -42,12 +42,12 @@ export interface SyncOptions {
|
|
|
42
42
|
* are merge-inserted by `id` (idempotent). Returns the row count + new
|
|
43
43
|
* watermark. A re-run with no new rows is a no-op (rows: 0, watermark unchanged).
|
|
44
44
|
*/
|
|
45
|
-
export declare const syncCollection: (cfg: UniversalConfig, collection: string, opts?: SyncOptions) => Promise<SyncResult>;
|
|
45
|
+
export declare const syncCollection: (cfg: UniversalConfig, collection: string, vaultPath: string, opts?: SyncOptions) => Promise<SyncResult>;
|
|
46
46
|
/**
|
|
47
47
|
* Sync every configured collection (or a caller-supplied list). Collections
|
|
48
48
|
* with no server-side table report `rows: 0` and are skipped harmlessly.
|
|
49
49
|
*/
|
|
50
|
-
export declare const syncAll: (cfg: UniversalConfig, collections?: string[], opts?: SyncOptions) => Promise<SyncResult[]>;
|
|
50
|
+
export declare const syncAll: (cfg: UniversalConfig, vaultPath: string, collections?: string[], opts?: SyncOptions) => Promise<SyncResult[]>;
|
|
51
51
|
export interface ReindexRemoteOptions {
|
|
52
52
|
/** Notified with the job status on each poll. */
|
|
53
53
|
onStatus?: (status: JobStatus) => void;
|
|
@@ -63,7 +63,7 @@ export interface ReindexRemoteOptions {
|
|
|
63
63
|
* Records map the JSONL entry's embeddable field (`fact`) to `text` and the
|
|
64
64
|
* remaining fields to `metadata`, so sync-down can reconstruct the local row.
|
|
65
65
|
*/
|
|
66
|
-
export declare const reindexRemote: (cfg: UniversalConfig, collections: string[], opts?: ReindexRemoteOptions) => Promise<Array<{
|
|
66
|
+
export declare const reindexRemote: (cfg: UniversalConfig, collections: string[], vaultPath: string, opts?: ReindexRemoteOptions) => Promise<Array<{
|
|
67
67
|
collection: string;
|
|
68
68
|
job?: JobStatus;
|
|
69
69
|
sync?: SyncResult;
|
package/dist/src/sync.js
CHANGED
|
@@ -13,11 +13,11 @@
|
|
|
13
13
|
import * as fs from "node:fs";
|
|
14
14
|
import * as path from "node:path";
|
|
15
15
|
import { upsertPrecomputed } from "./lance.js";
|
|
16
|
-
import { createModalClient, resolveDim, resolveModel } from "./modal-config.js";
|
|
16
|
+
import { createModalClient, resolveCollectionPrefix, resolveDim, resolveModel, serverCollectionName, } from "./modal-config.js";
|
|
17
17
|
const stateKey = (collection, model, dim) => `${collection}__${model}__${dim}`;
|
|
18
18
|
const resolveWatermarkFile = (cfg) => {
|
|
19
19
|
const vaultMind = cfg.vaultMind;
|
|
20
|
-
const rel = vaultMind.embedding.
|
|
20
|
+
const rel = vaultMind.embedding.sync?.watermarkPath || "_sync_state.json";
|
|
21
21
|
const dataDir = vaultMind.dataDir;
|
|
22
22
|
const absDir = path.isAbsolute(dataDir) ? dataDir : path.resolve(dataDir);
|
|
23
23
|
return path.isAbsolute(rel) ? rel : path.join(absDir, rel);
|
|
@@ -57,12 +57,12 @@ export const listRemoteCollections = async (cfg) => {
|
|
|
57
57
|
* server's table dim is discovered via `/sync/collections`. Returns null when
|
|
58
58
|
* the server has no table for that collection+model (nothing to sync).
|
|
59
59
|
*/
|
|
60
|
-
const resolveSyncDim = async (cfg, collection, model) => {
|
|
60
|
+
const resolveSyncDim = async (cfg, collection, remoteCollection, model) => {
|
|
61
61
|
const configured = resolveDim(cfg.vaultMind, collection);
|
|
62
62
|
if (configured != null)
|
|
63
63
|
return configured;
|
|
64
64
|
const remote = await listRemoteCollections(cfg);
|
|
65
|
-
const match = remote.find((c) => c.collection ===
|
|
65
|
+
const match = remote.find((c) => c.collection === remoteCollection && c.model === model);
|
|
66
66
|
return match?.dim ?? null;
|
|
67
67
|
};
|
|
68
68
|
/**
|
|
@@ -71,13 +71,15 @@ const resolveSyncDim = async (cfg, collection, model) => {
|
|
|
71
71
|
* are merge-inserted by `id` (idempotent). Returns the row count + new
|
|
72
72
|
* watermark. A re-run with no new rows is a no-op (rows: 0, watermark unchanged).
|
|
73
73
|
*/
|
|
74
|
-
export const syncCollection = async (cfg, collection, opts = {}) => {
|
|
74
|
+
export const syncCollection = async (cfg, collection, vaultPath, opts = {}) => {
|
|
75
75
|
const vaultMind = cfg.vaultMind;
|
|
76
76
|
const client = createModalClient(vaultMind);
|
|
77
77
|
if (!client)
|
|
78
78
|
throw new Error("Modal not configured (missing baseUrl or token).");
|
|
79
79
|
const model = resolveModel(vaultMind, collection);
|
|
80
|
-
const
|
|
80
|
+
const prefix = resolveCollectionPrefix(vaultMind, vaultPath);
|
|
81
|
+
const remoteCollection = serverCollectionName(prefix, collection);
|
|
82
|
+
const dim = await resolveSyncDim(cfg, collection, remoteCollection, model);
|
|
81
83
|
if (dim == null) {
|
|
82
84
|
return {
|
|
83
85
|
collection,
|
|
@@ -89,11 +91,11 @@ export const syncCollection = async (cfg, collection, opts = {}) => {
|
|
|
89
91
|
};
|
|
90
92
|
}
|
|
91
93
|
const state = loadSyncState(cfg);
|
|
92
|
-
const key = stateKey(
|
|
94
|
+
const key = stateKey(remoteCollection, model, dim);
|
|
93
95
|
const since = opts.full ? 0 : (state.watermarks[key] ?? 0);
|
|
94
|
-
const pageSize = vaultMind.embedding.
|
|
96
|
+
const pageSize = vaultMind.embedding.sync?.pageSize ?? 500;
|
|
95
97
|
let rows = 0;
|
|
96
|
-
const watermark = await client.exportAll(
|
|
98
|
+
const watermark = await client.exportAll(remoteCollection, async (page) => {
|
|
97
99
|
const withVectors = page.filter((r) => Array.isArray(r.vector) && r.vector.length > 0);
|
|
98
100
|
if (withVectors.length < page.length) {
|
|
99
101
|
console.warn(`[pi-vault-mind] sync: skipped ${page.length - withVectors.length} vector-less row(s) for "${collection}"`);
|
|
@@ -112,15 +114,15 @@ export const syncCollection = async (cfg, collection, opts = {}) => {
|
|
|
112
114
|
* Sync every configured collection (or a caller-supplied list). Collections
|
|
113
115
|
* with no server-side table report `rows: 0` and are skipped harmlessly.
|
|
114
116
|
*/
|
|
115
|
-
export const syncAll = async (cfg, collections, opts = {}) => {
|
|
116
|
-
const syncCfg = cfg.vaultMind.embedding.
|
|
117
|
+
export const syncAll = async (cfg, vaultPath, collections, opts = {}) => {
|
|
118
|
+
const syncCfg = cfg.vaultMind.embedding.sync;
|
|
117
119
|
const names = collections ?? syncCfg?.collections ?? Object.keys(cfg.collections);
|
|
118
120
|
const results = [];
|
|
119
121
|
for (const name of names) {
|
|
120
122
|
if (!cfg.collections[name])
|
|
121
123
|
continue;
|
|
122
124
|
try {
|
|
123
|
-
results.push(await syncCollection(cfg, name, opts));
|
|
125
|
+
results.push(await syncCollection(cfg, name, vaultPath, opts));
|
|
124
126
|
}
|
|
125
127
|
catch (err) {
|
|
126
128
|
console.warn(`[pi-vault-mind] sync "${name}" failed: ${err.message}`);
|
|
@@ -145,13 +147,14 @@ export const syncAll = async (cfg, collections, opts = {}) => {
|
|
|
145
147
|
* Records map the JSONL entry's embeddable field (`fact`) to `text` and the
|
|
146
148
|
* remaining fields to `metadata`, so sync-down can reconstruct the local row.
|
|
147
149
|
*/
|
|
148
|
-
export const reindexRemote = async (cfg, collections, opts = {}) => {
|
|
150
|
+
export const reindexRemote = async (cfg, collections, vaultPath, opts = {}) => {
|
|
149
151
|
const vaultMind = cfg.vaultMind;
|
|
150
152
|
const client = createModalClient(vaultMind);
|
|
151
153
|
if (!client)
|
|
152
154
|
throw new Error("Modal not configured (missing baseUrl or token).");
|
|
153
155
|
const model = resolveModel(vaultMind);
|
|
154
156
|
const dim = resolveDim(vaultMind);
|
|
157
|
+
const prefix = resolveCollectionPrefix(vaultMind, vaultPath);
|
|
155
158
|
const out = [];
|
|
156
159
|
for (const collection of collections) {
|
|
157
160
|
const def = cfg.collections[collection];
|
|
@@ -185,7 +188,8 @@ export const reindexRemote = async (cfg, collections, opts = {}) => {
|
|
|
185
188
|
out.push({ collection, error: "no records to embed" });
|
|
186
189
|
continue;
|
|
187
190
|
}
|
|
188
|
-
const
|
|
191
|
+
const remoteCollection = serverCollectionName(prefix, collection);
|
|
192
|
+
const submitted = await client.submitJob(remoteCollection, records, { model, dim });
|
|
189
193
|
let status;
|
|
190
194
|
try {
|
|
191
195
|
status = await client.waitForJob(submitted.job_id, opts.pollMs ?? 2000);
|
|
@@ -200,7 +204,7 @@ export const reindexRemote = async (cfg, collections, opts = {}) => {
|
|
|
200
204
|
continue;
|
|
201
205
|
}
|
|
202
206
|
// Pull the freshly-embedded vectors down (full, from seq 0).
|
|
203
|
-
const sync = await syncCollection(cfg, collection, { full: true });
|
|
207
|
+
const sync = await syncCollection(cfg, collection, vaultPath, { full: true });
|
|
204
208
|
out.push({ collection, job: status, sync });
|
|
205
209
|
}
|
|
206
210
|
catch (err) {
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-only tool catalog — enumerates all tools available to pi agents.
|
|
3
|
+
* Built-in tools are sourced from src/bridge.ts BuiltInTools type.
|
|
4
|
+
* vm_* tools are sourced from src/vm-handlers.ts registerAllExtensionTools.
|
|
5
|
+
*/
|
|
6
|
+
export interface ToolEntry {
|
|
7
|
+
name: string;
|
|
8
|
+
kind: "builtin" | "vm_search" | "vm_data" | "vm_config" | "vm_sync";
|
|
9
|
+
description: string;
|
|
10
|
+
/** JSON Schema for the tool's input parameters (vm_* tools only) */
|
|
11
|
+
params?: Record<string, unknown>;
|
|
12
|
+
}
|
|
13
|
+
/** Return all 30 tools (7 built-in + 23 extension tools). */
|
|
14
|
+
export declare function listAllTools(): ToolEntry[];
|
|
15
|
+
/** Return the 7 built-in tools. */
|
|
16
|
+
export declare function listBuiltinTools(): ToolEntry[];
|
|
17
|
+
/** Return the 23 extension tools. */
|
|
18
|
+
export declare function listExtensionTools(): ToolEntry[];
|