pi-vault-mind 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -2
- package/dist/src/autosync.d.ts +16 -0
- package/dist/src/autosync.js +43 -0
- package/dist/src/commands.d.ts +18 -0
- package/dist/src/commands.js +545 -12
- package/dist/src/embed-queue.d.ts +80 -0
- package/dist/src/embed-queue.js +163 -0
- package/dist/src/index.js +9 -0
- package/dist/src/lance.d.ts +7 -0
- package/dist/src/lance.js +432 -0
- package/dist/src/modal-client.d.ts +176 -0
- package/dist/src/modal-client.js +174 -0
- package/dist/src/modal-config.d.ts +54 -0
- package/dist/src/modal-config.js +113 -0
- package/dist/src/settings-ui.d.ts +9 -0
- package/dist/src/settings-ui.js +131 -5
- package/dist/src/sync.d.ts +71 -0
- package/dist/src/sync.js +211 -0
- package/dist/src/types.d.ts +107 -1
- package/dist/src/utils.d.ts +4 -0
- package/dist/src/utils.js +27 -0
- package/dist/src/watcher.js +2 -1
- package/dist/test/embed-queue.test.js +105 -0
- package/dist/test/index.test.js +35 -0
- package/dist/test/lance-modal.test.js +95 -0
- package/dist/test/modal-client.test.js +294 -0
- package/dist/test/modal-config.test.js +139 -0
- package/dist/test/sync.test.js +132 -0
- package/package.json +3 -2
- package/dist/test/index.test.d.ts +0 -1
package/dist/src/commands.js
CHANGED
|
@@ -7,8 +7,10 @@ import { DEFAULT_CONFIG } from "./types.js";
|
|
|
7
7
|
import { CONFIG_FILES, EXT_ROOT, collectionNames, ensureDir, findConfig, getPiContextConfig, hasPiContextTools, loadConfig, } from "./utils.js";
|
|
8
8
|
import { updateActiveCollectionWidget } from "./widget.js";
|
|
9
9
|
import { connect, pullOllamaModel, testOllamaConnection } from "./lance.js";
|
|
10
|
+
import { MODAL_TOKEN_ENV, createModalClient, isModalConfigured, resolveBaseUrl, resolveModalToken, resolveWorkspace, } from "./modal-config.js";
|
|
10
11
|
import { createServerState } from "./server.js";
|
|
11
12
|
import { createCollectionWizard, createInjectorWizard, openSettingsDashboard, setupWizard, } from "./settings-ui.js";
|
|
13
|
+
import { reindexRemote, syncAll, syncCollection } from "./sync.js";
|
|
12
14
|
import { createWatcherState, getWatcherStatus, startWatcher, stopWatcher } from "./watcher.js";
|
|
13
15
|
// ── Shared helpers ───────────────────────────────────────────────────────────
|
|
14
16
|
const WIKI_USAGE = [
|
|
@@ -26,10 +28,15 @@ const WIKI_USAGE = [
|
|
|
26
28
|
" /wiki context enable|disable Enable/disable pi-context integration",
|
|
27
29
|
" /wiki context status Show pi-context integration status",
|
|
28
30
|
" /wiki embedding status Show embedding config + Ollama models",
|
|
29
|
-
" /wiki embedding use Switch provider (ollama | transformers)",
|
|
31
|
+
" /wiki embedding use Switch provider (ollama | transformers | modal)",
|
|
30
32
|
" /wiki embedding model Set Ollama embedding model",
|
|
31
33
|
" /wiki embedding models List available Ollama models",
|
|
32
34
|
" /wiki embedding pull Pull a model from Ollama",
|
|
35
|
+
" /wiki modal status Show Modal config + health + remote collections",
|
|
36
|
+
" /wiki modal config Set Modal baseUrl/model/dim/sync/fallback",
|
|
37
|
+
" /wiki modal sync Pull server vectors into local LanceDB [--full]",
|
|
38
|
+
" /wiki modal jobs <id> Poll a Modal bulk job",
|
|
39
|
+
" /wiki modal migrate <model> Change canonical model + re-embed (remote)",
|
|
33
40
|
" /wiki watcher start Start passive file watcher",
|
|
34
41
|
" /wiki watcher stop Stop passive file watcher",
|
|
35
42
|
" /wiki watcher status Show watcher status",
|
|
@@ -139,9 +146,42 @@ export const selectActiveCollection = async (ctx) => {
|
|
|
139
146
|
}
|
|
140
147
|
};
|
|
141
148
|
// ── /wiki init ──────────────────────────────────────────────────────────────
|
|
149
|
+
/**
|
|
150
|
+
* Vault-root `.gitignore` entries that keep pi-vault-mind compatible with
|
|
151
|
+
* `obsidian-git` (and, transitively, Obsidian Sync setups that also back up via
|
|
152
|
+
* git): the LanceDB index is a large, per-device, rebuildable binary and must
|
|
153
|
+
* never be committed; Obsidian's workspace UI state churns constantly. See
|
|
154
|
+
* docs/OBSIDIAN_SETUP.md §6.
|
|
155
|
+
*/
|
|
156
|
+
export const GITIGNORE_ENTRIES = [".lancedb/", ".obsidian/workspace*.json"];
|
|
157
|
+
/**
|
|
158
|
+
* Decide what to do with the vault `.gitignore`, given its current contents
|
|
159
|
+
* (`null` if absent). Pure — does no I/O — so it is unit-testable. Creates the
|
|
160
|
+
* file when missing, appends only the entries that aren't already present
|
|
161
|
+
* (line-exact match, trimmed), and skips when everything is covered.
|
|
162
|
+
*/
|
|
163
|
+
export const planGitignore = (existing) => {
|
|
164
|
+
const header = "# pi-vault-mind: keep the rebuildable binary index out of git";
|
|
165
|
+
if (existing === null) {
|
|
166
|
+
return {
|
|
167
|
+
action: "create",
|
|
168
|
+
content: `${header}\n.lancedb/\n# Obsidian UI workspace state churns constantly\n.obsidian/workspace*.json\n`,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
// Match leniently: `.lancedb`, `.lancedb/`, and `/.lancedb/` are the same rule,
|
|
172
|
+
// so we don't append a duplicate when the user already ignores it differently.
|
|
173
|
+
const normalize = (s) => s.trim().replace(/^\/+|\/+$/g, "");
|
|
174
|
+
const present = new Set(existing.split(/\r?\n/).map(normalize));
|
|
175
|
+
const missing = GITIGNORE_ENTRIES.filter((e) => !present.has(normalize(e)));
|
|
176
|
+
if (missing.length === 0)
|
|
177
|
+
return { action: "skip", content: "" };
|
|
178
|
+
const prefix = existing.length === 0 ? "" : existing.endsWith("\n") ? "\n" : "\n\n";
|
|
179
|
+
return { action: "append", content: `${prefix}# pi-vault-mind\n${missing.join("\n")}\n` };
|
|
180
|
+
};
|
|
142
181
|
const handleInit = async (_args, ctx, pi) => {
|
|
143
182
|
const cfg = loadConfig(ctx.cwd);
|
|
144
183
|
const created = [];
|
|
184
|
+
const updated = [];
|
|
145
185
|
const skipped = [];
|
|
146
186
|
const ensureFile = (dest, tmpl) => {
|
|
147
187
|
if (fs.existsSync(dest)) {
|
|
@@ -203,12 +243,30 @@ const handleInit = async (_args, ctx, pi) => {
|
|
|
203
243
|
if (ij.artifactPath)
|
|
204
244
|
ensureFile(ij.artifactPath, "ARTIFACT.md");
|
|
205
245
|
}
|
|
246
|
+
// Keep the rebuildable binary index out of git so obsidian-git (and git-backed
|
|
247
|
+
// Obsidian Sync setups) never commit it. See docs/OBSIDIAN_SETUP.md §6.
|
|
248
|
+
const gitignoreDest = path.join(ctx.cwd, ".gitignore");
|
|
249
|
+
const plan = planGitignore(fs.existsSync(gitignoreDest) ? fs.readFileSync(gitignoreDest, "utf-8") : null);
|
|
250
|
+
if (plan.action === "create") {
|
|
251
|
+
fs.writeFileSync(gitignoreDest, plan.content, "utf-8");
|
|
252
|
+
created.push(gitignoreDest);
|
|
253
|
+
}
|
|
254
|
+
else if (plan.action === "append") {
|
|
255
|
+
fs.appendFileSync(gitignoreDest, plan.content);
|
|
256
|
+
updated.push(gitignoreDest);
|
|
257
|
+
}
|
|
258
|
+
else {
|
|
259
|
+
skipped.push(gitignoreDest);
|
|
260
|
+
}
|
|
206
261
|
const msg = [
|
|
207
262
|
"Wiki scaffolding complete.",
|
|
208
263
|
"",
|
|
209
264
|
"Created:",
|
|
210
265
|
...created.map((c) => ` • ${path.relative(ctx.cwd, c)}`),
|
|
211
266
|
];
|
|
267
|
+
if (updated.length) {
|
|
268
|
+
msg.push("", "Updated:", ...updated.map((u) => ` • ${path.relative(ctx.cwd, u)}`));
|
|
269
|
+
}
|
|
212
270
|
if (skipped.length) {
|
|
213
271
|
msg.push("", "Skipped (already exist):", ...skipped.map((s) => ` • ${path.relative(ctx.cwd, s)}`));
|
|
214
272
|
}
|
|
@@ -328,10 +386,47 @@ const handleApprove = async (args, ctx) => {
|
|
|
328
386
|
// ── /wiki reindex ────────────────────────────────────────────────────────────
|
|
329
387
|
const handleReindex = async (args, ctx, pi) => {
|
|
330
388
|
const cfg = loadConfig(ctx.cwd);
|
|
331
|
-
const
|
|
332
|
-
const
|
|
333
|
-
const
|
|
334
|
-
const
|
|
389
|
+
const tokens = args.trim().split(/\s+/).filter(Boolean);
|
|
390
|
+
const flags = new Set(tokens.map((t) => t.toLowerCase()));
|
|
391
|
+
const rebuildEmbeddings = flags.has("--reembed") || flags.has("--full");
|
|
392
|
+
const reindexAll = flags.has("--all");
|
|
393
|
+
const remote = flags.has("--remote");
|
|
394
|
+
const collectionFilter = reindexAll || rebuildEmbeddings ? null : (tokens.find((t) => !t.startsWith("--")) ?? null);
|
|
395
|
+
// Remote bulk re-index: read JSONL → submit Modal bulk job → poll → sync down.
|
|
396
|
+
if (remote) {
|
|
397
|
+
if (cfg.wiki.embedding.provider !== "modal") {
|
|
398
|
+
ctx.ui.notify("--remote requires the modal provider. Run /wiki embedding use modal.", "error");
|
|
399
|
+
return;
|
|
400
|
+
}
|
|
401
|
+
if (!isModalConfigured(cfg.wiki)) {
|
|
402
|
+
ctx.ui.notify(`Modal not configured. Set baseUrl (/wiki modal config baseUrl) and ${MODAL_TOKEN_ENV}.`, "error");
|
|
403
|
+
return;
|
|
404
|
+
}
|
|
405
|
+
const names = reindexAll
|
|
406
|
+
? Object.keys(cfg.collections)
|
|
407
|
+
: collectionFilter
|
|
408
|
+
? [collectionFilter]
|
|
409
|
+
: Object.keys(cfg.collections);
|
|
410
|
+
ctx.ui.notify(`Remote re-index: submitting Modal bulk job for ${names.join(", ")}...`, "info");
|
|
411
|
+
try {
|
|
412
|
+
const results = await reindexRemote(cfg, names, {
|
|
413
|
+
onStatus: (s) => ctx.ui.notify(` ${s.collection || ""}: ${s.status} ${s.processed}/${s.total}`, "info"),
|
|
414
|
+
});
|
|
415
|
+
const lines = ["**Remote Re-index Report:**", ""];
|
|
416
|
+
for (const r of results) {
|
|
417
|
+
if (r.error)
|
|
418
|
+
lines.push(` ❌ ${r.collection}: ${r.error}`);
|
|
419
|
+
else
|
|
420
|
+
lines.push(` ✅ ${r.collection}: job ${r.job?.status}, synced ${r.sync?.rows ?? 0} rows (wm ${r.sync?.watermark ?? 0})`);
|
|
421
|
+
}
|
|
422
|
+
lines.push("", "Old namespaces are left intact until the new one is verified.");
|
|
423
|
+
ctx.ui.notify(lines.join("\n"), "info");
|
|
424
|
+
}
|
|
425
|
+
catch (err) {
|
|
426
|
+
ctx.ui.notify(`Remote re-index failed: ${err.message}`, "error");
|
|
427
|
+
}
|
|
428
|
+
return;
|
|
429
|
+
}
|
|
335
430
|
ctx.ui.notify(rebuildEmbeddings
|
|
336
431
|
? "Reindexing: regenerating embeddings + rebuilding indexes..."
|
|
337
432
|
: "Reindexing: rebuilding FTS + vector indexes...", "info");
|
|
@@ -432,12 +527,27 @@ const handleEmbedding = async (args, ctx, pi) => {
|
|
|
432
527
|
`Provider: ${cfg.wiki.embedding.provider}`,
|
|
433
528
|
cfg.wiki.embedding.provider === "ollama"
|
|
434
529
|
? `Model: ${cfg.wiki.embedding.ollamaModel || "embeddinggemma"}`
|
|
435
|
-
:
|
|
530
|
+
: cfg.wiki.embedding.provider === "modal"
|
|
531
|
+
? `Model: ${cfg.wiki.embedding.modal?.model || "(default embeddinggemma)"}`
|
|
532
|
+
: "Model: all-MiniLM-L6-v2 (384 dims)",
|
|
436
533
|
`FTS: ${cfg.wiki.ftsEnabled !== false ? "enabled" : "disabled"}`,
|
|
437
534
|
`Graph: ${cfg.wiki.graph?.enabled !== false ? "enabled" : "disabled"}`,
|
|
438
535
|
`Data Dir: ${cfg.wiki.dataDir}`,
|
|
439
536
|
];
|
|
440
|
-
if (cfg.wiki.embedding.provider === "
|
|
537
|
+
if (cfg.wiki.embedding.provider === "modal") {
|
|
538
|
+
const modal = cfg.wiki.embedding.modal;
|
|
539
|
+
const tokenSrc = process.env[MODAL_TOKEN_ENV]
|
|
540
|
+
? `env ${MODAL_TOKEN_ENV} ✅`
|
|
541
|
+
: modal?.apiToken
|
|
542
|
+
? "config (set env PVM_API_TOKEN to override)"
|
|
543
|
+
: "❌ none (set PVM_API_TOKEN env)";
|
|
544
|
+
lines.push("", "**Modal:**", ` Base URL: ${modal?.baseUrl || "❌ not set"}`, ` Model: ${modal?.model || "(default embeddinggemma)"}`, ` Dim: ${modal?.dim ?? "(native)"}`, ` Token: ${tokenSrc}`, ` Fallback: ${modal?.fallback?.enabled === false ? "disabled" : modal?.fallback?.provider || "(none — degrade to FTS)"}`, ` Sync: auto=${modal?.sync?.autoSync ? "on" : "off"}, interval=${modal?.sync?.autoSyncIntervalMs ?? 300000}ms`);
|
|
545
|
+
const co = cfg.wiki.embedding.coalesce;
|
|
546
|
+
if (co) {
|
|
547
|
+
lines.push(` Coalesce: debounce=${co.debounceMs ?? 1000}ms, batch=${co.maxBatchSize ?? 64}, concurrency=${co.maxConcurrentFlushes ?? 2}, dedupe=${co.dedupe ?? true}, searchBypass=${co.searchBypass ?? true}`);
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
else if (cfg.wiki.embedding.provider === "ollama" || !cfg.wiki.embedding.provider) {
|
|
441
551
|
const conn = await testOllamaConnection(pi);
|
|
442
552
|
lines.push("", "**Ollama Status:**", ` Reachable: ${conn.reachable ? "✅ Yes" : "❌ No"}`);
|
|
443
553
|
if (conn.error)
|
|
@@ -462,8 +572,8 @@ const handleEmbedding = async (args, ctx, pi) => {
|
|
|
462
572
|
return;
|
|
463
573
|
}
|
|
464
574
|
case "use": {
|
|
465
|
-
if (!value || !["ollama", "transformers"].includes(value)) {
|
|
466
|
-
ctx.ui.notify("/wiki embedding use <ollama|transformers>", "error");
|
|
575
|
+
if (!value || !["ollama", "transformers", "modal"].includes(value)) {
|
|
576
|
+
ctx.ui.notify("/wiki embedding use <ollama|transformers|modal>", "error");
|
|
467
577
|
return;
|
|
468
578
|
}
|
|
469
579
|
if (value === "ollama") {
|
|
@@ -474,6 +584,12 @@ const handleEmbedding = async (args, ctx, pi) => {
|
|
|
474
584
|
return;
|
|
475
585
|
}
|
|
476
586
|
}
|
|
587
|
+
if (value === "modal") {
|
|
588
|
+
const modal = cfg.wiki.embedding.modal;
|
|
589
|
+
if (!modal?.baseUrl) {
|
|
590
|
+
ctx.ui.notify("Modal needs a base URL. Set it with: /wiki modal config baseUrl <url>\n(Token via PVM_API_TOKEN env, preferred.)", "warning");
|
|
591
|
+
}
|
|
592
|
+
}
|
|
477
593
|
const existing = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
|
|
478
594
|
existing.wiki = existing.wiki || {};
|
|
479
595
|
existing.wiki.embedding = existing.wiki.embedding || {};
|
|
@@ -528,6 +644,413 @@ const handleEmbedding = async (args, ctx, pi) => {
|
|
|
528
644
|
}
|
|
529
645
|
}
|
|
530
646
|
};
|
|
647
|
+
// ── /wiki modal ──────────────────────────────────────────────────────────────
|
|
648
|
+
const MODAL_CONFIG_USAGE = [
|
|
649
|
+
"**/wiki modal config**",
|
|
650
|
+
"",
|
|
651
|
+
" /wiki modal config baseUrl <url> Set the Modal ASGI base URL",
|
|
652
|
+
" /wiki modal config workspace <name> Derive the Modal URL from workspace slug",
|
|
653
|
+
" /wiki modal config model <name> Set the canonical embedder (default embeddinggemma)",
|
|
654
|
+
" /wiki modal config dim <n> Set output dimension (omit for native)",
|
|
655
|
+
" /wiki modal config fallback ollama|none Set offline fallback provider",
|
|
656
|
+
" /wiki modal config sync auto on|off Toggle auto-sync",
|
|
657
|
+
" /wiki modal config sync interval <ms> Auto-sync interval",
|
|
658
|
+
" /wiki modal config pageSize <n> Sync page size",
|
|
659
|
+
" /wiki modal config coalesce debounce <ms> Coalescer debounce window",
|
|
660
|
+
" /wiki modal config coalesce batch <n> Coalescer max batch size",
|
|
661
|
+
" /wiki modal config token Show token guidance (use PVM_API_TOKEN env)",
|
|
662
|
+
"",
|
|
663
|
+
" (no args) Show current Modal config",
|
|
664
|
+
].join("\n");
|
|
665
|
+
/** Read the raw project config object (mutable). */
|
|
666
|
+
const readProjectConfig = (cfgPath) => {
|
|
667
|
+
try {
|
|
668
|
+
return JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
|
|
669
|
+
}
|
|
670
|
+
catch {
|
|
671
|
+
return {};
|
|
672
|
+
}
|
|
673
|
+
};
|
|
674
|
+
const writeProjectConfig = (cfgPath, obj) => {
|
|
675
|
+
fs.writeFileSync(cfgPath, `${JSON.stringify(obj, null, 2)}\n`, "utf-8");
|
|
676
|
+
};
|
|
677
|
+
const modalSection = (obj) => {
|
|
678
|
+
obj.wiki = obj.wiki || {};
|
|
679
|
+
const wiki = obj.wiki;
|
|
680
|
+
wiki.embedding = wiki.embedding || {};
|
|
681
|
+
const emb = wiki.embedding;
|
|
682
|
+
emb.modal = emb.modal || {};
|
|
683
|
+
return emb.modal;
|
|
684
|
+
};
|
|
685
|
+
const handleModalConfig = async (args, ctx) => {
|
|
686
|
+
const { project: cfgPath } = findConfig(ctx.cwd);
|
|
687
|
+
if (!cfgPath) {
|
|
688
|
+
ctx.ui.notify("No config found. Run /wiki init first.", "error");
|
|
689
|
+
return;
|
|
690
|
+
}
|
|
691
|
+
const cfg = loadConfig(ctx.cwd);
|
|
692
|
+
const parts = args.trim().split(/\s+/).filter(Boolean);
|
|
693
|
+
const key = parts[0]?.toLowerCase();
|
|
694
|
+
const modal = cfg.wiki.embedding.modal ?? {};
|
|
695
|
+
if (!key) {
|
|
696
|
+
const tokenSrc = resolveModalToken(cfg.wiki)
|
|
697
|
+
? "env or ~/.pi/agent/vault-mind.env ✅"
|
|
698
|
+
: modal.apiToken
|
|
699
|
+
? "config (set PVM_API_TOKEN env/dotenv to override)"
|
|
700
|
+
: "❌ none (set PVM_API_TOKEN env or ~/.pi/agent/vault-mind.env)";
|
|
701
|
+
const derivedUrl = resolveBaseUrl(cfg.wiki);
|
|
702
|
+
const lines = [
|
|
703
|
+
"**Modal Config:**",
|
|
704
|
+
"",
|
|
705
|
+
` workspace: ${resolveWorkspace(cfg.wiki) || "(not set)"}`,
|
|
706
|
+
` baseUrl: ${modal.baseUrl || (derivedUrl ? `${derivedUrl} (derived)` : "❌ not set")}`,
|
|
707
|
+
` model: ${modal.model || "(default embeddinggemma)"}`,
|
|
708
|
+
` dim: ${modal.dim ?? "(native)"}`,
|
|
709
|
+
` token: ${tokenSrc}`,
|
|
710
|
+
` fallback: ${modal.fallback?.enabled === false ? "disabled" : modal.fallback?.provider || "(none)"}`,
|
|
711
|
+
` sync: ${JSON.stringify(modal.sync ?? {})}`,
|
|
712
|
+
` coalesce: ${JSON.stringify(cfg.wiki.embedding.coalesce ?? {})}`,
|
|
713
|
+
];
|
|
714
|
+
ctx.ui.notify(lines.join("\n"), "info");
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
const obj = readProjectConfig(cfgPath);
|
|
718
|
+
const m = modalSection(obj);
|
|
719
|
+
const setNum = (target, k, v) => {
|
|
720
|
+
const n = Number.parseInt(v, 10);
|
|
721
|
+
if (!Number.isFinite(n)) {
|
|
722
|
+
ctx.ui.notify(`Invalid number for ${k}: ${v}`, "error");
|
|
723
|
+
return false;
|
|
724
|
+
}
|
|
725
|
+
target[k] = n;
|
|
726
|
+
return true;
|
|
727
|
+
};
|
|
728
|
+
switch (key) {
|
|
729
|
+
case "baseurl":
|
|
730
|
+
case "url": {
|
|
731
|
+
const url = parts[1];
|
|
732
|
+
if (!url) {
|
|
733
|
+
ctx.ui.notify("/wiki modal config baseUrl <url>", "error");
|
|
734
|
+
return;
|
|
735
|
+
}
|
|
736
|
+
m.baseUrl = url.replace(/\/$/, "");
|
|
737
|
+
writeProjectConfig(cfgPath, obj);
|
|
738
|
+
ctx.ui.notify(`✅ Modal baseUrl set to ${m.baseUrl}`, "info");
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
case "model": {
|
|
742
|
+
if (!parts[1]) {
|
|
743
|
+
ctx.ui.notify("/wiki modal config model <name>", "error");
|
|
744
|
+
return;
|
|
745
|
+
}
|
|
746
|
+
m.model = parts[1];
|
|
747
|
+
writeProjectConfig(cfgPath, obj);
|
|
748
|
+
ctx.ui.notify(`✅ Modal model set to ${parts[1]}`, "info");
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
751
|
+
case "dim": {
|
|
752
|
+
if (!setNum(m, "dim", parts[1] ?? ""))
|
|
753
|
+
return;
|
|
754
|
+
writeProjectConfig(cfgPath, obj);
|
|
755
|
+
ctx.ui.notify(`✅ Modal dim set to ${m.dim}`, "info");
|
|
756
|
+
return;
|
|
757
|
+
}
|
|
758
|
+
case "fallback": {
|
|
759
|
+
const v = parts[1]?.toLowerCase();
|
|
760
|
+
if (v === "none") {
|
|
761
|
+
m.fallback = { enabled: false };
|
|
762
|
+
}
|
|
763
|
+
else if (v === "ollama" || v === "transformers") {
|
|
764
|
+
m.fallback = { enabled: true, provider: v };
|
|
765
|
+
}
|
|
766
|
+
else {
|
|
767
|
+
ctx.ui.notify("/wiki modal config fallback <ollama|transformers|none>", "error");
|
|
768
|
+
return;
|
|
769
|
+
}
|
|
770
|
+
writeProjectConfig(cfgPath, obj);
|
|
771
|
+
ctx.ui.notify(`✅ Modal fallback set to ${v}`, "info");
|
|
772
|
+
return;
|
|
773
|
+
}
|
|
774
|
+
case "sync": {
|
|
775
|
+
const sub = parts[1]?.toLowerCase();
|
|
776
|
+
const val = parts[2]?.toLowerCase();
|
|
777
|
+
const sync = m.sync || {};
|
|
778
|
+
if (sub === "auto") {
|
|
779
|
+
sync.autoSync = val === "on" || val === "true";
|
|
780
|
+
}
|
|
781
|
+
else if (sub === "interval") {
|
|
782
|
+
if (!setNum(sync, "autoSyncIntervalMs", parts[2] ?? ""))
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
else if (sub === "pagesize") {
|
|
786
|
+
if (!setNum(sync, "pageSize", parts[2] ?? ""))
|
|
787
|
+
return;
|
|
788
|
+
}
|
|
789
|
+
else if (sub === "collections") {
|
|
790
|
+
sync.collections = parts.slice(2);
|
|
791
|
+
}
|
|
792
|
+
else {
|
|
793
|
+
ctx.ui.notify("/wiki modal config sync <auto on|off|interval <ms>|pageSize <n>|collections ...>", "error");
|
|
794
|
+
return;
|
|
795
|
+
}
|
|
796
|
+
m.sync = sync;
|
|
797
|
+
writeProjectConfig(cfgPath, obj);
|
|
798
|
+
ctx.ui.notify(`✅ Modal sync.${sub} updated`, "info");
|
|
799
|
+
return;
|
|
800
|
+
}
|
|
801
|
+
case "coalesce": {
|
|
802
|
+
const sub = parts[1]?.toLowerCase();
|
|
803
|
+
const wiki = obj.wiki || {};
|
|
804
|
+
const emb = wiki.embedding || {};
|
|
805
|
+
const co = emb.coalesce || {};
|
|
806
|
+
if (sub === "debounce") {
|
|
807
|
+
if (!setNum(co, "debounceMs", parts[2] ?? ""))
|
|
808
|
+
return;
|
|
809
|
+
}
|
|
810
|
+
else if (sub === "batch") {
|
|
811
|
+
if (!setNum(co, "maxBatchSize", parts[2] ?? ""))
|
|
812
|
+
return;
|
|
813
|
+
}
|
|
814
|
+
else if (sub === "concurrency") {
|
|
815
|
+
if (!setNum(co, "maxConcurrentFlushes", parts[2] ?? ""))
|
|
816
|
+
return;
|
|
817
|
+
}
|
|
818
|
+
else {
|
|
819
|
+
ctx.ui.notify("/wiki modal config coalesce <debounce|batch|concurrency> <n>", "error");
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
emb.coalesce = co;
|
|
823
|
+
writeProjectConfig(cfgPath, obj);
|
|
824
|
+
ctx.ui.notify(`✅ Modal coalesce.${sub} updated`, "info");
|
|
825
|
+
return;
|
|
826
|
+
}
|
|
827
|
+
case "token": {
|
|
828
|
+
ctx.ui.notify(`Token resolution: env ${MODAL_TOKEN_ENV} is preferred (never committed).\nSet it in your shell: export ${MODAL_TOKEN_ENV}=...\nConfig wiki.embedding.modal.apiToken is a fallback only.`, "info");
|
|
829
|
+
return;
|
|
830
|
+
}
|
|
831
|
+
default:
|
|
832
|
+
ctx.ui.notify(MODAL_CONFIG_USAGE, "info");
|
|
833
|
+
}
|
|
834
|
+
};
|
|
835
|
+
const handleModal = async (args, ctx, pi) => {
|
|
836
|
+
const cfg = loadConfig(ctx.cwd);
|
|
837
|
+
const parts = args.trim().split(/\s+/).filter(Boolean);
|
|
838
|
+
const sub = parts[0]?.toLowerCase() || "status";
|
|
839
|
+
const rest = parts.slice(1).join(" ");
|
|
840
|
+
switch (sub) {
|
|
841
|
+
case "status": {
|
|
842
|
+
const modal = cfg.wiki.embedding.modal;
|
|
843
|
+
const tokenSrc = resolveModalToken(cfg.wiki)
|
|
844
|
+
? "env or ~/.pi/agent/vault-mind.env ✅"
|
|
845
|
+
: modal?.apiToken
|
|
846
|
+
? "config"
|
|
847
|
+
: "❌ none";
|
|
848
|
+
const derivedUrl = resolveBaseUrl(cfg.wiki);
|
|
849
|
+
const lines = [
|
|
850
|
+
"**Modal Status**",
|
|
851
|
+
"",
|
|
852
|
+
`Configured: ${isModalConfigured(cfg.wiki) ? "✅" : "❌"}`,
|
|
853
|
+
` workspace: ${resolveWorkspace(cfg.wiki) || "(not set)"}`,
|
|
854
|
+
` baseUrl: ${modal?.baseUrl || (derivedUrl ? `${derivedUrl} (derived)` : "(not set)")}`,
|
|
855
|
+
` model: ${modal?.model || "(default embeddinggemma)"}`,
|
|
856
|
+
` dim: ${modal?.dim ?? "(native)"}`,
|
|
857
|
+
` token: ${tokenSrc}`,
|
|
858
|
+
];
|
|
859
|
+
const client = createModalClient(cfg.wiki);
|
|
860
|
+
if (client) {
|
|
861
|
+
try {
|
|
862
|
+
const health = await client.health();
|
|
863
|
+
lines.push("", `Health: ✅ ok, default_model=${health.default_model}`);
|
|
864
|
+
}
|
|
865
|
+
catch (err) {
|
|
866
|
+
lines.push("", `Health: ❌ ${err.message}`);
|
|
867
|
+
}
|
|
868
|
+
try {
|
|
869
|
+
const cols = await client.syncCollections();
|
|
870
|
+
lines.push("", `Remote collections (${cols.length}):`);
|
|
871
|
+
for (const c of cols)
|
|
872
|
+
lines.push(` • ${c.collection} / ${c.model} / ${c.dim} — ${c.rows} rows (${c.table})`);
|
|
873
|
+
}
|
|
874
|
+
catch (err) {
|
|
875
|
+
lines.push("", `Remote collections: ❌ ${err.message}`);
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
ctx.ui.notify(lines.join("\n"), "info");
|
|
879
|
+
return;
|
|
880
|
+
}
|
|
881
|
+
case "config":
|
|
882
|
+
await handleModalConfig(rest, ctx);
|
|
883
|
+
return;
|
|
884
|
+
case "sync": {
|
|
885
|
+
if (!isModalConfigured(cfg.wiki)) {
|
|
886
|
+
ctx.ui.notify(`Modal not configured. Set baseUrl + ${MODAL_TOKEN_ENV} first.`, "error");
|
|
887
|
+
return;
|
|
888
|
+
}
|
|
889
|
+
const syncTokens = rest.split(/\s+/).filter(Boolean);
|
|
890
|
+
const full = syncTokens.includes("--full");
|
|
891
|
+
const colFlagIdx = syncTokens.indexOf("--collection");
|
|
892
|
+
const oneCollection = colFlagIdx >= 0 ? syncTokens[colFlagIdx + 1] : undefined;
|
|
893
|
+
ctx.ui.notify(`Syncing ${oneCollection ? `"${oneCollection}"` : "all collections"}${full ? " (full)" : ""}...`, "info");
|
|
894
|
+
try {
|
|
895
|
+
const results = oneCollection
|
|
896
|
+
? [await syncCollection(cfg, oneCollection, { full })]
|
|
897
|
+
: await syncAll(cfg, undefined, { full });
|
|
898
|
+
const lines = ["**Sync Report:**", ""];
|
|
899
|
+
for (const r of results)
|
|
900
|
+
lines.push(` ${r.rows > 0 ? "✅" : "•"} ${r.collection} / ${r.model} / ${r.dim}: ${r.rows} rows, watermark=${r.watermark}${r.full ? " (full)" : ""}`);
|
|
901
|
+
lines.push("", "Re-running with no new rows is a no-op.");
|
|
902
|
+
ctx.ui.notify(lines.join("\n"), "info");
|
|
903
|
+
}
|
|
904
|
+
catch (err) {
|
|
905
|
+
ctx.ui.notify(`Sync failed: ${err.message}`, "error");
|
|
906
|
+
}
|
|
907
|
+
return;
|
|
908
|
+
}
|
|
909
|
+
case "jobs": {
|
|
910
|
+
const jobId = parts[1];
|
|
911
|
+
if (!jobId) {
|
|
912
|
+
ctx.ui.notify("/wiki modal jobs <job_id> — poll a Modal bulk job.\n(Server-side job listing is pending upstream.)", "info");
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
const client = createModalClient(cfg.wiki);
|
|
916
|
+
if (!client) {
|
|
917
|
+
ctx.ui.notify("Modal not configured.", "error");
|
|
918
|
+
return;
|
|
919
|
+
}
|
|
920
|
+
try {
|
|
921
|
+
const status = await client.jobStatus(jobId);
|
|
922
|
+
ctx.ui.notify([
|
|
923
|
+
`Job ${jobId}:`,
|
|
924
|
+
` status: ${status.status}`,
|
|
925
|
+
` collection: ${status.collection}`,
|
|
926
|
+
` model: ${status.model} / dim ${status.dim}`,
|
|
927
|
+
` processed: ${status.processed}/${status.total}`,
|
|
928
|
+
...(status.error ? [` error: ${status.error}`] : []),
|
|
929
|
+
].join("\n"), "info");
|
|
930
|
+
}
|
|
931
|
+
catch (err) {
|
|
932
|
+
ctx.ui.notify(`Job poll failed: ${err.message}`, "error");
|
|
933
|
+
}
|
|
934
|
+
return;
|
|
935
|
+
}
|
|
936
|
+
case "auto": {
|
|
937
|
+
const cfgPath = findConfig(ctx.cwd).project;
|
|
938
|
+
if (!cfgPath) {
|
|
939
|
+
ctx.ui.notify("No project config found. Run /wiki init first.", "error");
|
|
940
|
+
return;
|
|
941
|
+
}
|
|
942
|
+
const workspace = resolveWorkspace(cfg.wiki);
|
|
943
|
+
if (!workspace && !cfg.wiki.embedding.modal?.baseUrl) {
|
|
944
|
+
ctx.ui.notify("Modal workspace or baseUrl not configured. Run:\n /wiki modal config workspace <name>", "error");
|
|
945
|
+
}
|
|
946
|
+
const token = resolveModalToken(cfg.wiki);
|
|
947
|
+
if (!token) {
|
|
948
|
+
ctx.ui.notify("Modal token not found. Set PVM_API_TOKEN env or write it to ~/.pi/agent/vault-mind.env", "error");
|
|
949
|
+
return;
|
|
950
|
+
}
|
|
951
|
+
ctx.ui.notify("🔄 Auto-configuring Modal...", "info");
|
|
952
|
+
const client = createModalClient(cfg.wiki);
|
|
953
|
+
if (!client) {
|
|
954
|
+
ctx.ui.notify("Could not create Modal client (missing workspace/baseUrl or token).", "error");
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
957
|
+
let health;
|
|
958
|
+
let models;
|
|
959
|
+
try {
|
|
960
|
+
health = await client.health();
|
|
961
|
+
ctx.ui.notify(`✅ Modal reachable (default model: ${health.default_model})`, "info");
|
|
962
|
+
}
|
|
963
|
+
catch (err) {
|
|
964
|
+
ctx.ui.notify(`❌ Modal health check failed: ${err.message}`, "error");
|
|
965
|
+
return;
|
|
966
|
+
}
|
|
967
|
+
try {
|
|
968
|
+
models = await client.models();
|
|
969
|
+
}
|
|
970
|
+
catch (err) {
|
|
971
|
+
ctx.ui.notify(`❌ Could not fetch model registry: ${err.message}`, "error");
|
|
972
|
+
return;
|
|
973
|
+
}
|
|
974
|
+
const defaultModel = models.default || health.default_model || "embeddinggemma";
|
|
975
|
+
const obj = readProjectConfig(cfgPath);
|
|
976
|
+
const wiki = obj.wiki || {};
|
|
977
|
+
const emb = wiki.embedding || {};
|
|
978
|
+
emb.provider = "modal";
|
|
979
|
+
const modalSec = emb.modal || {};
|
|
980
|
+
if (workspace)
|
|
981
|
+
modalSec.workspace = workspace;
|
|
982
|
+
modalSec.model = defaultModel;
|
|
983
|
+
if (models.default_dim != null)
|
|
984
|
+
modalSec.dim = models.default_dim;
|
|
985
|
+
modalSec.fallback = modalSec.fallback || {
|
|
986
|
+
enabled: true,
|
|
987
|
+
provider: "ollama",
|
|
988
|
+
};
|
|
989
|
+
modalSec.sync = modalSec.sync || {
|
|
990
|
+
autoSync: false,
|
|
991
|
+
autoSyncIntervalMs: 300000,
|
|
992
|
+
};
|
|
993
|
+
emb.modal = modalSec;
|
|
994
|
+
wiki.embedding = emb;
|
|
995
|
+
obj.wiki = wiki;
|
|
996
|
+
writeProjectConfig(cfgPath, obj);
|
|
997
|
+
const lines = [
|
|
998
|
+
"✅ Modal auto-configured:",
|
|
999
|
+
` baseUrl: ${resolveBaseUrl(cfg.wiki)}`,
|
|
1000
|
+
` model: ${defaultModel}`,
|
|
1001
|
+
` dim: ${models.default_dim ?? "native"}`,
|
|
1002
|
+
"",
|
|
1003
|
+
"Run /wiki modal status to verify.",
|
|
1004
|
+
];
|
|
1005
|
+
ctx.ui.notify(lines.join("\n"), "info");
|
|
1006
|
+
return;
|
|
1007
|
+
}
|
|
1008
|
+
case "migrate": {
|
|
1009
|
+
const newModel = parts[1];
|
|
1010
|
+
if (!newModel) {
|
|
1011
|
+
ctx.ui.notify("/wiki modal migrate <newModel> [dim] — change the canonical model + re-embed (remote).\nOld namespace is left intact until verified.", "info");
|
|
1012
|
+
return;
|
|
1013
|
+
}
|
|
1014
|
+
const newDim = parts[2] ? Number.parseInt(parts[2], 10) : undefined;
|
|
1015
|
+
if (parts[2] && !Number.isFinite(newDim)) {
|
|
1016
|
+
ctx.ui.notify(`Invalid dim: ${parts[2]}`, "error");
|
|
1017
|
+
return;
|
|
1018
|
+
}
|
|
1019
|
+
const { project: cfgPath } = findConfig(ctx.cwd);
|
|
1020
|
+
if (!cfgPath) {
|
|
1021
|
+
ctx.ui.notify("No config found.", "error");
|
|
1022
|
+
return;
|
|
1023
|
+
}
|
|
1024
|
+
const obj = readProjectConfig(cfgPath);
|
|
1025
|
+
const m = modalSection(obj);
|
|
1026
|
+
const oldModel = m.model || "embeddinggemma";
|
|
1027
|
+
const oldDim = m.dim;
|
|
1028
|
+
m.model = newModel;
|
|
1029
|
+
if (newDim != null)
|
|
1030
|
+
m.dim = newDim;
|
|
1031
|
+
writeProjectConfig(cfgPath, obj);
|
|
1032
|
+
ctx.ui.notify(`Canonical model → ${newModel}${newDim ? `@${newDim}` : ""} (was ${oldModel}${oldDim ? `@${oldDim}` : ""}). Old namespace kept. Starting remote re-embed...`, "info");
|
|
1033
|
+
const fresh = loadConfig(ctx.cwd);
|
|
1034
|
+
try {
|
|
1035
|
+
const results = await reindexRemote(fresh, Object.keys(fresh.collections));
|
|
1036
|
+
const lines = [`**Migration → ${newModel}:**`, ""];
|
|
1037
|
+
for (const r of results)
|
|
1038
|
+
if (r.error)
|
|
1039
|
+
lines.push(` ❌ ${r.collection}: ${r.error}`);
|
|
1040
|
+
else
|
|
1041
|
+
lines.push(` ✅ ${r.collection}: synced ${r.sync?.rows ?? 0} rows`);
|
|
1042
|
+
lines.push("", `Old col_*__${oldModel}__* tables are untouched. Verify the new space, then drop the old table(s) when ready.`);
|
|
1043
|
+
ctx.ui.notify(lines.join("\n"), "info");
|
|
1044
|
+
}
|
|
1045
|
+
catch (err) {
|
|
1046
|
+
ctx.ui.notify(`Migration re-embed failed: ${err.message}`, "error");
|
|
1047
|
+
}
|
|
1048
|
+
return;
|
|
1049
|
+
}
|
|
1050
|
+
default:
|
|
1051
|
+
ctx.ui.notify("Unknown /wiki modal subcommand. Try: status, config, sync, jobs, migrate", "error");
|
|
1052
|
+
}
|
|
1053
|
+
};
|
|
531
1054
|
// ── /wiki context ────────────────────────────────────────────────────────────
|
|
532
1055
|
const handleContext = async (args, ctx, pi) => {
|
|
533
1056
|
const parts = args.trim().split(/\s+/g);
|
|
@@ -687,7 +1210,7 @@ const handleWatcher = async (args, ctx, pi) => {
|
|
|
687
1210
|
};
|
|
688
1211
|
// ── Setup ────────────────────────────────────────────────────────────────────
|
|
689
1212
|
const handleSetup = async (args, ctx) => {
|
|
690
|
-
// Parse optional CLI-style args: --vault <path> --provider <name> --model <name>
|
|
1213
|
+
// Parse optional CLI-style args: --vault <path> --provider <name> --model <name> --workspace <name>
|
|
691
1214
|
const cliArgs = {};
|
|
692
1215
|
const parts = args.trim().split(/\s+--/);
|
|
693
1216
|
for (const part of parts) {
|
|
@@ -701,8 +1224,10 @@ const handleSetup = async (args, ctx) => {
|
|
|
701
1224
|
cliArgs.provider = trimmed.slice(9).trim();
|
|
702
1225
|
else if (trimmed.startsWith("model "))
|
|
703
1226
|
cliArgs.model = trimmed.slice(6).trim();
|
|
1227
|
+
else if (trimmed.startsWith("workspace "))
|
|
1228
|
+
cliArgs.workspace = trimmed.slice(10).trim();
|
|
704
1229
|
}
|
|
705
|
-
const hasCliArgs = cliArgs.vault || cliArgs.provider || cliArgs.model;
|
|
1230
|
+
const hasCliArgs = cliArgs.vault || cliArgs.provider || cliArgs.model || cliArgs.workspace;
|
|
706
1231
|
await setupWizard(ctx, hasCliArgs ? cliArgs : undefined);
|
|
707
1232
|
};
|
|
708
1233
|
// ── Main /wiki command ───────────────────────────────────────────────────────
|
|
@@ -723,6 +1248,7 @@ export const registerCommands = (pi) => {
|
|
|
723
1248
|
"injector",
|
|
724
1249
|
"context",
|
|
725
1250
|
"embedding",
|
|
1251
|
+
"modal",
|
|
726
1252
|
"watcher",
|
|
727
1253
|
"setup",
|
|
728
1254
|
];
|
|
@@ -756,10 +1282,15 @@ export const registerCommands = (pi) => {
|
|
|
756
1282
|
.map((c) => ({ label: c, value: c, description: `injector ${c}` }));
|
|
757
1283
|
}
|
|
758
1284
|
if (subcommand === "reindex") {
|
|
759
|
-
return ["--all", "--reembed"]
|
|
1285
|
+
return ["--all", "--reembed", "--remote"]
|
|
760
1286
|
.filter((c) => c.startsWith(prefix))
|
|
761
1287
|
.map((c) => ({ label: c, value: c, description: `reindex ${c}` }));
|
|
762
1288
|
}
|
|
1289
|
+
if (subcommand === "modal") {
|
|
1290
|
+
return ["status", "config", "sync", "jobs", "migrate"]
|
|
1291
|
+
.filter((c) => c.startsWith(prefix))
|
|
1292
|
+
.map((c) => ({ label: c, value: c, description: `modal ${c}` }));
|
|
1293
|
+
}
|
|
763
1294
|
if (subcommand === "watcher") {
|
|
764
1295
|
return ["start", "stop", "status"]
|
|
765
1296
|
.filter((c) => c.startsWith(prefix))
|
|
@@ -798,6 +1329,8 @@ export const registerCommands = (pi) => {
|
|
|
798
1329
|
return handleContext(rest, ctx, pi);
|
|
799
1330
|
case "embedding":
|
|
800
1331
|
return handleEmbedding(rest, ctx, pi);
|
|
1332
|
+
case "modal":
|
|
1333
|
+
return handleModal(rest, ctx, pi);
|
|
801
1334
|
case "server":
|
|
802
1335
|
return handleServer(ctx);
|
|
803
1336
|
case "watcher":
|