pi-vault-mind 0.7.3 → 0.7.5

package/README.md

@@ -182,7 +182,15 @@ The Broadcaster agent can generate podcasts, study guides, and slide decks from
 - **Embedding Provider** — choose one:
   - `@xenova/transformers` — built-in, no external deps (uses all-MiniLM-L6-v2, offline-capable)
   - `ollama` — requires Ollama running locally with `embeddinggemma` (higher quality)
-  - `modal` — optional: offload embedding + bulk re-index to a cloud GPU service and sync vectors down for offline search (see [docs/MODAL_EMBEDDING.md](docs/MODAL_EMBEDDING.md)). Default behavior is unchanged until you opt in.
+  - `modal` — optional, **bring-your-own deploy**: offload embedding + bulk re-index to a cloud GPU service and sync vectors down for offline search (see [docs/MODAL_EMBEDDING.md](docs/MODAL_EMBEDDING.md)). Default behavior is unchanged until you opt in.
+
+> **On the Modal provider:** the local providers above work for everyone with
+> zero infrastructure — that's the default. `modal` is an **optional, self-hosted
+> tier**: you deploy your own copy of the service (`uvx modal deploy modal/app.py`
+> from a clone of this repo) to your own Modal account and point the extension at
+> *your* URL + token. There is no shared/hosted endpoint — a deployment's URL and
+> bearer token are private to whoever owns it. (A standalone PyPI package for the
+> server may come later.)
 
 ### 1. Install with pi
 
@@ -375,10 +383,8 @@ Edit `pi-vault-mind.config.json` to match your domain:
 |---|---|
 | [docs/AGENTS.md](docs/AGENTS.md) | Agent Roster and Multi-Agent Architecture ("Fork & Review" model) |
 | [docs/EXTENSION_WIRING.md](docs/EXTENSION_WIRING.md) | Extension dependencies, runtime wiring, auto-install patterns |
-| [docs/DISPATCHER_SPEC.md](docs/DISPATCHER_SPEC.md) | Technical spec for the passive file-watcher and subagent routing |
-| [docs/PASSIVE_INTERACTION_MODEL.md](docs/PASSIVE_INTERACTION_MODEL.md) | "Fork & Dispatch" model — why we don't pollute the main chat session |
-| [docs/PASSIVE_INGESTION_WORKFLOW.md](docs/PASSIVE_INGESTION_WORKFLOW.md) | The "Drop & Forget" document ingestion pipeline |
-| [docs/NOTEBOOKLM_PIPELINE.md](docs/NOTEBOOKLM_PIPELINE.md) | NotebookLM automated podcast and study guide generation |
+| [docs/DISPATCHER_SPEC.md](docs/DISPATCHER_SPEC.md) | Technical spec for the passive file-watcher and subagent routing — incl. the "Fork & Dispatch" rationale and thread resume |
+| [docs/AGENTS.md](docs/AGENTS.md) | Agent roster + the Miner's "Drop & Forget" ingestion pipeline |
 | [docs/OBSIDIAN_SETUP.md](docs/OBSIDIAN_SETUP.md) | Recommended Obsidian vault structure, plugins, and CLI |
 
 ### Modal embedding service (local integration done)

package/dist/src/commands.js

@@ -1,3 +1,4 @@
+import { execFileSync } from "node:child_process";
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { Container, SelectList, Text } from "@earendil-works/pi-tui";
@@ -825,7 +826,7 @@ const handleModalConfig = async (args, ctx) => {
             return;
         }
         case "token": {
-            ctx.ui.notify(`Token resolution: env ${MODAL_TOKEN_ENV} is preferred (never committed).\nSet it in your shell: export ${MODAL_TOKEN_ENV}=...\nConfig wiki.embedding.modal.apiToken is a fallback only.`, "info");
+            await handleModalToken(ctx, parts[1]);
             return;
         }
         default:
@@ -1230,6 +1231,55 @@ const handleSetup = async (args, ctx) => {
     const hasCliArgs = cliArgs.vault || cliArgs.provider || cliArgs.model || cliArgs.workspace;
     await setupWizard(ctx, hasCliArgs ? cliArgs : undefined);
 };
+// ── /wiki modal token ────────────────────────────────────────────────────────
+/** Dotenv path where the Modal token is persisted. */
+const MODAL_TOKEN_ENV_PATH = path.join(process.env.HOME || process.env.USERPROFILE || "", ".pi", "agent", "vault-mind.env");
+const writeTokenEnv = (token) => {
+    const dir = path.dirname(MODAL_TOKEN_ENV_PATH);
+    if (!fs.existsSync(dir))
+        fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(MODAL_TOKEN_ENV_PATH, `PVM_API_TOKEN="${token}"\n`, { mode: 0o600 });
+};
+const fetchTokenFrom1Password = () => {
+    try {
+        return execFileSync("op", ["item", "get", "pi-vault-mind-auth", "--reveal", "--field", "password"], {
+            encoding: "utf-8",
+            stdio: ["ignore", "pipe", "pipe"],
+        }).trim();
+    }
+    catch {
+        return null;
+    }
+};
+const handleModalToken = async (ctx, cliToken) => {
+    if (cliToken) {
+        writeTokenEnv(cliToken);
+        ctx.ui.notify(`✅ Wrote token to ${MODAL_TOKEN_ENV_PATH}`, "info");
+        return;
+    }
+    const envToken = process.env[MODAL_TOKEN_ENV];
+    if (envToken) {
+        ctx.ui.notify("PVM_API_TOKEN is already exported in env. Skipping write so we don't clobber it.", "warning");
+        return;
+    }
+    const tokenFrom1Pass = fetchTokenFrom1Password();
+    if (tokenFrom1Pass) {
+        writeTokenEnv(tokenFrom1Pass);
+        ctx.ui.notify(`✅ Fetched token from 1Password item "pi-vault-mind-auth" and wrote it to ${MODAL_TOKEN_ENV_PATH}`, "info");
+        return;
+    }
+    if (!ctx.hasUI) {
+        ctx.ui.notify("No 1Password token found and no TUI available. Use:\n  /wiki modal token <token>", "error");
+        return;
+    }
+    const token = await ctx.ui.input("Modal API token:", "paste bearer token here (will be written to ~/.pi/agent/vault-mind.env)");
+    if (!token) {
+        ctx.ui.notify("Token setup cancelled.", "warning");
+        return;
+    }
+    writeTokenEnv(token);
+    ctx.ui.notify(`✅ Wrote token to ${MODAL_TOKEN_ENV_PATH}`, "info");
+};
 // ── Main /wiki command ───────────────────────────────────────────────────────
 export const registerCommands = (pi) => {
     pi.registerCommand("wiki", {
@@ -1287,7 +1337,7 @@ export const registerCommands = (pi) => {
                         .map((c) => ({ label: c, value: c, description: `reindex ${c}` }));
                 }
                 if (subcommand === "modal") {
-                    return ["status", "config", "sync", "jobs", "migrate"]
+                    return ["status", "config", "auto", "token", "sync", "jobs", "migrate"]
                         .filter((c) => c.startsWith(prefix))
                         .map((c) => ({ label: c, value: c, description: `modal ${c}` }));
                 }
@@ -1297,7 +1347,7 @@ export const registerCommands = (pi) => {
                         .map((c) => ({ label: c, value: c, description: `watcher ${c}` }));
                 }
                 if (subcommand === "setup") {
-                    return ["--vault", "--provider", "--model"]
+                    return ["--vault", "--provider", "--model", "--workspace"]
                         .filter((c) => c.startsWith(prefix))
                         .map((c) => ({ label: c, value: c, description: `setup ${c}` }));
                 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-vault-mind",
-  "version": "0.7.3",
+  "version": "0.7.5",
   "description": "Passive Obsidian vault extension for pi. Watches @agent markers, dispatches forked subagents (Miner, Broadcaster, Heavy-Lifter), stores in LanceDB with vector + FTS + graph. Multi-agent 'Drop & Forget' workflow for the pi agent ecosystem.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -8,6 +8,7 @@
   "files": [
     "dist",
     "skills",
+    "scripts",
     "CHANGELOG.md",
     "README.md",
     "LICENSE"

package/scripts/fetch-modal-token.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Fetch the Modal API token from 1Password and export or persist it.
+# Usage:
+#   ./scripts/fetch-modal-token.sh --export    # prints an export line
+#   ./scripts/fetch-modal-token.sh --write     # writes to ~/.pi/agent/vault-mind.env (default)
+#
+# The 1Password item must be titled "pi-vault-mind-auth" with the token in the
+# password field.
+
+set -euo pipefail
+
+OP_ITEM="pi-vault-mind-auth"
+OP_FIELD="password"
+ENV_DIR="${HOME}/.pi/agent"
+ENV_FILE="${ENV_DIR}/vault-mind.env"
+
+mode="write"
+if [ "${1:-}" = "--export" ]; then
+	mode="export"
+elif [ "${1:-}" = "--write" ]; then
+	mode="write"
+elif [ -n "${1:-}" ]; then
+	echo "Unknown option: $1" >&2
+	echo "Usage: $0 [--export | --write]" >&2
+	exit 1
+fi
+
+if ! command -v op >/dev/null 2>&1; then
+	echo "❌ 1Password CLI (op) not found. Install: brew install 1password-cli" >&2
+	exit 1
+fi
+
+# Check if already exported to avoid clobbering a live session.
+if [ -n "${PVM_API_TOKEN:-}" ]; then
+	echo "⚠️ PVM_API_TOKEN is already exported; skipping 1Password fetch." >&2
+	if [ "$mode" = "export" ]; then
+		echo "export PVM_API_TOKEN=\"${PVM_API_TOKEN}\""
+	else
+		echo "Token already available in environment."
+	fi
+	exit 0
+fi
+
+# Try to read without explicit account; op will use the default signed-in account.
+TOKEN=$(op item get "$OP_ITEM" --reveal --field "$OP_FIELD" 2>/dev/null) || {
+	echo "❌ Could not read 1Password item '$OP_ITEM' field '$OP_FIELD'." >&2
+	echo "   Make sure you are signed in (op signin) and the item exists." >&2
+	exit 1
+}
+
+if [ -z "$TOKEN" ]; then
+	echo "❌ Token is empty." >&2
+	exit 1
+fi
+
+if [ "$mode" = "export" ]; then
+	echo "export PVM_API_TOKEN=\"${TOKEN}\""
+else
+	mkdir -p "$ENV_DIR"
+	printf 'PVM_API_TOKEN="%s"\n' "$TOKEN" > "$ENV_FILE"
+	chmod 600 "$ENV_FILE"
+	echo "✅ Wrote PVM_API_TOKEN to ${ENV_FILE}"
+fi

package/scripts/modal-e2e-smoke.mjs

@@ -0,0 +1,171 @@
+#!/usr/bin/env node
+/**
+ * Live E2E smoke for the Modal embedding service — drives the *shipped* client.
+ *
+ * This is the TypeScript-client counterpart to `modal/client_example.py`: it
+ * imports the compiled `dist/src/modal-client.js` (the exact class the extension
+ * ships) and exercises the full HTTP contract against a real deployment, so a
+ * pass proves the production client ↔ live server round-trip — not a parallel
+ * reimplementation.
+ *
+ * It is HTTP-only (no LanceDB), so it runs anywhere Node can reach the deploy.
+ * The local-store half of sync (export → `.lancedb`) is exercised by the
+ * `/wiki modal sync` walkthrough in docs/E2E_MANUAL_TEST.md.
+ *
+ * Usage:
+ *   pnpm build                       # produces dist/src/modal-client.js
+ *   export PVM_MODAL_URL="https://<workspace>--pi-vault-mind-embed-embeddingservice-fastapi-app.modal.run"
+ *   export PVM_API_TOKEN="<the pi-vault-mind-auth secret>"
+ *   pnpm e2e:modal                   # or: node scripts/modal-e2e-smoke.mjs
+ *
+ * Optional env:
+ *   PVM_MODEL       embedder key (default: embeddinggemma)
+ *   PVM_COLLECTION  collection for the bulk-job + export (default: unique per run)
+ *
+ * Exit code 0 = all checks passed, 1 = a check failed or env is missing.
+ */
+
+import { ModalEmbeddingClient } from "../dist/src/modal-client.js";
+
+const URL = process.env.PVM_MODAL_URL?.replace(/\/$/, "");
+const TOKEN = process.env.PVM_API_TOKEN;
+const MODEL = process.env.PVM_MODEL || "embeddinggemma";
+// Unique per run by default: the server-side LanceDB is persistent on the
+// Volume, so a shared/static collection could carry leftover or concurrent
+// rows and break the exact row-count + paging assertions. Override with
+// PVM_COLLECTION to target a known collection.
+const COLLECTION =
+	process.env.PVM_COLLECTION || `e2e-smoke-${Math.random().toString(36).slice(2, 9)}`;
+
+if (!URL || !TOKEN) {
+	console.error(
+		"Missing env. Set PVM_MODAL_URL and PVM_API_TOKEN, then re-run.\n" +
+			'  export PVM_MODAL_URL="https://<workspace>--pi-vault-mind-embed-embeddingservice-fastapi-app.modal.run"\n' +
+			'  export PVM_API_TOKEN="<pi-vault-mind-auth secret>"'
+	);
+	process.exit(1);
+}
+
+const client = new ModalEmbeddingClient({ baseUrl: URL, apiToken: TOKEN });
+
+let failures = 0;
+const pass = (msg) => console.log(`  ✓ ${msg}`);
+const fail = (msg) => {
+	failures++;
+	console.log(`  ✗ ${msg}`);
+};
+const step = (msg) => console.log(`\n${msg}`);
+
+const eq = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+
+async function main() {
+	console.log(
+		`Modal live E2E smoke\n  url:        ${URL}\n  model:      ${MODEL}\n  collection: ${COLLECTION}`
+	);
+
+	// 1. health
+	step("1. health");
+	const health = await client.health();
+	health?.ok
+		? pass(`ok, default_model=${health.default_model}`)
+		: fail(`unexpected: ${JSON.stringify(health)}`);
+
+	// 2. models registry — resolve the native dim up front
+	step("2. models");
+	const reg = await client.models();
+	const info = reg.models.find((m) => m.key === MODEL);
+	if (!info) {
+		fail(`model "${MODEL}" not in registry (have: ${reg.models.map((m) => m.key).join(", ")})`);
+	} else {
+		pass(`${MODEL}: backend=${info.backend} native_dim=${info.native_dim} enabled=${info.enabled}`);
+	}
+	const expectedDim = info?.native_dim;
+
+	// 3. on-demand embed (query)
+	step("3. embed (task=query)");
+	const emb = await client.embed(["how long do tokens last?"], { model: MODEL, task: "query" });
+	const dimOk =
+		emb.dim === emb.vectors[0]?.length && (expectedDim == null || emb.dim === expectedDim);
+	dimOk
+		? pass(`model=${emb.model} dim=${emb.dim} (matches native + vector length)`)
+		: fail(`dim mismatch: dim=${emb.dim} vec.len=${emb.vectors[0]?.length} native=${expectedDim}`);
+
+	// 4. bulk job → wait → status
+	step("4. bulk job submit + wait");
+	const records = [
+		{ id: "e2e-1", text: "JWT tokens expire after one hour.", metadata: { tag: "auth" } },
+		{ id: "e2e-2", text: "Refresh tokens live for 30 days.", metadata: { tag: "auth" } },
+		{ id: "e2e-3", text: "Sessions are revoked on password change.", metadata: { tag: "auth" } },
+	];
+	const submit = await client.submitJob(COLLECTION, records, { model: MODEL });
+	pass(`submitted job_id=${submit.job_id} total=${submit.total}`);
+	const final = await client.waitForJob(submit.job_id);
+	final.status === "done" && final.processed === records.length
+		? pass(`job done ${final.processed}/${final.total} (model=${final.model} dim=${final.dim})`)
+		: fail(
+				`job ended status=${final.status} ${final.processed}/${final.total} err=${final.error ?? ""}`
+			);
+
+	// 5. listJobs — the job we just ran should be present
+	step("5. listJobs");
+	const list = await client.listJobs(10);
+	list.jobs.some((j) => j.collection === COLLECTION)
+		? pass(`listed ${list.count} job(s); collection present`)
+		: fail(`our collection "${COLLECTION}" not in job list`);
+
+	// 6. sync collections — the namespace should now exist
+	step("6. sync/collections");
+	const cols = await client.syncCollections();
+	const ns = cols.find((c) => c.collection === COLLECTION && c.model === final.model);
+	ns
+		? pass(`namespace ${ns.table} rows=${ns.rows} dim=${ns.dim}`)
+		: fail(`namespace for ${COLLECTION}/${final.model} missing`);
+
+	// 7. export drain — every row carries a vector of the right dim, watermark advances
+	step("7. sync/export (drain)");
+	const seen = new Map();
+	let pages = 0;
+	const finalWatermark = await client.exportAll(
+		COLLECTION,
+		(rows) => {
+			pages++;
+			for (const r of rows) seen.set(r.id, r);
+		},
+		{ model: final.model, dim: final.dim, limit: 2 }
+	);
+	const all = [...seen.values()];
+	const everyVector =
+		all.length > 0 && all.every((r) => Array.isArray(r.vector) && r.vector.length === final.dim);
+	const gotAll = all.length === records.length;
+	const idsOk = eq([...seen.keys()].sort(), records.map((r) => r.id).sort());
+	gotAll && idsOk
+		? pass(
+				`drained ${all.length} row(s) over ${pages} page(s); all ids present; watermark=${finalWatermark}`
+			)
+		: fail(
+				`expected ${records.length} ids ${JSON.stringify(records.map((r) => r.id))}, got ${JSON.stringify([...seen.keys()])}`
+			);
+	everyVector
+		? pass(`every row carries a ${final.dim}-dim vector`)
+		: fail("some rows missing a vector or wrong dim");
+
+	// 8. incremental re-export from the final watermark → nothing new
+	step("8. incremental re-export (idempotent)");
+	const inc = await client.exportSince(COLLECTION, {
+		model: final.model,
+		dim: final.dim,
+		since: finalWatermark,
+	});
+	inc.rows.length === 0 && inc.done
+		? pass("re-export from watermark returns 0 rows (idempotent)")
+		: fail(`expected 0 new rows, got ${inc.rows.length} (done=${inc.done})`);
+
+	// summary
+	console.log(`\n${failures === 0 ? "✅ ALL CHECKS PASSED" : `❌ ${failures} CHECK(S) FAILED`}`);
+	process.exit(failures === 0 ? 0 : 1);
+}
+
+main().catch((err) => {
+	console.error(`\n❌ smoke aborted: ${err?.message ?? err}`);
+	process.exit(1);
+});