pi-vault-mind 0.7.1 → 0.7.2

package/dist/src/modal-client.js

@@ -0,0 +1,174 @@
+/**
+ * Client for the pi-vault-mind Modal embedding service.
+ *
+ * This is the local (extension) side of the Modal app under `modal/`. It mirrors
+ * the HTTP contract documented in `docs/MODAL_EMBEDDING.md`:
+ *   - on-demand embedding         → POST /embed
+ *   - bulk background jobs        → POST /jobs, GET /jobs, GET /jobs/{id},
+ *                                  POST /jobs/{id}/cancel
+ *   - incremental vector sync     → GET /sync/collections, GET /sync/export
+ *                                  (format=json|arrow)
+ *   - model registry + stats      → GET /models, GET /stats
+ *
+ * This client is the typed mirror of the server contract. The server (Agent A)
+ * owns it; additive changes here are mirrored in the server's `modal/web.py`.
+ * The local wiring lives in `src/lance.ts` (provider), `src/sync.ts`, and
+ * `/wiki modal` commands (see docs/MODAL_EMBEDDING.md "Local integration").
+ */
+export class ModalEmbeddingClient {
+    baseUrl;
+    apiToken;
+    timeoutMs;
+    constructor(cfg) {
+        this.baseUrl = cfg.baseUrl.replace(/\/$/, "");
+        this.apiToken = cfg.apiToken;
+        this.timeoutMs = cfg.timeoutMs ?? 120_000;
+    }
+    async request(method, path, body) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this.timeoutMs);
+        try {
+            const resp = await fetch(`${this.baseUrl}${path}`, {
+                method,
+                headers: {
+                    Authorization: `Bearer ${this.apiToken}`,
+                    ...(body !== undefined ? { "Content-Type": "application/json" } : {}),
+                },
+                body: body !== undefined ? JSON.stringify(body) : undefined,
+                signal: controller.signal,
+            });
+            if (!resp.ok) {
+                const detail = await resp.text().catch(() => "");
+                throw new Error(`Modal ${method} ${path} failed: ${resp.status} ${detail}`);
+            }
+            return (await resp.json());
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    /** Liveness check; also returns the server's default model. */
+    health() {
+        return this.request("GET", "/health");
+    }
+    /** Registry of available embedders (public; no auth). Use native_dim to
+     * resolve a model's output dim up-front instead of waiting for the first
+     * /embed response. (Additive — Agent B request #2.) */
+    models() {
+        return this.request("GET", "/models");
+    }
+    /** Server-side store + compute stats (rows per namespace, index state, GPU). */
+    stats() {
+        return this.request("GET", "/stats");
+    }
+    /** Embed text on demand. Use task="query" for search, "document" for storage. */
+    embed(texts, opts = {}) {
+        return this.request("POST", "/embed", {
+            texts,
+            model: opts.model,
+            dim: opts.dim,
+            task: opts.task ?? "query",
+        });
+    }
+    /** Submit a bulk embedding job; embeds + stores server-side. */
+    submitJob(collection, records, opts = {}) {
+        return this.request("POST", "/jobs", {
+            collection,
+            records,
+            model: opts.model,
+            dim: opts.dim,
+        });
+    }
+    jobStatus(jobId) {
+        return this.request("GET", `/jobs/${encodeURIComponent(jobId)}`);
+    }
+    /** List recent jobs (newest first). Additive — surfaces GET /jobs so
+     * `/wiki modal jobs` can list, not just poll a known id. (Agent B request #1.) */
+    listJobs(limit) {
+        const p = new URLSearchParams();
+        if (limit != null)
+            p.set("limit", String(limit));
+        const qs = p.toString();
+        return this.request("GET", `/jobs${qs ? `?${qs}` : ""}`);
+    }
+    /** Cooperatively cancel a running/queued job. The worker stops after its
+     * current batch and writes status=cancelled. */
+    cancelJob(jobId) {
+        return this.request("POST", `/jobs/${encodeURIComponent(jobId)}/cancel`);
+    }
+    /** Poll a job until it reaches a terminal state. */
+    async waitForJob(jobId, pollMs = 2000) {
+        for (;;) {
+            const status = await this.jobStatus(jobId);
+            if (status.status === "done" || status.status === "error" || status.status === "cancelled")
+                return status;
+            await new Promise((r) => setTimeout(r, pollMs));
+        }
+    }
+    /** List the collections/tables held in the server-side vector store. */
+    async syncCollections() {
+        const out = await this.request("GET", "/sync/collections");
+        return out.collections;
+    }
+    /** Pull one page of rows with seq > since. Remember next_watermark. */
+    exportSince(collection, opts = {}) {
+        const p = new URLSearchParams({ collection });
+        if (opts.model)
+            p.set("model", opts.model);
+        if (opts.dim != null)
+            p.set("dim", String(opts.dim));
+        p.set("since", String(opts.since ?? 0));
+        p.set("limit", String(opts.limit ?? 500));
+        return this.request("GET", `/sync/export?${p.toString()}`);
+    }
+    /** Pull one page of rows with seq > since as an Arrow IPC stream.
+     * Vectors are always included (no include_vectors flag). The watermark /
+     * done / count come back as response headers (X-Next-Watermark, X-Done,
+     * X-Count) since the body is binary. Additive — the local sync path uses
+     * the JSON `exportSince`; this is for clients that want zero-copy rows. */
+    async exportSinceArrow(collection, opts = {}) {
+        const p = new URLSearchParams({ collection });
+        p.set("format", "arrow");
+        if (opts.model)
+            p.set("model", opts.model);
+        if (opts.dim != null)
+            p.set("dim", String(opts.dim));
+        p.set("since", String(opts.since ?? 0));
+        p.set("limit", String(opts.limit ?? 500));
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this.timeoutMs);
+        try {
+            const resp = await fetch(`${this.baseUrl}/sync/export?${p.toString()}`, {
+                method: "GET",
+                headers: { Authorization: `Bearer ${this.apiToken}` },
+                signal: controller.signal,
+            });
+            if (!resp.ok) {
+                const detail = await resp.text().catch(() => "");
+                throw new Error(`Modal GET /sync/export failed: ${resp.status} ${detail}`);
+            }
+            const nextWatermark = Number(resp.headers.get("X-Next-Watermark") ?? opts.since ?? 0);
+            const done = (resp.headers.get("X-Done") ?? "true") === "true";
+            const count = Number(resp.headers.get("X-Count") ?? 0);
+            return { data: await resp.arrayBuffer(), nextWatermark, done, count };
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    /**
+     * Drain every remaining page for a collection, invoking `onPage` for each.
+     * Returns the final watermark to persist for the next incremental sync.
+     */
+    async exportAll(collection, onPage, opts = {}) {
+        let watermark = opts.since ?? 0;
+        for (;;) {
+            const page = await this.exportSince(collection, { ...opts, since: watermark });
+            if (page.rows.length > 0)
+                await onPage(page.rows);
+            watermark = page.next_watermark;
+            if (page.done)
+                return watermark;
+        }
+    }
+}

package/dist/src/modal-config.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Config → Modal client resolution.
+ *
+ * Keeps `src/modal-client.ts` (the typed HTTP mirror of the contract) pristine
+ * and centralizes the pieces every consumer (lance.ts, sync.ts, commands.ts)
+ * shares: token resolution (env wins), the canonical `model__dim` pair, and
+ * client construction.
+ *
+ * No HTTP is reimplemented here — `ModalEmbeddingClient` owns that.
+ */
+import { ModalEmbeddingClient } from "./modal-client.js";
+import type { WikiConfig } from "./types.js";
+/** Env var name for the Modal bearer token (preferred over config). */
+export declare const MODAL_TOKEN_ENV = "PVM_API_TOKEN";
+/**
+ * Resolve the Modal API token. `PVM_API_TOKEN` env always wins and is
+ * preferred; config `wiki.embedding.modal.apiToken` is a fallback. Never log
+ * the resolved token.
+ */
+export declare const resolveModalToken: (cfg: WikiConfig) => string | undefined;
+/** True when Modal is usable: a base URL and a resolvable token are present. */
+export declare const isModalConfigured: (cfg: WikiConfig) => boolean;
+/**
+ * Build a `ModalEmbeddingClient` from config. Returns null when Modal is not
+ * configured (no base URL / token) so callers can degrade gracefully.
+ */
+export declare const createModalClient: (cfg: WikiConfig) => ModalEmbeddingClient | null;
+/**
+ * The canonical embedder key for a collection. Per-collection overrides win;
+ * otherwise the global Modal config model; otherwise "embeddinggemma"
+ * (the eval-confirmed baseline — kept as the default *model name*, never as a
+ * hard-coded dimension).
+ */
+export declare const resolveModel: (cfg: WikiConfig, collection?: string) => string;
+/**
+ * The configured output dimension for a collection, if known. Per-collection
+ * overrides win; otherwise the global Modal config `dim`. Returns undefined
+ * when unset — callers resolve the native dim lazily via `/models`.
+ */
+export declare const resolveDim: (cfg: WikiConfig, collection?: string) => number | undefined;
+/** Namespaced local table name mirroring the server's ADR-3 scheme. */
+export declare const namespacedTableName: (collection: string, model: string, dim: number) => string;

package/dist/src/modal-config.js

@@ -0,0 +1,60 @@
+/**
+ * Config → Modal client resolution.
+ *
+ * Keeps `src/modal-client.ts` (the typed HTTP mirror of the contract) pristine
+ * and centralizes the pieces every consumer (lance.ts, sync.ts, commands.ts)
+ * shares: token resolution (env wins), the canonical `model__dim` pair, and
+ * client construction.
+ *
+ * No HTTP is reimplemented here — `ModalEmbeddingClient` owns that.
+ */
+import { ModalEmbeddingClient } from "./modal-client.js";
+/** Env var name for the Modal bearer token (preferred over config). */
+export const MODAL_TOKEN_ENV = "PVM_API_TOKEN";
+/**
+ * Resolve the Modal API token. `PVM_API_TOKEN` env always wins and is
+ * preferred; config `wiki.embedding.modal.apiToken` is a fallback. Never log
+ * the resolved token.
+ */
+export const resolveModalToken = (cfg) => process.env[MODAL_TOKEN_ENV] || cfg.embedding.modal?.apiToken;
+/** True when Modal is usable: a base URL and a resolvable token are present. */
+export const isModalConfigured = (cfg) => cfg.embedding.provider === "modal" && !!cfg.embedding.modal?.baseUrl && !!resolveModalToken(cfg);
+/**
+ * Build a `ModalEmbeddingClient` from config. Returns null when Modal is not
+ * configured (no base URL / token) so callers can degrade gracefully.
+ */
+export const createModalClient = (cfg) => {
+    const modal = cfg.embedding.modal;
+    if (!modal?.baseUrl)
+        return null;
+    const apiToken = resolveModalToken(cfg);
+    if (!apiToken)
+        return null;
+    const clientCfg = { baseUrl: modal.baseUrl, apiToken };
+    return new ModalEmbeddingClient(clientCfg);
+};
+/**
+ * The canonical embedder key for a collection. Per-collection overrides win;
+ * otherwise the global Modal config model; otherwise "embeddinggemma"
+ * (the eval-confirmed baseline — kept as the default *model name*, never as a
+ * hard-coded dimension).
+ */
+export const resolveModel = (cfg, collection) => {
+    const override = collection ? cfg.embedding.collectionModels?.[collection] : undefined;
+    if (override?.model)
+        return override.model;
+    return cfg.embedding.modal?.model || "embeddinggemma";
+};
+/**
+ * The configured output dimension for a collection, if known. Per-collection
+ * overrides win; otherwise the global Modal config `dim`. Returns undefined
+ * when unset — callers resolve the native dim lazily via `/models`.
+ */
+export const resolveDim = (cfg, collection) => {
+    const override = collection ? cfg.embedding.collectionModels?.[collection] : undefined;
+    if (override?.dim != null)
+        return override.dim;
+    return cfg.embedding.modal?.dim;
+};
+/** Namespaced local table name mirroring the server's ADR-3 scheme. */
+export const namespacedTableName = (collection, model, dim) => `col_${collection}__${model}__${dim}`;

package/dist/src/settings-ui.d.ts

@@ -6,4 +6,11 @@ export declare const setupWizard: (ctx: ExtensionContext, cliArgs?: {
     provider?: string;
     model?: string;
 }) => Promise<void>;
+/**
+ * Interactive Modal embedding configuration + "Test connection" action.
+ * Walks base URL, canonical model, dim, offline fallback, and auto-sync
+ * (off by default). Token is read from `PVM_API_TOKEN` env (preferred) — not
+ * collected here; only an optional config fallback is offered.
+ */
+export declare const configureModalWizard: (ctx: ExtensionContext) => Promise<void>;
 export declare const openSettingsDashboard: (ctx: ExtensionContext) => Promise<void>;

package/dist/src/settings-ui.js

@@ -1,6 +1,7 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { testOllamaConnection } from "./lance.js";
+import { MODAL_TOKEN_ENV, createModalClient } from "./modal-config.js";
 import { GLOBAL_CONFIG_PATH, collectionNames, findConfig, loadConfig, } from "./utils.js";
 export const createCollectionWizard = async (ctx) => {
     const { project: cfgPath } = findConfig(ctx.cwd);
@@ -95,7 +96,7 @@ export const setupWizard = async (ctx, cliArgs) => {
         if (cliArgs.vault) {
             config.wiki.vaults.default = { path: cliArgs.vault };
         }
-        if (cliArgs.provider && ["ollama", "transformers"].includes(cliArgs.provider)) {
+        if (cliArgs.provider && ["ollama", "transformers", "modal"].includes(cliArgs.provider)) {
             config.wiki.embedding.provider = cliArgs.provider;
         }
         if (cliArgs.model) {
@@ -155,11 +156,18 @@ export const setupWizard = async (ctx, cliArgs) => {
     const provider = await ctx.ui.select("Embedding provider:", [
         "transformers  (all-MiniLM-L6-v2, offline, zero config)",
         "ollama        (embeddinggemma, higher quality, requires Ollama)",
+        "modal         (cloud GPU embeddings + server vector sync, requires Modal deploy)",
     ]);
     if (!provider) {
         ctx.ui.notify("Setup cancelled.", "warning");
         return;
     }
+    if (provider.startsWith("modal")) {
+        // Delegate to the dedicated Modal wizard (base URL, model, dim, fallback,
+        // auto-sync, test connection) and return — it writes the project config.
+        await configureModalWizard(ctx);
+        return;
+    }
     let ollamaModel = "embeddinggemma";
     if (provider.startsWith("ollama")) {
         const conn = await testOllamaConnection();
@@ -205,6 +213,102 @@ export const setupWizard = async (ctx, cliArgs) => {
         "Next: /wiki watcher start  (or restart pi for auto-start)",
     ].join("\n"), "info");
 };
+/**
+ * Interactive Modal embedding configuration + "Test connection" action.
+ * Walks base URL, canonical model, dim, offline fallback, and auto-sync
+ * (off by default). Token is read from `PVM_API_TOKEN` env (preferred) — not
+ * collected here; only an optional config fallback is offered.
+ */
+export const configureModalWizard = async (ctx) => {
+    const { project: cfgPath } = findConfig(ctx.cwd);
+    if (!cfgPath) {
+        ctx.ui.notify("No config found. Run /wiki init first.", "error");
+        return;
+    }
+    if (!ctx.hasUI) {
+        ctx.ui.notify("TUI required for the Modal wizard. Use /wiki modal config <key> <value>.", "error");
+        return;
+    }
+    const cfg = loadConfig(ctx.cwd);
+    const modal = cfg.wiki.embedding.modal ?? {};
+    const baseUrl = await ctx.ui.input("Modal base URL (deployed ASGI app):", modal.baseUrl ||
+        "https://<workspace>--pi-vault-mind-embed-embeddingservice-fastapi-app.modal.run");
+    if (!baseUrl)
+        return;
+    const model = await ctx.ui.input("Canonical embedder model:", modal.model || "embeddinggemma");
+    if (model === undefined)
+        return;
+    const dimStr = await ctx.ui.input("Output dimension (blank = model native, e.g. 768 for embeddinggemma):", modal.dim ? String(modal.dim) : "");
+    const dim = dimStr ? Number.parseInt(dimStr, 10) : undefined;
+    if (dimStr && !Number.isFinite(dim)) {
+        ctx.ui.notify(`Invalid dim: ${dimStr}`, "error");
+        return;
+    }
+    const fallback = await ctx.ui.select("Offline fallback (when Modal is unreachable):", [
+        "ollama (same canonical model only)",
+        "transformers (different space → degrades to FTS)",
+        "none (degrade to FTS, never fall back)",
+    ]);
+    let fallbackCfg;
+    if (fallback?.startsWith("ollama"))
+        fallbackCfg = { enabled: true, provider: "ollama" };
+    else if (fallback?.startsWith("transformers"))
+        fallbackCfg = { enabled: true, provider: "transformers" };
+    else if (fallback?.startsWith("none"))
+        fallbackCfg = { enabled: false };
+    const autoSync = await ctx.ui.confirm("Auto-sync", "Enable background vector sync (off by default)? Pulls new server vectors on an interval.");
+    let intervalMs = 300000;
+    if (autoSync) {
+        const intervalStr = await ctx.ui.input("Auto-sync interval (ms):", "300000");
+        intervalMs = Number.parseInt(intervalStr || "300000", 10) || 300000;
+    }
+    // Persist
+    const existing = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
+    existing.wiki = existing.wiki || {};
+    existing.wiki.embedding = existing.wiki.embedding || {};
+    existing.wiki.embedding.provider = "modal";
+    existing.wiki.embedding.modal = {
+        ...(existing.wiki.embedding.modal || {}),
+        baseUrl: baseUrl.replace(/\/$/, ""),
+        model: model || "embeddinggemma",
+        ...(dim != null ? { dim } : {}),
+        ...(fallbackCfg ? { fallback: fallbackCfg } : {}),
+        sync: {
+            ...(existing.wiki.embedding.modal?.sync || {}),
+            autoSync,
+            autoSyncIntervalMs: intervalMs,
+        },
+    };
+    fs.writeFileSync(cfgPath, `${JSON.stringify(existing, null, 2)}\n`, "utf-8");
+    // Token guidance + optional config fallback (env always wins)
+    const tokenEnv = process.env[MODAL_TOKEN_ENV];
+    if (!tokenEnv) {
+        ctx.ui.notify(`⚠️  No ${MODAL_TOKEN_ENV} env var detected. Set it in your shell:\n  export ${MODAL_TOKEN_ENV}=<bearer token>\n(Env is preferred; config apiToken is a fallback only.)`, "warning");
+    }
+    // Test connection against /health
+    const test = await ctx.ui.confirm("Test connection", "Call /health now to verify the Modal service?");
+    if (test) {
+        const client = createModalClient(loadConfig(ctx.cwd).wiki);
+        if (!client) {
+            ctx.ui.notify("❌ Modal not configured (missing baseUrl or token).", "error");
+            return;
+        }
+        try {
+            const health = await client.health();
+            ctx.ui.notify(`✅ Connected. Default model: ${health.default_model}`, "info");
+        }
+        catch (err) {
+            ctx.ui.notify(`❌ Health check failed: ${err.message}`, "error");
+        }
+    }
+    ctx.ui.notify([
+        "✅ Modal embedding configured.",
+        `   ${cfgPath}`,
+        `   Provider: modal • Model: ${model || "embeddinggemma"}${dim ? ` • dim ${dim}` : ""}`,
+        `   Auto-sync: ${autoSync ? `on (${intervalMs}ms)` : "off"}`,
+        "   Restart session for changes to take effect.",
+    ].join("\n"), "info");
+};
 export const openSettingsDashboard = async (ctx) => {
     if (!ctx.hasUI) {
         ctx.ui.notify("TUI required for settings dashboard.", "error");
@@ -220,6 +324,7 @@ export const openSettingsDashboard = async (ctx) => {
         const choice = await ctx.ui.select("pi-vault-mind Settings", [
             "Manage Collections",
             "Manage Injectors",
+            "Modal Embedding",
             "Extension Integrations",
             "Exit",
         ]);
@@ -233,6 +338,9 @@ export const openSettingsDashboard = async (ctx) => {
         else if (choice === "Manage Injectors") {
             await manageInjectors(ctx, cfgPath);
         }
+        else if (choice === "Modal Embedding") {
+            await configureModalWizard(ctx);
+        }
         else if (choice === "Extension Integrations") {
             await manageExtensions(ctx, cfgPath);
         }

package/dist/src/sync.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Modal vector sync engine.
+ *
+ * Drains `ModalEmbeddingClient.exportAll(collection, …)` into the local
+ * LanceDB via the precomputed-vector insert path (`upsertPrecomputed`), and
+ * persists a per-(collection, model, dim) `seq` watermark (ADR-4) so
+ * incremental syncs resume from where the last one left off.
+ *
+ * Idempotent: rows are keyed by `id` and applied with merge-insert, so
+ * re-fetching a boundary row is a no-op and re-running a sync with no new rows
+ * changes nothing but the (unchanged) watermark.
+ */
+import type { JobStatus } from "./modal-client.js";
+import type { UniversalConfig } from "./types.js";
+export interface SyncState {
+    /** `${collection}__${model}__${dim}` → last-seen `seq` watermark. */
+    watermarks: Record<string, number>;
+}
+export interface SyncResult {
+    collection: string;
+    model: string;
+    dim: number;
+    rows: number;
+    watermark: number;
+    full: boolean;
+}
+export declare const loadSyncState: (cfg: UniversalConfig) => SyncState;
+export declare const saveSyncState: (cfg: UniversalConfig, state: SyncState) => void;
+/** Current watermark for a (collection, model, dim), default 0. */
+export declare const getWatermark: (cfg: UniversalConfig, collection: string, model: string, dim: number) => number;
+/** List the collections/tables held in the server-side vector store. */
+export declare const listRemoteCollections: (cfg: UniversalConfig) => Promise<import("./modal-client.js").SyncCollection[]>;
+export interface SyncOptions {
+    /** Sync from watermark 0 (re-pull everything). Default false (incremental). */
+    full?: boolean;
+    /** Notified after each page is merged, with the running row count. */
+    onProgress?: (count: number) => void;
+}
+/**
+ * Sync a single collection from the server into the local LanceDB. Incremental
+ * by default (from the persisted watermark); `full` re-pulls from seq 0. Rows
+ * are merge-inserted by `id` (idempotent). Returns the row count + new
+ * watermark. A re-run with no new rows is a no-op (rows: 0, watermark unchanged).
+ */
+export declare const syncCollection: (cfg: UniversalConfig, collection: string, opts?: SyncOptions) => Promise<SyncResult>;
+/**
+ * Sync every configured collection (or a caller-supplied list). Collections
+ * with no server-side table report `rows: 0` and are skipped harmlessly.
+ */
+export declare const syncAll: (cfg: UniversalConfig, collections?: string[], opts?: SyncOptions) => Promise<SyncResult[]>;
+export interface ReindexRemoteOptions {
+    /** Notified with the job status on each poll. */
+    onStatus?: (status: JobStatus) => void;
+    /** Poll interval in ms (default 2000). */
+    pollMs?: number;
+}
+/**
+ * Read a collection's JSONL (the source of truth), submit a Modal bulk embedding
+ * job, poll it to completion, then sync the freshly-embedded vectors down into
+ * the local LanceDB. Used by `/wiki reindex --all --reembed --remote`. Local
+ * re-embed remains the default; this offloads bulk embedding to cloud GPUs.
+ *
+ * Records map the JSONL entry's embeddable field (`fact`) to `text` and the
+ * remaining fields to `metadata`, so sync-down can reconstruct the local row.
+ */
+export declare const reindexRemote: (cfg: UniversalConfig, collections: string[], opts?: ReindexRemoteOptions) => Promise<Array<{
+    collection: string;
+    job?: JobStatus;
+    sync?: SyncResult;
+    error?: string;
+}>>;