pi-vault-mind 0.7.1 → 0.7.2

package/dist/test/modal-client.test.js

@@ -0,0 +1,294 @@
+import { strict as assert } from "node:assert";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import { ModalEmbeddingClient } from "../src/modal-client.js";
+// Minimal typed helper to build a fetch mock returning canned responses.
+function mockFetch(responses = []) {
+    const calls = [];
+    const fetchMock = async (url, init) => {
+        const u = typeof url === "string" ? url : url.toString();
+        calls.push({ url: u, init });
+        const match = responses.find((r) => (r.match ? r.match(u, init) : true)) || responses[0];
+        if (!match)
+            throw new Error(`unexpected fetch ${u}`);
+        const status = match.status ?? 200;
+        return {
+            ok: status >= 200 && status < 300,
+            status,
+            json: async () => match.body,
+            text: async () => JSON.stringify(match.body ?? {}),
+        };
+    };
+    return { fetchMock, calls };
+}
+const baseClient = () => new ModalEmbeddingClient({ baseUrl: "https://pvm.modal.run", apiToken: "tok" });
+describe("ModalEmbeddingClient (mocked fetch)", () => {
+    let originalFetch;
+    beforeEach(() => {
+        originalFetch = globalThis.fetch;
+    });
+    afterEach(() => {
+        globalThis.fetch = originalFetch;
+    });
+    it("health() calls GET /health without auth body", async () => {
+        const { fetchMock, calls } = mockFetch([
+            { match: (u) => u.endsWith("/health"), body: { ok: true, default_model: "embeddinggemma" } },
+        ]);
+        globalThis.fetch = fetchMock;
+        const res = await baseClient().health();
+        assert.equal(res.ok, true);
+        assert.equal(res.default_model, "embeddinggemma");
+        assert.equal(calls[0].url, "https://pvm.modal.run/health");
+        assert.equal((calls[0].init?.headers).Authorization, "Bearer tok");
+    });
+    it("embed() posts texts with model/dim/task and returns vectors", async () => {
+        const { fetchMock, calls } = mockFetch([
+            {
+                match: (u) => u.endsWith("/embed"),
+                body: {
+                    model: "embeddinggemma",
+                    dim: 768,
+                    vectors: [
+                        [0.1, 0.2],
+                        [0.3, 0.4],
+                    ],
+                },
+            },
+        ]);
+        globalThis.fetch = fetchMock;
+        const res = await baseClient().embed(["a", "b"], { model: "embeddinggemma", task: "document" });
+        assert.deepEqual(res.vectors, [
+            [0.1, 0.2],
+            [0.3, 0.4],
+        ]);
+        const sent = JSON.parse(calls[0].init?.body ?? "{}");
+        assert.deepEqual(sent.texts, ["a", "b"]);
+        assert.equal(sent.task, "document");
+        assert.equal(sent.model, "embeddinggemma");
+    });
+    it("syncCollections() unwraps the collections array", async () => {
+        const { fetchMock } = mockFetch([
+            {
+                match: (u) => u.endsWith("/sync/collections"),
+                body: {
+                    collections: [
+                        {
+                            collection: "main",
+                            model: "embeddinggemma",
+                            dim: 768,
+                            rows: 5,
+                            table: "col_main__embeddinggemma__768",
+                        },
+                    ],
+                },
+            },
+        ]);
+        globalThis.fetch = fetchMock;
+        const cols = await baseClient().syncCollections();
+        assert.equal(cols.length, 1);
+        assert.equal(cols[0].collection, "main");
+        assert.equal(cols[0].rows, 5);
+    });
+    it("exportAll() drains pages and returns the final watermark", async () => {
+        const pages = [
+            {
+                rows: [
+                    {
+                        id: "1",
+                        text: "t1",
+                        vector: [0.1],
+                        metadata: {},
+                        model: "m",
+                        dim: 1,
+                        seq: 100,
+                        created_at: "",
+                    },
+                ],
+                next_watermark: 100,
+                count: 1,
+                done: false,
+            },
+            {
+                rows: [
+                    {
+                        id: "2",
+                        text: "t2",
+                        vector: [0.2],
+                        metadata: {},
+                        model: "m",
+                        dim: 1,
+                        seq: 200,
+                        created_at: "",
+                    },
+                ],
+                next_watermark: 200,
+                count: 1,
+                done: true,
+            },
+        ];
+        const calls = [];
+        // Return pages in sequence based on call order.
+        let callIdx = 0;
+        const seqMock = async (url, init) => {
+            const u = typeof url === "string" ? url : url.toString();
+            calls.push({ url: u, init });
+            const body = pages[Math.min(callIdx, pages.length - 1)];
+            callIdx++;
+            return {
+                ok: true,
+                status: 200,
+                json: async () => body,
+                text: async () => "",
+            };
+        };
+        globalThis.fetch = seqMock;
+        const seen = [];
+        const wm = await baseClient().exportAll("main", (rows) => {
+            seen.push(...rows.map((r) => r.id));
+        });
+        assert.deepEqual(seen, ["1", "2"]);
+        assert.equal(wm, 200);
+    });
+    it("waitForJob() polls until done", async () => {
+        const statuses = [
+            {
+                status: "running",
+                collection: "main",
+                model: "m",
+                dim: 1,
+                total: 2,
+                processed: 1,
+                updated_at: "",
+            },
+            {
+                status: "done",
+                collection: "main",
+                model: "m",
+                dim: 1,
+                total: 2,
+                processed: 2,
+                updated_at: "",
+            },
+        ];
+        let i = 0;
+        globalThis.fetch = (async () => {
+            const body = statuses[Math.min(i, statuses.length - 1)];
+            i++;
+            return {
+                ok: true,
+                status: 200,
+                json: async () => body,
+                text: async () => "",
+            };
+        });
+        const status = await baseClient().waitForJob("job1", 0);
+        assert.equal(status.status, "done");
+    });
+    it("throws on non-OK response", async () => {
+        globalThis.fetch = (async () => ({
+            ok: false,
+            status: 401,
+            json: async () => ({}),
+            text: async () => "unauthorized",
+        }));
+        await assert.rejects(baseClient().health(), /401/);
+    });
+    it("models() unwraps the registry with native_dim", async () => {
+        const { fetchMock } = mockFetch([
+            {
+                match: (u) => u.endsWith("/models"),
+                body: {
+                    default: "embeddinggemma",
+                    default_dim: null,
+                    models: [
+                        {
+                            key: "embeddinggemma",
+                            hf_id: "google/embeddinggemma-300m",
+                            backend: "sentence-transformers",
+                            native_dim: 768,
+                            matryoshka_dims: [512, 256, 128],
+                            gated: true,
+                            trust_remote_code: false,
+                            enabled: true,
+                            notes: "",
+                        },
+                    ],
+                },
+            },
+        ]);
+        globalThis.fetch = fetchMock;
+        const res = await baseClient().models();
+        assert.equal(res.default, "embeddinggemma");
+        assert.equal(res.models[0].native_dim, 768);
+        assert.equal(res.models[0].backend, "sentence-transformers");
+    });
+    it("listJobs() calls GET /jobs and returns the jobs array", async () => {
+        const { fetchMock, calls } = mockFetch([
+            {
+                match: (u) => u.includes("/jobs?"),
+                body: {
+                    jobs: [
+                        {
+                            status: "done",
+                            collection: "main",
+                            model: "m",
+                            dim: 1,
+                            total: 1,
+                            processed: 1,
+                            updated_at: "",
+                        },
+                    ],
+                    count: 1,
+                },
+            },
+        ]);
+        globalThis.fetch = fetchMock;
+        const res = await baseClient().listJobs(10);
+        assert.equal(res.count, 1);
+        assert.equal(res.jobs[0].status, "done");
+        assert.ok(calls[0].url.includes("limit=10"));
+    });
+    it("cancelJob() posts to /jobs/{id}/cancel", async () => {
+        const { fetchMock, calls } = mockFetch([
+            { match: (u) => u.endsWith("/cancel"), body: { job_id: "j1", cancel_requested: true } },
+        ]);
+        globalThis.fetch = fetchMock;
+        const res = await baseClient().cancelJob("j1");
+        assert.equal(res.cancel_requested, true);
+        assert.equal(calls[0].init?.method ?? "GET", "POST");
+    });
+    it("waitForJob() treats cancelled as terminal", async () => {
+        let i = 0;
+        const statuses = [
+            {
+                status: "running",
+                collection: "main",
+                model: "m",
+                dim: 1,
+                total: 2,
+                processed: 1,
+                updated_at: "",
+            },
+            {
+                status: "cancelled",
+                collection: "main",
+                model: "m",
+                dim: 1,
+                total: 2,
+                processed: 1,
+                updated_at: "",
+            },
+        ];
+        globalThis.fetch = (async () => {
+            const body = statuses[Math.min(i, statuses.length - 1)];
+            i++;
+            return {
+                ok: true,
+                status: 200,
+                json: async () => body,
+                text: async () => "",
+            };
+        });
+        const status = await baseClient().waitForJob("job1", 0);
+        assert.equal(status.status, "cancelled");
+    });
+});

package/dist/test/modal-config.test.js

@@ -0,0 +1,86 @@
+import { strict as assert } from "node:assert";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import { MODAL_TOKEN_ENV, createModalClient, isModalConfigured, namespacedTableName, resolveDim, resolveModalToken, resolveModel, } from "../src/modal-config.js";
+const wiki = (over = {}) => ({
+    dataDir: ".lancedb",
+    embedding: { provider: "modal" },
+    ...over,
+});
+describe("modal-config", () => {
+    let origToken;
+    beforeEach(() => {
+        origToken = process.env[MODAL_TOKEN_ENV];
+    });
+    afterEach(() => {
+        if (origToken === undefined)
+            delete process.env[MODAL_TOKEN_ENV];
+        else
+            process.env[MODAL_TOKEN_ENV] = origToken;
+    });
+    it("resolveModalToken: env wins over config apiToken", () => {
+        process.env[MODAL_TOKEN_ENV] = "env-token";
+        const cfg = wiki({
+            embedding: { provider: "modal", modal: { baseUrl: "https://x", apiToken: "cfg-token" } },
+        });
+        assert.equal(resolveModalToken(cfg), "env-token");
+    });
+    it("resolveModalToken: config apiToken is the fallback", () => {
+        delete process.env[MODAL_TOKEN_ENV];
+        const cfg = wiki({
+            embedding: { provider: "modal", modal: { baseUrl: "https://x", apiToken: "cfg-token" } },
+        });
+        assert.equal(resolveModalToken(cfg), "cfg-token");
+    });
+    it("resolveModalToken: undefined when neither set", () => {
+        delete process.env[MODAL_TOKEN_ENV];
+        const cfg = wiki({ embedding: { provider: "modal", modal: { baseUrl: "https://x" } } });
+        assert.equal(resolveModalToken(cfg), undefined);
+    });
+    it("createModalClient: null when not configured (no baseUrl or token)", () => {
+        delete process.env[MODAL_TOKEN_ENV];
+        assert.equal(createModalClient(wiki()), null);
+        assert.equal(createModalClient(wiki({ embedding: { provider: "modal", modal: { baseUrl: "https://x" } } })), null);
+    });
+    it("createModalClient: builds a client when baseUrl + token present", () => {
+        process.env[MODAL_TOKEN_ENV] = "tok";
+        const c = createModalClient(wiki({ embedding: { provider: "modal", modal: { baseUrl: "https://x/" } } }));
+        assert.ok(c);
+    });
+    it("isModalConfigured: requires provider modal + baseUrl + token", () => {
+        delete process.env[MODAL_TOKEN_ENV];
+        assert.equal(isModalConfigured(wiki()), false);
+        process.env[MODAL_TOKEN_ENV] = "tok";
+        assert.equal(isModalConfigured(wiki({ embedding: { provider: "modal", modal: { baseUrl: "https://x" } } })), true);
+        // ollama provider is not modal-configured
+        assert.equal(isModalConfigured(wiki({ embedding: { provider: "ollama", modal: { baseUrl: "https://x" } } })), false);
+    });
+    it("resolveModel: per-collection override wins, then modal.model, default embeddinggemma", () => {
+        const cfg = wiki({
+            embedding: {
+                provider: "modal",
+                modal: { baseUrl: "https://x", model: "global-model" },
+                collectionModels: { special: { model: "special-model", dim: 128 } },
+            },
+        });
+        assert.equal(resolveModel(cfg), "global-model");
+        assert.equal(resolveModel(cfg, "special"), "special-model");
+        assert.equal(resolveModel(cfg, "other"), "global-model");
+        assert.equal(resolveModel(wiki()), "embeddinggemma");
+    });
+    it("resolveDim: per-collection override wins, then modal.dim", () => {
+        const cfg = wiki({
+            embedding: {
+                provider: "modal",
+                modal: { baseUrl: "https://x", dim: 512 },
+                collectionModels: { special: { model: "m", dim: 128 } },
+            },
+        });
+        assert.equal(resolveDim(cfg), 512);
+        assert.equal(resolveDim(cfg, "special"), 128);
+        assert.equal(resolveDim(wiki()), undefined);
+    });
+    it("namespacedTableName mirrors the server's ADR-3 scheme", () => {
+        assert.equal(namespacedTableName("main", "embeddinggemma", 768), "col_main__embeddinggemma__768");
+        assert.equal(namespacedTableName("ctx", "minilm-l6", 384), "col_ctx__minilm-l6__384");
+    });
+});

package/dist/test/sync.test.js

@@ -0,0 +1,132 @@
+import { strict as assert } from "node:assert";
+import { randomUUID } from "node:crypto";
+import * as fs from "node:fs";
+import { tmpdir } from "node:os";
+import * as path from "node:path";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import { connect, resetConnection } from "../src/lance.js";
+import { MODAL_TOKEN_ENV } from "../src/modal-config.js";
+import { getWatermark, loadSyncState, saveSyncState, syncCollection } from "../src/sync.js";
+const MODEL = "testmodel";
+const DIM = 3;
+const makeCfg = (dataDir) => ({
+    version: 2,
+    collections: {
+        main: {
+            path: "collections/main.jsonl",
+            schema: ["id", "domain", "fact", "tag"],
+            dedupField: "fact",
+        },
+    },
+    injectors: [],
+    wiki: {
+        dataDir,
+        embedding: {
+            provider: "modal",
+            modal: { baseUrl: "https://pvm.modal.run", model: MODEL, dim: DIM },
+        },
+        ftsEnabled: true,
+    },
+});
+// Mock fetch that serves /sync/export pages based on the `since` query param.
+function exportPagesFetch(pages) {
+    return (async (url) => {
+        const u = typeof url === "string" ? url : url.toString();
+        const sp = new URL(u).searchParams;
+        const since = Number.parseInt(sp.get("since") || "0", 10);
+        const page = pages[since] ?? { rows: [], next: since, done: true };
+        const body = {
+            rows: page.rows,
+            next_watermark: page.next,
+            count: page.rows.length,
+            done: page.done,
+        };
+        return {
+            ok: true,
+            status: 200,
+            json: async () => body,
+            text: async () => "",
+        };
+    });
+}
+const row = (id, seq, vector) => ({
+    id,
+    text: `fact ${id}`,
+    vector,
+    metadata: { domain: "d", tag: "t" },
+    model: MODEL,
+    dim: DIM,
+    seq,
+    created_at: "",
+});
+describe("Modal sync engine", () => {
+    let dir;
+    let origToken;
+    let origFetch;
+    beforeEach(() => {
+        dir = path.join(tmpdir(), `pvm-sync-${randomUUID()}`);
+        fs.mkdirSync(dir, { recursive: true });
+        origToken = process.env[MODAL_TOKEN_ENV];
+        process.env[MODAL_TOKEN_ENV] = "tok";
+        origFetch = globalThis.fetch;
+        resetConnection();
+    });
+    afterEach(() => {
+        if (origToken === undefined)
+            delete process.env[MODAL_TOKEN_ENV];
+        else
+            process.env[MODAL_TOKEN_ENV] = origToken;
+        globalThis.fetch = origFetch;
+        resetConnection();
+    });
+    it("watermark state round-trips and defaults to 0", () => {
+        const cfg = makeCfg(dir);
+        const key = `main__${MODEL}__${DIM}`;
+        assert.equal(getWatermark(cfg, "main", MODEL, DIM), 0);
+        saveSyncState(cfg, { watermarks: { [key]: 123 } });
+        // Re-read through both the keyed accessor and the full state.
+        assert.equal(getWatermark(cfg, "main", MODEL, DIM), 123);
+        assert.equal(loadSyncState(cfg).watermarks[key], 123);
+    });
+    it("drains pages into local LanceDB and advances the watermark", async () => {
+        const cfg = makeCfg(dir);
+        globalThis.fetch = exportPagesFetch({
+            0: { rows: [row("1", 100, [1, 0, 0]), row("2", 150, [0, 1, 0])], next: 150, done: false },
+            150: { rows: [row("3", 200, [0, 0, 1])], next: 200, done: true },
+        });
+        const res = await syncCollection(cfg, "main");
+        assert.equal(res.rows, 3);
+        assert.equal(res.watermark, 200);
+        assert.equal(getWatermark(cfg, "main", MODEL, DIM), 200);
+        const conn = await connect(dir);
+        const table = await conn.openTable(`col_main__${MODEL}__${DIM}`);
+        assert.equal(await table.countRows(), 3);
+    });
+    it("re-running with no new rows is a no-op (idempotent)", async () => {
+        const cfg = makeCfg(dir);
+        globalThis.fetch = exportPagesFetch({
+            0: { rows: [row("1", 100, [1, 0, 0])], next: 100, done: false },
+            100: { rows: [row("2", 200, [0, 1, 0])], next: 200, done: true },
+            200: { rows: [], next: 200, done: true },
+        });
+        await syncCollection(cfg, "main");
+        const second = await syncCollection(cfg, "main"); // since=200 → empty, done
+        assert.equal(second.rows, 0);
+        assert.equal(second.watermark, 200);
+        const conn = await connect(dir);
+        const table = await conn.openTable(`col_main__${MODEL}__${DIM}`);
+        assert.equal(await table.countRows(), 2);
+    });
+    it("re-syncing the same ids merges (no duplicates)", async () => {
+        const cfg = makeCfg(dir);
+        globalThis.fetch = exportPagesFetch({
+            0: { rows: [row("1", 100, [1, 0, 0])], next: 100, done: false },
+            100: { rows: [row("1", 200, [1, 0, 0])], next: 200, done: true },
+        });
+        await syncCollection(cfg, "main"); // inserts id=1
+        await syncCollection(cfg, "main", { full: true }); // re-fetches id=1 → merge by id
+        const conn = await connect(dir);
+        const table = await conn.openTable(`col_main__${MODEL}__${DIM}`);
+        assert.equal(await table.countRows(), 1);
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-vault-mind",
-  "version": "0.7.1",
+  "version": "0.7.2",
   "description": "Passive Obsidian vault extension for pi. Watches @agent markers, dispatches forked subagents (Miner, Broadcaster, Heavy-Lifter), stores in LanceDB with vector + FTS + graph. Multi-agent 'Drop & Forget' workflow for the pi agent ecosystem.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -78,6 +78,7 @@
     "check": "biome check --write . && tsc --noEmit",
     "build": "tsc && tsc -p tsconfig.test.json",
     "dev": "tsc --watch",
-    "test": "node --test dist/test/index.test.js"
+    "test": "node --test dist/test/*.test.js",
+    "e2e:modal": "node scripts/modal-e2e-smoke.mjs"
   }
 }

package/dist/test/index.test.d.ts

@@ -1 +0,0 @@
-export {};