pi-vault-mind 0.7.0

package/dist/src/lance.js

@@ -0,0 +1,409 @@
+import * as fs from "node:fs";
+import { homedir } from "node:os";
+import * as path from "node:path";
+import * as lancedb from "@lancedb/lancedb";
+import { LanceSchema, TextEmbeddingFunction } from "@lancedb/lancedb/embedding";
+import * as arrow from "apache-arrow";
+let db = null;
+// ── Connection ──────────────────────────────────────────────────────────────
+export const connect = async (dataDir) => {
+    if (db)
+        return db;
+    const absDir = path.isAbsolute(dataDir) ? dataDir : path.resolve(dataDir);
+    if (!fs.existsSync(absDir))
+        fs.mkdirSync(absDir, { recursive: true });
+    db = await lancedb.connect(absDir);
+    return db;
+};
+export const resetConnection = () => {
+    db = null;
+    tables = {};
+};
+// ── Embedding Functions ─────────────────────────────────────────────────────
+/**
+ * Ollama embedding function that talks to a local Ollama instance.
+ * Uses the /api/embed endpoint with configurable model.
+ */
+class OllamaEmbeddingFunction extends TextEmbeddingFunction {
+    model;
+    host;
+    ndimsValue;
+    constructor(options = {}) {
+        super();
+        this.model = options.model || "embeddinggemma";
+        this.host = options.host || "http://127.0.0.1:11434";
+        this.ndimsValue = options.dims;
+    }
+    ndims() {
+        return this.ndimsValue;
+    }
+    embeddingDataType() {
+        return new arrow.Float32();
+    }
+    async init() {
+        // Verify Ollama is reachable and the model is available
+        try {
+            const resp = await fetch(`${this.host}/api/tags`);
+            if (!resp.ok)
+                throw new Error(`Ollama returned ${resp.status}`);
+            const data = (await resp.json());
+            const modelNames = data.models?.map((m) => m.name) || [];
+            if (!modelNames.some((n) => n.startsWith(this.model))) {
+                console.warn(`[pi-vault-mind] Ollama model "${this.model}" not found. Available: ${modelNames.join(", ")}. Pull it with: ollama pull ${this.model}`);
+            }
+        }
+        catch (err) {
+            console.warn(`[pi-vault-mind] Cannot reach Ollama at ${this.host}: ${err}`);
+        }
+    }
+    async generateEmbeddings(texts) {
+        const embeddings = [];
+        for (const text of texts) {
+            const resp = await fetch(`${this.host}/api/embed`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({ model: this.model, input: text }),
+            });
+            if (!resp.ok) {
+                const body = await resp.text().catch(() => "");
+                throw new Error(`Ollama embed failed: ${resp.status} ${body}`);
+            }
+            const data = (await resp.json());
+            if (!data.embeddings?.[0]) {
+                throw new Error("Ollama returned no embedding for input");
+            }
+            embeddings.push(data.embeddings[0]);
+        }
+        return embeddings;
+    }
+}
+/**
+ * Transformers.js fallback — fully offline, works on any Node.js.
+ */
+class TransformersEmbeddingFunction extends TextEmbeddingFunction {
+    extractor = null;
+    initPromise = null;
+    ndims() {
+        return 384;
+    }
+    embeddingDataType() {
+        return new arrow.Float32();
+    }
+    async init() {
+        if (!this.initPromise) {
+            this.initPromise = (async () => {
+                const { pipeline } = await import("@xenova/transformers");
+                this.extractor = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2");
+            })();
+        }
+        return this.initPromise;
+    }
+    async generateEmbeddings(texts) {
+        await this.init();
+        const embeddings = [];
+        for (const text of texts) {
+            const result = await this.extractor(text, { pooling: "mean", normalize: true });
+            embeddings.push(Array.from(result.data));
+        }
+        return embeddings;
+    }
+}
+const getModelsJsonPath = () => path.join(homedir(), ".pi", "agent", "models.json");
+/** Read Pi's models.json for already-registered Ollama models. */
+const loadPiModelsJson = () => {
+    const p = getModelsJsonPath();
+    if (!fs.existsSync(p))
+        return null;
+    try {
+        const data = JSON.parse(fs.readFileSync(p, "utf-8"));
+        return data?.providers?.ollama?.models || null;
+    }
+    catch {
+        return null;
+    }
+};
+/** Parse `ollama list` output (same format pi-model-router uses). */
+const parseOllamaList = (output) => {
+    const lines = output.trim().split("\n");
+    const models = [];
+    for (let i = 1; i < lines.length; i++) {
+        const line = lines[i].trim();
+        if (!line)
+            continue;
+        const parts = line.split(/\s{2,}/);
+        if (parts.length >= 3) {
+            models.push({
+                name: parts[0],
+                size: parts[2],
+                modified: parts.slice(3).join(" "),
+            });
+        }
+    }
+    return models;
+};
+/**
+ * Discover Ollama models using multiple paths (matching pi-model-router pattern):
+ * 1. Try `pi.exec('ollama', ['list'])` — Pi's managed shell execution
+ * 2. Fall back to HTTP /api/tags
+ * 3. Fall back to Pi's cached models.json
+ */
+export const discoverOllamaModels = async (piOrHost) => {
+    const host = typeof piOrHost === "string" ? piOrHost : "http://127.0.0.1:11434";
+    const pi = typeof piOrHost === "object" ? piOrHost : undefined;
+    // Path 1: pi.exec (preferred — uses Pi's managed process execution)
+    if (pi) {
+        try {
+            const result = await pi.exec("ollama", ["list"], { timeout: 10000 });
+            if (result.code === 0 && result.stdout) {
+                return parseOllamaList(result.stdout);
+            }
+        }
+        catch {
+            /* exec failed, try HTTP */
+        }
+    }
+    // Path 2: HTTP API
+    try {
+        const resp = await fetch(`${host}/api/tags`);
+        if (resp.ok) {
+            const data = (await resp.json());
+            return (data.models || []).map((m) => ({
+                name: m.name,
+                size: m.size ? `${(m.size / 1e9).toFixed(1)} GB` : "unknown",
+                modified: m.modified_at || "",
+                details: m.details,
+            }));
+        }
+    }
+    catch {
+        /* HTTP failed */
+    }
+    // Path 3: cached models.json (stale but no network needed)
+    const cached = loadPiModelsJson();
+    if (cached) {
+        return cached.map((m) => ({ name: m.id, size: "cached", modified: "" }));
+    }
+    return [];
+};
+/**
+ * Test Ollama connectivity. Uses pi.exec first, then HTTP.
+ */
+export const testOllamaConnection = async (hostOrPi) => {
+    const models = await discoverOllamaModels(hostOrPi);
+    if (models.length > 0)
+        return { reachable: true, models };
+    // Last-ditch: raw connectivity test
+    const host = typeof hostOrPi === "string" ? hostOrPi : undefined;
+    if (host) {
+        try {
+            const resp = await fetch(`${host}/api/tags`, { signal: AbortSignal.timeout(3000) });
+            if (resp.ok) {
+                const data = (await resp.json());
+                return {
+                    reachable: true,
+                    models: (data.models || []).map((m) => ({ name: m.name, size: "", modified: "" })),
+                };
+            }
+            return { reachable: false, models: [], error: `HTTP ${resp.status}` };
+        }
+        catch (err) {
+            return { reachable: false, models: [], error: err.message };
+        }
+    }
+    return { reachable: false, models: [], error: "No connectivity paths available" };
+};
+/**
+ * Pull a model from Ollama. Uses pi.exec for managed timeout, falls back to HTTP.
+ */
+export const pullOllamaModel = async (model, piOrHost) => {
+    const host = typeof piOrHost === "string" ? piOrHost : "http://127.0.0.1:11434";
+    const pi = typeof piOrHost === "object" ? piOrHost : undefined;
+    // Check if model already exists
+    const models = await discoverOllamaModels(piOrHost);
+    if (models.some((m) => m.name === model)) {
+        return { success: true, message: `Model "${model}" already exists` };
+    }
+    // Pull via pi.exec (preferred)
+    if (pi) {
+        try {
+            const result = await pi.exec("ollama", ["pull", model], { timeout: 120000 });
+            if (result.code === 0) {
+                return { success: true, message: `Successfully pulled "${model}"` };
+            }
+            return { success: false, message: `Pull exited with code ${result.code}: ${result.stderr}` };
+        }
+        catch (err) {
+            return { success: false, message: `Pull timed out or failed: ${err.message}` };
+        }
+    }
+    // Fall back to HTTP
+    try {
+        const resp = await fetch(`${host}/api/pull`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ name: model, stream: false }),
+        });
+        if (!resp.ok) {
+            const body = await resp.text().catch(() => "");
+            return { success: false, message: `Pull failed: ${resp.status} ${body}` };
+        }
+        return { success: true, message: `Successfully pulled "${model}"` };
+    }
+    catch (err) {
+        return { success: false, message: `Error: ${err.message}` };
+    }
+};
+// ── Embedding Provider Factory ──────────────────────────────────────────────
+const getEmbeddingFunction = async (cfg) => {
+    if (cfg.embedding.provider === "ollama") {
+        const fn = new OllamaEmbeddingFunction({
+            model: cfg.embedding.ollamaModel || "embeddinggemma",
+            host: cfg.embedding.ollamaHost || "http://127.0.0.1:11434",
+        });
+        await fn.init();
+        return fn;
+    }
+    const fn = new TransformersEmbeddingFunction();
+    await fn.init();
+    return fn;
+};
+// ── Schema Builders ─────────────────────────────────────────────────────────
+function makeCollectionSchema(embeddingFn) {
+    return LanceSchema({
+        id: new arrow.Utf8(),
+        domain: new arrow.Utf8(),
+        source: new arrow.Utf8(),
+        fact: embeddingFn.sourceField(),
+        tag: new arrow.Utf8(),
+        artifact: new arrow.Utf8(),
+        created_at: new arrow.Utf8(),
+        vector: embeddingFn.vectorField(),
+    });
+}
+function makeEntitySchema(embeddingFn) {
+    return LanceSchema({
+        id: new arrow.Utf8(),
+        name: embeddingFn.sourceField(),
+        type: new arrow.Utf8(),
+        aliases: new arrow.Utf8(),
+        summary: new arrow.Utf8(),
+        collection_ids: new arrow.Utf8(),
+        created_at: new arrow.Utf8(),
+        updated_at: new arrow.Utf8(),
+        vector: embeddingFn.vectorField(),
+    });
+}
+function makeRelationSchema(embeddingFn) {
+    return LanceSchema({
+        id: new arrow.Utf8(),
+        from_entity_id: new arrow.Utf8(),
+        to_entity_id: new arrow.Utf8(),
+        relation_type: new arrow.Utf8(),
+        fact: embeddingFn.sourceField(),
+        fact_strength: new arrow.Float32(),
+        source_entry_ids: new arrow.Utf8(),
+        valid_at: new arrow.Utf8(),
+        expired_at: new arrow.Utf8(),
+        created_at: new arrow.Utf8(),
+        vector: embeddingFn.vectorField(),
+    });
+}
+let tables = {};
+const getCollectionTable = async (dataDir, collectionName, cfg) => {
+    const conn = await connect(dataDir);
+    const tableName = `collection_${collectionName}`;
+    if (tables[tableName])
+        return tables[tableName];
+    const embeddingFn = await getEmbeddingFunction(cfg);
+    const schema = makeCollectionSchema(embeddingFn);
+    const existing = await conn.tableNames();
+    if (existing.includes(tableName)) {
+        tables[tableName] = await conn.openTable(tableName);
+        return tables[tableName];
+    }
+    tables[tableName] = await conn.createEmptyTable(tableName, schema);
+    if (cfg.ftsEnabled !== false) {
+        try {
+            await tables[tableName].createIndex("fact", { config: lancedb.Index.fts() });
+        }
+        catch {
+            /* already exists */
+        }
+    }
+    return tables[tableName];
+};
+const getEntityTable = async (dataDir, cfg) => {
+    const conn = await connect(dataDir);
+    const tableName = "entities";
+    if (tables[tableName])
+        return tables[tableName];
+    const embeddingFn = await getEmbeddingFunction(cfg);
+    const schema = makeEntitySchema(embeddingFn);
+    const existing = await conn.tableNames();
+    if (existing.includes(tableName)) {
+        tables[tableName] = await conn.openTable(tableName);
+        return tables[tableName];
+    }
+    tables[tableName] = await conn.createEmptyTable(tableName, schema);
+    return tables[tableName];
+};
+const getRelationTable = async (dataDir, cfg) => {
+    const conn = await connect(dataDir);
+    const tableName = "relations";
+    if (tables[tableName])
+        return tables[tableName];
+    const embeddingFn = await getEmbeddingFunction(cfg);
+    const schema = makeRelationSchema(embeddingFn);
+    const existing = await conn.tableNames();
+    if (existing.includes(tableName)) {
+        tables[tableName] = await conn.openTable(tableName);
+        return tables[tableName];
+    }
+    tables[tableName] = await conn.createEmptyTable(tableName, schema);
+    return tables[tableName];
+};
+// ── Public API ──────────────────────────────────────────────────────────────
+export const upsertEntry = async (dataDir, collectionName, entry, cfg) => {
+    const table = await getCollectionTable(dataDir, collectionName, cfg);
+    const row = {
+        id: entry.id || crypto.randomUUID(),
+        domain: entry.domain || "",
+        source: entry.source || "",
+        fact: entry.fact || "",
+        tag: entry.tag || "",
+        artifact: entry.artifact || "",
+        created_at: entry.created_at || new Date().toISOString(),
+    };
+    await table.add([row]);
+};
+export const searchHybrid = async (dataDir, collectionName, query, limit, cfg) => {
+    const table = await getCollectionTable(dataDir, collectionName, cfg);
+    const embeddingFn = await getEmbeddingFunction(cfg);
+    const queryVector = await embeddingFn.generateEmbeddings([query]);
+    const results = await table.query().nearestTo(queryVector[0]).limit(limit).toArray();
+    return results;
+};
+export const searchFts = async (dataDir, collectionName, query, limit, cfg) => {
+    const table = await getCollectionTable(dataDir, collectionName, cfg);
+    const results = await table.search(query).limit(limit).toArray();
+    return results;
+};
+export const getStatus = async (dataDir) => {
+    const conn = await connect(dataDir);
+    const names = await conn.tableNames();
+    const status = {
+        dataDir,
+        tables: {},
+    };
+    for (const name of names) {
+        try {
+            const table = await conn.openTable(name);
+            const count = await table.countRows();
+            status.tables[name] = { rows: count };
+        }
+        catch {
+            status.tables[name] = { error: "failed to open" };
+        }
+    }
+    return status;
+};

package/dist/src/server.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Lightweight HTTP server for bidirectional Obsidian ↔ pi communication.
+ *
+ * Endpoints:
+ *   GET  /vault-mind/status     → { running, dispatches, vaults, uptime }
+ *   POST /vault-mind/scan       → scan a file for @agent markers { file }
+ *   POST /vault-mind/dispatch   → fire a manual dispatch
+ *
+ * Binds 127.0.0.1 only — no network exposure. Port defaults to 11435,
+ * configurable via wiki.httpPort in pi-vault-mind.config.json.
+ *
+ * Used by the obsidian-shellcommands plugin to notify pi of file saves,
+ * replacing fs.watch reliance with explicit push notifications.
+ */
+import * as http from "node:http";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import type { WatcherState } from "./watcher.js";
+export interface ServerState {
+    server: http.Server | null;
+    port: number;
+    startTime: number;
+}
+export declare function createServerState(port?: number): ServerState;
+export declare function startServer(pi: ExtensionAPI, serverState: ServerState, watcherState: WatcherState): void;
+export declare function stopServer(serverState: ServerState): void;

package/dist/src/server.js

@@ -0,0 +1,180 @@
+/**
+ * Lightweight HTTP server for bidirectional Obsidian ↔ pi communication.
+ *
+ * Endpoints:
+ *   GET  /vault-mind/status     → { running, dispatches, vaults, uptime }
+ *   POST /vault-mind/scan       → scan a file for @agent markers { file }
+ *   POST /vault-mind/dispatch   → fire a manual dispatch
+ *
+ * Binds 127.0.0.1 only — no network exposure. Port defaults to 11435,
+ * configurable via wiki.httpPort in pi-vault-mind.config.json.
+ *
+ * Used by the obsidian-shellcommands plugin to notify pi of file saves,
+ * replacing fs.watch reliance with explicit push notifications.
+ */
+import * as fs from "node:fs";
+import * as http from "node:http";
+import { processQueue, scanFile } from "./watcher.js";
+export function createServerState(port = 11435) {
+    return { server: null, port, startTime: 0 };
+}
+export function startServer(pi, serverState, watcherState) {
+    if (serverState.server) {
+        console.warn("[pi-vault-mind] HTTP server already running.");
+        return;
+    }
+    serverState.startTime = Date.now();
+    serverState.server = http.createServer((req, res) => {
+        // CORS for localhost-only access
+        res.setHeader("Access-Control-Allow-Origin", "http://localhost:11435");
+        res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
+        res.setHeader("Access-Control-Allow-Headers", "Content-Type");
+        res.setHeader("Content-Type", "application/json");
+        if (req.method === "OPTIONS") {
+            res.writeHead(204);
+            res.end();
+            return;
+        }
+        const url = new URL(req.url || "/", `http://127.0.0.1:${serverState.port}`);
+        try {
+            switch (url.pathname) {
+                case "/vault-mind/status":
+                    handleStatus(res, serverState, watcherState);
+                    break;
+                case "/vault-mind/scan":
+                    if (req.method !== "POST") {
+                        res.writeHead(405);
+                        res.end(JSON.stringify({ error: "Method not allowed" }));
+                        return;
+                    }
+                    handleScan(req, res, pi, watcherState);
+                    break;
+                case "/vault-mind/dispatch":
+                    if (req.method !== "POST") {
+                        res.writeHead(405);
+                        res.end(JSON.stringify({ error: "Method not allowed" }));
+                        return;
+                    }
+                    handleDispatch(req, res, pi, watcherState);
+                    break;
+                default:
+                    res.writeHead(404);
+                    res.end(JSON.stringify({ error: "Not found" }));
+            }
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            res.writeHead(500);
+            res.end(JSON.stringify({ error: message }));
+        }
+    });
+    serverState.server.on("error", (err) => {
+        if (err.code === "EADDRINUSE") {
+            console.warn(`[pi-vault-mind] Port ${serverState.port} in use. HTTP server not started.`);
+        }
+        else {
+            console.error("[pi-vault-mind] HTTP server error:", err);
+        }
+    });
+    serverState.server.listen(serverState.port, "127.0.0.1", () => {
+        console.log(`[pi-vault-mind] HTTP server listening on http://127.0.0.1:${serverState.port}`);
+    });
+}
+export function stopServer(serverState) {
+    if (serverState.server) {
+        serverState.server.close();
+        serverState.server = null;
+        console.log("[pi-vault-mind] HTTP server stopped.");
+    }
+}
+function readBody(req) {
+    return new Promise((resolve, reject) => {
+        let body = "";
+        req.on("data", (chunk) => {
+            body += chunk.toString();
+        });
+        req.on("end", () => resolve(body));
+        req.on("error", reject);
+    });
+}
+function handleStatus(res, serverState, watcherState) {
+    const uptime = Date.now() - serverState.startTime;
+    const vaults = [...watcherState.watchers.keys()];
+    const recentDispatches = [...watcherState.activeDispatches_.values()].slice(-10).map((d) => ({
+        id: d.dispatchId,
+        file: d.filePath.split("/").slice(-3).join("/"),
+        role: d.role,
+        agent: d.agentName,
+        markers: d.markerCount,
+        when: d.dispatchedAt,
+    }));
+    res.writeHead(200);
+    res.end(JSON.stringify({
+        running: watcherState.running,
+        vaults,
+        activeDispatches: watcherState.activeDispatches,
+        pendingQueue: watcherState.pendingQueue.length,
+        maxConcurrent: watcherState.maxConcurrent,
+        dispatchRecords: watcherState.activeDispatches_.size,
+        recentDispatches,
+        uptimeMs: uptime,
+        uptime: `${Math.floor(uptime / 1000)}s`,
+        port: serverState.port,
+    }));
+}
+function handleScan(req, res, pi, watcherState) {
+    readBody(req)
+        .then((raw) => {
+        let parsed;
+        try {
+            parsed = JSON.parse(raw || "{}");
+        }
+        catch {
+            res.writeHead(400);
+            res.end(JSON.stringify({ error: "Invalid JSON body" }));
+            return;
+        }
+        if (!parsed.file) {
+            res.writeHead(400);
+            res.end(JSON.stringify({ error: "Missing 'file' field" }));
+            return;
+        }
+        const filePath = parsed.file;
+        if (!fs.existsSync(filePath)) {
+            res.writeHead(404);
+            res.end(JSON.stringify({ error: "File not found", file: filePath }));
+            return;
+        }
+        const groups = scanFile(filePath);
+        res.writeHead(200);
+        res.end(JSON.stringify({
+            file: filePath,
+            groups: groups.length,
+            details: groups.map((g) => ({
+                role: g.role,
+                markers: g.markers.length,
+                dispatchId: g.dispatchId,
+            })),
+        }));
+        // If markers found, queue them for dispatch
+        if (groups.length > 0) {
+            console.log(`[pi-vault-mind] HTTP scan: ${groups.length} group(s) in ${filePath}`);
+            for (const group of groups) {
+                watcherState.pendingQueue.push(group);
+            }
+            processQueue(pi, watcherState);
+        }
+    })
+        .catch((err) => {
+        if (!res.headersSent) {
+            res.writeHead(500);
+            res.end(JSON.stringify({ error: String(err) }));
+        }
+    });
+}
+function handleDispatch(req, res, _pi, _watcherState) {
+    res.writeHead(200);
+    res.end(JSON.stringify({
+        message: "Manual dispatch not yet implemented. Use /wiki watcher status for state.",
+    }));
+}

package/dist/src/settings-ui.d.ts

@@ -0,0 +1,9 @@
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+export declare const createCollectionWizard: (ctx: ExtensionContext) => Promise<void>;
+export declare const createInjectorWizard: (ctx: ExtensionContext) => Promise<void>;
+export declare const setupWizard: (ctx: ExtensionContext, cliArgs?: {
+    vault?: string;
+    provider?: string;
+    model?: string;
+}) => Promise<void>;
+export declare const openSettingsDashboard: (ctx: ExtensionContext) => Promise<void>;