pi-vault-mind 0.7.4 → 0.8.0

package/dist/src/auto-index.js

@@ -0,0 +1,122 @@
+/**
+ * Background auto-indexing for collection directories.
+ *
+ * Watches the directories that contain configured collection JSONL files.
+ * When a collection file is created, modified, or deleted, a debounced
+ * reindex is triggered for that collection.
+ *
+ * Controlled by `wiki.autoIndex` (default false). Started alongside the HTTP
+ * server and stopped on session shutdown.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { upsertEntry } from "./lance.js";
+export const DEFAULT_DEBOUNCE_MS = 2000;
+export function createAutoIndexState() {
+    return { running: false, watchers: new Map(), debounceTimers: new Map() };
+}
+/**
+ * Start watching configured collection directories.
+ * Returns a stop function that closes watchers and clears timers.
+ */
+export function startAutoIndex(cfg, options = {}) {
+    const state = options.state ?? createAutoIndexState();
+    if (state.running) {
+        console.warn("[pi-vault-mind] Auto-index already running.");
+        return { stop: () => stopAutoIndex(state), state };
+    }
+    state.running = true;
+    const debounceMs = options.debounceMs ?? DEFAULT_DEBOUNCE_MS;
+    const reindexFn = options.reindex ?? reindexCollection;
+    // Group collections by parent directory so a single watcher covers multiple
+    // collection files in the same directory.
+    const dirMap = new Map();
+    for (const [collection, def] of Object.entries(cfg.collections)) {
+        const dir = path.dirname(def.path);
+        const fileName = path.basename(def.path);
+        const entries = dirMap.get(dir) ?? [];
+        entries.push({ collection, fileName, def });
+        dirMap.set(dir, entries);
+    }
+    for (const [dir, entries] of dirMap) {
+        if (!fs.existsSync(dir)) {
+            console.warn(`[pi-vault-mind] Auto-index skipping missing directory: ${dir}`);
+            continue;
+        }
+        const watcher = fs.watch(dir, { recursive: false }, (_eventType, filename) => {
+            if (!filename || !state.running)
+                return;
+            const matched = entries.filter((e) => e.fileName === filename);
+            if (matched.length === 0)
+                return;
+            for (const m of matched) {
+                scheduleReindex(state, m.collection, m.def, cfg.wiki, reindexFn, debounceMs);
+            }
+        });
+        watcher.on("error", (err) => {
+            console.error(`[pi-vault-mind] Auto-index watcher error for ${dir}:`, err);
+        });
+        state.watchers.set(dir, watcher);
+    }
+    if (state.watchers.size > 0) {
+        console.log(`[pi-vault-mind] Auto-index started: watching ${state.watchers.size} collection dir(s).`);
+    }
+    return { stop: () => stopAutoIndex(state), state };
+}
+/**
+ * Stop all watchers and clear pending debounces.
+ */
+export function stopAutoIndex(state) {
+    state.running = false;
+    for (const [, timer] of state.debounceTimers) {
+        clearTimeout(timer);
+    }
+    state.debounceTimers.clear();
+    for (const [dir, watcher] of state.watchers) {
+        watcher.close();
+        console.log(`[pi-vault-mind] Auto-index stopped watching ${dir}.`);
+    }
+    state.watchers.clear();
+}
+function scheduleReindex(state, collection, def, wikiCfg, reindexFn, debounceMs) {
+    const existing = state.debounceTimers.get(collection);
+    if (existing)
+        clearTimeout(existing);
+    const timer = setTimeout(() => {
+        state.debounceTimers.delete(collection);
+        reindexFn(wikiCfg.dataDir, collection, def, wikiCfg).catch((err) => {
+            const message = err instanceof Error ? err.message : String(err);
+            console.warn(`[pi-vault-mind] Auto-index reindex failed for "${collection}": ${message}`);
+        });
+    }, debounceMs);
+    state.debounceTimers.set(collection, timer);
+}
+/**
+ * Re-index a single collection by reading its JSONL source file and upserting
+ * each valid entry into LanceDB. Malformed lines are skipped.
+ */
+export const reindexCollection = async (dataDir, collection, def, cfg) => {
+    if (!fs.existsSync(def.path)) {
+        console.log(`[pi-vault-mind] Auto-index skipped "${collection}" (file missing): ${def.path}`);
+        return;
+    }
+    const content = fs.readFileSync(def.path, "utf-8");
+    const lines = content.split("\n").filter(Boolean);
+    let indexed = 0;
+    let skipped = 0;
+    for (const line of lines) {
+        try {
+            const entry = JSON.parse(line);
+            if (typeof entry !== "object" || entry === null) {
+                skipped++;
+                continue;
+            }
+            await upsertEntry(dataDir, collection, entry, cfg);
+            indexed++;
+        }
+        catch {
+            skipped++;
+        }
+    }
+    console.log(`[pi-vault-mind] Auto-index reindexed "${collection}": ${indexed}/${lines.length} entries${skipped > 0 ? `, ${skipped} skipped` : ""}`);
+};

package/dist/src/commands.js

@@ -1,3 +1,4 @@
+import { execFileSync } from "node:child_process";
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { Container, SelectList, Text } from "@earendil-works/pi-tui";
@@ -6,11 +7,16 @@ import { getActiveCollection, setActiveCollection } from "./state.js";
 import { DEFAULT_CONFIG } from "./types.js";
 import { CONFIG_FILES, EXT_ROOT, collectionNames, ensureDir, findConfig, getPiContextConfig, hasPiContextTools, loadConfig, } from "./utils.js";
 import { updateActiveCollectionWidget } from "./widget.js";
+import { cancelJobHelper, listJobs, retryJob, } from "./agent-queue.js";
+import { handleDiscoverSchema } from "./discover-schema.js";
+import { runAskIntake } from "./intake.js";
 import { connect, pullOllamaModel, testOllamaConnection } from "./lance.js";
 import { MODAL_TOKEN_ENV, createModalClient, isModalConfigured, resolveBaseUrl, resolveModalToken, resolveWorkspace, } from "./modal-config.js";
 import { createServerState } from "./server.js";
 import { createCollectionWizard, createInjectorWizard, openSettingsDashboard, setupWizard, } from "./settings-ui.js";
 import { reindexRemote, syncAll, syncCollection } from "./sync.js";
+import { tombstoneEntryById } from "./tombstone.js";
+import { expandHome } from "./utils.js";
 import { createWatcherState, getWatcherStatus, startWatcher, stopWatcher } from "./watcher.js";
 // ── Shared helpers ───────────────────────────────────────────────────────────
 const WIKI_USAGE = [
@@ -19,11 +25,13 @@ const WIKI_USAGE = [
     "  /wiki init                  Scaffold config + collections",
     "  /wiki validate              Check LanceDB, config, and collection health",
     "  /wiki approve [collection]  Batch-review pending entries",
+    "  /wiki tombstone <collection> <entry-id>  Redact an entry in place",
     "  /wiki settings              Open interactive settings dashboard",
     "  /wiki audit                 Audit config for missing defaults",
     "  /wiki reindex               Rebuild FTS + vector indexes (replaces old /qmd-index)",
     "  /wiki collection select     Select active collection",
     "  /wiki collection create     Create a new collection (wizard)",
+    "  /wiki discover-schema       Infer schema from a JSONL file",
     "  /wiki injector create       Create a new injector (wizard)",
     "  /wiki context enable|disable  Enable/disable pi-context integration",
     "  /wiki context status        Show pi-context integration status",
@@ -40,6 +48,9 @@ const WIKI_USAGE = [
     "  /wiki watcher start          Start passive file watcher",
     "  /wiki watcher stop           Stop passive file watcher",
     "  /wiki watcher status         Show watcher status",
+    "  /wiki watcher queue [status] Show agent dispatch queue (pending|running|done|failed|cancelled|all)",
+    "  /wiki watcher retry <id>     Retry a failed or cancelled agent job",
+    "  /wiki watcher cancel <id>    Cancel a pending agent job",
     "  /wiki server status          Show HTTP server status + port",
     "  /wiki setup                  Interactive global config wizard",
     "  /wiki setup --vault <path>   CLI: set vault path",
@@ -383,6 +394,24 @@ const handleApprove = async (args, ctx) => {
     fs.writeFileSync(pending.path, rejected.join("\n") + (rejected.length > 0 ? "\n" : ""));
     ctx.ui.notify(`Approved ${approved} → "${targetName}". Rejected ${rejected.length}.`, "info");
 };
+// ── /wiki tombstone ─────────────────────────────────────────────────────────
+const handleTombstone = async (args, ctx) => {
+    const parts = args.trim().split(/\s+/).filter(Boolean);
+    if (parts.length < 2) {
+        ctx.ui.notify("Usage: /wiki tombstone <collection> <entry-id>", "error");
+        return;
+    }
+    const collection = parts[0];
+    const entryId = parts.slice(1).join(" ");
+    const cfg = loadConfig(ctx.cwd);
+    const def = cfg.collections[collection];
+    if (!def) {
+        ctx.ui.notify(`Unknown collection "${collection}".`, "error");
+        return;
+    }
+    const result = tombstoneEntryById(def.path, entryId);
+    ctx.ui.notify(result.message, result.success ? "info" : "error");
+};
 // ── /wiki reindex ────────────────────────────────────────────────────────────
 const handleReindex = async (args, ctx, pi) => {
     const cfg = loadConfig(ctx.cwd);
@@ -825,7 +854,7 @@ const handleModalConfig = async (args, ctx) => {
             return;
         }
         case "token": {
-            ctx.ui.notify(`Token resolution: env ${MODAL_TOKEN_ENV} is preferred (never committed).\nSet it in your shell: export ${MODAL_TOKEN_ENV}=...\nConfig wiki.embedding.modal.apiToken is a fallback only.`, "info");
+            await handleModalToken(ctx, parts[1]);
             return;
         }
         default:
@@ -1178,6 +1207,11 @@ const handleServer = (ctx) => {
 const handleWatcher = async (args, ctx, pi) => {
     const parts = args.trim().split(/\s+/g);
     const subcommand = parts[0]?.toLowerCase() || "status";
+    const defaultVaultPath = () => {
+        const cfg = loadConfig(ctx.cwd);
+        const p = cfg.wiki.vaults?.default?.path;
+        return p ? expandHome(p) : undefined;
+    };
     switch (subcommand) {
         case "start": {
             const cfg = loadConfig(ctx.cwd);
@@ -1202,6 +1236,92 @@ const handleWatcher = async (args, ctx, pi) => {
             ctx.ui.notify("Watcher stopped.", "info");
             return;
         }
+        case "queue": {
+            const vaultPath = defaultVaultPath();
+            if (!vaultPath) {
+                ctx.ui.notify("No default vault configured. Set one with /wiki setup --vault <path>.", "error");
+                return;
+            }
+            const statusArg = parts[1]?.toLowerCase();
+            const statuses = ["pending", "running", "done", "failed", "cancelled"];
+            const status = statusArg === "all"
+                ? undefined
+                : statuses.includes(statusArg || "")
+                    ? statusArg
+                    : undefined;
+            const jobs = listJobs(vaultPath, status);
+            if (jobs.length === 0) {
+                ctx.ui.notify(`No ${status || "agent"} jobs in queue for this vault.`, "info");
+                return;
+            }
+            const lines = [
+                `**Agent Queue (${status || "all"}) — ${jobs.length} job(s):**`,
+                "",
+                "| id | role | status | attempts | file |",
+                "|---|---|---|---|---|",
+                ...jobs.map((j) => {
+                    const rel = j.filePath.split("/").slice(-3).join("/");
+                    return `| ${j.id} | ${j.role} | ${j.status} | ${j.attempts} | ${rel} |`;
+                }),
+            ];
+            ctx.ui.notify(lines.join("\n"), "info");
+            return;
+        }
+        case "retry": {
+            const id = parts[1];
+            if (!id) {
+                ctx.ui.notify("Usage: /wiki watcher retry <job-id>", "error");
+                return;
+            }
+            const vaultPath = defaultVaultPath();
+            if (!vaultPath) {
+                ctx.ui.notify("No default vault configured.", "error");
+                return;
+            }
+            const result = retryJob(vaultPath, id);
+            if ("error" in result) {
+                ctx.ui.notify(result.error, "warning");
+                return;
+            }
+            const job = result.job;
+            // Re-create the dispatch by re-sending the task as a user message
+            const taskDescription = [
+                `Retry of dispatch ${id} for @agent-${job.role} in ${job.filePath}.`,
+                "",
+                "Please dispatch a subagent to handle it:",
+                "",
+                "```",
+                "subagent({",
+                `  agent: "${job.agentName}",`,
+                `  task: ${JSON.stringify(job.instruction)},`,
+                `  context: "fork",`,
+                "  async: true,",
+                "})",
+                "```",
+            ].join("\n");
+            pi.sendUserMessage(taskDescription, { deliverAs: "followUp" });
+            ctx.ui.notify(`✅ Retried job ${id} → ${job.agentName}`, "info");
+            return;
+        }
+        case "cancel": {
+            const id = parts[1];
+            if (!id) {
+                ctx.ui.notify("Usage: /wiki watcher cancel <job-id>", "error");
+                return;
+            }
+            const vaultPath = defaultVaultPath();
+            if (!vaultPath) {
+                ctx.ui.notify("No default vault configured.", "error");
+                return;
+            }
+            const result = cancelJobHelper(vaultPath, id);
+            if ("error" in result) {
+                ctx.ui.notify(result.error, "warning");
+                return;
+            }
+            ctx.ui.notify(`✅ Cancelled job ${id}`, "info");
+            return;
+        }
         default: {
             ctx.ui.notify(getWatcherStatus(watcherState), "info");
             return;
@@ -1230,6 +1350,88 @@ const handleSetup = async (args, ctx) => {
     const hasCliArgs = cliArgs.vault || cliArgs.provider || cliArgs.model || cliArgs.workspace;
     await setupWizard(ctx, hasCliArgs ? cliArgs : undefined);
 };
+// ── /wiki intake ─────────────────────────────────────────────────────────────
+const handleIntake = async (args, ctx) => {
+    const filePath = args.trim();
+    if (!filePath) {
+        ctx.ui.notify("Usage: /wiki intake <path-to-json-spec>", "error");
+        return;
+    }
+    if (!fs.existsSync(filePath)) {
+        ctx.ui.notify(`Intake spec not found: ${filePath}`, "error");
+        return;
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(fs.readFileSync(filePath, "utf-8"));
+    }
+    catch (err) {
+        ctx.ui.notify(`Failed to read intake spec: ${err.message}`, "error");
+        return;
+    }
+    if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.questions)) {
+        ctx.ui.notify("Invalid intake spec: expected { questions: [...], persist?, collection? }", "error");
+        return;
+    }
+    const result = await runAskIntake(ctx, parsed);
+    if (result.cancelled) {
+        ctx.ui.notify("Intake cancelled.", "warning");
+    }
+    else {
+        ctx.ui.notify("Intake complete. Answers stored.", "info");
+    }
+};
+// ── /wiki modal token ────────────────────────────────────────────────────────
+/** Dotenv path where the Modal token is persisted. */
+const MODAL_TOKEN_ENV_PATH = path.join(process.env.HOME || process.env.USERPROFILE || "", ".pi", "agent", "vault-mind.env");
+const writeTokenEnv = (token) => {
+    const dir = path.dirname(MODAL_TOKEN_ENV_PATH);
+    if (!fs.existsSync(dir))
+        fs.mkdirSync(dir, { recursive: true });
+    // fs mode bits are unsupported on Windows; skip them to avoid EINVAL.
+    const options = process.platform === "win32" ? {} : { mode: 0o600 };
+    fs.writeFileSync(MODAL_TOKEN_ENV_PATH, `PVM_API_TOKEN="${token}"\n`, options);
+};
+const fetchTokenFrom1Password = () => {
+    try {
+        return execFileSync("op", ["item", "get", "pi-vault-mind-auth", "--reveal", "--field", "password"], {
+            encoding: "utf-8",
+            stdio: ["ignore", "pipe", "pipe"],
+        }).trim();
+    }
+    catch {
+        return null;
+    }
+};
+const handleModalToken = async (ctx, cliToken) => {
+    if (cliToken) {
+        writeTokenEnv(cliToken);
+        ctx.ui.notify(`✅ Wrote token to ${MODAL_TOKEN_ENV_PATH}`, "info");
+        return;
+    }
+    const envToken = process.env[MODAL_TOKEN_ENV];
+    if (envToken) {
+        ctx.ui.notify("PVM_API_TOKEN is already exported in env. Skipping write so we don't clobber it.", "warning");
+        return;
+    }
+    const tokenFrom1Pass = fetchTokenFrom1Password();
+    if (tokenFrom1Pass) {
+        writeTokenEnv(tokenFrom1Pass);
+        ctx.ui.notify(`✅ Fetched token from 1Password item "pi-vault-mind-auth" and wrote it to ${MODAL_TOKEN_ENV_PATH}`, "info");
+        return;
+    }
+    if (!ctx.hasUI) {
+        ctx.ui.notify("No 1Password token found and no TUI available. Use:\n  /wiki modal token <token>", "error");
+        return;
+    }
+    const token = await ctx.ui.input("Modal API token:", "paste bearer token here (will be written to ~/.pi/agent/vault-mind.env)");
+    if (!token) {
+        ctx.ui.notify("Token setup cancelled.", "warning");
+        return;
+    }
+    writeTokenEnv(token);
+    ctx.ui.notify(`✅ Wrote token to ${MODAL_TOKEN_ENV_PATH}`, "info");
+};
 // ── Main /wiki command ───────────────────────────────────────────────────────
 export const registerCommands = (pi) => {
     pi.registerCommand("wiki", {
@@ -1240,11 +1442,13 @@ export const registerCommands = (pi) => {
                 "init",
                 "validate",
                 "approve",
+                "tombstone",
                 "settings",
                 "audit",
                 "help",
                 "reindex",
                 "collection",
+                "discover-schema",
                 "injector",
                 "context",
                 "embedding",
@@ -1271,6 +1475,11 @@ export const registerCommands = (pi) => {
                         .filter((c) => c.startsWith(prefix))
                         .map((c) => ({ label: c, value: c, description: `context ${c}` }));
                 }
+                if (subcommand === "discover-schema") {
+                    return ["--name", "--sample"]
+                        .filter((c) => c.startsWith(prefix))
+                        .map((c) => ({ label: c, value: c, description: `discover-schema ${c}` }));
+                }
                 if (subcommand === "collection") {
                     return ["select", "create"]
                         .filter((c) => c.startsWith(prefix))
@@ -1287,17 +1496,17 @@ export const registerCommands = (pi) => {
                         .map((c) => ({ label: c, value: c, description: `reindex ${c}` }));
                 }
                 if (subcommand === "modal") {
-                    return ["status", "config", "sync", "jobs", "migrate"]
+                    return ["status", "config", "auto", "token", "sync", "jobs", "migrate"]
                         .filter((c) => c.startsWith(prefix))
                         .map((c) => ({ label: c, value: c, description: `modal ${c}` }));
                 }
                 if (subcommand === "watcher") {
-                    return ["start", "stop", "status"]
+                    return ["start", "stop", "status", "queue", "retry", "cancel"]
                         .filter((c) => c.startsWith(prefix))
                         .map((c) => ({ label: c, value: c, description: `watcher ${c}` }));
                 }
                 if (subcommand === "setup") {
-                    return ["--vault", "--provider", "--model"]
+                    return ["--vault", "--provider", "--model", "--workspace"]
                         .filter((c) => c.startsWith(prefix))
                         .map((c) => ({ label: c, value: c, description: `setup ${c}` }));
                 }
@@ -1315,12 +1524,16 @@ export const registerCommands = (pi) => {
                     return handleValidate(rest, ctx, pi);
                 case "approve":
                     return handleApprove(rest, ctx);
+                case "tombstone":
+                    return handleTombstone(rest, ctx);
                 case "settings":
                     return openSettingsDashboard(ctx);
                 case "audit":
                     return auditConfig(ctx);
                 case "reindex":
                     return handleReindex(rest, ctx, pi);
+                case "discover-schema":
+                    return handleDiscoverSchema(rest, ctx);
                 case "collection":
                     return handleCollection(rest, ctx);
                 case "injector":
@@ -1343,4 +1556,52 @@ export const registerCommands = (pi) => {
         },
     });
     // Legacy aliases — /wiki-* commands delegate to /wiki subcommand handlers
+    /* ── /intake ───────────────────────────────────────────────────────────── */
+    pi.registerCommand("intake", {
+        description: "Run an intake questionnaire from a JSON spec file. Optionally write the result to --output.",
+        handler: async (args, ctx) => {
+            const parts = args.trim().split(/\s+/);
+            const outputIdx = parts.indexOf("--output");
+            const outputPath = outputIdx >= 0 ? parts[outputIdx + 1] : undefined;
+            const specPath = outputIdx === 0 ? parts[2] : parts[0];
+            if (!specPath || specPath === "--output") {
+                ctx.ui.notify("Usage: /intake <path-to-json-spec> [--output <result-json-path>]", "error");
+                return;
+            }
+            const resolved = path.isAbsolute(specPath) ? specPath : path.join(ctx.cwd, specPath);
+            if (!fs.existsSync(resolved)) {
+                ctx.ui.notify(`Intake spec not found: ${resolved}`, "error");
+                return;
+            }
+            let parsed;
+            try {
+                parsed = JSON.parse(fs.readFileSync(resolved, "utf-8"));
+            }
+            catch (err) {
+                ctx.ui.notify(`Failed to read intake spec: ${err.message}`, "error");
+                return;
+            }
+            if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.questions)) {
+                ctx.ui.notify("Invalid intake spec: expected { questions: [...], persist?, collection? }", "error");
+                return;
+            }
+            const result = await runAskIntake(ctx, parsed);
+            if (outputPath) {
+                try {
+                    const out = path.isAbsolute(outputPath) ? outputPath : path.join(ctx.cwd, outputPath);
+                    ensureDir(out);
+                    fs.writeFileSync(out, `${JSON.stringify(result, null, 2)}\n`, "utf-8");
+                }
+                catch (err) {
+                    ctx.ui.notify(`Failed to write intake result: ${err.message}`, "error");
+                }
+            }
+            if (result.cancelled) {
+                ctx.ui.notify("Intake cancelled.", "warning");
+            }
+            else {
+                ctx.ui.notify(`Intake complete. ${Object.keys(result.answers).length} answer(s).${result.ledgerEntry ? ` Ledger: ${result.ledgerEntry.path}.` : ""}${result.vaultEntry ? ` Vault: ${result.vaultEntry.path}.` : ""}`, "info");
+            }
+        },
+    });
 };

package/dist/src/discover-schema.d.ts

@@ -0,0 +1,27 @@
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { CollectionDef } from "./types.js";
+/** Scalar type family that can be inferred from JSON values. */
+export type InferredType = "string" | "number" | "boolean" | "array" | "object";
+/** Result returned by the discover_schema tool and helper functions. */
+export interface DiscoverSchemaResult {
+    collectionName: string;
+    path: string;
+    schema: string[];
+    types: Record<string, InferredType | "mixed">;
+    rowCount: number;
+    sampled: number;
+}
+/** Infer a single JSON value's type. */
+export declare const inferType: (value: unknown) => InferredType | null;
+/** Read the first `sample` JSON lines from a .jsonl file and infer schema. */
+export declare const discoverSchemaFromFile: (filePath: string, sample?: number) => Omit<DiscoverSchemaResult, "collectionName" | "path"> & {
+    allKeys: string[];
+};
+/** Register (or update) a collection in the project config. */
+export declare const writeCollectionToConfig: (cwd: string, name: string, def: CollectionDef) => string;
+/** Run the full discover_schema flow and update config. */
+export declare const runDiscoverSchema: (cwd: string, file: string, name?: string, sample?: number) => DiscoverSchemaResult;
+/** Register the `discover_schema` Pi tool. */
+export declare const registerDiscoverSchemaTool: (pi: ExtensionAPI) => void;
+/** Handler for the /wiki discover-schema slash command. */
+export declare const handleDiscoverSchema: (args: string, ctx: ExtensionContext) => Promise<void>;