pi-vault-mind 0.7.4 → 0.8.0

package/dist/src/discover-schema.js

@@ -0,0 +1,255 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { Type } from "typebox";
+import { ensureDir, findConfig } from "./utils.js";
+/** Infer a single JSON value's type. */
+export const inferType = (value) => {
+    if (value === null || value === undefined)
+        return null;
+    switch (typeof value) {
+        case "string":
+            return "string";
+        case "number":
+            return Number.isFinite(value) ? "number" : null;
+        case "boolean":
+            return "boolean";
+        case "object":
+            return Array.isArray(value) ? "array" : "object";
+        default:
+            return null;
+    }
+};
+/** Pick the representative type from a set of observed types. */
+const resolveType = (seen) => {
+    if (seen.size === 0)
+        return "string";
+    if (seen.size === 1)
+        return [...seen][0];
+    return "mixed";
+};
+/** Make an absolute path relative to `cwd` when it lives underneath it. */
+const shrinkPath = (cwd, p) => {
+    const abs = path.resolve(p);
+    const root = path.resolve(cwd);
+    if (abs === root)
+        return ".";
+    if (abs.startsWith(`${root}${path.sep}`)) {
+        return path.relative(root, abs);
+    }
+    return abs;
+};
+/** Read the first `sample` JSON lines from a .jsonl file and infer schema. */
+export const discoverSchemaFromFile = (filePath, sample = 100) => {
+    if (!fs.existsSync(filePath)) {
+        throw new Error(`File not found: ${filePath}`);
+    }
+    const raw = fs.readFileSync(filePath, "utf-8");
+    const lines = raw.split(/\r?\n/).filter((line) => line.trim().length > 0);
+    const rowCount = lines.length;
+    const sampled = Math.min(sample, rowCount);
+    const typeSets = new Map();
+    for (let i = 0; i < sampled; i++) {
+        let parsed;
+        try {
+            parsed = JSON.parse(lines[i]);
+        }
+        catch {
+            continue;
+        }
+        if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+            continue;
+        }
+        const record = parsed;
+        for (const [key, value] of Object.entries(record)) {
+            const type = inferType(value);
+            if (type) {
+                let set = typeSets.get(key);
+                if (!set) {
+                    set = new Set();
+                    typeSets.set(key, set);
+                }
+                set.add(type);
+            }
+        }
+    }
+    const allKeys = [...typeSets.keys()];
+    const types = {};
+    for (const key of allKeys) {
+        const set = typeSets.get(key);
+        types[key] = set ? resolveType(set) : "string";
+    }
+    return {
+        schema: allKeys,
+        types,
+        rowCount,
+        sampled,
+        allKeys,
+    };
+};
+/** Read the mutable project config object, or an empty object if none exists. */
+const readProjectConfig = (cwd) => {
+    const cfgPath = findConfig(cwd).project || path.join(cwd, "pi-vault-mind.config.json");
+    if (!fs.existsSync(cfgPath))
+        return {};
+    try {
+        return JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
+    }
+    catch {
+        return {};
+    }
+};
+/** Persist the project config with standard formatting. */
+const writeProjectConfig = (cwd, obj) => {
+    const cfgPath = findConfig(cwd).project || path.join(cwd, "pi-vault-mind.config.json");
+    ensureDir(cfgPath);
+    fs.writeFileSync(cfgPath, `${JSON.stringify(obj, null, 2)}\n`, "utf-8");
+    return cfgPath;
+};
+/** Register (or update) a collection in the project config. */
+export const writeCollectionToConfig = (cwd, name, def) => {
+    const cfg = readProjectConfig(cwd);
+    const collections = (cfg.collections || {});
+    cfg.collections = {
+        ...collections,
+        [name]: {
+            path: def.path,
+            schema: def.schema,
+            ...(def.dedupField ? { dedupField: def.dedupField } : {}),
+            ...(def.dedupMode ? { dedupMode: def.dedupMode } : {}),
+            ...(def.dedupThreshold !== undefined ? { dedupThreshold: def.dedupThreshold } : {}),
+        },
+    };
+    return writeProjectConfig(cwd, cfg);
+};
+/** Run the full discover_schema flow and update config. */
+export const runDiscoverSchema = (cwd, file, name, sample = 100) => {
+    const resolvedFile = path.isAbsolute(file) ? file : path.join(cwd, file);
+    const parsed = discoverSchemaFromFile(resolvedFile, sample);
+    const collectionName = name?.trim() ||
+        path.basename(resolvedFile, path.extname(resolvedFile)).replace(/\W+/g, "_") ||
+        "discovered";
+    const collectionPath = shrinkPath(cwd, resolvedFile);
+    const def = {
+        path: collectionPath,
+        schema: parsed.schema,
+    };
+    writeCollectionToConfig(cwd, collectionName, def);
+    return {
+        collectionName,
+        path: collectionPath,
+        schema: parsed.schema,
+        types: parsed.types,
+        rowCount: parsed.rowCount,
+        sampled: parsed.sampled,
+    };
+};
+/** Register the `discover_schema` Pi tool. */
+export const registerDiscoverSchemaTool = (pi) => {
+    pi.registerTool({
+        name: "discover_schema",
+        label: "Discover JSONL Schema",
+        description: "Infer the schema of an unconfigured .jsonl file by sampling its first N rows and register it as a wiki collection.",
+        promptSnippet: 'discover_schema(file="data/events.jsonl", name="events", sample=100)',
+        promptGuidelines: [
+            "Use when you have a .jsonl file that is not yet mapped in pi-vault-mind.config.json.",
+            "The tool reads the first N rows, infers field names and JSON types, and writes a CollectionDef to the project config.",
+        ],
+        parameters: Type.Object({
+            file: Type.String({
+                description: "Path to the .jsonl file (absolute or relative to the project root)",
+            }),
+            name: Type.Optional(Type.String({ description: "Collection name to register (defaults to the file basename)" })),
+            sample: Type.Optional(Type.Number({
+                description: "Number of rows to sample for schema inference (default: 100)",
+                default: 100,
+            })),
+        }),
+        async execute(_id, params, _signal, _onUpdate, ctx) {
+            try {
+                const result = runDiscoverSchema(ctx.cwd, params.file, params.name, params.sample ?? 100);
+                const report = [
+                    `Discovered schema for "${result.collectionName}"`,
+                    `Path: ${result.path}`,
+                    `Rows read: ${result.rowCount}`,
+                    `Sampled: ${result.sampled}`,
+                    "",
+                    "Schema:",
+                    ...result.schema.map((field) => `  - ${field}: ${result.types[field]}`),
+                ].join("\n");
+                return { content: [{ type: "text", text: report }], details: { result } };
+            }
+            catch (error) {
+                return { content: [{ type: "text", text: `Error: ${error.message}` }], details: {} };
+            }
+        },
+    });
+};
+/** Parse CLI-style arguments for /wiki discover-schema. */
+const parseDiscoverSchemaArgs = (args) => {
+    const tokens = args.trim().split(/\s+/).filter(Boolean);
+    if (tokens.length === 0) {
+        return { error: "Usage: /wiki discover-schema <file> [--name <collection>] [--sample <n>]" };
+    }
+    let file;
+    let name;
+    let sample;
+    for (let i = 0; i < tokens.length; i++) {
+        const token = tokens[i];
+        if (token === "--name") {
+            name = tokens[++i];
+            if (!name)
+                return { error: "--name requires a value" };
+        }
+        else if (token === "--sample") {
+            const raw = tokens[++i];
+            if (!raw)
+                return { error: "--sample requires a value" };
+            const n = Number.parseInt(raw, 10);
+            if (!Number.isFinite(n) || n <= 0)
+                return { error: `--sample must be a positive integer, got "${raw}"` };
+            sample = n;
+        }
+        else if (!token.startsWith("--")) {
+            if (file)
+                return { error: `Unexpected extra argument: ${token}` };
+            file = token;
+        }
+        else {
+            return { error: `Unknown flag: ${token}` };
+        }
+    }
+    if (!file) {
+        return { error: "Missing .jsonl file path" };
+    }
+    return { file, name, sample };
+};
+/** Handler for the /wiki discover-schema slash command. */
+export const handleDiscoverSchema = async (args, ctx) => {
+    const parsed = parseDiscoverSchemaArgs(args);
+    if (parsed.error) {
+        ctx.ui.notify(parsed.error, "error");
+        return;
+    }
+    const { file, name, sample } = parsed;
+    if (!file) {
+        ctx.ui.notify("Missing .jsonl file path", "error");
+        return;
+    }
+    try {
+        const result = runDiscoverSchema(ctx.cwd, file, name, sample ?? 100);
+        const lines = [
+            `**Discovered schema: "${result.collectionName}"**`,
+            "",
+            `Path: ${result.path}`,
+            `Rows read: ${result.rowCount}`,
+            `Sampled: ${result.sampled}`,
+            "",
+            "Schema:",
+            ...result.schema.map((field) => `  • ${field}: ${result.types[field]}`),
+        ];
+        ctx.ui.notify(lines.join("\n"), "info");
+    }
+    catch (err) {
+        ctx.ui.notify(`discover-schema failed: ${err.message}`, "error");
+    }
+};

package/dist/src/index.js

@@ -3,6 +3,7 @@ import * as path from "node:path";
 import { startAutoSync } from "./autosync.js";
 import { registerCommands, selectActiveCollection, serverState, watcherState } from "./commands.js";
 import { handleBeforeAgentStart, handleTurnEnd } from "./events.js";
+import { registerIntakeTool } from "./intake.js";
 import { startServer, stopServer } from "./server.js";
 import { registerTools } from "./tools.js";
 import { EXT_ROOT } from "./utils.js";
@@ -30,6 +31,7 @@ export default function (pi) {
     registerCommands(pi);
     /* register tools */
     registerTools(pi);
+    registerIntakeTool(pi);
     /* before_agent_start: dynamic injectors */
     pi.on("before_agent_start", async (event, ctx) => {
         return handleBeforeAgentStart(pi, event, ctx);

package/dist/src/intake.d.ts

@@ -0,0 +1,4 @@
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { AskIntakeInput, AskIntakeResult } from "./types.js";
+export declare const runAskIntake: (ctx: ExtensionContext, params: AskIntakeInput) => Promise<AskIntakeResult>;
+export declare const registerIntakeTool: (pi: ExtensionAPI) => void;

package/dist/src/intake.js

@@ -0,0 +1,248 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { Type } from "typebox";
+import { ensureDir, expandHome, loadConfig } from "./utils.js";
+/**
+ * ask_intake — interactive questionnaire tool.
+ *
+ * Implementation notes (answers to the ambiguities in docs/plans/intake-tui.md):
+ *
+ * 1. Multi-select UI primitive: ctx.ui.custom() is unavailable in RPC mode
+ *    (returns undefined), so we build multi-select by looping ctx.ui.select()
+ *    with a control item ("✓ Done") and a "Cancel" item. This works in both
+ *    TUI and RPC and gives us an explicit cancel signal.
+ * 2. Vault path resolution: loadConfig() already expands wiki.vaults.default.path
+ *    (see src/utils.ts expandVaultPaths). We use cfg.wiki.vaults.default.path.
+ * 3. Atomic vault write: VaultWriter already ensures Agent/Inbox/ exists, so we
+ *    reuse it when available; fall back to mkdirSync + writeFileSync.
+ * 4. Progress indicator: ctx.ui.setStatus("ask_intake", "Question 2 of 7...")
+ *    works in both TUI and RPC (fire-and-forget in RPC).
+ * 5. Build verification: pnpm build.
+ */
+const OptionSchema = Type.Object({
+    label: Type.String({ description: "Display label for the option" }),
+    description: Type.Optional(Type.String({ description: "Description shown below or beside the label" })),
+});
+const QuestionSchema = Type.Object({
+    question: Type.String({ description: "Full prompt shown to the user" }),
+    header: Type.String({ description: "Short label used in the progress indicator" }),
+    options: Type.Optional(Type.Array(OptionSchema, { description: "Present = select; absent = free text" })),
+    multi_select: Type.Boolean({ description: "true = multi-select, false = single-select" }),
+});
+const AskIntakeParameters = Type.Object({
+    questions: Type.Array(QuestionSchema, { description: "At least one question" }),
+    persist: Type.Optional(Type.Union([Type.Literal("none"), Type.Literal("ledger"), Type.Literal("vault"), Type.Literal("both")], { description: "Where to persist answers (default: none)" })),
+    collection: Type.Optional(Type.String({ description: "Ledger collection (default: main)" })),
+});
+const CONTROL_DONE = "✓ Done";
+const CONTROL_CANCEL = "✗ Cancel";
+/** Render a single option label, optionally with its description. */
+const formatOption = (opt) => {
+    if (opt.description)
+        return `${opt.label} — ${opt.description}`;
+    return opt.label;
+};
+/** Recover the original option label from a formatted select value. */
+const findOptionLabel = (value, options) => {
+    for (const opt of options) {
+        if (formatOption(opt) === value)
+            return opt.label;
+    }
+    return undefined;
+};
+/** Random suffix for generated IDs. */
+const randomSuffix = () => `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
+export const runAskIntake = async (ctx, params) => {
+    const persist = params.persist ?? "none";
+    const collection = params.collection ?? "main";
+    const answers = {};
+    let cancelled = false;
+    try {
+        for (let i = 0; i < params.questions.length; i++) {
+            if (cancelled)
+                break;
+            const q = params.questions[i];
+            ctx.ui.setStatus("ask_intake", `Question ${i + 1} of ${params.questions.length}: ${q.header}`);
+            if (q.options && q.options.length > 0) {
+                if (q.multi_select) {
+                    const selected = [];
+                    let choosing = true;
+                    while (choosing) {
+                        const remaining = q.options
+                            .filter((opt) => !selected.includes(opt.label))
+                            .map((opt) => formatOption(opt));
+                        const items = [...remaining, CONTROL_DONE, CONTROL_CANCEL];
+                        const pick = await ctx.ui.select(`${q.question}\n(Selected: ${selected.length ? selected.join(", ") : "none"})`, items);
+                        if (pick === undefined || pick === CONTROL_CANCEL) {
+                            cancelled = true;
+                            choosing = false;
+                            break;
+                        }
+                        if (pick === CONTROL_DONE) {
+                            choosing = false;
+                            break;
+                        }
+                        const label = findOptionLabel(pick, q.options) ?? pick;
+                        if (!selected.includes(label))
+                            selected.push(label);
+                    }
+                    answers[q.question] = selected;
+                }
+                else {
+                    const items = q.options.map((opt) => formatOption(opt));
+                    const pick = await ctx.ui.select(q.question, items);
+                    if (pick === undefined) {
+                        cancelled = true;
+                    }
+                    else {
+                        answers[q.question] = findOptionLabel(pick, q.options) ?? pick;
+                    }
+                }
+            }
+            else {
+                const value = await ctx.ui.input(q.question);
+                if (value === undefined) {
+                    cancelled = true;
+                }
+                else {
+                    answers[q.question] = value;
+                }
+            }
+        }
+    }
+    finally {
+        ctx.ui.setStatus("ask_intake", undefined);
+    }
+    const partial = cancelled && Object.keys(answers).length > 0;
+    const result = { answers, cancelled, partial };
+    if (!cancelled && (persist === "ledger" || persist === "both")) {
+        const ledgerResult = persistToLedger(ctx, answers, collection);
+        if (ledgerResult)
+            result.ledgerEntry = ledgerResult;
+    }
+    if (!cancelled && (persist === "vault" || persist === "both")) {
+        const vaultResult = persistToVault(ctx, answers, params.questions);
+        if (vaultResult)
+            result.vaultEntry = vaultResult;
+    }
+    return result;
+};
+export const registerIntakeTool = (pi) => {
+    pi.registerTool({
+        name: "ask_intake",
+        label: "Ask Intake",
+        description: "Ask the user a structured questionnaire (single or multi-select, free text) and optionally persist answers to the wiki ledger, the Obsidian vault, or both.",
+        promptSnippet: 'ask_intake(questions=[{question:"...", header:"...", options:[{label:"...", description:"..."}], multi_select:false}], persist="none")',
+        promptGuidelines: [
+            "Use ask_intake when you need structured answers from the user and want them persisted.",
+            "persist='ledger' appends a row to the JSONL collection; persist='vault' writes a markdown note under Agent/Inbox/.",
+            "Answers are returned keyed by the question string.",
+        ],
+        parameters: AskIntakeParameters,
+        async execute(_id, params, _signal, _onUpdate, ctx) {
+            if (!ctx.hasUI) {
+                return {
+                    content: [{ type: "text", text: "ask_intake requires an interactive UI session." }],
+                    details: {
+                        answers: {},
+                        cancelled: true,
+                        partial: true,
+                    },
+                };
+            }
+            if (!params.questions || params.questions.length === 0) {
+                return {
+                    content: [{ type: "text", text: "ask_intake requires at least one question." }],
+                    details: {
+                        answers: {},
+                        cancelled: true,
+                        partial: true,
+                    },
+                };
+            }
+            const result = await runAskIntake(ctx, params);
+            const summary = result.cancelled
+                ? `Intake cancelled. Answered ${Object.keys(result.answers).length}/${params.questions.length} question(s).`
+                : `Intake complete. Answered ${params.questions.length} question(s).`;
+            return {
+                content: [{ type: "text", text: summary }],
+                details: result,
+            };
+        },
+    });
+};
+const persistToLedger = (ctx, answers, collection) => {
+    const cfg = loadConfig(ctx.cwd);
+    const def = cfg.collections[collection];
+    if (!def) {
+        ctx.ui.notify(`ask_intake: unknown collection "${collection}" — skipping ledger persist.`, "warning");
+        return null;
+    }
+    const id = `intake-${new Date().toISOString()}-${randomSuffix()}`;
+    const entry = {
+        id,
+        domain: "intake",
+        source: "intake_tui",
+        fact: JSON.stringify(answers),
+        tag: "intake",
+        artifact: "",
+    };
+    try {
+        ensureDir(def.path);
+        fs.appendFileSync(def.path, `${JSON.stringify(entry)}\n`, "utf-8");
+        return { id, path: def.path, tag: "intake" };
+    }
+    catch (err) {
+        ctx.ui.notify(`ask_intake: ledger write failed: ${err.message}`, "error");
+        return null;
+    }
+};
+const persistToVault = (ctx, answers, questions) => {
+    const cfg = loadConfig(ctx.cwd);
+    const vaultCfg = cfg.wiki.vaults?.default;
+    if (!vaultCfg?.path) {
+        ctx.ui.notify("ask_intake: no default vault configured — skipping vault persist.", "warning");
+        return null;
+    }
+    const vaultPath = expandHome(vaultCfg.path);
+    const inbox = path.join(vaultPath, "Agent", "Inbox");
+    const filename = `intake-${new Date().toISOString()}.md`;
+    const filepath = path.join(inbox, filename);
+    const lines = [
+        "---",
+        `created: ${new Date().toISOString()}`,
+        "source: intake_tui",
+        "tag: intake",
+        "---",
+        "",
+        "# Intake Answers",
+        "",
+    ];
+    for (const q of questions) {
+        const answer = answers[q.question];
+        lines.push(`## ${q.header || q.question}`);
+        lines.push("");
+        if (answer === undefined) {
+            lines.push("_Not answered_");
+        }
+        else if (Array.isArray(answer)) {
+            for (const a of answer) {
+                lines.push(`- ${a}`);
+            }
+        }
+        else {
+            lines.push(`- ${answer}`);
+        }
+        lines.push("");
+    }
+    try {
+        if (!fs.existsSync(inbox))
+            fs.mkdirSync(inbox, { recursive: true });
+        fs.writeFileSync(filepath, lines.join("\n"), "utf-8");
+        return { path: filepath };
+    }
+    catch (err) {
+        ctx.ui.notify(`ask_intake: vault write failed: ${err.message}`, "error");
+        return null;
+    }
+};

package/dist/src/server.d.ts

@@ -5,21 +5,52 @@
  *   GET  /vault-mind/status     → { running, dispatches, vaults, uptime }
  *   POST /vault-mind/scan       → scan a file for @agent markers { file }
  *   POST /vault-mind/dispatch   → fire a manual dispatch
+ *   POST /vault-mind/context    → push active editor context from Obsidian { filePath, selection, cursor }
+ *   GET  /vault-mind/context    → latest editor context (file, cursor, selection) for pi agents
  *
  * Binds 127.0.0.1 only — no network exposure. Port defaults to 11435,
  * configurable via wiki.httpPort in pi-vault-mind.config.json.
  *
  * Used by the obsidian-shellcommands plugin to notify pi of file saves,
  * replacing fs.watch reliance with explicit push notifications.
+ *
+ * New Stage-1 routes (auth-gated by `PVM_API_TOKEN`):
+ *   GET  /wiki/status, POST /wiki/init, POST /wiki/setup, POST /server/start,
+ *   GET  /vault-mind/config, GET /agent/queue, GET /agent/jobs/:id,
+ *   POST /agent/jobs/:id/retry, POST /agent/jobs/:id/cancel,
+ *   POST /wiki/search, POST /wiki/append, WS /agent/stream.
  */
 import * as http from "node:http";
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { WebSocketServer } from "ws";
 import type { WatcherState } from "./watcher.js";
 export interface ServerState {
     server: http.Server | null;
+    /** WebSocket server for live queue streaming. */
+    wss: WebSocketServer | null;
     port: number;
     startTime: number;
+    /** The pi ExtensionAPI, stored so REST routes can call pi.sendUserMessage etc. */
+    pi?: ExtensionAPI;
+    /** The watcher state, stored so REST routes can read queue / status. */
+    watcherState?: WatcherState;
+    /** Latest editor context pushed from Obsidian (active file, cursor, selection). */
+    latestEditorContext?: {
+        filePath: string | null;
+        selection: string | null;
+        cursor: number | null;
+    };
+    /** The vault path this server is serving (default vault from global config, or cwd). */
+    vaultPath?: string;
 }
 export declare function createServerState(port?: number): ServerState;
 export declare function startServer(pi: ExtensionAPI, serverState: ServerState, watcherState: WatcherState): void;
 export declare function stopServer(serverState: ServerState): void;
+/**
+ * Wrap a request handler with bearer-token auth. Use this for every
+ * new Stage-1 route so auth is consistent and testable in one place.
+ * The handler is only invoked if `authoriseRequest` passes.
+ */
+export declare const withAuth: (req: http.IncomingMessage, res: http.ServerResponse, handler: () => void) => void;
+/** Read the full request body as UTF-8. Already used by handleScan; exposed for new routes. */
+export declare const readJsonBody: (req: http.IncomingMessage) => Promise<unknown>;