pdf-brain 1.2.0 → 2.0.0

package/README.md

@@ -26,9 +26,12 @@ Local **PDF & Markdown** knowledge base with semantic search and AI-powered enri
 
 ## Quick Start
 
+> Note: `pdf-brain` is agent-first and emits a single JSON envelope to stdout by default.  
+> Use `--format text` for human-readable output (and TUI/progress rendering), or inspect the machine contract via `pdf-brain capabilities`.
+
 ```bash
 # 1. Install (standalone binary, no runtime needed)
-curl -fsSL https://raw.githubusercontent.com/joelhooks/pdf-library/main/scripts/install.sh | bash
+curl -fsSL https://raw.githubusercontent.com/joelhooks/pdf-brain/main/scripts/install.sh | bash
 
 # 2. Install Ollama (macOS)
 brew install ollama
@@ -81,7 +84,7 @@ ollama serve
 
 ```bash
 # Standalone binary (no runtime needed)
-curl -fsSL https://raw.githubusercontent.com/joelhooks/pdf-library/main/scripts/install.sh | bash
+curl -fsSL https://raw.githubusercontent.com/joelhooks/pdf-brain/main/scripts/install.sh | bash
 
 # or via npm
 npm install -g pdf-brain
@@ -89,6 +92,21 @@ npm install -g pdf-brain
 
 ## CLI Reference
 
+### Agent Output (Default)
+
+`pdf-brain` is optimized for agentic workflows: stdout is machine-readable by default.
+
+- `--format json|ndjson|text` (default: `json`)
+- `--pretty` pretty-print JSON
+- `--quiet` (alias: `--no-hints`) omit `nextActions`
+- `--log-level silent|error|info|debug` (logs go to stderr)
+
+Discover the full command/tool contract (including JSON Schemas) at runtime:
+
+```bash
+pdf-brain capabilities
+```
+
 ### Basic Commands
 
 ```bash
@@ -382,6 +400,8 @@ pdf-brain config set enrichment.model anthropic/claude-haiku-4-5
 | `PDF_LIBRARY_PATH`   | `~/Documents/.pdf-library` | Library storage location |
 | `OLLAMA_HOST`        | `http://localhost:11434`   | Ollama API endpoint      |
 | `AI_GATEWAY_API_KEY` | -                          | API key for AI Gateway   |
+| `PDF_BRAIN_LOG_LEVEL` | `silent`                  | stderr logging verbosity |
+| `PDF_BRAIN_QUERY_EMBED_CACHE_SIZE` | `256`        | Query embedding LRU cache size (0 disables) |
 
 ### AI Gateway
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pdf-brain",
-  "version": "1.2.0",
+  "version": "2.0.0",
   "description": "Local PDF & Markdown knowledge base with semantic search, AI enrichment, and SKOS taxonomy",
   "type": "module",
   "main": "src/index.ts",
@@ -29,6 +29,7 @@
     "@effect/schema": "^0.75.0",
     "@electric-sql/pglite": "^0.3.0",
     "@libsql/client": "^0.15.15",
+    "@modelcontextprotocol/sdk": "1.26.0",
     "ai": "^5.0.115",
     "dotenv": "^17.2.3",
     "effect": "^3.12.0",

package/scripts/install.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-REPO="joelhooks/pdf-library"
+REPO="joelhooks/pdf-brain"
 BINARY="pdf-brain"
 INSTALL_DIR="${PDF_BRAIN_INSTALL_DIR:-/usr/local/bin}"
 

package/src/agent/hints.ts

@@ -4,15 +4,22 @@
  * Pure function: CommandResult discriminated union in, string[] hints out.
  */
 
+import type { NextAction } from "./protocol.js";
+
 export type CommandResult =
   | {
       _tag: "search";
       query: string;
-      results: { title: string; docId: string; score: number }[];
+      results: { title: string; docId: string; chunkId?: string; score: number }[];
       concepts: { id: string; prefLabel: string }[];
       hadExpand: boolean;
       wasFts: boolean;
     }
+  | {
+      _tag: "searchPack";
+      queries: string[];
+      results: { title: string; docId: string; chunkId?: string; score: number }[];
+    }
   | { _tag: "read"; title: string; id: string; tags: string[] }
   | {
       _tag: "list";
@@ -37,12 +44,29 @@ export type CommandResult =
   | { _tag: "remove"; title: string }
   | { _tag: "noResults"; query: string; wasFts: boolean }
   | { _tag: "error"; command: string; message: string }
-  | { _tag: "doctor"; healthy: boolean }
+  | {
+      _tag: "doctor";
+      healthy: boolean;
+      chunkerOutdated?: number;
+      chunkerMissing?: number;
+      chunkerMismatch?: number;
+    }
   | { _tag: "config"; subcommand: string }
   | { _tag: "tag"; title: string; tags: string[] }
   | { _tag: "check"; reachable: boolean }
   | { _tag: "repair"; orphanedChunks: number; orphanedEmbeddings: number }
-  | { _tag: "reindex"; count: number; errors: number };
+  | { _tag: "reindex"; count: number; errors: number }
+  | {
+      _tag: "rechunk";
+      dryRun: boolean;
+      planned: number;
+      succeeded: number;
+      failed: number;
+      includeMissing?: boolean;
+      skippedMissing?: number;
+      plannedMissing?: number;
+      plannedMismatch?: number;
+    };
 
 /**
  * Generate contextual next-action hints from a command result.
@@ -84,6 +108,26 @@ export function generateHints(result: CommandResult): string[] {
       return hints;
     }
 
+    case "searchPack": {
+      const hints: string[] = [];
+      if (result.results.length > 0) {
+        const top = result.results[0];
+        hints.push(
+          `\`pdf-brain read "${top.title}"\` -- Full metadata for top result`
+        );
+        if (top.chunkId) {
+          hints.push(
+            `\`pdf-brain chunk get "${top.chunkId}"\` -- Fetch exact top chunk text`
+          );
+        }
+      }
+      hints.push(
+        `\`pdf-brain search "<query>"\` -- Drill into a single query`,
+        `\`pdf-brain search-pack --with-content "${result.queries[0] ?? "query"}"\` -- Include chunk text in pack output`,
+      );
+      return hints;
+    }
+
     case "noResults": {
       const hints: string[] = [];
       if (!result.wasFts) {
@@ -219,6 +263,22 @@ export function generateHints(result: CommandResult): string[] {
           `\`pdf-brain doctor --fix\` -- Auto-repair detected issues`
         );
       }
+      const missing = result.chunkerMissing ?? 0;
+      const mismatch = result.chunkerMismatch ?? 0;
+
+      if (mismatch > 0) {
+        hints.push(
+          `\`pdf-brain rechunk --dry-run\` -- Preview docs with stale chunker metadata`,
+          `\`pdf-brain rechunk\` -- Apply rechunk (rebuild chunks + embeddings)`,
+        );
+      }
+
+      if (missing > 0) {
+        hints.push(
+          `\`pdf-brain rechunk --dry-run --include-missing\` -- Preview docs missing chunker metadata (upgrade sweep)`,
+          `\`pdf-brain rechunk --include-missing --max-docs 25\` -- Rechunk a small batch (expensive)`,
+        );
+      }
       hints.push(
         `\`pdf-brain stats\` -- Check library statistics`,
         `\`pdf-brain search "<query>"\` -- Search documents`
@@ -257,6 +317,32 @@ export function generateHints(result: CommandResult): string[] {
       ];
     }
 
+    case "rechunk": {
+      const hints: string[] = [];
+      if (result.dryRun) {
+        if (result.includeMissing) {
+          hints.push(
+            `\`pdf-brain rechunk --include-missing --max-docs 25\` -- Rechunk a small batch (rebuild chunks + embeddings)`,
+          );
+        } else {
+          hints.push(
+            `\`pdf-brain rechunk\` -- Apply rechunk (rebuild chunks + embeddings)`,
+          );
+          if ((result.skippedMissing ?? 0) > 0) {
+            hints.push(
+              `\`pdf-brain rechunk --dry-run --include-missing\` -- Include missing-metadata docs in the plan`,
+            );
+          }
+        }
+      } else {
+        hints.push(
+          `\`pdf-brain stats\` -- Verify counts after rechunk`,
+          `\`pdf-brain search "<query>"\` -- Sanity-check retrieval quality`,
+        );
+      }
+      return hints;
+    }
+
     case "reindex": {
       return [
         `\`pdf-brain stats\` -- Check updated statistics`,
@@ -278,3 +364,340 @@ export function generateHints(result: CommandResult): string[] {
     }
   }
 }
+
+/**
+ * Structured follow-up actions for agent workflows.
+ * These are equivalent to `generateHints`, but machine-friendly.
+ */
+export function generateNextActions(result: CommandResult): NextAction[] {
+  switch (result._tag) {
+    case "search": {
+      const actions: NextAction[] = [];
+      if (result.results.length > 0) {
+        const top = result.results[0];
+        actions.push({
+          kind: "shell",
+          argv: ["pdf-brain", "read", top.docId],
+          description: "Full metadata for top result",
+        });
+
+        if (top.chunkId) {
+          actions.push({
+            kind: "shell",
+            argv: ["pdf-brain", "chunk", "get", top.chunkId],
+            description: "Fetch exact top chunk text",
+          });
+        }
+
+        if (!result.hadExpand) {
+          actions.push({
+            kind: "shell",
+            argv: ["pdf-brain", "search", result.query, "--expand", "2000"],
+            description: "Get expanded context around matches",
+          });
+        }
+      }
+
+      if (result.concepts.length > 0) {
+        const topConcept = result.concepts[0];
+        actions.push({
+          kind: "shell",
+          argv: ["pdf-brain", "taxonomy", "tree", topConcept.id],
+          description: "Navigate concept hierarchy",
+        });
+      }
+
+      if (result.results.length > 0 && !result.wasFts) {
+        actions.push({
+          kind: "shell",
+          argv: ["pdf-brain", "search", result.query, "--fts"],
+          description: "Try keyword matching instead",
+        });
+      }
+
+      if (result.results.length === 0 && result.concepts.length === 0) {
+        return generateNextActions({
+          _tag: "noResults",
+          query: result.query,
+          wasFts: result.wasFts,
+        });
+      }
+
+      return actions;
+    }
+
+    case "searchPack": {
+      const actions: NextAction[] = [];
+      if (result.results.length > 0) {
+        const top = result.results[0];
+        actions.push({
+          kind: "shell",
+          argv: ["pdf-brain", "read", top.docId],
+          description: "Read top document metadata",
+        });
+        if (top.chunkId) {
+          actions.push({
+            kind: "shell",
+            argv: ["pdf-brain", "chunk", "get", top.chunkId],
+            description: "Fetch exact top chunk text",
+          });
+        }
+      }
+      actions.push({
+        kind: "shell",
+        argv: ["pdf-brain", "search", "your query here"],
+        description: "Drill into a single query",
+      });
+      return actions;
+    }
+
+    case "noResults": {
+      const actions: NextAction[] = [];
+      if (!result.wasFts) {
+        actions.push({
+          kind: "shell",
+          argv: ["pdf-brain", "search", result.query, "--fts"],
+          description: "Try full-text keyword search",
+        });
+      } else {
+        actions.push({
+          kind: "shell",
+          argv: ["pdf-brain", "search", result.query],
+          description: "Try semantic vector search",
+        });
+      }
+      actions.push(
+        { kind: "shell", argv: ["pdf-brain", "list"], description: "Browse all documents" },
+        {
+          kind: "shell",
+          argv: ["pdf-brain", "taxonomy", "search", result.query],
+          description: "Search taxonomy concepts",
+        },
+      );
+      return actions;
+    }
+
+    case "read": {
+      const actions: NextAction[] = [];
+      actions.push({
+        kind: "shell",
+        argv: ["pdf-brain", "search", result.title, "--expand", "2000"],
+        description: "Search within this document's content",
+      });
+      if (result.tags.length > 0) {
+        actions.push({
+          kind: "shell",
+          argv: ["pdf-brain", "list", "--tag", result.tags[0]],
+          description: "Browse documents with same tag",
+        });
+      }
+      actions.push({
+        kind: "shell",
+        argv: ["pdf-brain", "taxonomy", "search", result.title],
+        description: "Find related concepts",
+      });
+      return actions;
+    }
+
+    case "list": {
+      const actions: NextAction[] = [];
+      if (result.firstDoc) {
+        actions.push({
+          kind: "shell",
+          argv: ["pdf-brain", "read", result.firstDoc.id],
+          description: "Read the first listed document",
+        });
+      }
+      actions.push({
+        kind: "shell",
+        argv: ["pdf-brain", "search", "your query here"],
+        description: "Search the library",
+      });
+      return actions;
+    }
+
+    case "stats": {
+      return [
+        { kind: "shell", argv: ["pdf-brain", "search", "your question here"], description: "Search the library" },
+        { kind: "shell", argv: ["pdf-brain", "list"], description: "Browse all documents" },
+        { kind: "shell", argv: ["pdf-brain", "taxonomy", "list"], description: "Browse taxonomy concepts" },
+        { kind: "shell", argv: ["pdf-brain", "doctor"], description: "Check database health" },
+      ];
+    }
+
+    case "taxonomySearch": {
+      const actions: NextAction[] = [];
+      if (result.matches.length > 0) {
+        actions.push({
+          kind: "shell",
+          argv: ["pdf-brain", "taxonomy", "tree", result.matches[0].id],
+          description: "Navigate concept hierarchy",
+        });
+      } else {
+        actions.push({
+          kind: "shell",
+          argv: ["pdf-brain", "taxonomy", "list"],
+          description: "Browse all concepts",
+        });
+      }
+      actions.push({
+        kind: "shell",
+        argv: ["pdf-brain", "search", result.query],
+        description: "Search documents for this concept",
+      });
+      return actions;
+    }
+
+    case "taxonomyList": {
+      return [
+        { kind: "shell", argv: ["pdf-brain", "taxonomy", "tree"], description: "View full concept tree" },
+        { kind: "shell", argv: ["pdf-brain", "taxonomy", "search", "your query"], description: "Search concepts" },
+      ];
+    }
+
+    case "taxonomyTree": {
+      return [
+        { kind: "shell", argv: ["pdf-brain", "taxonomy", "tree"], description: "View full concept tree" },
+      ];
+    }
+
+    case "add": {
+      return [
+        { kind: "shell", argv: ["pdf-brain", "read", result.id], description: "Read the new document" },
+        { kind: "shell", argv: ["pdf-brain", "search", result.title], description: "Search for related content" },
+        { kind: "shell", argv: ["pdf-brain", "tag", result.id, "tag1,tag2"], description: "Apply tags" },
+      ];
+    }
+
+    case "remove": {
+      return [
+        { kind: "shell", argv: ["pdf-brain", "list"], description: "Browse remaining documents" },
+        { kind: "shell", argv: ["pdf-brain", "stats"], description: "Verify counts" },
+      ];
+    }
+
+    case "tag": {
+      const actions: NextAction[] = [
+        { kind: "shell", argv: ["pdf-brain", "read", result.title], description: "Read document metadata" },
+        { kind: "shell", argv: ["pdf-brain", "list", "--tag", result.tags[0] ?? ""], description: "Browse by tag" },
+      ];
+      return actions.filter((a) => a.argv[a.argv.length - 1] !== "");
+    }
+
+    case "doctor": {
+      const actions: NextAction[] = [];
+      if (!result.healthy) {
+        actions.push({
+          kind: "shell",
+          argv: ["pdf-brain", "doctor", "--fix"],
+          description: "Attempt auto-repair",
+        });
+      }
+      const missing = result.chunkerMissing ?? 0;
+      const mismatch = result.chunkerMismatch ?? 0;
+
+      if (mismatch > 0) {
+        actions.push(
+          {
+            kind: "shell",
+            argv: ["pdf-brain", "rechunk", "--dry-run"],
+            description: "Preview docs with stale chunker metadata",
+          },
+          {
+            kind: "shell",
+            argv: ["pdf-brain", "rechunk"],
+            description: "Apply rechunk (rebuild chunks + embeddings)",
+          },
+        );
+      }
+
+      if (missing > 0) {
+        actions.push(
+          {
+            kind: "shell",
+            argv: ["pdf-brain", "rechunk", "--dry-run", "--include-missing"],
+            description: "Preview docs missing chunker metadata (upgrade sweep)",
+          },
+          {
+            kind: "shell",
+            argv: ["pdf-brain", "rechunk", "--include-missing", "--max-docs", "25"],
+            description: "Rechunk a small batch (expensive)",
+          },
+        );
+      }
+      actions.push({
+        kind: "shell",
+        argv: ["pdf-brain", "stats"],
+        description: "Verify counts",
+      });
+      return actions;
+    }
+
+    case "config": {
+      return [
+        { kind: "shell", argv: ["pdf-brain", "config", "show"], description: "Show config" },
+      ];
+    }
+
+    case "check": {
+      return [
+        { kind: "shell", argv: ["pdf-brain", "stats"], description: "Check library stats" },
+      ];
+    }
+
+    case "repair": {
+      return [
+        { kind: "shell", argv: ["pdf-brain", "doctor"], description: "Re-run health check" },
+      ];
+    }
+
+    case "reindex": {
+      return [
+        { kind: "shell", argv: ["pdf-brain", "stats"], description: "Verify counts" },
+      ];
+    }
+
+    case "rechunk": {
+      if (result.dryRun) {
+        if (result.includeMissing) {
+          return [
+            {
+              kind: "shell",
+              argv: ["pdf-brain", "rechunk", "--include-missing", "--max-docs", "25"],
+              description: "Rechunk a small batch (rebuild chunks + embeddings)",
+            },
+          ];
+        }
+
+        const actions: NextAction[] = [
+          {
+            kind: "shell",
+            argv: ["pdf-brain", "rechunk"],
+            description: "Apply rechunk (rebuild chunks + embeddings)",
+          },
+        ];
+
+        if ((result.skippedMissing ?? 0) > 0) {
+          actions.push({
+            kind: "shell",
+            argv: ["pdf-brain", "rechunk", "--dry-run", "--include-missing"],
+            description: "Include missing-metadata docs in the plan",
+          });
+        }
+
+        return actions;
+      }
+      return [
+        { kind: "shell", argv: ["pdf-brain", "stats"], description: "Verify counts" },
+      ];
+    }
+
+    case "error": {
+      return [
+        { kind: "shell", argv: ["pdf-brain", "doctor"], description: "Check database health" },
+        { kind: "shell", argv: ["pdf-brain", "check"], description: "Check embedding provider connectivity" },
+        { kind: "shell", argv: ["pdf-brain", "--help"], description: "Show available commands" },
+      ];
+    }
+  }
+}

package/src/agent/manifest.ts

@@ -33,11 +33,23 @@ ${docCount} documents indexed. Every command returns contextual next-action hint
     --docs-only          Search documents only
     --include-clusters   Include multi-scale cluster summaries
 
+  pdf-brain search-pack "<q1>" "<q2>" ... [options]
+    --limit <n>          Max results per query (default 10)
+    --global-limit <n>   Max deduped results across all queries (optional)
+    --fts                Full-text search only (keyword matching)
+    --expand <chars>     Surrounding context (up to 4000 chars)
+    --with-content       Include chunk text in pack output (default: handles only)
+
 ### Read & Browse
   pdf-brain read "<id|title>"       Document metadata (title, pages, tags, path)
   pdf-brain list [--tag <tag>]      All documents, optionally filtered by tag
   pdf-brain stats                   Library statistics (doc/chunk/embedding counts)
 
+### Progressive Disclosure (agent primitives)
+  pdf-brain chunk get <chunkId>           Fetch a single chunk's full text
+  pdf-brain doc chunks <docId> [--page N] List chunk IDs for a document (optionally by page)
+  pdf-brain page get <docId> <page>       Reconstruct full page text by concatenating chunks
+
 ### Taxonomy (concept navigation)
   pdf-brain taxonomy search "<q>"   Find concepts by keyword or semantic similarity
   pdf-brain taxonomy tree [id]      Visual hierarchy tree from a concept
@@ -51,17 +63,23 @@ ${docCount} documents indexed. Every command returns contextual next-action hint
   pdf-brain ingest <dir> [--enrich] [--auto-tag] [--recursive]
 
 ### Maintenance
+  pdf-brain capabilities            Self-describing command list + JSON Schemas
+  pdf-brain mcp                     Start MCP server (stdio) for tool-based agent access
+  pdf-brain update                  Self-update to latest release
   pdf-brain doctor [--fix]          Health check (WAL, orphans, connectivity)
   pdf-brain config show|get|set     View/modify configuration
   pdf-brain reindex [--clean]       Re-embed all documents
+  pdf-brain rechunk [--dry-run] [--include-missing] [--max-docs N] [--max-chunks N]  Rebuild chunks + embeddings when the chunker changes
   pdf-brain export / import         Backup and restore
 
 ## Agent Workflow
 1. \`search\` -> find relevant chunks with similarity scores
-2. \`search --expand 2000\` -> get full surrounding context for deeper reading
-3. \`read\` -> get document metadata (title, tags, page count)
-4. \`taxonomy search\` -> find concept categories, then \`taxonomy tree\` to navigate
-5. \`list --tag\` -> discover documents by topic area
+2. Copy chunk IDs from \`search\` output -> \`chunk get\` to pull full text precisely
+3. Use \`doc chunks\` / \`page get\` to expand context only when needed
+4. \`search --expand 2000\` -> get full surrounding context for deeper reading
+5. \`read\` -> get document metadata (title, tags, page count)
+6. \`taxonomy search\` -> find concept categories, then \`taxonomy tree\` to navigate
+7. \`list --tag\` -> discover documents by topic area
 
 ## Tips
 - Scores closer to 1.0 = stronger semantic match
@@ -74,5 +92,8 @@ ${docCount} documents indexed. Every command returns contextual next-action hint
 ## Options
   --help, -h            Show this help
   --version, -v         Show version
+  --format <mode>       Output mode: json (default), ndjson, text
+  --pretty              Pretty-print JSON
+  --log-level <level>   stderr logs: silent (default), error, info, debug
   --quiet, --no-hints   Suppress next-action hints`;
 }

package/src/agent/protocol.ts

@@ -0,0 +1,52 @@
+/**
+ * Agent-first output protocol for pdf-brain.
+ *
+ * Design goals:
+ * - stdout is machine-readable (JSON by default)
+ * - stderr is diagnostics only (opt-in via log-level)
+ * - stable envelope so agents can reliably parse responses and chain next actions
+ */
+
+export const PDF_BRAIN_PROTOCOL_VERSION = 1 as const;
+
+export type OutputFormat = "json" | "ndjson" | "text";
+export type LogLevel = "silent" | "error" | "info" | "debug";
+
+export interface NextAction {
+  kind: "shell";
+  argv: string[];
+  description?: string;
+}
+
+export interface AgentErrorShape {
+  code: string;
+  message: string;
+  details?: unknown;
+}
+
+export type AgentEnvelope<T> =
+  | {
+      ok: true;
+      command: string;
+      protocolVersion: typeof PDF_BRAIN_PROTOCOL_VERSION;
+      result: T;
+      nextActions?: NextAction[];
+      meta?: Record<string, unknown>;
+    }
+  | {
+      ok: false;
+      command: string;
+      protocolVersion: typeof PDF_BRAIN_PROTOCOL_VERSION;
+      error: AgentErrorShape;
+      nextActions?: NextAction[];
+      meta?: Record<string, unknown>;
+    };
+
+export function toJsonLine(
+  value: unknown,
+  opts?: { pretty?: boolean }
+): string {
+  const pretty = opts?.pretty === true;
+  return JSON.stringify(value, null, pretty ? 2 : 0) + "\n";
+}
+