pdf-brain 1.3.0 → 2.0.0

package/src/chunking.ts

@@ -0,0 +1,130 @@
+import type { Document, LibraryConfig } from "./types.js";
+
+export type ChunkerMetadata = {
+  /** Stable identifier for the chunking algorithm implementation */
+  id: string;
+  /** Monotonic version for breaking chunker changes */
+  version: number;
+  /** Unit used for chunk sizing and overlap */
+  unit: "chars";
+  chunkSize: number;
+  chunkOverlap: number;
+};
+
+export const CURRENT_CHUNKER: Record<
+  "pdf" | "markdown",
+  { id: string; version: number }
+> = {
+  // v2: paragraph-preserving normalization + hyphenation fix
+  pdf: { id: "pdf-extractor:paragraphs-v2", version: 2 },
+  // v1: section-aware markdown parsing + placeholder-preserving chunking
+  markdown: { id: "markdown-extractor:sections+placeholders-v1", version: 1 },
+};
+
+export function inferFileTypeFromPath(path: string): "pdf" | "markdown" {
+  const lower = path.toLowerCase();
+  if (lower.endsWith(".md") || lower.endsWith(".markdown")) return "markdown";
+  return "pdf";
+}
+
+export function buildChunkerMetadata(
+  fileType: "pdf" | "markdown",
+  config: Pick<LibraryConfig, "chunkSize" | "chunkOverlap">,
+): ChunkerMetadata {
+  const base = CURRENT_CHUNKER[fileType];
+  return {
+    id: base.id,
+    version: base.version,
+    unit: "chars",
+    chunkSize: config.chunkSize,
+    chunkOverlap: config.chunkOverlap,
+  };
+}
+
+export function parseChunkerMetadata(value: unknown): ChunkerMetadata | null {
+  if (!value || typeof value !== "object") return null;
+  const v: any = value;
+
+  const id = typeof v.id === "string" ? v.id : null;
+  const version = typeof v.version === "number" ? v.version : null;
+  const unit = v.unit === "chars" ? ("chars" as const) : null;
+  const chunkSize = typeof v.chunkSize === "number" ? v.chunkSize : null;
+  const chunkOverlap = typeof v.chunkOverlap === "number" ? v.chunkOverlap : null;
+
+  if (!id || version === null || !unit || chunkSize === null || chunkOverlap === null) {
+    return null;
+  }
+
+  return { id, version, unit, chunkSize, chunkOverlap };
+}
+
+export function getDocChunkerMetadata(doc: Document): ChunkerMetadata | null {
+  const meta = doc.metadata;
+  if (!meta || typeof meta !== "object") return null;
+  const chunker = (meta as any).chunker;
+  return parseChunkerMetadata(chunker);
+}
+
+export function assessDocChunker(
+  doc: Document,
+  config: Pick<LibraryConfig, "chunkSize" | "chunkOverlap">,
+): {
+  needsRechunk: boolean;
+  code:
+    | "ok"
+    | "missing_metadata"
+    | "id_version_mismatch"
+    | "config_mismatch"
+    | "unit_mismatch";
+  reason: string;
+  expected: ChunkerMetadata;
+  actual: ChunkerMetadata | null;
+} {
+  const fileType =
+    doc.fileType ??
+    (doc.path ? inferFileTypeFromPath(doc.path) : ("pdf" as const));
+  const expected = buildChunkerMetadata(fileType, config);
+  const actual = getDocChunkerMetadata(doc);
+
+  if (!actual) {
+    return {
+      needsRechunk: true,
+      code: "missing_metadata",
+      reason: "missing chunker metadata",
+      expected,
+      actual: null,
+    };
+  }
+
+  if (actual.id !== expected.id || actual.version !== expected.version) {
+    return {
+      needsRechunk: true,
+      code: "id_version_mismatch",
+      reason: `chunker id/version mismatch (${actual.id}@${actual.version} != ${expected.id}@${expected.version})`,
+      expected,
+      actual,
+    };
+  }
+
+  if (actual.chunkSize !== expected.chunkSize || actual.chunkOverlap !== expected.chunkOverlap) {
+    return {
+      needsRechunk: true,
+      code: "config_mismatch",
+      reason: `chunkSize/chunkOverlap mismatch (${actual.chunkSize}/${actual.chunkOverlap} != ${expected.chunkSize}/${expected.chunkOverlap})`,
+      expected,
+      actual,
+    };
+  }
+
+  if (actual.unit !== expected.unit) {
+    return {
+      needsRechunk: true,
+      code: "unit_mismatch",
+      reason: `chunk unit mismatch (${actual.unit} != ${expected.unit})`,
+      expected,
+      actual,
+    };
+  }
+
+  return { needsRechunk: false, code: "ok", reason: "ok", expected, actual };
+}

package/src/cli.contract.test.ts

@@ -0,0 +1,239 @@
+import { describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import { Client } from "@modelcontextprotocol/sdk/client";
+import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
+
+function runCli(
+  argv: string[],
+  opts?: { env?: Record<string, string | undefined> },
+): { exitCode: number; stdout: string; stderr: string } {
+  const env: Record<string, string> = {
+    ...process.env,
+    ...(opts?.env ?? {}),
+  } as any;
+
+  const proc = Bun.spawnSync([process.execPath, "run", "src/cli.ts", ...argv], {
+    env,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+
+  return {
+    exitCode: proc.exitCode ?? 0,
+    stdout: new TextDecoder().decode(proc.stdout),
+    stderr: new TextDecoder().decode(proc.stderr),
+  };
+}
+
+function withTempLibraryPath<T>(fn: (libraryPath: string) => T): T {
+  const dir = mkdtempSync(join(tmpdir(), "pdf-brain-cli-contract-"));
+  try {
+    return fn(dir);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+async function withTempLibraryPathAsync<T>(
+  fn: (libraryPath: string) => Promise<T>,
+): Promise<T> {
+  const dir = mkdtempSync(join(tmpdir(), "pdf-brain-cli-contract-"));
+  try {
+    return await fn(dir);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+describe("CLI JSON Envelope Contract", () => {
+  test("stats emits a single JSON envelope with nextActions when not --quiet", () =>
+    withTempLibraryPath((libraryPath) => {
+      const res = runCli(["stats", "--format", "json"], {
+        env: {
+          PDF_LIBRARY_PATH: libraryPath,
+          // Avoid touching any real local Ollama instance during tests.
+          OLLAMA_HOST: "http://127.0.0.1:1",
+        },
+      });
+
+      expect(res.exitCode).toBe(0);
+      expect(res.stdout.trim().startsWith("{")).toBe(true);
+
+      const obj = JSON.parse(res.stdout);
+      expect(obj.ok).toBe(true);
+      expect(obj.command).toBe("stats");
+      expect(obj.protocolVersion).toBe(1);
+      expect(obj.result).toBeDefined();
+      expect(obj.result.libraryPath).toBe(libraryPath);
+      expect(obj.result.documents).toBe(0);
+      expect(obj.result.chunks).toBe(0);
+      expect(obj.result.embeddings).toBe(0);
+
+      // Agent mode: nextActions should exist by default (unless --quiet)
+      expect(Array.isArray(obj.nextActions)).toBe(true);
+      expect(obj.nextActions.length).toBeGreaterThan(0);
+    }));
+
+  test("stats with --quiet omits nextActions", () =>
+    withTempLibraryPath((libraryPath) => {
+      const res = runCli(["stats", "--format", "json", "--quiet"], {
+        env: {
+          PDF_LIBRARY_PATH: libraryPath,
+          OLLAMA_HOST: "http://127.0.0.1:1",
+        },
+      });
+
+      expect(res.exitCode).toBe(0);
+      const obj = JSON.parse(res.stdout);
+      expect(obj.ok).toBe(true);
+      expect(obj.command).toBe("stats");
+      expect("nextActions" in obj).toBe(false);
+    }));
+
+  test("invalid --format returns a structured error envelope and non-zero exit code", () =>
+    withTempLibraryPath((libraryPath) => {
+      const res = runCli(["--format", "wat", "stats"], {
+        env: {
+          PDF_LIBRARY_PATH: libraryPath,
+          OLLAMA_HOST: "http://127.0.0.1:1",
+        },
+      });
+
+      expect(res.exitCode).not.toBe(0);
+      const obj = JSON.parse(res.stdout);
+      expect(obj.ok).toBe(false);
+      expect(obj.protocolVersion).toBe(1);
+      expect(obj.error).toBeDefined();
+      expect(obj.error.code).toBe("INVALID_FLAG");
+    }));
+
+  test("rechunk flag validation: --max-docs requires a numeric value", () =>
+    withTempLibraryPath((libraryPath) => {
+      const res = runCli(["rechunk", "--max-docs", "--format", "json"], {
+        env: {
+          PDF_LIBRARY_PATH: libraryPath,
+          OLLAMA_HOST: "http://127.0.0.1:1",
+        },
+      });
+
+      expect(res.exitCode).not.toBe(0);
+      const obj = JSON.parse(res.stdout);
+      expect(obj.ok).toBe(false);
+      expect(obj.error.code).toBe("INVALID_ARGS");
+      expect(String(obj.error.message)).toContain("--max-docs");
+    }));
+
+  test("capabilities is self-describing and includes JSON Schemas", () =>
+    withTempLibraryPath((libraryPath) => {
+      const res = runCli(["capabilities", "--format", "json", "--quiet"], {
+        env: {
+          PDF_LIBRARY_PATH: libraryPath,
+          OLLAMA_HOST: "http://127.0.0.1:1",
+        },
+      });
+
+      expect(res.exitCode).toBe(0);
+      const obj = JSON.parse(res.stdout);
+      expect(obj.ok).toBe(true);
+      expect(obj.command).toBe("capabilities");
+      expect(obj.protocolVersion).toBe(1);
+
+      const result = obj.result;
+      expect(result).toBeDefined();
+      expect(result.protocolVersion).toBe(1);
+      expect(typeof result.pdfBrainVersion).toBe("string");
+
+      // Command list invariants (agent discovery depends on these names)
+      const commandNames = new Set(
+        (result.commands as Array<any>).map((c) => String(c.name)),
+      );
+      expect(commandNames.has("search")).toBe(true);
+      expect(commandNames.has("search-pack")).toBe(true);
+      expect(commandNames.has("chunk")).toBe(true);
+      expect(commandNames.has("doc")).toBe(true);
+      expect(commandNames.has("page")).toBe(true);
+      expect(commandNames.has("stats")).toBe(true);
+      expect(commandNames.has("rechunk")).toBe(true);
+      expect(commandNames.has("reindex")).toBe(true);
+      expect(commandNames.has("mcp")).toBe(true);
+
+      // Schema invariants (agents can validate/parses these)
+      expect(result.schemas).toBeDefined();
+      expect(result.schemas.Document).toBeDefined();
+      expect(result.schemas.PDFChunk).toBeDefined();
+      expect(result.schemas.SearchResult).toBeDefined();
+      expect(result.schemas.Config).toBeDefined();
+
+      // Lightweight stability assertions: required field names shouldn't drift.
+      const docSchema = result.schemas.Document as any;
+      expect(docSchema.type).toBe("object");
+      expect(Array.isArray(docSchema.required)).toBe(true);
+      expect(docSchema.required).toContain("id");
+      expect(docSchema.required).toContain("title");
+      expect(docSchema.required).toContain("path");
+      expect(docSchema.required).toContain("tags");
+    }));
+});
+
+describe("MCP Tool Output Contract", () => {
+  test(
+    "mcp tools return structuredContent matching the agent envelope schema",
+    async () =>
+      withTempLibraryPathAsync(async (libraryPath) => {
+        const transport = new StdioClientTransport({
+          command: process.execPath,
+          args: ["run", "src/cli.ts", "mcp", "--quiet"],
+          cwd: process.cwd(),
+          stderr: "pipe",
+          env: {
+            ...process.env,
+            PDF_LIBRARY_PATH: libraryPath,
+            // Avoid hitting any real local Ollama instance during tests.
+            OLLAMA_HOST: "http://127.0.0.1:1",
+            // Keep noise down if something logs unexpectedly.
+            PDF_BRAIN_LOG_LEVEL: "silent",
+          } as any,
+        });
+
+        const client = new Client({
+          name: "pdf-brain-contract-test",
+          version: "0.0.0",
+        });
+
+        try {
+          await client.connect(transport);
+
+          const tools = await client.listTools();
+          const toolNames = new Set(tools.tools.map((t) => t.name));
+          expect(toolNames.has("capabilities")).toBe(true);
+          expect(toolNames.has("stats")).toBe(true);
+          expect(toolNames.has("search")).toBe(true);
+
+          const call = await client.callTool({ name: "stats", arguments: {} });
+          expect(Boolean(call.isError)).toBe(false);
+          expect(call.structuredContent).toBeDefined();
+
+          const envelope: any = call.structuredContent;
+          expect(envelope.ok).toBe(true);
+          expect(envelope.command).toBe("stats");
+          expect(envelope.protocolVersion).toBe(1);
+          expect(envelope.result).toBeDefined();
+          expect(envelope.result.libraryPath).toBe(libraryPath);
+        } finally {
+          try {
+            await client.close();
+          } catch {
+            // ignore
+          }
+          try {
+            await transport.close();
+          } catch {
+            // ignore
+          }
+        }
+      }),
+    { timeout: 20000 },
+  );
+});