pi-docparser 1.0.0

package/extensions/docparser/doctor.ts

@@ -0,0 +1,291 @@
+import type { ExtensionAPI, ExtensionCommandContext } from "@mariozechner/pi-coding-agent";
+
+import { DOCTOR_COMMAND, DOCTOR_COMMAND_NAME, INSTALL_COMMAND_TIMEOUT_MS } from "./constants.ts";
+import {
+  buildInstallStrategies,
+  diagnoseDependencies,
+  getInputCategoryLabel,
+  getPlatformLabel,
+  getPreferredStrategies,
+  getRelevantDependencyNames,
+  summarizeInstallOutput,
+} from "./deps.ts";
+import { resolveDocumentTarget } from "./input.ts";
+import type {
+  DependencyDiagnosis,
+  InputInspection,
+  InstallStrategy,
+  UiMessageLevel,
+} from "./types.ts";
+
+function normalizeDoctorArgument(input: string): string | undefined {
+  const trimmed = input.trim();
+  if (!trimmed) {
+    return undefined;
+  }
+
+  const isQuoted =
+    (trimmed.startsWith('"') && trimmed.endsWith('"')) ||
+    (trimmed.startsWith("'") && trimmed.endsWith("'"));
+
+  return isQuoted ? trimmed.slice(1, -1) : trimmed;
+}
+
+function formatDoctorReport(options: {
+  inspection?: InputInspection;
+  sourcePath?: string;
+  resolvedPath?: string;
+  diagnoses: DependencyDiagnosis[];
+  strategies: InstallStrategy[];
+  installSummary?: string[];
+}): { text: string; level: UiMessageLevel } {
+  const missing = options.diagnoses.filter((diagnosis) => !diagnosis.installed);
+  const relevantMissing = missing.filter((diagnosis) => diagnosis.relevant);
+  const level: UiMessageLevel =
+    relevantMissing.length > 0 ? "error" : missing.length > 0 ? "warning" : "info";
+  const lines = ["docparser doctor", `Platform: ${getPlatformLabel()}`];
+
+  if (options.sourcePath) {
+    lines.push(`Target: ${options.sourcePath}`);
+  }
+
+  if (options.resolvedPath && options.resolvedPath !== options.sourcePath) {
+    lines.push(`Resolved path: ${options.resolvedPath}`);
+  }
+
+  if (options.inspection) {
+    lines.push(`Detected input type: ${getInputCategoryLabel(options.inspection.category)}`);
+    if (options.inspection.extension) {
+      lines.push(`Detected extension: ${options.inspection.extension}`);
+    }
+  }
+
+  if (options.inspection && getRelevantDependencyNames(options.inspection).size === 0) {
+    lines.push(
+      "This input type does not require extra host conversion packages for normal parsing.",
+    );
+  }
+
+  lines.push("Dependency status:");
+  for (const diagnosis of options.diagnoses) {
+    const status = diagnosis.installed
+      ? diagnosis.detectedCommand
+        ? `installed (${diagnosis.detectedCommand})`
+        : "installed"
+      : diagnosis.relevant
+        ? options.inspection
+          ? "missing — required for this input"
+          : "missing — relevant"
+        : "missing — optional";
+
+    lines.push(`- ${diagnosis.label}: ${status}`);
+    lines.push(`  ${diagnosis.summary}`);
+  }
+
+  if (options.installSummary?.length) {
+    lines.push("Installation attempt:");
+    for (const line of options.installSummary) {
+      const [firstLine, ...rest] = line.split("\n");
+      lines.push(`- ${firstLine}`);
+      for (const continuation of rest) {
+        lines.push(`  ${continuation}`);
+      }
+    }
+  }
+
+  if (missing.length > 0) {
+    const preferredStrategies = getPreferredStrategies(options.strategies);
+    if (preferredStrategies.length > 0) {
+      lines.push("Suggested setup commands:");
+      for (const strategy of preferredStrategies.slice(0, 2)) {
+        lines.push(`- ${strategy.label}:`);
+        for (const command of strategy.commands) {
+          lines.push(`  ${command.display}`);
+        }
+        if (strategy.autoRunBlockedReason) {
+          lines.push(`  Note: ${strategy.autoRunBlockedReason}`);
+        }
+      }
+    } else if (process.platform === "linux") {
+      lines.push(
+        "Suggested setup: install the missing packages with your distribution package manager.",
+      );
+    } else if (process.platform === "darwin") {
+      lines.push(
+        "Suggested setup: install Homebrew, then run the appropriate brew install commands.",
+      );
+    } else if (process.platform === "win32") {
+      lines.push("Suggested setup: use winget or Chocolatey to install the missing packages.");
+    }
+  }
+
+  return {
+    text: lines.join("\n"),
+    level,
+  };
+}
+
+async function collectDoctorState(inspection?: InputInspection): Promise<{
+  diagnoses: DependencyDiagnosis[];
+  missingDependencies: DependencyDiagnosis[];
+  installCandidates: DependencyDiagnosis[];
+  strategies: InstallStrategy[];
+}> {
+  const diagnoses = await diagnoseDependencies(inspection);
+  const missingDependencies = diagnoses.filter((diagnosis) => !diagnosis.installed);
+  const installCandidates = inspection
+    ? missingDependencies.filter((diagnosis) => diagnosis.relevant)
+    : missingDependencies;
+  const strategyInput = installCandidates.length > 0 ? installCandidates : missingDependencies;
+
+  return {
+    diagnoses,
+    missingDependencies,
+    installCandidates,
+    strategies: await buildInstallStrategies(strategyInput),
+  };
+}
+
+async function selectInstallStrategy(
+  strategies: InstallStrategy[],
+  ctx: ExtensionCommandContext,
+): Promise<InstallStrategy | undefined> {
+  if (strategies.length === 1) {
+    return strategies[0];
+  }
+
+  const labels = strategies.map((strategy) => strategy.label);
+  const selectedLabel = await ctx.ui.select("Choose an install strategy", labels);
+  if (!selectedLabel) {
+    return undefined;
+  }
+
+  return strategies.find((strategy) => strategy.label === selectedLabel);
+}
+
+export function registerDoctorCommand(pi: ExtensionAPI) {
+  pi.registerCommand(DOCTOR_COMMAND_NAME, {
+    description:
+      "Diagnose docparser host dependencies and optionally try to install missing packages",
+    handler: async (args, ctx) => {
+      if (!ctx.hasUI) {
+        return;
+      }
+
+      const normalizedArg = normalizeDoctorArgument(args);
+
+      if (normalizedArg === "help" || normalizedArg === "--help") {
+        ctx.ui.notify(
+          [
+            `${DOCTOR_COMMAND} usage`,
+            `- ${DOCTOR_COMMAND}`,
+            `- ${DOCTOR_COMMAND} @path/to/file.docx`,
+            "",
+            "With a file path, the doctor focuses on the dependencies relevant to that input.",
+          ].join("\n"),
+          "info",
+        );
+        return;
+      }
+
+      try {
+        let target: Awaited<ReturnType<typeof resolveDocumentTarget>> | undefined;
+        try {
+          target = normalizedArg ? await resolveDocumentTarget(normalizedArg, ctx.cwd) : undefined;
+        } catch {
+          if (normalizedArg) {
+            ctx.ui.notify(`Document file not found or not readable: ${normalizedArg}`, "error");
+          }
+          return;
+        }
+
+        const initialState = await collectDoctorState(target?.inspection);
+        const initialReport = formatDoctorReport({
+          inspection: target?.inspection,
+          sourcePath: target?.sourcePath,
+          resolvedPath: target?.resolvedPath,
+          diagnoses: initialState.diagnoses,
+          strategies: initialState.strategies,
+        });
+        ctx.ui.notify(initialReport.text, initialReport.level);
+
+        if (
+          initialState.missingDependencies.length === 0 ||
+          initialState.installCandidates.length === 0
+        ) {
+          return;
+        }
+
+        const autoRunnableStrategies = getPreferredStrategies(
+          initialState.strategies.filter((strategy) => strategy.autoRunnable),
+        );
+        if (autoRunnableStrategies.length === 0) {
+          ctx.ui.notify(
+            `No automatic install strategy is safely available right now. Follow the suggested commands above or install the packages manually, then rerun ${DOCTOR_COMMAND}.`,
+            "warning",
+          );
+          return;
+        }
+
+        const selectedStrategy = await selectInstallStrategy(autoRunnableStrategies, ctx);
+        if (!selectedStrategy) {
+          return;
+        }
+
+        const installList = initialState.installCandidates
+          .map((dependency) => dependency.label)
+          .join(", ");
+        const confirmText = [
+          `Missing packages: ${installList}`,
+          `Installer: ${selectedStrategy.label}`,
+          "",
+          "Commands that will be attempted:",
+          ...selectedStrategy.commands.map((command) => `- ${command.display}`),
+          "",
+          "Try running them now?",
+        ].join("\n");
+
+        const confirmed = await ctx.ui.confirm("docparser doctor", confirmText);
+        if (!confirmed) {
+          return;
+        }
+
+        const installSummary: string[] = [];
+        for (const command of selectedStrategy.commands) {
+          ctx.ui.notify(`Running: ${command.display}`, "info");
+          const result = await pi.exec(command.command, command.args, {
+            timeout: command.timeoutMs ?? INSTALL_COMMAND_TIMEOUT_MS,
+          });
+          const success = result.code === 0 && !result.killed;
+
+          installSummary.push(
+            `${command.description}: ${success ? "ok" : `failed (exit ${result.code}${result.killed ? ", killed" : ""})`}`,
+          );
+
+          if (!success) {
+            const outputSummary = summarizeInstallOutput(result.stdout, result.stderr);
+            if (outputSummary) {
+              installSummary.push(`Command output:\n${outputSummary}`);
+            }
+          }
+        }
+
+        const finalState = await collectDoctorState(target?.inspection);
+        const finalReport = formatDoctorReport({
+          inspection: target?.inspection,
+          sourcePath: target?.sourcePath,
+          resolvedPath: target?.resolvedPath,
+          diagnoses: finalState.diagnoses,
+          strategies: finalState.strategies,
+          installSummary,
+        });
+        ctx.ui.notify(finalReport.text, finalReport.level);
+      } catch (error) {
+        ctx.ui.notify(
+          `docparser doctor failed: ${error instanceof Error ? error.message : String(error)}`,
+          "error",
+        );
+      }
+    },
+  });
+}

package/extensions/docparser/index.ts

@@ -0,0 +1,9 @@
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+
+import { registerDoctorCommand } from "./doctor.ts";
+import { registerDocumentParseTool } from "./tool.ts";
+
+export default function parseDocumentExtension(pi: ExtensionAPI) {
+  registerDocumentParseTool(pi);
+  registerDoctorCommand(pi);
+}

package/extensions/docparser/input.ts

@@ -0,0 +1,230 @@
+import { constants as fsConstants } from "node:fs";
+import { access, open } from "node:fs/promises";
+import { extname, isAbsolute, resolve as resolvePath } from "node:path";
+import { homedir } from "node:os";
+
+import { IMAGE_EXTENSIONS, OFFICE_EXTENSIONS, SPREADSHEET_EXTENSIONS } from "./constants.ts";
+import type { InputCategory, InputInspection, ScreenshotSelection } from "./types.ts";
+
+const UNICODE_SPACES = /[\u00A0\u2000-\u200A\u202F\u205F\u3000]/g;
+const NARROW_NO_BREAK_SPACE = "\u202F";
+
+function normalizeDocumentPathInput(input: string): string {
+  return input.trim().replace(/^@/, "").replace(UNICODE_SPACES, " ");
+}
+
+function expandHomeDirectory(filePath: string): string {
+  if (filePath === "~") {
+    return homedir();
+  }
+
+  if (filePath.startsWith("~/")) {
+    return `${homedir()}${filePath.slice(1)}`;
+  }
+
+  return filePath;
+}
+
+function tryMacOsAmPmVariant(filePath: string): string {
+  return filePath.replace(/ (AM|PM)\./g, `${NARROW_NO_BREAK_SPACE}$1.`);
+}
+
+function tryNfdVariant(filePath: string): string {
+  return filePath.normalize("NFD");
+}
+
+function tryCurlyQuoteVariant(filePath: string): string {
+  return filePath.replace(/'/g, "\u2019");
+}
+
+async function pathExists(filePath: string): Promise<boolean> {
+  try {
+    await access(filePath, fsConstants.F_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function resolveExistingPath(filePath: string, cwd: string): Promise<string> {
+  const expanded = expandHomeDirectory(filePath);
+  const resolved = isAbsolute(expanded) ? expanded : resolvePath(cwd, expanded);
+  const nfdVariant = tryNfdVariant(resolved);
+
+  for (const candidate of new Set([
+    resolved,
+    tryMacOsAmPmVariant(resolved),
+    nfdVariant,
+    tryCurlyQuoteVariant(resolved),
+    tryCurlyQuoteVariant(nfdVariant),
+  ])) {
+    if (await pathExists(candidate)) {
+      return candidate;
+    }
+  }
+
+  return resolved;
+}
+
+async function ensureReadableFile(filePath: string, sourcePath: string): Promise<void> {
+  try {
+    await access(filePath, fsConstants.R_OK);
+  } catch {
+    throw new Error(`Document file not found or not readable: ${sourcePath}`);
+  }
+}
+
+function getInputCategory(extension: string): InputCategory | undefined {
+  if (extension === ".pdf") {
+    return "pdf";
+  }
+
+  if (OFFICE_EXTENSIONS.has(extension)) {
+    return "office";
+  }
+
+  if (SPREADSHEET_EXTENSIONS.has(extension)) {
+    return "spreadsheet";
+  }
+
+  if (IMAGE_EXTENSIONS.has(extension)) {
+    return "image";
+  }
+
+  return undefined;
+}
+
+async function readFileHeader(filePath: string, length: number): Promise<Buffer> {
+  const handle = await open(filePath, "r");
+
+  try {
+    const buffer = Buffer.alloc(length);
+    const { bytesRead } = await handle.read(buffer, 0, length, 0);
+    return buffer.subarray(0, bytesRead);
+  } finally {
+    await handle.close();
+  }
+}
+
+function isPdfHeader(header: Buffer): boolean {
+  return header.length >= 4 && header.toString("utf8", 0, 4) === "%PDF";
+}
+
+function isPngHeader(header: Buffer): boolean {
+  return (
+    header.length >= 4 &&
+    header[0] === 0x89 &&
+    header[1] === 0x50 &&
+    header[2] === 0x4e &&
+    header[3] === 0x47
+  );
+}
+
+function isJpegHeader(header: Buffer): boolean {
+  return header.length >= 3 && header[0] === 0xff && header[1] === 0xd8 && header[2] === 0xff;
+}
+
+async function inspectInputFile(filePath: string): Promise<InputInspection> {
+  const extension = extname(filePath).toLowerCase();
+  const category = getInputCategory(extension);
+
+  if (category) {
+    return { extension, category };
+  }
+
+  try {
+    const header = await readFileHeader(filePath, 16);
+
+    if (isPdfHeader(header)) {
+      return { extension: extension || ".pdf", category: "pdf" };
+    }
+
+    if (!extension && isPngHeader(header)) {
+      return { extension: ".png", category: "image" };
+    }
+
+    if (!extension && isJpegHeader(header)) {
+      return { extension: ".jpg", category: "image" };
+    }
+  } catch {
+    // Best-effort inspection only. Readability is validated separately.
+  }
+
+  return { extension, category: "other" };
+}
+
+export async function resolveDocumentTarget(
+  input: string,
+  cwd: string,
+): Promise<{
+  sourcePath: string;
+  resolvedPath: string;
+  inspection: InputInspection;
+}> {
+  const sourcePath = normalizeDocumentPathInput(input);
+  const resolvedPath = await resolveExistingPath(sourcePath, cwd);
+
+  await ensureReadableFile(resolvedPath, sourcePath);
+
+  return {
+    sourcePath,
+    resolvedPath,
+    inspection: await inspectInputFile(resolvedPath),
+  };
+}
+
+export function parsePageSelection(selection: string): number[] {
+  const pages = new Set<number>();
+
+  for (const rawPart of selection.split(",")) {
+    const part = rawPart.trim();
+    if (!part) continue;
+
+    if (part.includes("-")) {
+      const [rawStart, rawEnd] = part.split("-", 2).map((value) => value.trim());
+      const start = Number.parseInt(rawStart, 10);
+      const end = Number.parseInt(rawEnd, 10);
+
+      if (!Number.isInteger(start) || !Number.isInteger(end) || start < 1 || end < start) {
+        throw new Error(`Invalid page range: ${part}`);
+      }
+
+      for (let page = start; page <= end; page++) {
+        pages.add(page);
+      }
+      continue;
+    }
+
+    const page = Number.parseInt(part, 10);
+    if (!Number.isInteger(page) || page < 1) {
+      throw new Error(`Invalid page number: ${part}`);
+    }
+    pages.add(page);
+  }
+
+  const result = Array.from(pages).sort((a, b) => a - b);
+  if (result.length === 0) {
+    throw new Error("No valid page numbers were provided.");
+  }
+
+  return result;
+}
+
+export function resolveScreenshotSelection(selection: string): ScreenshotSelection {
+  const trimmedSelection = selection.trim();
+  if (!trimmedSelection) {
+    throw new Error("Screenshot page selection must not be empty.");
+  }
+
+  if (["all", "*"].includes(trimmedSelection.toLowerCase())) {
+    return {
+      pageNumbers: undefined,
+      description: "all pages",
+    };
+  }
+
+  return {
+    pageNumbers: parsePageSelection(trimmedSelection),
+    description: `pages ${trimmedSelection}`,
+  };
+}

package/extensions/docparser/request.ts

@@ -0,0 +1,67 @@
+import { DEFAULT_DPI, DEFAULT_MAX_PAGES, DEFAULT_NUM_WORKERS } from "./constants.ts";
+import { resolveScreenshotSelection } from "./input.ts";
+import type { DocumentParseParams, DocumentParsePlan, LiteParseToolConfig } from "./types.ts";
+
+function normalizeOptionalString(value: string | undefined): string | undefined {
+  const trimmed = value?.trim();
+  return trimmed ? trimmed : undefined;
+}
+
+function resolveOcrLanguage(
+  params: DocumentParseParams,
+  ocrServerUrl: string | undefined,
+  warnings: string[],
+): LiteParseToolConfig["ocrLanguage"] | undefined {
+  const singleOcrLanguage = normalizeOptionalString(params.ocrLanguage);
+  const ocrLanguages = (params.ocrLanguages ?? [])
+    .map((language) => language.trim())
+    .filter(Boolean);
+
+  if (singleOcrLanguage && ocrLanguages.length > 0) {
+    warnings.push("Both ocrLanguage and ocrLanguages were provided. Using ocrLanguages.");
+  }
+
+  if (ocrLanguages.length === 0) {
+    return singleOcrLanguage;
+  }
+
+  if (ocrServerUrl) {
+    if (ocrLanguages.length > 1) {
+      warnings.push(
+        "Multiple OCR languages were provided, but HTTP OCR servers currently receive only the first language code.",
+      );
+    }
+
+    return ocrLanguages[0];
+  }
+
+  return ocrLanguages.join("+");
+}
+
+export function buildDocumentParsePlan(params: DocumentParseParams): DocumentParsePlan {
+  const warnings: string[] = [];
+  const ocrServerUrl = normalizeOptionalString(params.ocrServerUrl);
+  const ocrLanguage = resolveOcrLanguage(params, ocrServerUrl, warnings);
+
+  const parserConfig: LiteParseToolConfig = {
+    outputFormat: params.format ?? "text",
+    ocrEnabled: (params.ocr ?? "auto") !== "off",
+    ocrLanguage,
+    ocrServerUrl,
+    numWorkers: params.numWorkers ?? DEFAULT_NUM_WORKERS,
+    maxPages: params.maxPages ?? DEFAULT_MAX_PAGES,
+    targetPages: normalizeOptionalString(params.targetPages),
+    dpi: params.dpi ?? DEFAULT_DPI,
+    preciseBoundingBox: params.preciseBoundingBox ?? true,
+    preserveVerySmallText: params.preserveSmallText ?? false,
+    preserveLayoutAlignmentAcrossPages: params.preserveLayoutAlignmentAcrossPages ?? false,
+  };
+
+  return {
+    parserConfig,
+    screenshotSelection: params.screenshotPages
+      ? resolveScreenshotSelection(params.screenshotPages)
+      : undefined,
+    warnings,
+  };
+}

package/extensions/docparser/schema.ts

@@ -0,0 +1,82 @@
+import { StringEnum, Type } from "@mariozechner/pi-ai";
+
+export const DocumentParseSchema = Type.Object({
+  path: Type.String({
+    description:
+      "Path to the document file to parse (PDF, DOCX, PPTX, XLSX, CSV, PNG, JPG, TIFF, WebP, etc.)",
+  }),
+  format: Type.Optional(
+    StringEnum(["text", "json"] as const, {
+      description: "Output format for the parsed document (default: text)",
+    }),
+  ),
+  targetPages: Type.Optional(
+    Type.String({
+      description: 'Optional page selection for parsing, e.g. "1-5,10,15-20"',
+    }),
+  ),
+  screenshotPages: Type.Optional(
+    Type.String({
+      description:
+        'Optional PDF page selection for screenshots, e.g. "1-3,8" or "all". Screenshots are currently generated only for PDF inputs and are saved as PNG files.',
+    }),
+  ),
+  ocr: Type.Optional(
+    StringEnum(["auto", "off"] as const, {
+      description:
+        "OCR mode: auto uses LiteParse OCR behavior, off disables OCR for faster parsing",
+    }),
+  ),
+  ocrLanguage: Type.Optional(
+    Type.String({
+      description:
+        "Optional single OCR language code. Built-in Tesseract typically uses ISO 639-3 codes such as eng, deu, fra, jpn. Many HTTP OCR servers instead expect ISO 639-1 codes such as en, de, fr, ja.",
+    }),
+  ),
+  ocrLanguages: Type.Optional(
+    Type.Array(Type.String(), {
+      minItems: 1,
+      description:
+        "Optional multiple OCR language codes. For built-in Tesseract they are joined into a multilingual language string. For HTTP OCR servers, only the first code is forwarded.",
+    }),
+  ),
+  ocrServerUrl: Type.Optional(
+    Type.String({
+      description: "Optional HTTP OCR server URL implementing the LiteParse OCR API",
+    }),
+  ),
+  numWorkers: Type.Optional(
+    Type.Integer({
+      minimum: 1,
+      description: "Optional OCR worker count (default: CPU cores - 1)",
+    }),
+  ),
+  maxPages: Type.Optional(
+    Type.Integer({
+      minimum: 1,
+      description: "Maximum number of pages to parse (default: 10000)",
+    }),
+  ),
+  dpi: Type.Optional(
+    Type.Integer({
+      minimum: 72,
+      description: "Rendering DPI for OCR and screenshots (default: 150)",
+    }),
+  ),
+  preciseBoundingBox: Type.Optional(
+    Type.Boolean({
+      description: "Whether to compute precise bounding boxes (default: true)",
+    }),
+  ),
+  preserveSmallText: Type.Optional(
+    Type.Boolean({
+      description: "Whether to preserve very small text that would otherwise be filtered out",
+    }),
+  ),
+  preserveLayoutAlignmentAcrossPages: Type.Optional(
+    Type.Boolean({
+      description:
+        "Whether to preserve text alignment consistently across page boundaries (default: false)",
+    }),
+  ),
+});