parse-bench 1.0.0__py3-none-any.whl

parse_bench/cli.py

@@ -0,0 +1,267 @@
+"""Unified command-line interface for parse-bench."""
+
+import sys
+from pathlib import Path
+
+import fire
+from dotenv import find_dotenv, load_dotenv
+
+from parse_bench import __version__
+from parse_bench.analysis.cli import AnalysisCLI
+from parse_bench.data.cli import DataCLI
+from parse_bench.evaluation.cli import EvaluationCLI
+from parse_bench.inference.cli import InferenceCLI
+from parse_bench.pipeline.cli import PipelineCLI
+
+
+# Load .env file if it exists. Search the current directory and its parents so
+# the CLI works both from a repo checkout and as an installed package.
+def _load_env() -> None:
+    """Load environment variables from the nearest .env file."""
+    env_path = find_dotenv(usecwd=True)
+    if env_path:
+        load_dotenv(env_path, override=False)  # Don't override existing env vars
+
+
+def _resolve_pipeline_dir(name_or_path: str | Path) -> Path:
+    """Resolve a pipeline name or path to a directory.
+
+    If the input is an existing directory, use it as-is.
+    Otherwise, try ./output/<name>.
+    """
+    p = Path(name_or_path)
+    if p.exists():
+        return p
+    candidate = Path("./output") / p
+    if candidate.exists():
+        return candidate
+    return p  # Return original; caller will handle the error
+
+
+class BenchCLI:
+    """Unified CLI for parse-bench.
+
+    Top-level commands (recommended):
+        run          Run end-to-end benchmark pipeline
+        download     Download dataset from HuggingFace
+        status       Check if dataset is ready
+        pipelines    List available pipeline configurations
+        compare      Compare two pipeline results
+        serve        View reports in browser with PDF support
+        version      Print the installed parse-bench version
+
+    Advanced subcommands:
+        inference    Run inference only
+        evaluation   Run evaluation only
+        analysis     Generate reports, dashboards, comparisons
+        pipeline     End-to-end pipeline (same as 'run')
+        data         Dataset management (same as 'download'/'status')
+    """
+
+    def __init__(self) -> None:
+        self.inference = InferenceCLI()
+        self.evaluation = EvaluationCLI()
+        self.analysis = AnalysisCLI()
+        self.pipeline = PipelineCLI()
+        self.data = DataCLI()
+
+    # ── Top-level convenience commands ──────────────────────────────
+
+    def version(self) -> str:
+        """Print the installed parse-bench version."""
+        return __version__
+
+    def run(
+        self,
+        pipeline: str,
+        input_dir: str | Path | None = None,
+        file: str | Path | None = None,
+        output_dir: str | Path | None = None,
+        max_concurrent: int = 20,
+        force: bool = False,
+        verbose: bool = False,
+        group: str | None = None,
+        tags: str | tuple[str, ...] | list[str] | None = None,
+        open_report: bool = True,
+        skip_inference: bool = False,
+        test: bool = False,
+    ) -> int:
+        """Run end-to-end benchmark: inference -> evaluation -> report.
+
+        Args:
+            pipeline: Pipeline name (e.g., 'llamaparse_agentic', 'llamaparse_cost_effective')
+            input_dir: Directory containing test cases/PDFs (default: ./data)
+            file: Single file to run (PDF/image)
+            output_dir: Directory to save results (default: ./output)
+            max_concurrent: Maximum concurrent inference requests (default: 20)
+            force: Force regeneration even if results exist (default: False)
+            verbose: Enable verbose output (default: False)
+            group: Filter by category (e.g., 'chart', 'table')
+            tags: Tags for this run
+            open_report: Open HTML report in browser (default: True)
+            skip_inference: Skip inference, only re-evaluate (default: False)
+            test: Download and run on the small test dataset (3 files per category)
+
+        Example:
+            parse-bench run llamaparse_agentic
+            parse-bench run llamaparse_agentic --group chart
+            parse-bench run llamaparse_agentic --skip_inference
+            parse-bench run llamaparse_agentic --test
+        """
+        return self.pipeline.run(
+            pipeline=pipeline,
+            input_dir=input_dir,
+            file=file,
+            output_dir=output_dir,
+            max_concurrent=max_concurrent,
+            force=force,
+            verbose=verbose,
+            group=group,
+            tags=tags,
+            open_report=open_report,
+            skip_inference=skip_inference,
+            test=test,
+        )
+
+    def download(
+        self,
+        data_dir: str | Path | None = None,
+        force: bool = False,
+        test: bool = False,
+    ) -> int:
+        """Download the benchmark dataset from HuggingFace.
+
+        Args:
+            data_dir: Directory to store dataset (default: ./data)
+            force: Force re-download even if data exists
+            test: Download the small test dataset (3 files per category)
+
+        Example:
+            parse-bench download
+            parse-bench download --test
+        """
+        return self.data.download(data_dir=data_dir, force=force, test=test)
+
+    def status(
+        self,
+        data_dir: str | Path | None = None,
+        test: bool = False,
+    ) -> int:
+        """Check if the benchmark dataset is downloaded and ready.
+
+        Args:
+            data_dir: Data directory to check
+                (default: ./data, or ./data/test when --test is set)
+            test: Check the small test dataset instead of the full dataset
+
+        Example:
+            parse-bench status
+            parse-bench status --test
+            parse-bench status data/
+        """
+        return self.data.status(data_dir=data_dir, test=test)
+
+    def pipelines(self) -> None:
+        """List all available pipeline configurations."""
+        return self.inference.list_pipelines()
+
+    def compare(
+        self,
+        pipeline_a: str | Path,
+        pipeline_b: str | Path,
+        test_cases_dir: str | Path | None = None,
+        output_file: str | Path | None = None,
+    ) -> int:
+        """Compare results from two pipelines.
+
+        Pipeline names are auto-resolved to ./output/<name> if the path
+        doesn't exist as-is.
+
+        Args:
+            pipeline_a: Pipeline A name or directory (e.g., 'llamaparse_agentic')
+            pipeline_b: Pipeline B name or directory (e.g., 'llamaparse_cost_effective')
+            test_cases_dir: Directory containing test cases (default: auto-detect)
+            output_file: Path to save comparison report (default: auto)
+
+        Example:
+            parse-bench compare llamaparse_agentic llamaparse_cost_effective
+            parse-bench compare ./output/llamaparse_agentic ./output/llamaparse_cost_effective
+        """
+        return self.analysis.compare_pipelines(
+            pipeline_a_dir=_resolve_pipeline_dir(pipeline_a),
+            pipeline_b_dir=_resolve_pipeline_dir(pipeline_b),
+            test_cases_dir=test_cases_dir,
+            output_file=output_file,
+        )
+
+    def leaderboard(
+        self,
+        *pipelines: str,
+        output_dir: str | Path = "./output",
+        output_file: str | Path | None = None,
+    ) -> int:
+        """Generate a leaderboard comparing all pipelines side-by-side.
+
+        If no pipeline names are given, auto-discovers all pipelines in the
+        output directory.
+
+        Args:
+            *pipelines: Optional pipeline names to include (e.g., 'llamaparse_agentic')
+            output_dir: Parent directory containing pipeline subdirectories
+            output_file: Path to save the leaderboard HTML
+
+        Example:
+            parse-bench leaderboard
+            parse-bench leaderboard llamaparse_agentic llamaparse_cost_effective
+        """
+        pipeline_list = list(pipelines) if pipelines else None
+        return self.analysis.generate_leaderboard(
+            output_dir=output_dir,
+            pipelines=pipeline_list,
+            output_file=output_file,
+        )
+
+    def serve(
+        self,
+        pipeline: str | Path | None = None,
+        port: int = 8080,
+        root: str | Path = ".",
+    ) -> int:
+        """Start a local server to view reports with PDF rendering support.
+
+        Pipeline names are auto-resolved to ./output/<name> if the path
+        doesn't exist as-is.
+
+        Args:
+            pipeline: Pipeline name or directory (e.g., 'llamaparse_agentic')
+            port: Port number (default: 8080)
+            root: Root directory to serve (default: current directory)
+
+        Example:
+            parse-bench serve llamaparse_agentic
+            parse-bench serve ./output/llamaparse_agentic
+            parse-bench serve
+        """
+        pipeline_dir = _resolve_pipeline_dir(pipeline) if pipeline else None
+        return self.analysis.serve(
+            pipeline_dir=pipeline_dir,
+            port=port,
+            root=root,
+        )
+
+
+def main() -> int:
+    """Main entry point for the unified CLI."""
+    # Load .env file before any commands run
+    _load_env()
+    cli = BenchCLI()
+    result = fire.Fire(cli)
+    # Fire returns the result of the called method
+    # If it's an integer (exit code), use it; otherwise default to 0
+    if isinstance(result, int):
+        return result
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

parse_bench/data/__init__.py

@@ -0,0 +1 @@
+"""Data management for parse-bench datasets."""

parse_bench/data/cli.py

@@ -0,0 +1,118 @@
+"""Command-line interface for data management."""
+
+import sys
+from pathlib import Path
+
+from parse_bench.data.download import default_data_dir, download_dataset, is_dataset_ready
+
+
+class DataCLI:
+    """Command-line interface for managing benchmark datasets."""
+
+    def download(
+        self,
+        data_dir: str | Path | None = None,
+        force: bool = False,
+        test: bool = False,
+    ) -> int:
+        """Download the parse-bench dataset from HuggingFace.
+
+        Args:
+            data_dir: Local directory to store the dataset
+                (default: ./data, or ./data/test when --test is set)
+            force: Force re-download even if data already exists
+            test: Download the small test dataset (3 files per category)
+
+        Returns:
+            Exit code (0 for success, non-zero for failure)
+        """
+        try:
+            data_path = Path(data_dir) if data_dir else default_data_dir(test=test)
+            download_dataset(data_dir=data_path, force=force, test=test)
+            return 0
+        except Exception as e:
+            print(f"Error downloading dataset: {e}", file=sys.stderr)
+            import traceback
+
+            traceback.print_exc()
+            return 1
+
+    def status(
+        self,
+        data_dir: str | Path | None = None,
+        test: bool = False,
+    ) -> int:
+        """Check if the dataset is downloaded and show summary statistics.
+
+        Args:
+            data_dir: Data directory to check
+                (default: ./data, or ./data/test when --test is set)
+            test: Check the small test dataset instead of the full dataset
+
+        Returns:
+            Exit code (0 if ready, 1 if not)
+        """
+        import json
+
+        data_path = Path(data_dir) if data_dir else Path.cwd() / default_data_dir(test=test)
+        ready = is_dataset_ready(data_path)
+
+        if not ready:
+            print(f"Dataset is NOT ready at: {data_path}")
+            print("Run 'parse-bench download' to download it.")
+            return 1
+
+        print(f"Dataset: {data_path}")
+        print()
+
+        # Gather per-category stats from JSONL files
+        jsonl_files = sorted(data_path.glob("*.jsonl"))
+        total_cases = 0
+        total_pdfs = 0
+        all_pdfs: set[str] = set()  # track unique PDFs across all categories
+        rows: list[tuple[str, int, int]] = []
+
+        for jf in jsonl_files:
+            category = jf.stem
+            lines = jf.read_text().strip().splitlines()
+            n_cases = len(lines)
+            pdfs: set[str] = set()
+            for line in lines:
+                rec = json.loads(line)
+                pdfs.add(rec.get("pdf", ""))
+            n_pdfs = len(pdfs)
+            rows.append((category, n_cases, n_pdfs))
+            total_cases += n_cases
+            total_pdfs += n_pdfs
+            all_pdfs.update(pdfs)
+
+        # Count docs on disk per category
+        doc_counts: dict[str, int] = {}
+        docs_dir = data_path / "docs"
+        if docs_dir.exists():
+            for cat_dir in sorted(docs_dir.iterdir()):
+                if cat_dir.is_dir():
+                    doc_counts[cat_dir.name] = sum(1 for _ in cat_dir.rglob("*") if _.is_file())
+
+        # Print table
+        hdr = f"{'Category':<20} {'Test Cases':>12} {'PDFs':>8}"
+        print(hdr)
+        print("-" * len(hdr))
+        for category, n_cases, n_pdfs in rows:
+            print(f"{category:<20} {n_cases:>12,} {n_pdfs:>8,}")
+        print("-" * len(hdr))
+        print(f"{'Total':<20} {total_cases:>12,} {total_pdfs:>8,}")
+        n_unique = len(all_pdfs)
+        if n_unique < total_pdfs:
+            print(f"{'Unique documents':<20} {'':>12} {n_unique:>8,}")
+            print("  (text_content and text_formatting share the same PDF files)")
+        print()
+
+        # Docs on disk
+        if doc_counts:
+            print("Documents on disk:")
+            for cat, count in doc_counts.items():
+                print(f"  {cat:<18} {count:>6,} files")
+            print(f"  {'total':<18} {sum(doc_counts.values()):>6,} files")
+
+        return 0

parse_bench/data/download.py

@@ -0,0 +1,127 @@
+"""Download parse-bench dataset from HuggingFace.
+
+Dataset: https://huggingface.co/datasets/llamaindex/ParseBench
+
+Structure after download:
+    <local_dir>/
+    ├── chart.jsonl
+    ├── layout.jsonl
+    ├── table.jsonl
+    ├── text.jsonl
+    ├── expected_markdown.json
+    └── pdfs/{chart,layout,table,text}/*.pdf
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+DATASET_REPO = "llamaindex/ParseBench"
+DATASET_REPO_TYPE = "dataset"
+TEST_DATA_REVISION = "test-data"
+
+# Default on-disk locations. Test data lives in a sibling subdirectory so the
+# two datasets coexist and `--test` does not silently overlay or get masked
+# by an existing full download.
+DEFAULT_DATA_DIR = Path("./data")
+DEFAULT_TEST_DATA_DIR = Path("./data/test")
+
+
+def default_data_dir(test: bool = False) -> Path:
+    """Return the default on-disk dataset path for the given mode.
+
+    Used by every CLI surface that takes ``--test`` so the routing is
+    consistent between ``download``, ``run`` and ``status``.
+    """
+    return DEFAULT_TEST_DATA_DIR if test else DEFAULT_DATA_DIR
+
+
+# Files that must exist for the dataset to be considered complete
+_REQUIRED_FILES = [
+    "chart.jsonl",
+    "layout.jsonl",
+    "table.jsonl",
+    "text_content.jsonl",
+    "text_formatting.jsonl",
+]
+
+# At least one document must exist per category
+_REQUIRED_DOC_DIRS = ["docs/chart", "docs/layout", "docs/table", "docs/text"]
+
+
+def is_dataset_ready(data_dir: Path) -> bool:
+    """Check if the dataset is already downloaded and complete.
+
+    Args:
+        data_dir: Path to the data directory.
+
+    Returns:
+        True if all required files and at least one PDF per category exist.
+    """
+    if not data_dir.exists():
+        return False
+
+    for f in _REQUIRED_FILES:
+        if not (data_dir / f).exists():
+            return False
+
+    for d in _REQUIRED_DOC_DIRS:
+        doc_dir = data_dir / d
+        if not doc_dir.exists():
+            return False
+        # Check for any supported document file (PDF, image, etc.)
+        if not any(doc_dir.rglob("*.*")):
+            return False
+
+    return True
+
+
+def download_dataset(
+    data_dir: Path | None = None,
+    force: bool = False,
+    test: bool = False,
+) -> Path:
+    """Download the parse-bench dataset from HuggingFace.
+
+    Uses huggingface_hub's snapshot_download to fetch the full dataset,
+    including JSONL files and PDFs.
+
+    Args:
+        data_dir: Local directory to store the dataset.
+            Defaults to ./data in the current working directory.
+        force: Force re-download even if data already exists.
+        test: Download the small test dataset (3 files per category)
+            instead of the full dataset.
+
+    Returns:
+        Path to the downloaded dataset directory.
+    """
+    from huggingface_hub import snapshot_download
+
+    if data_dir is None:
+        data_dir = Path.cwd() / "data"
+
+    revision = TEST_DATA_REVISION if test else None
+
+    if not force and is_dataset_ready(data_dir):
+        print(f"Dataset already downloaded at: {data_dir}")
+        return data_dir
+
+    label = "test dataset" if test else "dataset"
+    print(f"Downloading {label} from HuggingFace: {DATASET_REPO}")
+    if test:
+        print(f"Branch: {TEST_DATA_REVISION}")
+    print(f"Destination: {data_dir}")
+
+    snapshot_download(
+        repo_id=DATASET_REPO,
+        repo_type=DATASET_REPO_TYPE,
+        local_dir=str(data_dir),
+        revision=revision,
+    )
+
+    if not is_dataset_ready(data_dir):
+        raise RuntimeError(f"Dataset download completed but validation failed. Check {data_dir} for missing files.")
+
+    print(f"Dataset ready at: {data_dir}")
+    return data_dir

parse_bench/evaluation/__init__.py

@@ -0,0 +1,11 @@
+"""Evaluation package exports."""
+
+__all__ = ["EvaluationRunner"]
+
+
+def __getattr__(name: str) -> object:  # pragma: no cover - simple lazy import shim.
+    if name == "EvaluationRunner":
+        from .runner import EvaluationRunner
+
+        return EvaluationRunner
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")