langparse 0.1.0__py3-none-any.whl

langparse/cli.py

@@ -0,0 +1,329 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from langparse import __version__
+from langparse.chunkers.registry import available_chunkers
+from langparse.errors import classify_exception
+from langparse.progress import ProgressEvent
+from langparse.services.batch_service import BatchParseService
+from langparse.services.benchmark_service import BenchmarkService
+from langparse.services.parse_service import ParseService
+
+
+def build_parser():
+    parser = argparse.ArgumentParser(
+        prog="langparse",
+        description="Parse documents and evaluate parsing quality.",
+    )
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    parse_cmd = subparsers.add_parser(
+        "parse",
+        help="parse one document or a batch of documents",
+    )
+    parse_cmd.add_argument("inputs", nargs="+")
+    parse_cmd.add_argument("--engine", default=None)
+    parse_cmd.add_argument("--device", default=None)
+    parse_cmd.add_argument("--model-dir", default=None)
+    parse_cmd.add_argument("--download-dir", default=None)
+    parse_cmd.add_argument("--api-url", default=None)
+    parse_cmd.add_argument("--api-host", default=None)
+    parse_cmd.add_argument("--api-port", type=int, default=None)
+    parse_cmd.add_argument("--api-command", default=None)
+    parse_cmd.add_argument("--api-start-timeout", type=float, default=None)
+    parse_cmd.add_argument("--mineru-request-timeout", type=float, default=None)
+    parse_cmd.add_argument("--mineru-backend", default=None)
+    parse_cmd.add_argument("--mineru-server-url", default=None)
+    parse_cmd.add_argument(
+        "--model-policy", choices=["download_if_missing", "require_existing"], default=None
+    )
+    parse_cmd.add_argument("--model-source", default=None)
+    parse_cmd.add_argument("--auto-install-runtime", action="store_true")
+    parse_cmd.add_argument("--runtime-package", default=None)
+    parse_cmd.add_argument("--format", default="markdown", help="markdown, json, or workbook-json")
+    parse_cmd.add_argument("--batch", action="store_true")
+    parse_cmd.add_argument("--output", default=None)
+    parse_cmd.add_argument("--output-dir", default=None)
+    parse_cmd.add_argument("--max-workers", type=int, default=None)
+    parse_cmd.add_argument("--skip-existing", action="store_true")
+    parse_cmd.add_argument("--metrics", action="store_true")
+    parse_cmd.add_argument("--progress", action="store_true", help="write progress to stderr")
+    parse_cmd.add_argument(
+        "--chunk",
+        action="store_true",
+        help="semantically chunk the parsed document and include chunks in the output",
+    )
+    parse_cmd.add_argument("--chunk-strategy", choices=available_chunkers(), default=None)
+    parse_cmd.add_argument(
+        "--chunk-size",
+        type=int,
+        default=None,
+        help="strategy size budget (lexical tokens for fixed-token, characters otherwise)",
+    )
+    parse_cmd.add_argument("--chunk-overlap", type=int, default=None)
+    parse_cmd.add_argument(
+        "--chunk-profile",
+        choices=["retrieval", "analysis"],
+        default="retrieval",
+        help="choose retrieval-oriented or analysis-oriented chunks",
+    )
+    parse_cmd.add_argument(
+        "--model",
+        nargs="?",
+        const="",
+        default=None,
+        help="enable model-assisted parsing (reads OPENAI_MODEL from env if model name omitted)",
+    )
+    parse_cmd.add_argument("--base-url", default=None, help="override OPENAI_BASE_URL")
+    parse_cmd.add_argument(
+        "--disambiguation",
+        choices=["off", "auto", "required"],
+        default=None,
+        help="workbook ambiguity resolution mode",
+    )
+
+    benchmark_cmd = subparsers.add_parser(
+        "benchmark",
+        help="run the general parsing benchmark",
+    )
+    benchmark_cmd.add_argument("manifest")
+    benchmark_cmd.add_argument("--engine", default=None)
+    benchmark_cmd.add_argument("--output-dir", default="reports")
+    benchmark_cmd.add_argument("--format", default="json")
+    benchmark_cmd.add_argument("--max-workers", type=int, default=1)
+    benchmark_cmd.add_argument("--api-url", default=None)
+    benchmark_cmd.add_argument("--mineru-request-timeout", type=float, default=None)
+    benchmark_cmd.add_argument("--mineru-backend", default=None)
+    benchmark_cmd.add_argument("--mineru-server-url", default=None)
+    benchmark_cmd.add_argument("--device", default=None)
+    benchmark_cmd.add_argument("--model-dir", default=None)
+    benchmark_cmd.add_argument("--download-dir", default=None)
+    benchmark_cmd.add_argument("--auto-install-runtime", action="store_true")
+    benchmark_cmd.add_argument("--runtime-package", default=None)
+
+    eval_cmd = subparsers.add_parser(
+        "benchmark-workbook-ambiguity",
+        aliases=["eval", "eval-excel"],
+        help="evaluate workbook ambiguity handling from a manifest",
+        description=(
+            "Evaluate deterministic and optional live-model workbook ambiguity handling. "
+            "API keys are read from OPENAI_API_KEY, never command-line arguments."
+        ),
+    )
+    eval_cmd.add_argument("manifest")
+    eval_cmd.add_argument("--output-dir", default="reports/workbook-ambiguity")
+    eval_cmd.add_argument(
+        "--no-markdown", action="store_true", help="disable markdown summary output"
+    )
+    eval_cmd.add_argument(
+        "--model",
+        nargs="?",
+        const="",
+        default=None,
+        help="enable live model evaluation (reads OPENAI_MODEL from env if name omitted)",
+    )
+    eval_cmd.add_argument("--base-url", default=None, help="override OPENAI_BASE_URL")
+
+    quality_cmd = subparsers.add_parser(
+        "benchmark-workbook-quality",
+        aliases=["eval-workbook"],
+        help="evaluate complete workbook structure from a versioned manifest",
+    )
+    quality_cmd.add_argument("manifest")
+    quality_cmd.add_argument("--output-dir", default="reports/workbook-quality")
+    quality_cmd.add_argument(
+        "--no-markdown", action="store_true", help="disable markdown summary output"
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        return _run(args, parser)
+    except Exception as exc:  # noqa: BLE001 - CLI boundary: report, never traceback
+        classified = classify_exception(exc)
+        print(f"langparse: {classified.error_type.value}: {classified.message}", file=sys.stderr)
+        return 2
+
+
+def _run(args, parser) -> int:
+    if args.command in ("benchmark-workbook-quality", "eval-workbook"):
+        from langparse.services.workbook_quality_benchmark import (
+            WorkbookQualityBenchmarkService,
+        )
+
+        report = WorkbookQualityBenchmarkService().run(
+            args.manifest,
+            output_dir=args.output_dir,
+            markdown=not args.no_markdown,
+        )
+        print(
+            "langparse: workbook quality benchmark completed "
+            f"with status={report.summary['status']} ({report.run_digest})"
+        )
+        return 0 if report.summary["status"] == "passed" else 1
+
+    if args.command in ("benchmark-workbook-ambiguity", "eval", "eval-excel"):
+        from langparse.services.workbook_ambiguity_benchmark import (
+            WorkbookAmbiguityBenchmarkService,
+        )
+
+        report = WorkbookAmbiguityBenchmarkService().run(
+            args.manifest,
+            output_dir=args.output_dir,
+            markdown=not args.no_markdown,
+            model=args.model,
+            base_url=args.base_url,
+        )
+        print(f"langparse: workbook ambiguity benchmark completed ({report.run_digest})")
+        return 0
+
+    if args.command == "benchmark":
+        benchmark_kwargs = {
+            key: value
+            for key, value in {
+                "api_url": args.api_url,
+                "request_timeout": args.mineru_request_timeout,
+                "backend": args.mineru_backend,
+                "server_url": args.mineru_server_url,
+                "device": args.device,
+                "model_dir": args.model_dir,
+                "download_dir": args.download_dir,
+                "auto_install_runtime": args.auto_install_runtime,
+                "runtime_package": args.runtime_package,
+            }.items()
+            if value is not None and value is not False
+        }
+        BenchmarkService().run(
+            args.manifest,
+            output_dir=args.output_dir,
+            engine_name=args.engine,
+            fmt=args.format,
+            max_workers=args.max_workers,
+            **benchmark_kwargs,
+        )
+        return 0
+
+    if args.command != "parse":
+        parser.error(f"Unsupported command: {args.command}")
+
+    service = ParseService()
+    engine_name = args.engine or "simple"
+    disambiguation_mode = args.disambiguation or ("auto" if args.model is not None else None)
+    model_kwargs = {
+        key: value
+        for key, value in {
+            "workbook_disambiguation": disambiguation_mode,
+            "model": args.model,
+            "base_url": args.base_url,
+        }.items()
+        if value is not None
+    }
+
+    parse_kwargs = {
+        key: value
+        for key, value in {
+            "device": args.device,
+            "model_dir": args.model_dir,
+            "download_dir": args.download_dir,
+            "api_url": args.api_url,
+            "api_host": args.api_host,
+            "api_port": args.api_port,
+            "api_command": args.api_command,
+            "api_start_timeout": args.api_start_timeout,
+            "request_timeout": args.mineru_request_timeout,
+            "backend": args.mineru_backend,
+            "server_url": args.mineru_server_url,
+            "model_policy": args.model_policy,
+            "model_source": args.model_source,
+            "auto_install_runtime": args.auto_install_runtime,
+            "runtime_package": args.runtime_package,
+            **model_kwargs,
+        }.items()
+        if value is not None and value is not False
+    }
+    chunk_kwargs = {"chunk_profile": args.chunk_profile} if args.chunk else {}
+    if any(
+        value is not None for value in (args.chunk_strategy, args.chunk_size, args.chunk_overlap)
+    ):
+        if not args.chunk:
+            parser.error("chunk strategy/size/overlap options require --chunk")
+        chunk_kwargs["chunk_strategy"] = args.chunk_strategy or "semantic"
+        chunk_kwargs["chunk_options"] = {
+            key: value
+            for key, value in {
+                "max_chunk_size": args.chunk_size,
+                "overlap": args.chunk_overlap,
+            }.items()
+            if value is not None
+        }
+    if args.progress:
+        parse_kwargs["progress_callback"] = _print_progress
+
+    if args.batch:
+        # One implementation regardless of flags. Without --output-dir the run
+        # renders to memory and prints; with it, outputs and reports are written.
+        result = BatchParseService().run(
+            args.inputs,
+            engine_name=engine_name,
+            output_dir=args.output_dir,
+            fmt=args.format,
+            max_workers=args.max_workers,
+            skip_existing=args.skip_existing,
+            collect_metrics=args.metrics,
+            chunk=args.chunk,
+            **chunk_kwargs,
+            **parse_kwargs,
+        )
+        for rendered in result.rendered_outputs:
+            print(rendered)
+        return 0
+
+    if len(args.inputs) != 1:
+        parser.error(
+            "Single parse mode accepts exactly one input. Use --batch for multiple inputs."
+        )
+
+    rendered = service.parse_output(
+        args.inputs[0],
+        engine_name=engine_name,
+        fmt=args.format,
+        chunk=args.chunk,
+        **chunk_kwargs,
+        **parse_kwargs,
+    )
+
+    if args.output:
+        service.write_output(rendered, Path(args.output))
+    else:
+        print(rendered)
+
+    return 0
+
+
+def _print_progress(event: ProgressEvent) -> None:
+    counts = ""
+    if event.completed_units is not None:
+        total = event.total_units if event.total_units is not None else "?"
+        counts = f" {event.completed_units}/{total} {event.unit or ''}"
+    percent = f" {event.percent:.0f}%" if event.percent is not None else ""
+    # Escape embedded line breaks/control characters from untrusted filenames/messages.
+    source = repr(event.source) if event.source else "batch"
+    message = f" {event.message!r}" if event.message else ""
+    print(
+        f"langparse: {source} {event.phase} {event.state}{counts}{percent}{message}",
+        file=sys.stderr,
+        flush=True,
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

langparse/config.py

@@ -0,0 +1,169 @@
+import copy
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from langparse.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class Config:
+    """
+    Global configuration manager for LangParse.
+    Priorities:
+    1. Runtime kwargs (passed to functions)
+    2. Environment variables (LANGPARSE_*)
+    3. Config file (~/.langparse/config.json)
+    4. Defaults
+    """
+
+    ENV_MAP = {
+        "LANGPARSE_DEFAULT_PDF_ENGINE": "default_pdf_engine",
+        "LANGPARSE_MINERU_DEVICE": "engines.mineru.device",
+        "LANGPARSE_MINERU_MODEL_DIR": "engines.mineru.model_dir",
+        "LANGPARSE_MINERU_DOWNLOAD_DIR": "engines.mineru.download_dir",
+        "LANGPARSE_MINERU_ENABLE_OCR": "engines.mineru.enable_ocr",
+        "LANGPARSE_MINERU_API_URL": "engines.mineru.api_url",
+        "LANGPARSE_MINERU_API_HOST": "engines.mineru.api_host",
+        "LANGPARSE_MINERU_API_PORT": "engines.mineru.api_port",
+        "LANGPARSE_MINERU_API_COMMAND": "engines.mineru.api_command",
+        "LANGPARSE_MINERU_API_START_TIMEOUT": "engines.mineru.api_start_timeout",
+        "LANGPARSE_MINERU_REQUEST_TIMEOUT": "engines.mineru.request_timeout",
+        "LANGPARSE_MINERU_BACKEND": "engines.mineru.backend",
+        "LANGPARSE_MINERU_SERVER_URL": "engines.mineru.server_url",
+        "LANGPARSE_MINERU_MODEL_POLICY": "engines.mineru.model_policy",
+        "LANGPARSE_MINERU_MODEL_SOURCE": "engines.mineru.model_source",
+        "LANGPARSE_MINERU_AUTO_INSTALL_RUNTIME": "engines.mineru.auto_install_runtime",
+        "LANGPARSE_MINERU_RUNTIME_PACKAGE": "engines.mineru.runtime_package",
+        "LANGPARSE_DEEPDOC_DEVICE": "engines.deepdoc.device",
+        "LANGPARSE_DEEPDOC_MODEL_DIR": "engines.deepdoc.model_dir",
+        "LANGPARSE_DEEPDOC_DOWNLOAD_DIR": "engines.deepdoc.download_dir",
+        "LANGPARSE_DEEPDOC_MODEL_POLICY": "engines.deepdoc.model_policy",
+    }
+
+    DEFAULT_CONFIG = {
+        "default_pdf_engine": "simple",
+        "engines": {
+            "mineru": {
+                "device": "auto",
+                "model_dir": None,
+                "download_dir": None,
+                "enable_ocr": True,
+                "api_url": None,
+                "api_host": "127.0.0.1",
+                "api_port": 8000,
+                "api_command": "mineru-api",
+                "api_start_timeout": 30.0,
+                "request_timeout": 300.0,
+                "backend": None,
+                "server_url": None,
+                "model_policy": "download_if_missing",
+                "model_source": None,
+                "auto_install_runtime": False,
+                "runtime_package": "mineru>=3.4,<4",
+                "extra_options": {},
+            },
+            "vision_llm": {
+                "provider": "openai",
+                "model": "gpt-4o",
+                "api_key": None,
+            },
+            "deepdoc": {
+                "device": "cpu",
+                "model_dir": None,
+                "download_dir": None,
+                "model_policy": "download_if_missing",
+            },
+        },
+    }
+
+    def __init__(self):
+        self._config = copy.deepcopy(self.DEFAULT_CONFIG)
+        self._load_from_file()
+        self._load_from_env()
+
+    def _load_from_file(self):
+        config_path = Path.home() / ".langparse" / "config.json"
+        if config_path.exists():
+            try:
+                with open(config_path, encoding="utf-8") as f:
+                    user_config = json.load(f)
+                    self._merge_dict(self._config, user_config)
+            except Exception as e:
+                logger.warning("Failed to load config file %s: %s", config_path, e)
+
+    def _load_from_env(self):
+        for env_key, config_key in self.ENV_MAP.items():
+            if env_key not in os.environ:
+                continue
+            current_value = self.get(config_key)
+            parsed_value = self._parse_env_value(os.environ[env_key], current_value)
+            self._set_nested_value(config_key, parsed_value)
+
+    def _parse_env_value(self, value: str, current_value: Any = None) -> Any:
+        normalized = value.strip()
+        lowered = normalized.lower()
+
+        if isinstance(current_value, bool):
+            if lowered in {"true", "1", "yes", "on"}:
+                return True
+            if lowered in {"false", "0", "no", "off"}:
+                return False
+
+        if current_value is None and lowered in {"null", "none", ""}:
+            return None
+
+        try:
+            return json.loads(normalized)
+        except json.JSONDecodeError:
+            return normalized
+
+    def _set_nested_value(self, key: str, value: Any) -> None:
+        keys = key.split(".")
+        target = self._config
+        for part in keys[:-1]:
+            if part not in target or not isinstance(target[part], dict):
+                target[part] = {}
+            target = target[part]
+        target[keys[-1]] = value
+
+    def _merge_dict(self, base: dict, update: dict):
+        for k, v in update.items():
+            if k in base and isinstance(base[k], dict) and isinstance(v, dict):
+                self._merge_dict(base[k], v)
+            else:
+                base[k] = v
+
+    def resolve_engine_config(
+        self, engine_name: str, runtime_kwargs: dict[str, Any]
+    ) -> dict[str, Any]:
+        config_key = f"engines.{engine_name}"
+        engine_config = self.get(config_key, {})
+        resolved_config = {**engine_config, **runtime_kwargs}
+
+        config_extra_options = engine_config.get("extra_options")
+        runtime_extra_options = runtime_kwargs.get("extra_options")
+        if isinstance(config_extra_options, dict) and isinstance(runtime_extra_options, dict):
+            resolved_config["extra_options"] = {
+                **config_extra_options,
+                **runtime_extra_options,
+            }
+
+        return resolved_config
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Get a config value using dot notation, e.g. 'engines.mineru.model_dir'"""
+        keys = key.split(".")
+        val = self._config
+        for k in keys:
+            if isinstance(val, dict) and k in val:
+                val = val[k]
+            else:
+                return default
+        return val
+
+
+# Singleton instance
+settings = Config()

langparse/core/chunker.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+
+from langparse.types import Chunk, Document
+
+
+class BaseChunker(ABC):
+    """
+    Abstract base class for all text chunkers.
+    """
+
+    @abstractmethod
+    def chunk(self, document: Document, **kwargs) -> list[Chunk]:
+        """
+        Split a Document into a list of Chunks.
+        """
+        pass

langparse/core/engine.py

@@ -0,0 +1,37 @@
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from langparse.types import ParsedElement, StructuredData
+
+
+@dataclass
+class PageResult:
+    """
+    Engine-facing iterative page result yielded during parsing.
+    Mirrors the normalized parsed page shape so engines can stream page data
+    before document assembly without carrying a second incompatible contract.
+    """
+
+    page_number: int
+    markdown_content: str
+    plain_text: str = ""
+    elements: list[ParsedElement] = field(default_factory=list)
+    tables: list[StructuredData] = field(default_factory=list)
+    images: list[StructuredData] = field(default_factory=list)
+    metadata: StructuredData = field(default_factory=dict)
+
+
+class BaseEngine(ABC):
+    """
+    Abstract base class for all parsing engines.
+    """
+
+    @abstractmethod
+    def process(self, file_path: Path, **kwargs) -> Iterator[PageResult]:
+        """
+        Process a file and yield results page by page.
+        This allows for streaming processing of large documents.
+        """
+        pass

langparse/core/parser.py

@@ -0,0 +1,35 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from langparse.core.rendering import document_from_result
+from langparse.types import Document, ParsedDocumentResult
+
+
+class BaseParser(ABC):
+    """
+    Abstract base class for all document parsers.
+
+    Parsers implement `parse_result`, which returns the structured
+    `ParsedDocumentResult` that metrics, quality checks and batch reporting all
+    read. `parse` is the flat Markdown view rendered from that same result, so
+    the two can never disagree.
+    """
+
+    @abstractmethod
+    def parse_result(self, file_path: str | Path, **kwargs) -> ParsedDocumentResult:
+        """
+        Parse a file into its structured page/element/table representation.
+        """
+
+    def parse(self, file_path: str | Path, **kwargs) -> Document:
+        """
+        Parse a file and return the rendered Markdown Document.
+        """
+        return document_from_result(self.parse_result(file_path, **kwargs))
+
+    @staticmethod
+    def _resolve_existing_path(file_path: str | Path) -> Path:
+        path = Path(file_path)
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {path}")
+        return path

langparse/core/rendering.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from langparse.types import Document, ParsedDocumentResult
+
+PAGE_MARKER_TEMPLATE = "<!-- page_number: {page_number} -->"
+
+
+def document_from_result(parsed: ParsedDocumentResult) -> Document:
+    """
+    Render the structured parse result into the flat Markdown ``Document`` that
+    chunkers and end users consume.
+
+    Page markers are injected only for paginated results. A flow format such as
+    plain Markdown has no page boundaries to mark, and injecting a fake one
+    would corrupt any markers the source already carries.
+    """
+    if parsed.paginated:
+        blocks: list[str] = []
+        for page in parsed.pages:
+            blocks.append(f"\n{PAGE_MARKER_TEMPLATE.format(page_number=page.page_number)}\n")
+            blocks.append(page.markdown_content)
+        content = "\n".join(blocks)
+    else:
+        content = "\n".join(page.markdown_content for page in parsed.pages)
+
+    return Document(content=content, metadata=document_metadata(parsed))
+
+
+def document_metadata(parsed: ParsedDocumentResult) -> dict:
+    """
+    Build the Document metadata.
+
+    Engine-specific detail stays nested under ``parsed_metadata`` on purpose:
+    SemanticChunker copies this dict into every chunk, so flattening MinerU's
+    runtime fields here would duplicate them across every chunk written to a
+    vector store.
+    """
+    return {
+        "source": parsed.source,
+        "filename": parsed.filename,
+        "extension": parsed.metadata.get("extension") or Path(parsed.filename).suffix,
+        "engine": parsed.engine,
+        # Copied, not aliased: SemanticChunker shallow-copies this dict into
+        # every chunk, so sharing one instance would let a later mutation of the
+        # parse result reach through into already-emitted chunks.
+        "parsed_metadata": dict(parsed.metadata),
+    }

langparse/engines/__init__.py

@@ -0,0 +1 @@
+# Engines package

langparse/engines/pdf/__init__.py

@@ -0,0 +1 @@
+# PDF Engines package