langparse 0.1.0__py3-none-any.whl

langparse/engines/pdf/simple.py

@@ -0,0 +1,134 @@
+import threading
+from collections.abc import Iterator
+from pathlib import Path
+
+from langparse.core.engine import BaseEngine, PageResult
+from langparse.engines.pdf.ocr import (
+    DEFAULT_MIN_CHARS,
+    DEFAULT_RESOLUTION,
+    load_recogniser,
+    needs_ocr,
+    ocr_page_text,
+)
+from langparse.progress import ProgressCallback, ProgressReporter
+
+
+class BasePDFEngine(BaseEngine):
+    """
+    Base class specifically for PDF engines.
+    """
+
+    def __init__(self, **kwargs):
+        pass
+
+
+class SimplePDFEngine(BasePDFEngine):
+    """
+    A lightweight, dependency-free (except pdfplumber) engine.
+    Good for simple, native PDFs.
+
+    Scanned pages fall back to OCR. Without it a scanned document parses
+    "successfully" into a handful of watermark characters, which is harder for a
+    caller to notice than an outright failure.
+    """
+
+    def __init__(
+        self,
+        enable_ocr: bool = True,
+        ocr_min_chars: int = DEFAULT_MIN_CHARS,
+        ocr_resolution: int = DEFAULT_RESOLUTION,
+        recogniser=None,
+        **kwargs,
+    ):
+        self.enable_ocr = enable_ocr
+        self.ocr_min_chars = ocr_min_chars
+        self.ocr_resolution = ocr_resolution
+        self._recogniser = recogniser
+        # Batch runs share one engine across worker threads. The lock keeps
+        # model loading (tens of seconds) from happening once per thread, and
+        # serialises recognition because rapidocr states no thread-safety
+        # guarantee -- correctness over throughput on an already slow path.
+        self._ocr_lock = threading.Lock()
+
+    def process(
+        self, file_path: Path, *, progress_callback: ProgressCallback | None = None, **kwargs
+    ) -> Iterator[PageResult]:
+        reporter = ProgressReporter(str(file_path), progress_callback)
+        try:
+            import pdfplumber
+        except ImportError:
+            raise ImportError("Please install `pdfplumber` to use the 'simple' engine.") from None
+
+        with pdfplumber.open(file_path) as pdf:
+            for i, page in enumerate(pdf.pages):
+                text = page.extract_text() or ""
+                ocr_applied = False
+
+                if self.enable_ocr and needs_ocr(page, min_chars=self.ocr_min_chars):
+                    recovered = self._run_ocr(page)
+                    if recovered:
+                        text = recovered
+                        ocr_applied = True
+
+                tables, table_markdown = self._extract_tables(page)
+
+                markdown_content = text
+                if table_markdown:
+                    markdown_content = "\n\n".join([text, "\n".join(table_markdown)]).strip()
+
+                reporter.emit(
+                    "parsing", completed_units=i + 1, total_units=len(pdf.pages), unit="pages"
+                )
+                yield PageResult(
+                    page_number=i + 1,
+                    markdown_content=markdown_content,
+                    plain_text=text,
+                    elements=[],
+                    tables=tables,
+                    images=[],
+                    metadata={
+                        "engine_name": "simple",
+                        "ocr_applied": ocr_applied,
+                        "ocr_text_chars": len(text) if ocr_applied else 0,
+                    },
+                )
+
+    def _run_ocr(self, page) -> str:
+        """Recognise a page, degrading to no text rather than failing the parse."""
+        try:
+            with self._ocr_lock:
+                recogniser = self._build_recogniser_if_needed()
+                return ocr_page_text(page, recogniser, resolution=self.ocr_resolution)
+        except ImportError:
+            # rapidocr absent: the page keeps whatever thin text layer it had.
+            return ""
+
+    def _resolve_recogniser(self):
+        with self._ocr_lock:
+            return self._build_recogniser_if_needed()
+
+    def _build_recogniser_if_needed(self):
+        """Caller must hold `_ocr_lock`."""
+        if self._recogniser is None:
+            self._recogniser = load_recogniser()
+        return self._recogniser
+
+    def _extract_tables(self, page):
+        tables = []
+        table_markdown = []
+        extract_tables = getattr(page, "extract_tables", None)
+        for table in (extract_tables() if callable(extract_tables) else []) or []:
+            cleaned_table = [
+                ["" if cell is None else str(cell).strip().replace("\n", " ") for cell in row]
+                for row in table
+            ]
+            if not cleaned_table:
+                continue
+
+            tables.append({"rows": cleaned_table})
+            headers = cleaned_table[0]
+            table_markdown.append(f"| {' | '.join(headers)} |")
+            table_markdown.append(f"| {' | '.join(['---'] * len(headers))} |")
+            for row in cleaned_table[1:]:
+                table_markdown.append(f"| {' | '.join(row)} |")
+        return tables, table_markdown

langparse/engines/pdf/vision_llm.py

@@ -0,0 +1,27 @@
+from collections.abc import Iterator
+from pathlib import Path
+
+from langparse.core.engine import PageResult
+from langparse.engines.pdf.simple import BasePDFEngine
+from langparse.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class VisionLLMEngine(BasePDFEngine):
+    """
+    Uses Vision LLMs (GPT-4o, Gemini 1.5 Pro) to parse pages.
+    Extremely slow/expensive but handles EVERYTHING (handwriting, complex charts).
+    """
+
+    def __init__(self, model_name: str = "gpt-4o", api_key: str = None):
+        self.model_name = model_name
+        self.api_key = api_key
+
+    def process(self, file_path: Path, **kwargs) -> Iterator[PageResult]:
+        # 1. Convert PDF to Images (using pdf2image)
+        # 2. Send each image to LLM with a prompt like "Transcribe this page to Markdown"
+
+        logger.debug("VisionLLM processing %s with %s", file_path, self.model_name)
+
+        raise NotImplementedError("Vision LLM integration is pending.")

langparse/errors.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from enum import Enum
+from subprocess import TimeoutExpired
+from urllib.error import URLError
+
+
+class ErrorType(str, Enum):
+    DEPENDENCY_MISSING = "dependency_missing"
+    FILE_NOT_FOUND = "file_not_found"
+    UNSUPPORTED_FORMAT = "unsupported_format"
+    ENGINE_UNAVAILABLE = "engine_unavailable"
+    ENGINE_TIMEOUT = "engine_timeout"
+    PARSE_FAILED = "parse_failed"
+    QUALITY_CHECK_FAILED = "quality_check_failed"
+    OCR_UNAVAILABLE = "ocr_unavailable"
+    LAYOUT_QUALITY_WARNING = "layout_quality_warning"
+    TABLE_EXTRACTION_FAILED = "table_extraction_failed"
+    WORKBOOK_EVALUATION_FAILED = "workbook_evaluation_failed"
+
+
+@dataclass
+class ClassifiedError:
+    error_type: ErrorType
+    message: str
+
+
+def classify_exception(exc: BaseException) -> ClassifiedError:
+    message = str(exc)
+    lowered = message.lower()
+
+    from langparse.workbooks.evaluation.schema import WorkbookEvaluationError
+
+    if isinstance(exc, WorkbookEvaluationError):
+        return ClassifiedError(ErrorType.WORKBOOK_EVALUATION_FAILED, message)
+    if isinstance(exc, FileNotFoundError):
+        return ClassifiedError(ErrorType.FILE_NOT_FOUND, message)
+    if isinstance(exc, ImportError):
+        return ClassifiedError(ErrorType.DEPENDENCY_MISSING, message)
+    if "unsupported file extension" in lowered:
+        return ClassifiedError(ErrorType.UNSUPPORTED_FORMAT, message)
+    if "cuda" in lowered and "not available" in lowered:
+        return ClassifiedError(ErrorType.ENGINE_UNAVAILABLE, message)
+    if "unable to start local mineru-api" in lowered:
+        return ClassifiedError(ErrorType.ENGINE_UNAVAILABLE, message)
+    if _is_timeout(exc) or re.search(r"\btimed out\b", lowered):
+        return ClassifiedError(ErrorType.ENGINE_TIMEOUT, message)
+    if "ocr" in lowered and "unavailable" in lowered:
+        return ClassifiedError(ErrorType.OCR_UNAVAILABLE, message)
+    if "table" in lowered and "failed" in lowered:
+        return ClassifiedError(ErrorType.TABLE_EXTRACTION_FAILED, message)
+
+    return ClassifiedError(ErrorType.PARSE_FAILED, message)
+
+
+def _is_timeout(exc: BaseException) -> bool:
+    """Recognize standard timeout types through explicit backend wrappers."""
+    seen: set[int] = set()
+    current: BaseException | None = exc
+    while current is not None and id(current) not in seen:
+        seen.add(id(current))
+        if isinstance(current, (TimeoutError, TimeoutExpired)):
+            return True
+        if isinstance(current, URLError) and isinstance(current.reason, BaseException):
+            current = current.reason
+        else:
+            current = current.__cause__
+    return False

langparse/logging.py

@@ -0,0 +1,27 @@
+"""
+Logging for the library.
+
+A library should not choose a logging framework for the application embedding
+it, nor write to stderr uninvited. The stdlib logger with a NullHandler stays
+silent until the host configures logging, and costs no dependency -- which
+leaves langparse installable with nothing but the extras a caller actually
+needs for their formats.
+"""
+
+from __future__ import annotations
+
+import logging
+
+ROOT_LOGGER_NAME = "langparse"
+
+_root = logging.getLogger(ROOT_LOGGER_NAME)
+_root.addHandler(logging.NullHandler())
+
+
+def get_logger(name: str | None = None) -> logging.Logger:
+    """Return a logger under the `langparse` namespace."""
+    if not name:
+        return _root
+    if name.startswith(f"{ROOT_LOGGER_NAME}."):
+        return logging.getLogger(name)
+    return logging.getLogger(f"{ROOT_LOGGER_NAME}.{name}")

langparse/metrics.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from langparse.types import Chunk, ParsedDocumentResult
+
+
+def pages_per_second(page_count: int, elapsed_seconds: float) -> float:
+    if elapsed_seconds <= 0:
+        return 0.0
+    return round(page_count / elapsed_seconds, 4)
+
+
+def count_markdown_tables(markdown: str) -> int:
+    separator_pattern = re.compile(r"^\s*\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$")
+    return sum(1 for line in markdown.splitlines() if separator_pattern.match(line))
+
+
+@dataclass
+class ParseMetrics:
+    elapsed_seconds: float = 0.0
+    page_count: int = 0
+    pages_per_second: float = 0.0
+    output_bytes: int = 0
+    markdown_chars: int = 0
+    table_count: int = 0
+    image_count: int = 0
+    chunk_count: int = 0
+    chunks_with_page_numbers_ratio: float = 0.0
+    page_marker_coverage: float = 0.0
+    ocr_applied: bool = False
+    ocr_text_chars: int = 0
+    multi_column_detected: bool = False
+    reading_order_warnings: int = 0
+    header_footer_removed_count: int = 0
+    caption_count: int = 0
+    images_with_caption_ratio: float = 0.0
+
+
+@dataclass
+class BatchItemResult:
+    source: str
+    status: str
+    output_path: str | None = None
+    metrics: ParseMetrics | None = None
+    error_type: str | None = None
+    error_message: str | None = None
+    engine: str | None = None
+    started_at: str | None = None
+    finished_at: str | None = None
+
+
+@dataclass
+class BatchRunResult:
+    items: list[BatchItemResult] = field(default_factory=list)
+    summary: dict[str, Any] = field(default_factory=dict)
+    #: Rendered documents, populated only when the run had no output directory
+    #: and therefore rendered to memory instead of to disk. Kept off
+    #: BatchItemResult so it never bloats the JSONL report.
+    rendered_outputs: list[str] = field(default_factory=list)
+
+    @property
+    def total_files(self) -> int:
+        return len(self.items)
+
+    @property
+    def success_count(self) -> int:
+        return sum(1 for item in self.items if item.status == "success")
+
+    @property
+    def failed_count(self) -> int:
+        return sum(1 for item in self.items if item.status == "failed")
+
+    @property
+    def skipped_count(self) -> int:
+        return sum(1 for item in self.items if item.status == "skipped")
+
+
+def page_marker_coverage(parsed: ParsedDocumentResult) -> float:
+    """
+    Fraction of pages carrying a usable page number.
+
+    This is what makes a chunk citable back to a page, so the
+    ``require_page_markers`` quality check reads it directly.
+    """
+    if not getattr(parsed, "paginated", True) or not parsed.pages:
+        return 0.0
+    numbered = sum(1 for page in parsed.pages if page.page_number and page.page_number >= 1)
+    return round(numbered / len(parsed.pages), 4)
+
+
+def collect_parse_metrics(
+    parsed: ParsedDocumentResult,
+    elapsed_seconds: float,
+    chunks: list[Chunk] | None = None,
+) -> ParseMetrics:
+    markdown = parsed.markdown_content or ""
+    page_count = len(parsed.pages)
+    chunk_count = len(chunks) if chunks is not None else 0
+    chunks_with_pages = (
+        sum(1 for chunk in chunks if chunk.metadata.get("page_numbers")) if chunks else 0
+    )
+    image_count = sum(len(page.images) for page in parsed.pages)
+    table_count = sum(len(page.tables) for page in parsed.pages) or count_markdown_tables(markdown)
+    caption_count = sum(1 for page in parsed.pages for image in page.images if image.get("caption"))
+
+    return ParseMetrics(
+        elapsed_seconds=round(elapsed_seconds, 4),
+        page_count=page_count,
+        pages_per_second=pages_per_second(page_count, elapsed_seconds),
+        output_bytes=len(markdown.encode("utf-8")),
+        markdown_chars=len(markdown),
+        table_count=table_count,
+        image_count=image_count,
+        chunk_count=chunk_count,
+        chunks_with_page_numbers_ratio=round(chunks_with_pages / chunk_count, 4)
+        if chunk_count
+        else 0.0,
+        page_marker_coverage=page_marker_coverage(parsed),
+        ocr_applied=bool(parsed.metadata.get("ocr_applied", False)),
+        ocr_text_chars=int(parsed.metadata.get("ocr_text_chars", 0) or 0),
+        multi_column_detected=bool(parsed.metadata.get("multi_column_detected", False)),
+        reading_order_warnings=int(parsed.metadata.get("reading_order_warnings", 0) or 0),
+        header_footer_removed_count=int(parsed.metadata.get("header_footer_removed_count", 0) or 0),
+        caption_count=caption_count,
+        images_with_caption_ratio=round(caption_count / image_count, 4) if image_count else 0.0,
+    )

langparse/parsers/docx_parser.py

@@ -0,0 +1,114 @@
+from pathlib import Path
+
+from langparse.core.parser import BaseParser
+from langparse.types import ParsedDocumentResult, ParsedElement, ParsedPageResult
+
+_HEADING_PREFIXES = (
+    ("heading 1", "# "),
+    ("title", "# "),
+    ("heading 2", "## "),
+    ("heading 3", "### "),
+)
+
+
+class DocxParser(BaseParser):
+    """
+    Parses .docx files to Markdown.
+    Note: DOCX is a flow format, so 'page numbers' are not strictly defined.
+    We treat the entire document as Page 1 for now, unless we convert to PDF first.
+    """
+
+    def parse_result(self, file_path: str | Path, **kwargs) -> ParsedDocumentResult:
+        path = self._resolve_existing_path(file_path)
+
+        try:
+            import docx
+        except ImportError:
+            raise ImportError(
+                "python-docx is required. Install with `pip install python-docx`."
+            ) from None
+
+        doc = docx.Document(path)
+        markdown_lines: list[str] = []
+        elements: list[ParsedElement] = []
+        tables: list[dict] = []
+
+        for block in self._iter_block_items(docx, doc):
+            if isinstance(block, docx.text.paragraph.Paragraph):
+                rendered = self._render_paragraph(block)
+                if rendered is None:
+                    continue
+                kind, markdown = rendered
+                markdown_lines.extend([markdown, ""])
+                elements.append(ParsedElement(kind=kind, text=block.text.strip()))
+                continue
+
+            rows = self._table_rows(block)
+            if not rows:
+                continue
+            table_markdown = self._rows_to_markdown(rows)
+            tables.append({"rows": rows})
+            markdown_lines.extend([table_markdown, ""])
+            elements.append(ParsedElement(kind="table", text=table_markdown))
+
+        markdown_content = "\n".join(markdown_lines)
+        return ParsedDocumentResult(
+            source=str(path),
+            filename=path.name,
+            engine="docx",
+            pages=[
+                ParsedPageResult(
+                    page_number=1,
+                    markdown_content=markdown_content,
+                    plain_text="\n".join(
+                        element.text for element in elements if element.kind != "table"
+                    ),
+                    elements=elements,
+                    tables=tables,
+                )
+            ],
+            markdown_content=markdown_content,
+            metadata={"extension": ".docx"},
+        )
+
+    def _iter_block_items(self, docx, parent):
+        """
+        Yield paragraphs and tables in document order.
+
+        python-docx exposes paragraphs and tables as separate collections, which
+        loses their interleaving, so walk the body XML directly.
+        """
+        from docx.oxml.table import CT_Tbl
+        from docx.oxml.text.paragraph import CT_P
+
+        parent_elm = (
+            parent.element.body if isinstance(parent, docx.document.Document) else parent._element
+        )
+        for child in parent_elm.iterchildren():
+            if isinstance(child, CT_P):
+                yield docx.text.paragraph.Paragraph(child, parent)
+            elif isinstance(child, CT_Tbl):
+                yield docx.table.Table(child, parent)
+
+    def _render_paragraph(self, paragraph) -> tuple[str, str] | None:
+        text = paragraph.text.strip()
+        if not text:
+            return None
+
+        style_name = paragraph.style.name.lower()
+        for needle, prefix in _HEADING_PREFIXES:
+            if needle in style_name:
+                return "heading", f"{prefix}{text}"
+        if "list" in style_name:
+            return "list_item", f"- {text}"
+        return "paragraph", text
+
+    def _table_rows(self, table) -> list[list[str]]:
+        return [[cell.text.strip().replace("\n", " ") for cell in row.cells] for row in table.rows]
+
+    def _rows_to_markdown(self, rows: list[list[str]]) -> str:
+        width = max(len(row) for row in rows)
+        padded = [row + [""] * (width - len(row)) for row in rows]
+        lines = [f"| {' | '.join(padded[0])} |", f"| {' | '.join(['---'] * width)} |"]
+        lines.extend(f"| {' | '.join(row)} |" for row in padded[1:])
+        return "\n".join(lines)