langparse 0.1.0__py3-none-any.whl

langparse/parsers/excel_parser.py

@@ -0,0 +1,220 @@
+import csv
+from pathlib import Path
+
+from langparse.core.parser import BaseParser
+from langparse.parsers.sniff import looks_like_ole_binary, looks_like_zip_ooxml
+from langparse.progress import ProgressCallback, ProgressReporter
+from langparse.types import ParsedDocumentResult, ParsedElement, ParseDiagnostics, ParsedPageResult
+from langparse.workbooks.modeling import (
+    RequiredWorkbookDisambiguationError,
+    WorkbookDisambiguation,
+)
+
+
+class ExcelParser(BaseParser):
+    """
+    Parses workbook facts and exposes one compatibility part per sheet.
+
+    Sheets keep stable ordinals but are not pages, so Excel results are always
+    marked ``paginated=False``.
+    """
+
+    def __init__(
+        self,
+        disambiguation: str | WorkbookDisambiguation | None = None,
+        *,
+        model: str | None = None,
+        api_key: str | None = None,
+        base_url: str | None = None,
+    ) -> None:
+        if isinstance(disambiguation, WorkbookDisambiguation):
+            self.disambiguation = disambiguation
+        elif isinstance(disambiguation, str):
+            mode = disambiguation.lower()
+            if mode == "off":
+                self.disambiguation = WorkbookDisambiguation.off()
+            elif mode in ("auto", "required"):
+                from langparse.workbooks.modeling.openai_adapter import (
+                    OpenAIWorkbookStructureAdapter,
+                )
+
+                adapter = OpenAIWorkbookStructureAdapter.from_env(
+                    cli_model=model,
+                    cli_api_key=api_key,
+                    cli_base_url=base_url,
+                )
+                if mode == "auto":
+                    self.disambiguation = WorkbookDisambiguation.auto(adapter)
+                else:
+                    self.disambiguation = WorkbookDisambiguation.required(adapter)
+            else:
+                raise ValueError(f"Invalid disambiguation mode: {disambiguation}")
+        elif model is not None:
+            from langparse.workbooks.modeling.openai_adapter import (
+                OpenAIWorkbookStructureAdapter,
+            )
+
+            adapter = OpenAIWorkbookStructureAdapter.from_env(
+                cli_model=model,
+                cli_api_key=api_key,
+                cli_base_url=base_url,
+            )
+            self.disambiguation = WorkbookDisambiguation.auto(adapter)
+        else:
+            self.disambiguation = (
+                WorkbookDisambiguation.off() if disambiguation is None else disambiguation
+            )
+
+    def parse_result(
+        self, file_path: str | Path, *, progress_callback: ProgressCallback | None = None, **kwargs
+    ) -> ParsedDocumentResult:
+        path = self._resolve_existing_path(file_path)
+        reporter = ProgressReporter(path, progress_callback)
+
+        if looks_like_zip_ooxml(path):
+            return self._parse_ooxml(path, reporter)
+
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ImportError(
+                "pandas and openpyxl are required. Install with `pip install pandas openpyxl`."
+            ) from None
+
+        # The extension names one of .csv/.xls/.xlsx, but it can lie -- a
+        # workbook re-exported or renamed to .csv (or vice versa) would
+        # otherwise be handed to the wrong pandas reader. Content decides:
+        # a real workbook is either a ZIP-OOXML or legacy-OLE container;
+        # anything else is read as delimited text regardless of its label.
+        reporter.emit("extracting")
+        is_legacy_workbook = looks_like_ole_binary(path)
+        if is_legacy_workbook:
+            sheets = pd.read_excel(path, sheet_name=None)
+        else:
+            with path.open(encoding="utf-8-sig", newline="") as stream:
+                sample = stream.read(65536)
+            try:
+                delimiter = csv.Sniffer().sniff(sample, delimiters=",\t;|").delimiter
+            except csv.Error:
+                # A sample can end inside a quoted multiline field. The header
+                # still provides a delimiter without counting commas in prose.
+                try:
+                    delimiter = (
+                        csv.Sniffer().sniff(sample.splitlines()[0], delimiters=",\t;|").delimiter
+                    )
+                except (csv.Error, IndexError):
+                    delimiter = ","
+            sheets = {
+                None: pd.read_csv(
+                    path, sep=delimiter, encoding="utf-8-sig", dtype=str, keep_default_na=False
+                )
+            }
+
+        reporter.emit("rendering")
+        pages = [
+            self._page_for_sheet(index + 1, sheet_name, frame)
+            for index, (sheet_name, frame) in enumerate(sheets.items())
+        ]
+
+        return ParsedDocumentResult(
+            source=str(path),
+            filename=path.name,
+            engine="excel",
+            pages=pages,
+            markdown_content="\n".join(page.markdown_content for page in pages),
+            metadata={"extension": path.suffix, "sheet_count": len(pages)},
+            paginated=False,
+            diagnostics=(
+                ParseDiagnostics(
+                    status="partial",
+                    reconstruction_passed=False,
+                    unsupported_features=[
+                        "Legacy OLE workbooks use the pandas compatibility adapter; "
+                        "rich workbook facts are not available in Phase 1."
+                    ],
+                )
+                if is_legacy_workbook
+                else None
+            ),
+        )
+
+    def _parse_ooxml(self, path: Path, reporter: ProgressReporter) -> ParsedDocumentResult:
+        try:
+            from langparse.workbooks.adapters import OOXMLWorkbookAdapter
+            from langparse.workbooks.assembly import assemble_baseline, assemble_workbook
+            from langparse.workbooks.rendering import (
+                compatibility_pages,
+                render_workbook_markdown,
+            )
+        except ImportError:
+            raise ImportError(
+                "openpyxl is required for OOXML workbooks. "
+                "Install with `pip install langparse[excel]`."
+            ) from None
+
+        reporter.emit("extracting")
+        snapshot = OOXMLWorkbookAdapter().snapshot(path)
+        reporter.emit("assembling")
+        try:
+            structure, diagnostics = assemble_workbook(
+                snapshot,
+                disambiguation=self.disambiguation,
+            )
+        except RequiredWorkbookDisambiguationError:
+            raise
+        except Exception as exc:
+            structure, diagnostics = assemble_baseline(snapshot)
+            diagnostics.status = "partial"
+            diagnostics.warnings.append(
+                f"Semantic workbook assembly failed; retained raw-grid fallback: {type(exc).__name__}"
+            )
+        from langparse.workbooks.objects import attach_objects
+
+        attach_objects(snapshot, structure, diagnostics)
+        reporter.emit("rendering")
+        pages = compatibility_pages(snapshot, structure)
+        markdown = render_workbook_markdown(snapshot, structure)
+        return ParsedDocumentResult(
+            source=str(path),
+            filename=path.name,
+            engine="excel",
+            pages=pages,
+            markdown_content=markdown,
+            metadata={"extension": path.suffix, "sheet_count": len(snapshot.sheets)},
+            paginated=False,
+            structure=structure,
+            diagnostics=diagnostics,
+        )
+
+    def _page_for_sheet(self, page_number: int, sheet_name, frame) -> ParsedPageResult:
+        table_markdown = frame.to_markdown(index=False, disable_numparse=True)
+        heading = f"## Sheet: {sheet_name}\n" if sheet_name is not None else ""
+        markdown_content = f"{heading}\n{table_markdown}\n" if heading else table_markdown
+
+        rows = [[str(column) for column in frame.columns]]
+        rows.extend([[self._cell_text(value) for value in row] for row in frame.values])
+
+        return ParsedPageResult(
+            page_number=page_number,
+            markdown_content=markdown_content,
+            plain_text=table_markdown,
+            elements=[ParsedElement(kind="table", text=table_markdown)],
+            tables=[{"rows": rows, "sheet_name": sheet_name}],
+            metadata={"part_kind": "sheet", "sheet_name": sheet_name, "source_range": None},
+        )
+
+    def _cell_text(self, value) -> str:
+        """
+        Render one cell for the structured table.
+
+        Blank cells arrive as NaN rather than None, and a single blank promotes
+        an integer column to float -- so a naive str() yields "nan" and "1.0"
+        where the source held an empty cell and 1.
+        """
+        import pandas as pd
+
+        if value is None or pd.isna(value):
+            return ""
+        if isinstance(value, float) and value.is_integer():
+            return str(int(value))
+        return str(value)

langparse/parsers/markdown_parser.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+
+from langparse.core.parser import BaseParser
+from langparse.types import ParsedDocumentResult, ParsedPageResult
+
+
+class MarkdownParser(BaseParser):
+    """
+    A simple parser for Markdown files.
+
+    Markdown is a flow format with no page boundaries, so the whole file is one
+    unpaginated page and the content passes through verbatim -- including any
+    page markers the source already carries.
+    """
+
+    def parse_result(self, file_path: str | Path, **kwargs) -> ParsedDocumentResult:
+        path = self._resolve_existing_path(file_path)
+        content = path.read_text(encoding="utf-8")
+
+        return ParsedDocumentResult(
+            source=str(path),
+            filename=path.name,
+            engine="markdown",
+            pages=[
+                ParsedPageResult(
+                    page_number=1,
+                    markdown_content=content,
+                    plain_text=content,
+                )
+            ],
+            markdown_content=content,
+            metadata={"extension": path.suffix},
+            paginated=False,
+        )

langparse/parsers/pdf_parser.py

@@ -0,0 +1,31 @@
+from pathlib import Path
+from typing import Literal
+
+from langparse.config import settings
+from langparse.core.parser import BaseParser
+from langparse.services.parse_service import ParseService
+from langparse.types import ParsedDocumentResult
+
+
+class PDFParser(BaseParser):
+    """
+    A universal PDF parser that delegates to specific engines.
+    """
+
+    def __init__(
+        self,
+        engine: Literal["simple", "mineru", "deepdoc"] = None,
+        **engine_kwargs,
+    ):
+        # Engine name resolves as: argument > config > default. Construction is
+        # delegated so selection errors read the same here as from the CLI.
+        self.engine_name = engine or settings.get("default_pdf_engine", "simple")
+        self.engine = ParseService().create_engine(self.engine_name, **engine_kwargs)
+
+    def parse_result(self, file_path: str | Path, **kwargs) -> ParsedDocumentResult:
+        return ParseService().parse_result(
+            file_path,
+            engine_name=self.engine_name,
+            engine=self.engine,
+            **kwargs,
+        )

langparse/parsers/registry.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from langparse.parsers.sniff import sniff_kind
+
+#: Single source of truth for which extensions LangParse handles and which
+#: parser family owns each one. AutoParser, ParseService routing, directory
+#: expansion and CLI error messages all read from here so they cannot drift.
+PARSER_KIND_BY_EXTENSION: dict[str, str] = {
+    ".pdf": "pdf",
+    ".docx": "docx",
+    ".doc": "docx",
+    ".xlsx": "excel",
+    ".xlsm": "excel",
+    ".xls": "excel",
+    ".csv": "excel",
+    ".md": "markdown",
+    ".txt": "markdown",
+}
+
+SUPPORTED_EXTENSIONS = frozenset(PARSER_KIND_BY_EXTENSION)
+
+
+def parser_kind_for(file_path) -> str | None:
+    """Return the parser family for a path, or None when unsupported.
+
+    Extensions can lie -- a renamed or re-exported file can carry the wrong
+    suffix. When the path exists, its content is sniffed first (magic bytes /
+    OOXML zip contents); a conclusive sniff overrides the extension. An
+    inconclusive sniff (plain text, legacy OLE binary, missing file) falls
+    back to the extension, exactly as before.
+    """
+    path = Path(file_path)
+    sniffed = sniff_kind(path)
+    if sniffed is not None:
+        return sniffed
+    return PARSER_KIND_BY_EXTENSION.get(path.suffix.lower())
+
+
+def is_supported(file_path) -> bool:
+    return parser_kind_for(file_path) is not None
+
+
+def unsupported_extension_error(file_path) -> ValueError:
+    suffix = Path(file_path).suffix.lower() or "(no extension)"
+    supported = ", ".join(sorted(SUPPORTED_EXTENSIONS))
+    return ValueError(f"Unsupported file extension: {suffix}. Supported: {supported}")

langparse/parsers/sniff.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import zipfile
+from pathlib import Path
+
+#: Magic-byte signatures for formats we can identify without a dependency.
+_PDF_SIGNATURE = b"%PDF-"
+_ZIP_SIGNATURE = b"PK\x03\x04"
+_OLE_CFB_SIGNATURE = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
+
+#: OOXML zip packages are only distinguishable by which top-level folder
+#: their parts live under -- the file extension plays no part in this.
+_OOXML_KIND_BY_NAMELIST_PREFIX = {
+    "word/": "docx",
+    "xl/": "excel",
+}
+
+
+def sniff_kind(file_path) -> str | None:
+    """Best-effort parser-kind detection from file content, independent of extension.
+
+    Extensions can lie -- a renamed or re-exported file can carry the wrong
+    suffix. Returns a `PARSER_KIND_BY_EXTENSION` value when the signature is
+    conclusive (PDF, OOXML docx/xlsx). Returns None when it isn't -- legacy
+    OLE binaries, plain text (csv/md/txt), a missing file, or an unrecognized
+    layout -- so the caller falls back to the extension.
+    """
+    header = _read_header(file_path)
+    if header is None:
+        return None
+    if header.startswith(_PDF_SIGNATURE):
+        return "pdf"
+    if header.startswith(_ZIP_SIGNATURE):
+        return _sniff_ooxml_kind(file_path)
+    return None
+
+
+def looks_like_zip_ooxml(file_path) -> bool:
+    """True when the file is any ZIP-based Office document (docx/xlsx/...),
+    regardless of extension. Used to catch a workbook mislabeled `.csv`."""
+    header = _read_header(file_path)
+    return header is not None and header.startswith(_ZIP_SIGNATURE)
+
+
+def looks_like_ole_binary(file_path) -> bool:
+    """True when the file is a legacy OLE Compound File Binary container
+    (pre-2007 .doc/.xls), regardless of extension. The signature alone can't
+    tell a Word doc from a workbook -- callers only check this after routing
+    has already settled the family via the extension."""
+    header = _read_header(file_path)
+    return header is not None and header.startswith(_OLE_CFB_SIGNATURE)
+
+
+def _read_header(file_path, size: int = 8) -> bytes | None:
+    path = Path(file_path)
+    try:
+        with path.open("rb") as handle:
+            return handle.read(size)
+    except OSError:
+        return None
+
+
+def _sniff_ooxml_kind(file_path) -> str | None:
+    try:
+        with zipfile.ZipFile(file_path) as archive:
+            names = archive.namelist()
+    except zipfile.BadZipFile:
+        return None
+    for prefix, kind in _OOXML_KIND_BY_NAMELIST_PREFIX.items():
+        if any(name.startswith(prefix) for name in names):
+            return kind
+    return None

langparse/progress.py

@@ -0,0 +1,77 @@
+"""Optional, synchronous progress observations; no task storage or dependencies."""
+
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from langparse.logging import get_logger
+
+logger = get_logger(__name__)
+ProgressState = Literal["running", "completed", "failed", "skipped"]
+
+
+@dataclass(frozen=True)
+class ProgressEvent:
+    """Percent is 0–100 within phase, never an estimate of remaining time.
+
+    A file's lifecycle ends only at phase='file', state='completed'/'failed'.
+    Completion does not imply that the result's quality diagnostics are successful.
+    Unknown amounts remain None. An empty source identifies the batch itself.
+    """
+
+    source: str
+    phase: str
+    state: ProgressState = "running"
+    completed_units: int | None = None
+    total_units: int | None = None
+    unit: str | None = None
+    percent: float | None = None
+    message: str = ""
+
+
+ProgressCallback = Callable[[ProgressEvent], None]
+
+
+class ProgressReporter:
+    """Internal adapter keeping observer failures out of parsing and fallback paths."""
+
+    def __init__(self, source: str | Path, callback: ProgressCallback | None = None):
+        self.source = str(source)
+        self.callback = callback
+
+    def emit(
+        self,
+        phase: str,
+        *,
+        state: ProgressState = "running",
+        completed_units: int | None = None,
+        total_units: int | None = None,
+        unit: str | None = None,
+        percent: float | None = None,
+        message: str = "",
+    ) -> None:
+        if self.callback is None:
+            return
+        if percent is None and completed_units is not None and total_units:
+            percent = 100.0 * completed_units / total_units
+        event = ProgressEvent(
+            self.source, phase, state, completed_units, total_units, unit, percent, message
+        )
+        try:
+            self.callback(event)
+        except Exception as exc:
+            # Do not include observer exceptions' text: it may contain credentials.
+            logger.warning("Progress callback failed (%s)", type(exc).__name__)
+
+    @contextmanager
+    def operation(self, phase: str) -> Iterator[None]:
+        self.emit(phase)
+        try:
+            yield
+        except Exception as exc:
+            self.emit(phase, state="failed", message=type(exc).__name__)
+            raise
+        else:
+            self.emit(phase, state="completed")

langparse/services/__init__.py

@@ -0,0 +1,11 @@
+from langparse.services.batch_service import BatchParseService
+from langparse.services.benchmark_service import BenchmarkService
+from langparse.services.parse_service import ParseService
+from langparse.services.workbook_quality_benchmark import WorkbookQualityBenchmarkService
+
+__all__ = [
+    "BatchParseService",
+    "BenchmarkService",
+    "ParseService",
+    "WorkbookQualityBenchmarkService",
+]