universal-doc-parser 1.0.0__py3-none-any.whl

universal_parser/extractors/pdf/native.py

@@ -0,0 +1,331 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+from typing import ClassVar
+
+import numpy as np
+import pdfplumber
+import pypdfium2 as pdfium
+from rapidocr_onnxruntime import RapidOCR
+
+from universal_parser.core.router import register
+from universal_parser.core.schema import BBox, Element
+from universal_parser.core.sniffer import FileType
+from universal_parser.extractors.base import BaseExtractor
+from universal_parser.extractors.pdf.tables import PDFTableExtractor
+
+
+@register
+class NativePDFExtractor(BaseExtractor):
+    """Extractor for PDF files (both native text and scanned pages).
+
+    Handles:
+        - Multi-column reading order via column-band clustering
+        - Dynamic header hierarchy using font-size percentiles
+        - Bordered & borderless table extraction integrated into the reading flow
+        - Automatic OCR fallback (via RapidOCR ONNX) for scanned pages with no text
+    """
+
+    supported_types: ClassVar[list[FileType]] = [FileType.PDF]
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._ocr = RapidOCR()
+
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        """Stream elements from a PDF page by page."""
+        path_str = str(path)
+
+        try:
+            plumber_doc = pdfplumber.open(path_str)
+            pdfium_doc = pdfium.PdfDocument(path_str)
+        except Exception:  # noqa: BLE001
+            return
+
+        try:
+            # Collect font sizes for header hierarchy
+            font_sizes = self._collect_font_sizes(plumber_doc, max_pages=10)
+            thresholds = self._compute_header_thresholds(font_sizes)
+
+            for page_num in range(len(plumber_doc.pages)):
+                plumber_page = plumber_doc.pages[page_num]
+                page_width = float(plumber_page.width)
+
+                # A. Extract tables and their bounding boxes using pdfplumber
+                try:
+                    table_extractor = PDFTableExtractor(plumber_page)
+                    page_tables = table_extractor.extract_tables()
+                except Exception:  # noqa: BLE001
+                    page_tables = []
+
+                table_bboxes = [table["bbox"] for table in page_tables]
+
+                # B. Extract native text words/spans
+                words = plumber_page.extract_words(extra_attrs=["size"], keep_blank_chars=False)
+                spans = []
+
+                # Group adjacent words on same line into phrase spans
+                current_line: list[dict] = []
+                for word in words:
+                    bbox = (
+                        float(word["x0"]),
+                        float(word["top"]),
+                        float(word["x1"]),
+                        float(word["bottom"]),
+                    )
+                    if self._is_inside_any_bbox(bbox, table_bboxes):
+                        continue
+
+                    if not current_line:
+                        current_line.append(word)
+                    else:
+                        prev = current_line[-1]
+                        # Check if on same line (vertical overlap) and close horizontal gap
+                        same_line = abs(float(word["top"]) - float(prev["top"])) < 4.0
+                        gap = float(word["x0"]) - float(prev["x1"])
+                        if same_line and gap < 12.0:
+                            current_line.append(word)
+                        else:
+                            span_text = " ".join(w["text"] for w in current_line).strip()
+                            if span_text:
+                                spans.append(
+                                    {
+                                        "is_table": False,
+                                        "text": span_text,
+                                        "size": round(float(current_line[0].get("size", 10.0)), 1),
+                                        "bbox": (
+                                            float(current_line[0]["x0"]),
+                                            float(min(w["top"] for w in current_line)),
+                                            float(current_line[-1]["x1"]),
+                                            float(max(w["bottom"] for w in current_line)),
+                                        ),
+                                    }
+                                )
+                            current_line = [word]
+
+                if current_line:
+                    span_text = " ".join(w["text"] for w in current_line).strip()
+                    if span_text:
+                        spans.append(
+                            {
+                                "is_table": False,
+                                "text": span_text,
+                                "size": round(float(current_line[0].get("size", 10.0)), 1),
+                                "bbox": (
+                                    float(current_line[0]["x0"]),
+                                    float(min(w["top"] for w in current_line)),
+                                    float(current_line[-1]["x1"]),
+                                    float(max(w["bottom"] for w in current_line)),
+                                ),
+                            }
+                        )
+
+                # C. Wrap extracted tables
+                for table in page_tables:
+                    spans.append(
+                        {
+                            "is_table": True,
+                            "table_data": table["data"],
+                            "confidence": table["confidence"],
+                            "bbox": table["bbox"],
+                        }
+                    )
+
+                # D. SCANNED PAGE FALLBACK: If page has NO native text or tables, run OCR
+                if not spans:
+                    if page_num < len(pdfium_doc):
+                        yield from self._ocr_scanned_page(
+                            pdfium_doc[page_num],
+                            page_num + 1,
+                            page_width,
+                            float(plumber_page.height),
+                        )
+                    continue
+
+                # Step 3: Sort in natural reading order
+                ordered_elements = self._sort_reading_order(spans, page_width)
+
+                # Step 4: Yield sorted elements
+                for el in ordered_elements:
+                    bbox_coords = el["bbox"]
+
+                    if el["is_table"]:
+                        headers = el["table_data"].headers
+                        rows = el["table_data"].rows
+                        md_header = "| " + " | ".join(headers) + " |"
+                        md_separator = "| " + " | ".join(["---"] * len(headers)) + " |"
+                        md_rows = ["| " + " | ".join(row) + " |" for row in rows]
+                        markdown_repr = "\n".join([md_header, md_separator] + md_rows)
+
+                        yield Element(
+                            type="table",
+                            page=page_num + 1,
+                            bbox=BBox(
+                                x0=bbox_coords[0],
+                                y0=bbox_coords[1],
+                                x1=bbox_coords[2],
+                                y1=bbox_coords[3],
+                            ),
+                            data=el["table_data"],
+                            markdown_repr=markdown_repr,
+                            confidence=el["confidence"],
+                        )
+                    else:
+                        size = el["size"]
+                        text = el["text"]
+
+                        el_type = "paragraph"
+                        level = None
+
+                        if size >= thresholds["h1"]:
+                            el_type = "heading"
+                            level = 1
+                        elif size >= thresholds["h2"]:
+                            el_type = "heading"
+                            level = 2
+                        elif size >= thresholds["h3"]:
+                            el_type = "heading"
+                            level = 3
+
+                        yield Element(
+                            type=el_type,
+                            level=level,
+                            text=text,
+                            page=page_num + 1,
+                            bbox=BBox(
+                                x0=bbox_coords[0],
+                                y0=bbox_coords[1],
+                                x1=bbox_coords[2],
+                                y1=bbox_coords[3],
+                            ),
+                        )
+        finally:
+            pdfium_doc.close()
+            plumber_doc.close()
+
+    def _ocr_scanned_page(
+        self, page: pdfium.PdfPage, page_num: int, page_w: float, page_h: float
+    ) -> Iterator[Element]:
+        """Render a scanned PDF page to an image and run RapidOCR."""
+        try:
+            scale = 200.0 / 72.0
+            bitmap = page.render(scale=scale)
+            pil_img = bitmap.to_pil().convert("RGB")
+            img_np = np.array(pil_img)
+            bitmap.close()
+
+            ocr_results, _ = self._ocr(img_np)
+            if not ocr_results:
+                return
+
+            scale_x = page_w / pil_img.width
+            scale_y = page_h / pil_img.height
+
+            for item in ocr_results:
+                dt_boxes, text, score = item
+                clean_text = text.strip()
+                if not clean_text:
+                    continue
+
+                pts = np.array(dt_boxes, dtype=np.float32)
+                x0 = float(np.min(pts[:, 0])) * scale_x
+                y0 = float(np.min(pts[:, 1])) * scale_y
+                x1 = float(np.max(pts[:, 0])) * scale_x
+                y1 = float(np.max(pts[:, 1])) * scale_y
+
+                yield Element(
+                    type="paragraph",
+                    text=clean_text,
+                    page=page_num,
+                    bbox=BBox(x0=x0, y0=y0, x1=x1, y1=y1),
+                    markdown_repr=clean_text,
+                    confidence=round(float(score), 3),
+                )
+        except Exception:  # noqa: BLE001
+            return
+
+    def _is_inside_any_bbox(
+        self,
+        span_bbox: tuple[float, float, float, float],
+        table_bboxes: list[tuple[float, float, float, float]],
+    ) -> bool:
+        """Check if a text span falls inside any table bounding box."""
+        sx0, sy0, sx1, sy1 = span_bbox
+        for tx0, ty0, tx1, ty1 in table_bboxes:
+            if sx0 >= (tx0 - 2) and sy0 >= (ty0 - 2) and sx1 <= (tx1 + 2) and sy1 <= (ty1 + 2):
+                return True
+        return False
+
+    def _collect_font_sizes(self, doc: pdfplumber.PDF, max_pages: int) -> list[float]:
+        """Collect all font sizes from the start of the document."""
+        sizes = []
+        pages_to_scan = min(len(doc.pages), max_pages)
+        for i in range(pages_to_scan):
+            page = doc.pages[i]
+            words = page.extract_words(extra_attrs=["size"])
+            for w in words:
+                sizes.append(float(w.get("size", 10.0)))
+        return sizes
+
+    def _compute_header_thresholds(self, font_sizes: list[float]) -> dict[str, float]:
+        """Compute font size cutoffs for H1, H2, H3 using percentiles."""
+        if not font_sizes:
+            return {"h1": 16.0, "h2": 14.0, "h3": 12.0}
+
+        arr = np.array(font_sizes)
+        h1_val = float(np.percentile(arr, 95))
+        h2_val = float(np.percentile(arr, 85))
+        h3_val = float(np.percentile(arr, 75))
+
+        median = float(np.median(arr))
+
+        h1_val = max(h1_val, median + 3.0)
+        h2_val = max(h2_val, median + 1.5)
+        h3_val = max(h3_val, median + 0.5)
+
+        return {"h1": h1_val, "h2": h2_val, "h3": h3_val}
+
+    def _sort_reading_order(self, elements: list[dict], page_width: float) -> list[dict]:
+        """Sort elements to preserve natural multi-column reading order."""
+        if len(elements) < 5:
+            return sorted(elements, key=lambda e: (e["bbox"][1], e["bbox"][0]))
+
+        midpoint = page_width / 2.0
+
+        left_col = []
+        right_col = []
+        full_width = []
+
+        for el in elements:
+            x0, _, x1, _ = el["bbox"]
+            if x1 <= (midpoint + 15):
+                left_col.append(el)
+            elif x0 >= (midpoint - 15):
+                right_col.append(el)
+            else:
+                full_width.append(el)
+
+        def key_y(e):
+            return (e["bbox"][1], e["bbox"][0])
+
+        left_sorted = sorted(left_col, key=key_y)
+        right_sorted = sorted(right_col, key=key_y)
+
+        if len(left_sorted) >= 2 and len(right_sorted) >= 2:
+            col_top = min(left_sorted[0]["bbox"][1], right_sorted[0]["bbox"][1])
+            col_bottom = max(left_sorted[-1]["bbox"][3], right_sorted[-1]["bbox"][3])
+
+            top_full = [e for e in full_width if e["bbox"][3] <= col_top + 10]
+            bottom_full = [e for e in full_width if e["bbox"][1] >= col_bottom - 10]
+            mid_full = [e for e in full_width if e not in top_full and e not in bottom_full]
+
+            return (
+                sorted(top_full, key=key_y)
+                + sorted(mid_full, key=key_y)
+                + left_sorted
+                + right_sorted
+                + sorted(bottom_full, key=key_y)
+            )
+
+        return sorted(elements, key=key_y)

universal_parser/extractors/pdf/tables.py

@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+import pdfplumber
+
+from universal_parser.core.schema import TableData
+
+
+class PDFTableExtractor:
+    """
+    Handles bordered (Lattice) and borderless (Stream) table extraction from PDF pages.
+
+    Provides:
+        - Bordered table extraction via coordinate grid mapping
+        - Borderless table extraction via whitespace column clustering
+        - Confidence scoring based on layout density and cell consistency
+    """
+
+    def __init__(self, page: pdfplumber.page.Page):
+        self.page = page
+
+    def extract_tables(self) -> list[dict]:
+        """
+        Extract all tables from the page.
+
+        Returns:
+            list[dict]: A list of tables found, each formatted as:
+                {
+                    "data": TableData,
+                    "bbox": tuple(x0, y0, x1, y1),
+                    "confidence": float (0.0 to 1.0)
+                }
+        """
+
+        tables = []
+
+        # 1. Try Lattice extraction first (bordered tables)
+        lattice_tables = self._extract_lattice()
+        if lattice_tables:
+            tables.extend(lattice_tables)
+
+        # 2. Try Stream extraction (borderless tables)
+        # We only run stream extraction if we don't find lattice tables
+        # to prevent duplicate extractions on the same area.
+        if not lattice_tables:
+            stream_tables = self._extract_stream()
+            if stream_tables:
+                tables.extend(stream_tables)
+
+        return tables
+
+    def _extract_lattice(self) -> list[dict]:
+        """Extract tables using vertical and horizontal vector lines."""
+        extracted = []
+        # pdfplumber vertical and horizontal line settings
+        table_settings = {
+            "vertical_strategy": "lines",
+            "horizontal_strategy": "lines",
+            "snap_tolerance": 3,
+            "join_tolerance": 3,
+        }
+
+        plumber_tables = self.page.find_tables(table_settings=table_settings)
+        for table in plumber_tables:
+            raw_data = table.extract()
+            if not raw_data or len(raw_data) < 2:
+                continue
+
+            clean_rows = []
+            for row in raw_data:
+                # Convert None to empty string
+                clean_row = [str(cell).strip() if cell is not None else "" for cell in row]
+                clean_rows.append(clean_row)
+
+            headers = clean_rows[0]
+            data_rows = clean_rows[1:]
+
+            # Lattice table confidence is high (0.95+) since lines physically define cells
+            confidence = 0.98 if all(len(row) == len(headers) for row in data_rows) else 0.90
+
+            extracted.append(
+                {
+                    "data": TableData(headers=headers, rows=data_rows),
+                    "bbox": table.bbox,  # (x0, y0, x1, y1)
+                    "confidence": confidence,
+                }
+            )
+
+        return extracted
+
+    def _extract_stream(self) -> list[dict]:
+        """Extract borderless tables using whitespace distance clustering and text alignment."""
+        extracted = []
+        table_settings = {
+            "vertical_strategy": "text",
+            "horizontal_strategy": "text",
+            "snap_tolerance": 4,
+            "join_tolerance": 4,
+            "min_words_vertical": 3,
+            "min_words_horizontal": 2,
+        }
+
+        try:
+            plumber_tables = self.page.find_tables(table_settings=table_settings)
+            for table in plumber_tables:
+                raw_data = table.extract()
+                if not raw_data or len(raw_data) < 2:
+                    continue
+
+                clean_rows = []
+                total_words = 0
+                non_empty_cells = 0
+
+                for row in raw_data:
+                    clean_row = [str(cell).strip() if cell is not None else "" for cell in row]
+                    if any(clean_row):
+                        clean_rows.append(clean_row)
+                        for cell in clean_row:
+                            if cell:
+                                word_count = len(cell.split())
+                                total_words += word_count
+                                non_empty_cells += 1
+
+                if len(clean_rows) < 2 or non_empty_cells < 4:
+                    continue
+
+                # Heuristic: Genuine tables contain concise cell values (labels, numbers, codes <= 2.5 words/cell).
+                # If cells contain full paragraph sentences, it is multi-column text, not a table.
+                avg_words_per_cell = total_words / max(1, non_empty_cells)
+                if avg_words_per_cell > 2.5:
+                    continue
+
+                # Filter out tables that swallow more than 40% of page height (multi-column text pages)
+                page_h = float(self.page.height)
+                table_h = float(table.bbox[3] - table.bbox[1])
+                if table_h > (0.40 * page_h):
+                    continue
+
+                max_cols = max(len(r) for r in clean_rows)
+                if max_cols < 2:
+                    continue
+
+                headers = clean_rows[0]
+                data_rows = clean_rows[1:]
+
+                extracted.append(
+                    {
+                        "data": TableData(headers=headers, rows=data_rows),
+                        "bbox": table.bbox,  # (x0, y0, x1, y1)
+                        "confidence": 0.85,
+                    }
+                )
+        except Exception:  # noqa: BLE001
+            return []
+
+        return extracted

universal_parser/extractors/structured/csv_extractor.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import csv
+from collections.abc import Iterator
+from pathlib import Path
+from typing import ClassVar
+
+from universal_parser.core.router import register
+from universal_parser.core.schema import Element, TableData
+from universal_parser.core.sniffer import FileType
+from universal_parser.extractors.base import BaseExtractor
+
+
+@register
+class CSVExtractor(BaseExtractor):
+    """
+    Extractor for delimited text files (.csv, .tsv).
+    Handles:
+        - Auto-dialect detection (delimiter, quotechar) via csv.Sniffer
+        - UTF-8, Latin-1, and Windows-1252 encodings
+        - Streams rows into structured TableData
+    """
+
+    supported_types: ClassVar[list[FileType]] = [FileType.CSV, FileType.TSV]
+
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        """Stream table element from a CSV/TSV file."""
+        path_obj = Path(path)
+        path_str = str(path)
+
+        # Step 1: Detect encoding safely
+        encoding = self._detect_encoding(path_str)
+
+        try:
+            with open(path_str, encoding=encoding, errors="replace") as f:
+                sample = f.read(4096)
+                f.seek(0)
+
+                if not sample.strip():
+                    return
+
+                # Step 2: Auto-detect delimiter using Sniffer, fallback to extension
+                try:
+                    dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|")
+                    delimiter = dialect.delimiter
+                except Exception:  # noqa: BLE001
+                    delimiter = "\t" if path_obj.suffix.lower() == ".tsv" else ","
+
+                reader = csv.reader(f, delimiter=delimiter)
+                rows = []
+                for row in reader:
+                    clean_row = [cell.strip() for cell in row]
+                    if any(clean_row):
+                        rows.append(clean_row)
+
+                if not rows:
+                    return
+
+                headers = rows[0]
+                data_rows = rows[1:]
+
+                # Generate markdown representation
+                md_header = "| " + " | ".join(headers) + " |"
+                md_separator = "| " + " | ".join(["---"] * len(headers)) + " |"
+                md_rows = ["| " + " | ".join(r) + " |" for r in data_rows]
+                markdown_repr = "\n".join([md_header, md_separator] + md_rows)
+
+                yield Element(
+                    type="table",
+                    text=path_obj.name,
+                    data=TableData(headers=headers, rows=data_rows),
+                    markdown_repr=markdown_repr,
+                    confidence=1.0,
+                )
+
+        except Exception:  # noqa: BLE001
+            return
+
+    def _detect_encoding(self, path: str) -> str:
+        """Try decoding a small chunk with UTF-8, fallback to latin-1."""
+        try:
+            with open(path, "rb") as f:
+                f.read(2048).decode("utf-8")
+            return "utf-8"
+        except UnicodeDecodeError:
+            return "latin-1"

universal_parser/extractors/structured/json_xml_extractor.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import json
+import xml.etree.ElementTree as ET
+from collections.abc import Iterator
+from pathlib import Path
+from typing import ClassVar
+
+from universal_parser.core.router import register
+from universal_parser.core.schema import Element, TableData
+from universal_parser.core.sniffer import FileType
+from universal_parser.extractors.base import BaseExtractor
+
+
+@register
+class JSONXMLExtractor(BaseExtractor):
+    """
+    Extractor for JSON (.json) and XML (.xml) files.
+
+    Handles:
+        - List of objects in JSON -> Table element (if tabular schema)
+        - Nested JSON objects -> formatted code block element
+        - XML -> element nodes parsed into headings, paragraphs, and code blocks
+    """
+
+    supported_types: ClassVar[list[FileType]] = [FileType.JSON, FileType.XML]
+
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        path_obj = Path(path)
+        ext = path_obj.suffix.lower()
+
+        if ext == ".json":
+            yield from self._stream_json(path_obj)
+        elif ext == ".xml":
+            yield from self._stream_xml(path_obj)
+
+    def _stream_json(self, path: Path) -> Iterator[Element]:
+        try:
+            with open(path, encoding="utf-8", errors="replace") as f:
+                data = json.load(f)
+
+            # Check if it's a list of uniform dictionaries (tabular JSON)
+            if isinstance(data, list) and data and all(isinstance(row, dict) for row in data):
+                headers = list(data[0].keys())
+                rows = [[str(row.get(h, "")).strip() for h in headers] for row in data]
+
+                md_header = "| " + " | ".join(headers) + " |"
+                md_separator = "| " + " | ".join(["---"] * len(headers)) + " |"
+                md_rows = ["| " + " | ".join(r) + " |" for r in rows]
+                markdown_repr = "\n".join([md_header, md_separator] + md_rows)
+
+                yield Element(
+                    type="table",
+                    text=path.name,
+                    data=TableData(headers=headers, rows=rows),
+                    markdown_repr=markdown_repr,
+                    confidence=1.0,
+                )
+            else:
+                # General JSON: render formatted code block
+                formatted_json = json.dumps(data, indent=2)
+                yield Element(
+                    type="code_block",
+                    text=formatted_json,
+                    markdown_repr=f"```json\n{formatted_json}\n```",
+                    confidence=1.0,
+                )
+        except Exception:  # noqa: BLE001
+            return
+
+    def _stream_xml(self, path: Path) -> Iterator[Element]:
+        try:
+            tree = ET.parse(path)
+            root = tree.getroot()
+
+            # Yield root tag as level 1 heading
+            yield Element(
+                type="heading",
+                level=1,
+                text=root.tag,
+                markdown_repr=f"# {root.tag}",
+                confidence=1.0,
+            )
+
+            for child in root:
+                text_val = (child.text or "").strip()
+                if text_val:
+                    yield Element(
+                        type="paragraph",
+                        text=f"{child.tag}: {text_val}",
+                        markdown_repr=f"**{child.tag}**: {text_val}",
+                        confidence=1.0,
+                    )
+                else:
+                    child_str = ET.tostring(child, encoding="unicode").strip()
+                    if child_str:
+                        yield Element(
+                            type="code_block",
+                            text=child_str,
+                            markdown_repr=f"```xml\n{child_str}\n```",
+                            confidence=1.0,
+                        )
+        except Exception:  # noqa: BLE001
+            return