universal-doc-parser 1.0.0__py3-none-any.whl

universal_parser/core/sniffer.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+from enum import Enum, auto
+from pathlib import Path
+
+try:
+    import magic
+except (ImportError, Exception):
+    magic = None
+
+
+
+class FileType(Enum):
+    """All document types this parser understands."""
+
+    PDF = auto()
+    DOCX = auto()
+    XLSX = auto()
+    PPTX = auto()
+    DOC = auto()  # legacy binary — Phase 6
+    XLS = auto()  # legacy binary — Phase 6
+    PPT = auto()  # legacy binary — Phase 6
+    HTML = auto()
+    EPUB = auto()
+    CSV = auto()
+    TSV = auto()
+    PARQUET = auto()
+    JSON = auto()
+    XML = auto()
+    EML = auto()
+    MSG = auto()
+    MBOX = auto()
+    IMAGE = auto()  # tiff, bmp, webp, jpg, png
+    UNKNOWN = auto()  # never crash — return this for anything unrecognized
+
+
+_EXT_MAP: dict[str, FileType] = {
+    ".pdf": FileType.PDF,
+    ".docx": FileType.DOCX,
+    ".xlsx": FileType.XLSX,
+    ".pptx": FileType.PPTX,
+    ".doc": FileType.DOC,
+    ".xls": FileType.XLS,
+    ".ppt": FileType.PPT,
+    ".html": FileType.HTML,
+    ".htm": FileType.HTML,
+    ".xhtml": FileType.HTML,
+    ".epub": FileType.EPUB,
+    ".csv": FileType.CSV,
+    ".tsv": FileType.TSV,
+    ".parquet": FileType.PARQUET,
+    ".json": FileType.JSON,
+    ".xml": FileType.XML,
+    ".eml": FileType.EML,
+    ".msg": FileType.MSG,
+    ".mbox": FileType.MBOX,
+    ".tiff": FileType.IMAGE,
+    ".tif": FileType.IMAGE,
+    ".bmp": FileType.IMAGE,
+    ".webp": FileType.IMAGE,
+    ".jpg": FileType.IMAGE,
+    ".jpeg": FileType.IMAGE,
+    ".png": FileType.IMAGE,
+}
+
+_MIME_MAP: dict[str, FileType] = {
+    "application/pdf": FileType.PDF,
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document": FileType.DOCX,
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": FileType.XLSX,
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation": FileType.PPTX,
+    "application/msword": FileType.DOC,
+    "application/vnd.ms-excel": FileType.XLS,
+    "application/vnd.ms-powerpoint": FileType.PPT,
+    "text/html": FileType.HTML,
+    "application/epub+zip": FileType.EPUB,
+    "text/csv": FileType.CSV,
+    "text/plain": FileType.CSV,  # resolved by extension
+    "application/json": FileType.JSON,
+    "text/xml": FileType.XML,
+    "application/xml": FileType.XML,
+    "message/rfc822": FileType.EML,
+    "image/tiff": FileType.IMAGE,
+    "image/bmp": FileType.IMAGE,
+    "image/webp": FileType.IMAGE,
+    "image/jpeg": FileType.IMAGE,
+    "image/png": FileType.IMAGE,
+}
+
+
+def sniff(path: str | Path) -> FileType:
+    """
+    Detect the file type of the given path.
+    Strategy:
+        1. Read magic bytes via python-magic -> look up MIME in _MIME_MAP
+        2. If ambiguous (e.g. text/plain could be CSV or TSV), fall back to extension
+        3. If still unknown, return FileType.UNKNOWN — never raise
+    Args:
+        path: path to any file
+    Returns:
+        FileType enum value
+    """
+    path = Path(path)
+    ext = path.suffix.lower()
+
+    if magic is not None:
+        try:
+            mime = magic.from_file(str(path), mime=True)
+            file_type = _MIME_MAP.get(mime)
+
+            # MIME was recognized but ambiguous — let extension break the tie
+            if file_type in (FileType.CSV, FileType.HTML, None):
+                ext_type = _EXT_MAP.get(ext)
+                if ext_type is not None:
+                    return ext_type
+
+            if file_type is not None:
+                return file_type
+
+        except Exception:  # noqa: BLE001, S110
+            # magic can fail on locked files, permission errors, etc.
+            # fall through to extension lookup
+            pass
+
+    # Last resort: extension only
+    return _EXT_MAP.get(ext, FileType.UNKNOWN)

universal_parser/exports/__init__.py

@@ -0,0 +1 @@
+# Export package

universal_parser/exports/to_chunks.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from universal_parser.core.schema import Document
+
+
+@dataclass
+class Chunk:
+    """A token-budgeted chunk ready for vector database embeddings."""
+
+    chunk_id: str
+    text: str
+    headings: list[str] = field(default_factory=list)
+    element_types: list[str] = field(default_factory=list)
+    page_numbers: list[int] = field(default_factory=list)
+    estimated_tokens: int = 0
+
+
+def to_chunks(
+    doc: Document,
+    max_tokens: int = 512,
+    overlap_tokens: int = 50,
+) -> list[Chunk]:
+    """
+    Hierarchical token-aware chunker for RAG pipelines.
+    Rules:
+        - Tracks active H1, H2, H3 hierarchy across elements
+        - Prefixes chunk with current heading path context
+        - Bundles elements until max_tokens budget is reached
+        - Keeps tables intact within a single chunk where possible
+    """
+    chunks: list[Chunk] = []
+    current_headings: dict[int, str] = {}  # level -> heading text
+    current_elements: list[str] = []
+    current_types: list[int] = []
+    current_pages: list[int] = []
+    current_tokens = 0
+    chunk_index = 1
+
+    for el in doc.content_tree:
+        # 1. Update heading stack
+        if el.type == "heading":
+            level = el.level or 1
+            current_headings = {lvl: txt for lvl, txt in current_headings.items() if lvl < level}
+            current_headings[level] = el.text or ""
+
+        # 2. Get text representation
+        el_text = el.markdown_repr or el.text or ""
+        if not el_text.strip():
+            continue
+
+        # Fast token estimation (~4 chars per token)
+        el_tokens = max(1, len(el_text) // 4)
+
+        # 3. Check if adding exceeds budget
+        if current_tokens + el_tokens > max_tokens and current_elements:
+            heading_hierarchy = [txt for _, txt in sorted(current_headings.items())]
+            heading_prefix = " > ".join(heading_hierarchy)
+            header_str = f"Context: {heading_prefix}\n\n" if heading_prefix else ""
+            chunk_text = header_str + "\n\n".join(current_elements)
+            total_est = len(chunk_text) // 4
+
+            chunks.append(
+                Chunk(
+                    chunk_id=f"{doc.doc_id}-chunk-{chunk_index}",
+                    text=chunk_text,
+                    headings=heading_hierarchy,
+                    element_types=list(set(current_types)),
+                    page_numbers=sorted(set(current_pages)),
+                    estimated_tokens=total_est,
+                )
+            )
+            chunk_index += 1
+
+            current_elements = []
+            current_types = []
+            current_pages = []
+            current_tokens = 0
+
+        # 4. Append element to current buffer
+        current_elements.append(el_text)
+        current_types.append(el.type)
+        if el.page is not None:
+            current_pages.append(el.page)
+        current_tokens += el_tokens
+
+    # Flush remaining buffer
+    if current_elements:
+        heading_hierarchy = [txt for _, txt in sorted(current_headings.items())]
+        heading_prefix = " > ".join(heading_hierarchy)
+        header_str = f"Context: {heading_prefix}\n\n" if heading_prefix else ""
+
+        chunk_text = header_str + "\n\n".join(current_elements)
+        total_est = len(chunk_text) // 4
+
+        chunks.append(
+            Chunk(
+                chunk_id=f"{doc.doc_id}-chunk-{chunk_index}",
+                text=chunk_text,
+                headings=heading_hierarchy,
+                element_types=list(set(current_types)),
+                page_numbers=sorted(set(current_pages)),
+                estimated_tokens=total_est,
+            )
+        )
+
+    return chunks

universal_parser/exports/to_graph.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from universal_parser.core.schema import Document
+
+
+@dataclass
+class GraphNode:
+    id: str
+    type: str  # "document", "section", "table", "content"
+    properties: dict[str, str | int | float] = field(default_factory=dict)
+
+
+@dataclass
+class GraphEdge:
+    source_id: str
+    target_id: str
+    relationship: str  # "CONTAINS_SECTION", "CONTAINS", "FOLLOWS"
+
+
+@dataclass
+class KnowledgeGraph:
+    nodes: list[GraphNode] = field(default_factory=list)
+    edges: list[GraphEdge] = field(default_factory=list)
+
+
+def to_graph(doc: Document) -> KnowledgeGraph:
+    """
+    Export Document into a Knowledge Graph structure for GraphRAG & Neo4j.
+
+    Creates:
+        - Document Root Node
+        - Heading / Section Nodes with hierarchical edges
+        - Paragraph & Table Content Nodes linked to their parent sections
+    """
+    nodes: list[GraphNode] = []
+    edges: list[GraphEdge] = []
+
+    # 1. Document Root Node
+    doc_node_id = f"doc:{doc.doc_id}"
+    nodes.append(
+        GraphNode(
+            id=doc_node_id,
+            type="document",
+            properties={
+                "file_name": doc.metadata.file_name,
+                "file_type": doc.metadata.file_type,
+            },
+        )
+    )
+
+    heading_stack: list[tuple[int, str]] = []  # (level, node_id)
+    prev_node_id: str | None = None
+
+    for idx, el in enumerate(doc.content_tree, start=1):
+        node_id = f"el:{doc.doc_id}:{idx}"
+
+        if el.type == "heading":
+            level = el.level or 1
+            while heading_stack and heading_stack[-1][0] >= level:
+                heading_stack.pop()
+
+            parent_id = heading_stack[-1][1] if heading_stack else doc_node_id
+
+            nodes.append(
+                GraphNode(
+                    id=node_id,
+                    type="section",
+                    properties={"title": el.text or "", "level": level},
+                )
+            )
+            edges.append(
+                GraphEdge(
+                    source_id=parent_id,
+                    target_id=node_id,
+                    relationship="CONTAINS_SECTION",
+                )
+            )
+            heading_stack.append((level, node_id))
+
+        else:
+            parent_id = heading_stack[-1][1] if heading_stack else doc_node_id
+            node_type = "table" if el.type == "table" else "content"
+
+            nodes.append(
+                GraphNode(
+                    id=node_id,
+                    type=node_type,
+                    properties={
+                        "text": el.text or el.markdown_repr or "",
+                        "element_type": el.type,
+                    },
+                )
+            )
+            edges.append(
+                GraphEdge(
+                    source_id=parent_id,
+                    target_id=node_id,
+                    relationship="CONTAINS",
+                )
+            )
+
+        if prev_node_id:
+            edges.append(
+                GraphEdge(
+                    source_id=prev_node_id,
+                    target_id=node_id,
+                    relationship="FOLLOWS",
+                )
+            )
+        prev_node_id = node_id
+
+    return KnowledgeGraph(nodes=nodes, edges=edges)

universal_parser/exports/to_markdown.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from universal_parser.core.schema import Document
+
+
+def to_markdown(doc: Document) -> str:
+    """
+    Convert a parsed Document into a clean, unified Markdown string.
+    Rules:
+        - Headings render as # H1, ## H2, ### H3
+        - Paragraphs render as clean prose
+        - List items render with "- "
+        - Tables render with structured Markdown grids
+        - Code blocks render within ``` fences
+    """
+    blocks: list[str] = []
+
+    for el in doc.content_tree:
+        if el.markdown_repr:
+            blocks.append(el.markdown_repr)
+        elif el.type == "heading":
+            level = el.level or 1
+            blocks.append(f"{'#' * level} {el.text or ''}")
+        elif el.type == "list_item":
+            blocks.append(f"- {el.text or ''}")
+        elif el.type == "code_block":
+            blocks.append(f"```\n{el.text or ''}\n```")
+        elif el.text:
+            blocks.append(el.text)
+
+    return "\n\n".join(blocks)

universal_parser/extractors/base.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from pathlib import Path
+from typing import ClassVar
+
+from universal_parser.core.schema import Element
+from universal_parser.core.sniffer import FileType
+
+
+class BaseExtractor(ABC):
+    """
+    Abstract base class for all format extractors.
+    Every extractor in universal_parser.extractors must:
+        1. Inherit from BaseExtractor
+        2. Declare which FileTypes it handles via supported_types
+        3. Implement stream() — yield Elements one at a time, never return a list
+    Adding a new format = one new file inheriting this + one entry in router.py.
+    Nothing else in core/ needs to change.
+    """
+
+    supported_types: ClassVar[list[FileType]] = []
+
+    @abstractmethod
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        """
+        Parse the document at the given path and yield Element objects.
+        Rules:
+            - Must yield, never return a full list (streaming = memory safe)
+            - Must never raise on a recoverable error — yield a low-confidence
+              element or skip, log the issue, keep going
+            - One Element at a time, in reading order
+        Args:
+            path: absolute or relative path to the source file
+        Yields:
+            Element objects in document reading order
+        """
+        ...

universal_parser/extractors/images/scan_extractor.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+from typing import ClassVar
+
+import cv2
+import numpy as np
+from rapidocr_onnxruntime import RapidOCR
+
+from universal_parser.core.router import register
+from universal_parser.core.schema import BBox, Element
+from universal_parser.core.sniffer import FileType
+from universal_parser.extractors.base import BaseExtractor
+
+
+@register
+class ImageScanExtractor(BaseExtractor):
+    """
+    Extractor for image and scanned files (.png, .jpg, .tiff, .bmp, .webp).
+    Handles:
+        - Image preprocessing (deskewing / contrast enhancement via OpenCV)
+        - CPU-based text and bounding box detection via RapidOCR (ONNX)
+        - Confidence scoring per text element
+    """
+
+    supported_types: ClassVar[list[FileType]] = [FileType.IMAGE]
+
+    def __init__(self) -> None:
+        super().__init__()
+        # Initialize RapidOCR engine (cached in memory)
+        self._ocr = RapidOCR()
+
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        path_str = str(path)
+
+        try:
+            # Step 1: Read image with OpenCV
+            img = cv2.imread(path_str)
+            if img is None:
+                return
+
+            # Step 2: Preprocess / deskew
+            preprocessing_img = self._preprocess_image(img)
+
+            # Step 3: Run RapidOCR
+            ocr_results, _ = self._ocr(preprocessing_img)
+            if not ocr_results:
+                return
+            # Step 4: Stream extracted elements
+            for item in ocr_results:
+                dt_boxes, text, score = item
+                clean_text = text.strip()
+                if not clean_text:
+                    continue
+
+                # Compute bounding box (x0, y0, x1, y1)
+                pts = np.array(dt_boxes, dtype=np.float32)
+                x0 = float(np.min(pts[:, 0]))
+                y0 = float(np.min(pts[:, 1]))
+                x1 = float(np.max(pts[:, 0]))
+                y1 = float(np.max(pts[:, 1]))
+
+                yield Element(
+                    type="paragraph",
+                    text=clean_text,
+                    page=1,
+                    bbox=BBox(x0=x0, y0=y0, x1=x1, y1=y1),
+                    markdown_repr=clean_text,
+                    confidence=round(float(score), 3),
+                )
+
+        except Exception:  # noqa: BLE001
+            return
+
+    def _preprocess_image(self, img: np.ndarray) -> np.ndarray:
+        """Basic contrast & grayscale cleanup for sharper OCR."""
+        # RapidOCR handles RGB internally; return original if valid
+        return img