universal-doc-parser 1.0.0__py3-none-any.whl

universal_parser/extractors/mail/mail_extractor.py

@@ -0,0 +1,206 @@
+from __future__ import annotations
+
+import email
+import mailbox
+import tempfile
+from collections.abc import Iterator
+from email import policy
+from pathlib import Path
+from typing import ClassVar
+
+import extract_msg
+from selectolax.parser import HTMLParser
+
+from universal_parser.core.router import get_extractor, register
+from universal_parser.core.schema import Element
+from universal_parser.core.sniffer import FileType, sniff
+from universal_parser.extractors.base import BaseExtractor
+
+
+@register
+class MailExtractor(BaseExtractor):
+    """
+    Extractor for email formats (.eml, .msg, .mbox).
+
+    Handles:
+        - Header metadata (Subject, From, To, Date)
+        - Body extraction (HTML / Plaintext via selectolax)
+        - Recursive attachment parsing through the sniffer/router
+    """
+
+    supported_types: ClassVar[list[FileType]] = [
+        FileType.EML,
+        FileType.MSG,
+        FileType.MBOX,
+    ]
+
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        path_obj = Path(path)
+        ext = path_obj.suffix.lower()
+
+        if ext == ".eml":
+            yield from self._stream_eml(path_obj)
+        elif ext == ".msg":
+            yield from self._stream_msg(path_obj)
+        elif ext == ".mbox":
+            yield from self._stream_mbox(path_obj)
+
+    def _stream_eml(self, path: Path) -> Iterator[Element]:
+        try:
+            with open(path, "rb") as f:
+                msg = email.message_from_binary_file(f, policy=policy.default)
+
+            yield from self._process_email_message(msg)
+        except Exception:  # noqa: BLE001
+            return
+
+    def _stream_msg(self, path: Path) -> Iterator[Element]:
+        try:
+            msg = extract_msg.Message(str(path))
+            subject = msg.subject or "No Subject"
+            sender = msg.sender or "Unknown Sender"
+            to = msg.to or "Unknown Recipient"
+            date = str(msg.date) if msg.date else ""
+
+            yield Element(
+                type="heading",
+                level=1,
+                text=f"Subject: {subject}",
+                markdown_repr=f"# Subject: {subject}",
+                confidence=1.0,
+            )
+
+            meta_text = f"From: {sender} | To: {to}" + (f" | Date: {date}" if date else "")
+            yield Element(
+                type="paragraph",
+                text=meta_text,
+                markdown_repr=f"**{meta_text}**",
+                confidence=1.0,
+            )
+
+            body_text = msg.body
+            if body_text:
+                for paragraph in body_text.split("\n\n"):
+                    clean_p = paragraph.strip()
+                    if clean_p:
+                        yield Element(
+                            type="paragraph",
+                            text=clean_p,
+                            markdown_repr=clean_p,
+                            confidence=1.0,
+                        )
+
+            # Process attachments recursively
+            for att in msg.attachments:
+                att_data = att.data
+                att_name = att.longFilename or att.shortFilename or "attachment"
+                if att_data:
+                    yield from self._process_attachment_bytes(att_name, att_data)
+
+            msg.close()
+        except Exception:  # noqa: BLE001
+            return
+
+    def _stream_mbox(self, path: Path) -> Iterator[Element]:
+        try:
+            mbox = mailbox.mbox(str(path))
+            for msg in mbox.values():
+                yield from self._process_email_message(msg)
+        except Exception:  # noqa: BLE001
+            return
+
+    def _process_email_message(self, msg: email.message.EmailMessage) -> Iterator[Element]:
+        subject = str(msg.get("Subject", "No Subject"))
+        sender = str(msg.get("From", "Unknown Sender"))
+        to = str(msg.get("To", "Unknown Recipient"))
+        date = str(msg.get("Date", ""))
+
+        yield Element(
+            type="heading",
+            level=1,
+            text=f"Subject: {subject}",
+            markdown_repr=f"# Subject: {subject}",
+            confidence=1.0,
+        )
+
+        meta_text = f"From: {sender} | To: {to}" + (f" | Date: {date}" if date else "")
+        yield Element(
+            type="paragraph",
+            text=meta_text,
+            markdown_repr=f"**{meta_text}**",
+            confidence=1.0,
+        )
+
+        # Body extraction
+        body_part = msg.get_body(preferencelist=("html", "plain"))
+        if body_part:
+            content = body_part.get_content()
+            if body_part.get_content_type() == "text/html":
+                parser = HTMLParser(content)
+                for tag in parser.css("script, style, noscript"):
+                    tag.decompose()
+                body_elem = parser.body or parser.root
+                if body_elem:
+                    for node in body_elem.iter():
+                        tag_name = node.tag.lower() if node.tag else ""
+                        if tag_name in ("h1", "h2", "h3", "h4", "h5", "h6"):
+                            h_text = node.text(strip=True)
+                            if h_text:
+                                yield Element(
+                                    type="heading",
+                                    level=int(tag_name[1]),
+                                    text=h_text,
+                                    markdown_repr=f"{'#' * int(tag_name[1])} {h_text}",
+                                    confidence=1.0,
+                                )
+                        elif tag_name in ("p", "li"):
+                            p_text = node.text(strip=True)
+                            if p_text:
+                                yield Element(
+                                    type="paragraph",
+                                    text=p_text,
+                                    markdown_repr=p_text,
+                                    confidence=1.0,
+                                )
+            else:
+                for paragraph in str(content).split("\n\n"):
+                    clean_p = paragraph.strip()
+                    if clean_p:
+                        yield Element(
+                            type="paragraph",
+                            text=clean_p,
+                            markdown_repr=clean_p,
+                            confidence=1.0,
+                        )
+
+        # Attachment extraction (recursive)
+        for part in msg.iter_attachments():
+            filename = part.get_filename() or "attachment"
+            payload = part.get_payload(decode=True)
+            if payload:
+                yield from self._process_attachment_bytes(filename, payload)
+
+    def _process_attachment_bytes(self, filename: str, data: bytes) -> Iterator[Element]:
+        """Save attachment to temp file, sniff type, and route to sub-extractor."""
+        try:
+            suffix = Path(filename).suffix
+            with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
+                tmp.write(data)
+                tmp_path = Path(tmp.name)
+
+            file_type = sniff(tmp_path)
+            extractor = get_extractor(file_type)
+
+            if extractor:
+                yield Element(
+                    type="heading",
+                    level=2,
+                    text=f"Attachment: {filename}",
+                    markdown_repr=f"## Attachment: {filename}",
+                    confidence=1.0,
+                )
+                yield from extractor.stream(tmp_path)
+
+            tmp_path.unlink(missing_ok=True)
+        except Exception:  # noqa: BLE001
+            return

universal_parser/extractors/office/docx_extractor.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+from typing import ClassVar
+
+from docx import Document as DocxDocument
+from docx.oxml.ns import qn
+from docx.table import Table
+from docx.text.paragraph import Paragraph
+
+from universal_parser.core.router import register
+from universal_parser.core.schema import Element, TableData
+from universal_parser.core.sniffer import FileType
+from universal_parser.extractors.base import BaseExtractor
+
+# Map Word built-in heading style names to heading levels
+_HEADING_STYLE_MAP: dict[str, int] = {
+    "heading 1": 1,
+    "heading 2": 2,
+    "heading 3": 3,
+    "heading 4": 4,
+    "heading 5": 5,
+    "heading 6": 6,
+    "title": 1,  # Word's "Title" style treated as H1
+    "subtitle": 2,  # Word's "Subtitle" style treated as H2
+}
+
+
+@register
+class DocxExtractor(BaseExtractor):
+    """
+    Extractor for .docx files using python-docx.
+    Handles:
+        - Paragraphs and headings in correct document order
+        - Tables (headers inferred from first row)
+        - Preserves reading order by iterating document body directly
+
+    Does NOT handle:
+        - Legacy .doc binary format (needs olefile — Phase 6)
+        - Embedded images/charts (Phase 6)
+    """
+
+    supported_types: ClassVar[list[FileType]] = [FileType.DOCX]
+
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        """Stream elements from a DOCX file in document reading order."""
+        try:
+            doc = DocxDocument(str(path))
+        except Exception:  # noqa: BLE001
+            # Corrupted or password-protected DOCX — exit cleanly
+            return
+
+        try:
+            # Iterate body children directly to preserve paragraph + table order.
+            # doc.paragraphs and doc.tables are separate lists — using them
+            # would give all paragraphs first then all tables, losing real order.
+            for child in doc.element.body:
+                tag = child.tag
+
+                # Paragraph element
+                if tag == qn("w:p"):
+                    element = self._process_paragraph(Paragraph(child, doc))
+                    if element is not None:
+                        yield element
+
+                # Table element
+                elif tag == qn("w:tbl"):
+                    element = self._process_table(Table(child, doc))
+                    if element is not None:
+                        yield element
+
+        except Exception:  # noqa: BLE001
+            # If anything fails mid-document, we stop cleanly.
+            # We do not crash — whatever was yielded before the error is valid.
+            return
+
+    def _process_paragraph(self, para: Paragraph) -> Element | None:
+        """Convert a python-docx Paragraph into an Element."""
+        text = para.text.strip()
+
+        # Skip empty paragraphs — they carry no information for RAG
+        if not text:
+            return None
+
+        style_name = para.style.name.lower() if para.style else ""
+        heading_level = _HEADING_STYLE_MAP.get(style_name)
+
+        if heading_level is not None:
+            return Element(
+                type="heading",
+                level=heading_level,
+                text=text,
+            )
+
+        # Check for list items (Word list styles contain "list" in the name)
+        if "list" in style_name:
+            return Element(
+                type="list_item",
+                text=text,
+            )
+        return Element(
+            type="paragraph",
+            text=text,
+        )
+
+    def _process_table(self, table: Table) -> Element | None:
+        """Convert a python-docx Table into an Element with TableData."""
+        rows = []
+        for row in table.rows:
+            row_data = [cell.text.strip() for cell in row.cells]
+            rows.append(row_data)
+
+        if not rows:
+            return None
+
+        # Treat the first row as headers
+        headers = rows[0]
+        data_rows = rows[1:]
+
+        # Build a markdown representation for quick use in RAG prompts
+        md_header = "| " + " | ".join(headers) + " |"
+        md_separator = "| " + " | ".join(["---"] * len(headers)) + " |"
+        md_rows = ["| " + " | ".join(row) + " |" for row in data_rows]
+        markdown_repr = "\n".join([md_header, md_separator] + md_rows)
+
+        return Element(
+            type="table",
+            data=TableData(headers=headers, rows=data_rows),
+            markdown_repr=markdown_repr,
+        )

universal_parser/extractors/office/legacy_extractor.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator
+from pathlib import Path
+from typing import ClassVar
+
+import olefile
+import xlrd
+
+from universal_parser.core.router import register
+from universal_parser.core.schema import Element, TableData
+from universal_parser.core.sniffer import FileType
+from universal_parser.extractors.base import BaseExtractor
+
+
+@register
+class LegacyOfficeExtractor(BaseExtractor):
+    """
+    Extractor for legacy Microsoft Office binary formats (.doc, .xls, .ppt).
+    Handles:
+        - .xls: Streams sheets into structured TableData & Markdown using xlrd
+        - .doc / .ppt: Decodes text streams from OLE2 binary containers via olefile
+    """
+
+    supported_types: ClassVar[list[FileType]] = [
+        FileType.DOC,
+        FileType.XLS,
+        FileType.PPT,
+    ]
+
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        path_obj = Path(path)
+        ext = path_obj.suffix.lower()
+
+        if ext == ".xls":
+            yield from self._stream_xls(path_obj)
+        elif ext in (".doc", ".ppt"):
+            yield from self._stream_ole_text(path_obj)
+
+    def _stream_xls(self, path: Path) -> Iterator[Element]:
+        """Stream sheets from legacy .xls files using xlrd."""
+        try:
+            wb = xlrd.open_workbook(str(path))
+            for sheet in wb.sheets():
+                rows = []
+                for row_idx in range(sheet.nrows):
+                    rows_vals = [
+                        str(sheet.cell_value(row_idx, col_idx)).strip()
+                        for col_idx in range(sheet.ncols)
+                    ]
+                    if any(rows_vals):
+                        rows.append(rows_vals)
+
+                if not rows:
+                    continue
+
+                headers = rows[0]
+                data_rows = rows[1:]
+
+                md_header = "| " + " | ".join(headers) + " |"
+                md_separator = "| " + " | ".join(["---"] * len(headers)) + " |"
+                md_rows = ["| " + " | ".join(r) + " |" for r in data_rows]
+                markdown_repr = f"### Sheet: {sheet.name}\n\n" + "\n".join(
+                    [md_header, md_separator] + md_rows
+                )
+
+                yield Element(
+                    type="table",
+                    text=f"Sheet: {sheet.name}",
+                    data=TableData(headers=headers, rows=data_rows),
+                    markdown_repr=markdown_repr,
+                    confidence=1.0,
+                )
+
+        except Exception:  # noqa: BLE001
+            return
+
+    def _stream_ole_text(self, path: Path) -> Iterator[Element]:
+        """Extract readable text streams from OLE2 compound files (.doc / .ppt)."""
+        try:
+            if not olefile.isOleFile(str(path)):
+                return
+
+            ole = olefile.OleFileIO(str(path))
+            extracted_text = []
+
+            for stream_name in ole.listdir():
+                try:
+                    stream_data = ole.openstream(stream_name).read()
+                    # Filter ASCII string sequences from binary stream
+                    raw_strings = re.findall(rb"[\x20-\x7E]{4,}", stream_data)
+                    for s in raw_strings:
+                        decoded = s.decode("ascii", errors="ignore").strip()
+                        if len(decoded) > 3 and not decoded.startswith("Microsoft"):
+                            extracted_text.append(decoded)
+
+                except Exception:  # noqa: BLE001, S112
+                    continue
+
+            ole.close()
+
+            for text_chunk in extracted_text:
+                yield Element(
+                    type="Paragraph",
+                    text=text_chunk,
+                    markdown_repr=text_chunk,
+                    confidence=0.8,
+                )
+
+        except Exception:  # noqa: BLE001
+            return

universal_parser/extractors/office/pptx_extractor.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+from typing import ClassVar
+
+from pptx import Presentation
+
+from universal_parser.core.router import register
+from universal_parser.core.schema import Element, TableData
+from universal_parser.core.sniffer import FileType
+from universal_parser.extractors.base import BaseExtractor
+
+
+@register
+class PPTXExtractor(BaseExtractor):
+    """
+    Extractor for PowerPoint presentations (.pptx).
+    Handles:
+        - Slide titles -> Level 1 Headings with slide numbers
+        - Text boxes and bullet points -> Paragraphs and List Items
+        - Presentation tables -> Structured TableData & Markdown tables
+    """
+
+    supported_types: ClassVar[list[FileType]] = [FileType.PPTX]
+
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        path_str = str(path)
+
+        try:
+            prs = Presentation(path_str)
+
+        except Exception:  # noqa: BLE001
+            return
+
+        for slide_num, slide in enumerate(prs.slides, start=1):
+            # 1. Slide Title (Heading 1)
+            if slide.shapes.title and slide.shapes.title.text.strip():
+                title = slide.shapes.title.text.strip()
+                yield Element(
+                    type="heading",
+                    level=1,
+                    text=f"Slide {slide_num}: {title}",
+                    page=slide_num,
+                    markdown_repr=f"# Slide {slide_num}: {title}",
+                    confidence=1.0,
+                )
+            else:
+                yield Element(
+                    type="heading",
+                    level=1,
+                    text=f"Slide {slide_num}",
+                    page=slide_num,
+                    markdown_repr=f"# Slide {slide_num}",
+                    confidence=1.0,
+                )
+
+            # 2. Iterate through shapes on the slide
+            for shape in slide.shapes:
+                # Skip title shape since already extracted
+                if shape == slide.shapes.title:
+                    continue
+                # A. Handle Tables in slides
+                if shape.has_table:
+                    table = shape.table
+                    rows = []
+                    for row in table.rows:
+                        row_cells = [cell.text.strip() for cell in row.cells]
+                        if any(row_cells):
+                            rows.append(row_cells)
+
+                    if not rows:
+                        continue
+
+                    headers = rows[0]
+                    data_rows = rows[1:]
+
+                    md_header = "| " + " | ".join(headers) + " |"
+                    md_separator = "| " + " | ".join(["---"] * len(headers)) + " |"
+                    md_rows = ["| " + " | ".join(r) + " |" for r in data_rows]
+                    markdown_repr = "\n".join([md_header, md_separator] + md_rows)
+
+                    yield Element(
+                        type="table",
+                        page=slide_num,
+                        text=f"Slide {slide_num} Table",
+                        data=TableData(headers=headers, rows=data_rows),
+                        markdown_repr=markdown_repr,
+                        confidence=1.0,
+                    )
+
+                # B. Handle Text Frames (paragraphs and bullet points)
+                elif shape.has_text_frame:
+                    for paragraph in shape.text_frame.paragraphs:
+                        text = paragraph.text.strip()
+                        if not text:
+                            continue
+
+                        is_bullet = paragraph.level > 0
+                        elem_type = "list_item" if is_bullet else "paragraph"
+                        prefix = "  " * paragraph.level + "- " if is_bullet else ""
+
+                        yield Element(
+                            type=elem_type,
+                            page=slide_num,
+                            text=text,
+                            markdown_repr=f"{prefix} {text}",
+                            confidence=1.0,
+                        )

universal_parser/extractors/office/xlsx_extractor.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+from typing import ClassVar
+
+from openpyxl import load_workbook
+
+from universal_parser.core.router import register
+from universal_parser.core.schema import Element, TableData
+from universal_parser.core.sniffer import FileType
+from universal_parser.extractors.base import BaseExtractor
+
+
+@register
+class XlsxExtractor(BaseExtractor):
+    """
+    Extractor for .xlsx files using openpyxl.
+    Handles:
+        - Memory-safe streaming using read_only=True
+        - Multiple sheets (each sheet is extracted as a Table element)
+        - Converts formula cells to values automatically using data_only=True
+        - Forward-fills empty/merged header names
+    """
+
+    supported_types: ClassVar[list[FileType]] = [FileType.XLSX]
+
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        """Stream sheets from an Excel workbook as Table elements."""
+        path_str = str(path)
+
+        try:
+            # read_only=True streams cells instead of loading the whole document.
+            # data_only=True evaluates Excel formulas and returns values.
+            wb = load_workbook(path_str, read_only=True, data_only=True)
+
+        except Exception:  # noqa: BLE001
+            # Corrupted or password-protected file
+            return
+
+        try:
+            for sheet_name in wb.sheetnames:
+                sheet = wb[sheet_name]
+
+                # Extract rows from sheet
+                rows = []
+                for row in sheet.iter_rows(values_only=True):
+                    # Clean the row data (convert None values to empty strings)
+                    clean_row = [str(cell).strip() if cell is not None else "" for cell in row]
+
+                    # Skip completely empty rows
+                    if any(clean_row):
+                        rows.append(clean_row)
+
+                if not rows:
+                    continue
+
+                # Clean headers: handle merged cells by filling empty header slots
+                raw_headers = rows[0]
+                headers = []
+                last_seen_header = ""
+                for idx, h in enumerate(raw_headers):
+                    if h:
+                        headers.append(h)
+                        last_seen_header = h
+                    elif last_seen_header:
+                        # Merged column continuation: e.g. "Q1 Revenue_2"
+                        headers.append(f"{last_seen_header}_col{idx + 1}")
+                    else:
+                        headers.append(f"Column_{idx + 1}")
+
+                data_rows = rows[1:]
+
+                # Build a markdown representation of the spreadsheet
+                md_header = "| " + " | ".join(headers) + " |"
+                md_separator = "| " + " | ".join(["---"] * len(headers)) + " |"
+                md_rows = ["| " + " | ".join(r) + " |" for r in data_rows]
+                markdown_repr = f"### Sheet: {sheet_name}\n\n" + "\n".join(
+                    [md_header, md_separator] + md_rows
+                )
+
+                yield Element(
+                    type="table",
+                    text=f"Sheet: {sheet_name}",
+                    data=TableData(headers=headers, rows=data_rows),
+                    markdown_repr=markdown_repr,
+                    confidence=1.0,
+                )
+
+        except Exception:  # noqa: BLE001
+            return
+        finally:
+            wb.close()