universal-doc-parser 1.0.0__py3-none-any.whl

universal_parser/extractors/structured/parquet_extractor.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+from typing import ClassVar
+
+import pyarrow.parquet as pq
+
+from universal_parser.core.router import register
+from universal_parser.core.schema import Element, TableData
+from universal_parser.core.sniffer import FileType
+from universal_parser.extractors.base import BaseExtractor
+
+
+@register
+class ParquetExtractor(BaseExtractor):
+    """
+    Extractor for Apache Parquet files (.parquet).
+    Handles:
+        - Reads schema headers and row data using pyarrow
+        - Converts columnar data into structured TableData & Markdown
+    """
+
+    supported_types: ClassVar[list[FileType]] = [FileType.PARQUET]
+
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        path_obj = Path(path)
+
+        try:
+            parquet_file = pq.ParquetFile(str(path_obj))
+            schema = parquet_file.schema.to_arrow_schema()
+            headers = schema.names
+
+            # Read table data
+            table = parquet_file.read()
+            df_dict = table.to_pydict()
+
+            num_rows = table.num_rows
+            rows = []
+            for i in range(num_rows):
+                row = [str(df_dict[col][i]).strip() for col in headers]
+                rows.append(row)
+
+            # Generate markdown table representation
+            md_header = "| " + " | ".join(headers) + " |"
+            md_separator = "| " + " | ".join(["---"] * len(headers)) + " |"
+            md_rows = ["| " + " | ".join(r) + " |" for r in rows]
+            markdown_repr = "\n".join([md_header, md_separator] + md_rows)
+
+            yield Element(
+                type="table",
+                text=path_obj.name,
+                data=TableData(headers=headers, rows=rows),
+                markdown_repr=markdown_repr,
+                confidence=1.0,
+            )
+
+        except Exception:  # noqa: BLE001
+            return

universal_parser/extractors/web/__init__.py

@@ -0,0 +1 @@
+# Web extractors

universal_parser/extractors/web/epub_extractor.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+from typing import ClassVar
+
+import ebooklib
+from ebooklib import epub
+from selectolax.parser import HTMLParser
+
+from universal_parser.core.router import register
+from universal_parser.core.schema import Element
+from universal_parser.core.sniffer import FileType
+from universal_parser.extractors.base import BaseExtractor
+
+
+@register
+class EPUBExtractor(BaseExtractor):
+    """
+    Extractor for EPUB e-books (.epub) using ebooklib & selectolax.
+
+    Handles:
+        - Iterates over document items (chapters/sections)
+        - Converts chapter titles to Headings
+        - Extracts chapter paragraphs and list items
+    """
+
+    supported_types: ClassVar[list[FileType]] = [FileType.EPUB]
+
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        path_obj = Path(path)
+
+        try:
+            book = epub.read_epub(str(path_obj))
+
+            for item in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
+                html_content = item.get_content().decode("utf-8", errors="replace")
+                if not html_content.strip():
+                    continue
+
+                parser = HTMLParser(html_content)
+
+                # Clean noise tags
+                for tag in parser.css("script, style, noscript, svg"):
+                    tag.decompose()
+
+                body = parser.body or parser.root
+                if body is None:
+                    continue
+
+                for node in body.iter():
+                    tag_name = node.tag.lower() if node.tag else ""
+
+                    # Headings
+                    if tag_name in ("h1", "h2", "h3", "h4", "h5", "h6"):
+                        level = int(tag_name[1])
+                        text = node.text(strip=True)
+                        if text:
+                            yield Element(
+                                type="heading",
+                                level=level,
+                                text=text,
+                                markdown_repr=f"{'#' * level} {text}",
+                                confidence=1.0,
+                            )
+
+                    # Paragraphs & Lists
+                    elif tag_name in ("p", "li"):
+                        text = node.text(strip=True)
+                        if text:
+                            prefix = "- " if tag_name == "li" else ""
+                            elem_type = "list_item" if tag_name == "li" else "paragraph"
+                            yield Element(
+                                type=elem_type,
+                                text=text,
+                                markdown_repr=f"{prefix}{text}",
+                                confidence=1.0,
+                            )
+
+        except Exception:  # noqa: BLE001
+            return

universal_parser/extractors/web/html_extractor.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+from typing import ClassVar
+
+from selectolax.parser import HTMLParser
+
+from universal_parser.core.router import register
+from universal_parser.core.schema import Element, TableData
+from universal_parser.core.sniffer import FileType
+from universal_parser.extractors.base import BaseExtractor
+
+
+@register
+class HTMLExtractor(BaseExtractor):
+    """
+    Ultra-fast HTML/XHTML extractor using selectolax.
+
+    Handles:
+        - Headings (h1 - h6) preserving document tree hierarchy
+        - Text paragraphs (<p>, <li>)
+        - HTML <table> tags -> TableData + Markdown tables
+        - Strips script, style, and navigation junk tags
+    """
+
+    supported_types: ClassVar[list[FileType]] = [FileType.HTML]
+
+    def stream(self, path: str | Path) -> Iterator[Element]:
+        path_obj = Path(path)
+
+        try:
+            with open(path_obj, encoding="utf-8", errors="replace") as f:
+                html_content = f.read()
+
+            if not html_content.strip():
+                return
+
+            parser = HTMLParser(html_content)
+
+            # Strip noise (script, style, svg)
+            for tag in parser.css("script, style, noscript, svg"):
+                tag.decompose()
+
+            body = parser.body or parser.root
+            if body is None:
+                return
+
+            for node in body.iter():
+                tag_name = node.tag.lower() if node.tag else ""
+
+                # Headings
+                if tag_name in ("h1", "h2", "h3", "h4", "h5", "h6"):
+                    level = int(tag_name[1])
+                    text = node.text(strip=True)
+                    if text:
+                        yield Element(
+                            type="heading",
+                            level=level,
+                            text=text,
+                            markdown_repr=f"{'#' * level} {text}",
+                            confidence=1.0,
+                        )
+
+                # Paragraphs and Lists
+                elif tag_name in ("p", "li"):
+                    text = node.text(strip=True)
+                    if text:
+                        prefix = "- " if tag_name == "li" else ""
+                        elem_type = "list_item" if tag_name == "li" else "paragraph"
+                        yield Element(
+                            type=elem_type,
+                            text=text,
+                            markdown_repr=f"{prefix}{text}",
+                            confidence=1.0,
+                        )
+
+                # Tables
+                elif tag_name == "table":
+                    headers = []
+                    rows = []
+
+                    for th in node.css("th"):
+                        h_text = th.text(strip=True)
+                        if h_text:
+                            headers.append(h_text)
+
+                    for tr in node.css("tr"):
+                        tds = [td.text(strip=True) for td in tr.css("td")]
+                        if any(tds):
+                            rows.append(tds)
+
+                    if not headers and rows:
+                        headers = [f"Column_{i + 1}" for i in range(len(rows[0]))]
+
+                    if headers or rows:
+                        md_header = "| " + " | ".join(headers) + " |"
+                        md_separator = "| " + " | ".join(["---"] * len(headers)) + " |"
+                        md_rows = ["| " + " | ".join(r) + " |" for r in rows]
+                        markdown_repr = "\n".join([md_header, md_separator] + md_rows)
+
+                        yield Element(
+                            type="table",
+                            text=path_obj.name,
+                            data=TableData(headers=headers, rows=rows),
+                            markdown_repr=markdown_repr,
+                            confidence=1.0,
+                        )
+
+        except Exception:  # noqa: BLE001
+            return

universal_parser/mcp/__init__.py

@@ -0,0 +1 @@
+# MCP package

universal_parser/mcp/server.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from mcp.server.fastmcp import FastMCP
+
+from universal_parser.core.engine import parse
+from universal_parser.core.router import FORMAT_REGISTRY
+from universal_parser.exports.to_chunks import to_chunks
+from universal_parser.exports.to_markdown import to_markdown
+
+# Initialize FastMCP Server
+mcp = FastMCP(
+    "universal-parser",
+    instructions="A zero-GPU universal document ingestion engine for RAG and AI agents.",
+)
+
+
+@mcp.tool()
+def parse_document(file_path: str, output_format: str = "markdown") -> str:
+    """
+    Parse any local document (PDF, DOCX, XLSX, PPTX, HTML, EPUB, CSV, JSON, XML, Parquet, Email, Image).
+    Args:
+        file_path: Path to the document.
+        output_format: 'markdown', 'chunks', or 'json'.
+    """
+    path = Path(file_path)
+    if not path.exists():
+        return f"Error: File not found: {file_path}"
+
+    try:
+        doc = parse(path)
+
+        if output_format == "markdown":
+            return to_markdown(doc)
+        elif output_format == "chunks":
+            chunks = to_chunks(doc, max_tokens=512)
+            return json.dumps([c.__dict__ for c in chunks], indent=2)
+        elif output_format == "json":
+            return doc.model_dump_json(indent=2)
+        else:
+            return f"Error: Unknown output format '{output_format}'. Use 'markdown', 'chunks' or 'json'."
+
+    except Exception as e:  # noqa: BLE001
+        return f"Error parsing document: {e!s}"
+
+
+@mcp.tool()
+def list_supported_formats() -> list[str]:
+    """List all document file formats supported by Universal Parser."""
+    return sorted({ft.name.lower() for ft in FORMAT_REGISTRY})
+
+
+def run_server() -> None:
+    """Run the MCP server over standard I/O (stdio)."""
+    mcp.run(transport="studio")
+
+
+if __name__ == "__main__":
+    run_server()

universal_parser/observability/__init__.py

@@ -0,0 +1,12 @@
+from universal_parser.observability.dashboard import (
+    export_dashboard,
+    generate_dashboard_html,
+)
+from universal_parser.observability.metrics import MetricsCollector, ParseEventMetric
+
+__all__ = [
+    "MetricsCollector",
+    "ParseEventMetric",
+    "export_dashboard",
+    "generate_dashboard_html",
+]

universal_parser/observability/dashboard.py

@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from universal_parser.observability.metrics import MetricsCollector
+
+
+def generate_dashboard_html(collector: MetricsCollector | None = None) -> str:
+    """Generates a self-contained, responsive HTML observability dashboard."""
+    col = collector or MetricsCollector()
+    summary = col.get_summary()
+    events = col.get_events()
+
+    # Generate format table rows
+    format_rows = ""
+    for fmt, count in summary["format_breakdown"].items():
+        format_rows += f"<tr><td><strong>{fmt.upper()}</strong></td><td>{count}</td></tr>\n"
+
+    if not format_rows:
+        format_rows = "<tr><td colspan='2' style='text-align:center; color:#888;'>No events recorded yet</td></tr>"
+
+    # Generate recent events table rows
+    event_rows = ""
+    for e in reversed(events[-20:]):
+        anomaly_badge = (
+            f"<span class='badge badge-warn'>{len(e.anomalies)} anomalies</span>"
+            if e.anomalies
+            else "<span class='badge badge-ok'>clean</span>"
+        )
+        event_rows += f"""
+        <tr>
+            <td>{e.file_name}</td>
+            <td>{e.file_type.upper()}</td>
+            <td>{e.page_count}</td>
+            <td>{e.element_count}</td>
+            <td>{e.duration_ms} ms</td>
+            <td>{anomaly_badge}</td>
+        </tr>
+        """
+
+    if not event_rows:
+        event_rows = "<tr><td colspan='6' style='text-align:center; color:#888;'>No events recorded yet</td></tr>"
+
+    html = f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Universal Parser — Observability & Drift Dashboard</title>
+    <style>
+        :root {{
+            --bg-color: #0f172a;
+            --card-bg: #1e293b;
+            --text-main: #f8fafc;
+            --text-muted: #94a3b8;
+            --primary: #38bdf8;
+            --accent: #818cf8;
+            --success: #4ade80;
+            --warning: #fbbf24;
+            --border: #334155;
+        }}
+        body {{
+            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
+            background-color: var(--bg-color);
+            color: var(--text-main);
+            margin: 0;
+            padding: 24px;
+        }}
+        .container {{
+            max-width: 1200px;
+            margin: 0 auto;
+        }}
+        header {{
+            margin-bottom: 32px;
+            border-bottom: 1px solid var(--border);
+            padding-bottom: 16px;
+        }}
+        h1 {{
+            margin: 0 0 8px 0;
+            font-size: 28px;
+            color: var(--primary);
+        }}
+        .subtitle {{
+            color: var(--text-muted);
+            margin: 0;
+        }}
+        .grid {{
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
+            gap: 16px;
+            margin-bottom: 32px;
+        }}
+        .card {{
+            background-color: var(--card-bg);
+            border: 1px solid var(--border);
+            border-radius: 8px;
+            padding: 20px;
+        }}
+        .card-title {{
+            color: var(--text-muted);
+            font-size: 13px;
+            text-transform: uppercase;
+            letter-spacing: 0.5px;
+            margin-bottom: 8px;
+        }}
+        .card-value {{
+            font-size: 32px;
+            font-weight: bold;
+            color: var(--text-main);
+        }}
+        table {{
+            width: 100%;
+            border-collapse: collapse;
+            margin-top: 12px;
+            background-color: var(--card-bg);
+            border-radius: 8px;
+            overflow: hidden;
+            border: 1px solid var(--border);
+        }}
+        th, td {{
+            padding: 12px 16px;
+            text-align: left;
+            border-bottom: 1px solid var(--border);
+            font-size: 14px;
+        }}
+        th {{
+            background-color: rgba(0, 0, 0, 0.2);
+            color: var(--primary);
+            font-weight: 600;
+        }}
+        tr:last-child td {{
+            border-bottom: none;
+        }}
+        .badge {{
+            padding: 4px 8px;
+            border-radius: 4px;
+            font-size: 12px;
+            font-weight: 600;
+        }}
+        .badge-ok {{
+            background-color: rgba(74, 222, 128, 0.15);
+            color: var(--success);
+        }}
+        .badge-warn {{
+            background-color: rgba(251, 191, 36, 0.15);
+            color: var(--warning);
+        }}
+        .layout-two-col {{
+            display: grid;
+            grid-template-columns: 1fr 2fr;
+            gap: 24px;
+        }}
+        @media (max-width: 768px) {{
+            .layout-two-col {{
+                grid-template-columns: 1fr;
+            }}
+        }}
+    </style>
+</head>
+<body>
+    <div class="container">
+        <header>
+            <h1>Universal Parser — Observability Dashboard</h1>
+            <p class="subtitle">Real-time performance telemetry, anomaly tracking, and multi-format throughput.</p>
+        </header>
+
+        <div class="grid">
+            <div class="card">
+                <div class="card-title">Total Documents</div>
+                <div class="card-value">{summary["total_documents"]}</div>
+            </div>
+            <div class="card">
+                <div class="card-title">Total Pages</div>
+                <div class="card-value">{summary["total_pages"]}</div>
+            </div>
+            <div class="card">
+                <div class="card-title">Avg Latency</div>
+                <div class="card-value">{summary["avg_duration_ms"]}<span style="font-size:16px; color:var(--text-muted);"> ms</span></div>
+            </div>
+            <div class="card">
+                <div class="card-title">Drift / Anomalies</div>
+                <div class="card-value" style="color: {"var(--warning)" if summary["anomaly_count"] > 0 else "var(--success)"};">{summary["anomaly_count"]}</div>
+            </div>
+        </div>
+
+        <div class="layout-two-col">
+            <div>
+                <h2>Formats Handled</h2>
+                <table>
+                    <thead>
+                        <tr><th>Format</th><th>Documents</th></tr>
+                    </thead>
+                    <tbody>
+                        {format_rows}
+                    </tbody>
+                </table>
+            </div>
+
+            <div>
+                <h2>Recent Ingestion Stream</h2>
+                <table>
+                    <thead>
+                        <tr>
+                            <th>File</th>
+                            <th>Type</th>
+                            <th>Pages</th>
+                            <th>Elements</th>
+                            <th>Latency</th>
+                            <th>Status</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {event_rows}
+                    </tbody>
+                </table>
+            </div>
+        </div>
+    </div>
+</body>
+</html>
+"""
+    return html
+
+
+def export_dashboard(output_path: str | Path, collector: MetricsCollector | None = None) -> Path:
+    """Exports the observability dashboard to an HTML file on disk."""
+    path = Path(output_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    html = generate_dashboard_html(collector)
+    path.write_text(html, encoding="utf-8")
+    return path

universal_parser/observability/metrics.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from threading import Lock
+from typing import Any, Self
+
+from universal_parser.core.schema import Document
+
+
+@dataclass
+class ParseEventMetric:
+    """Telemetry record for a single document parse execution."""
+
+    file_name: str
+    file_type: str
+    page_count: int
+    element_count: int
+    duration_ms: float
+    timestamp: float = field(default_factory=time.time)
+    has_scanned_pages: bool = False
+    anomalies: list[str] = field(default_factory=list)
+
+
+class MetricsCollector:
+    """Thread-safe collector for document ingestion telemetry and anomaly signals."""
+
+    _instance: MetricsCollector | None = None
+    _lock = Lock()
+
+    def __new__(cls) -> Self:
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = super().__new__(cls)
+                cls._instance._events = []
+            return cls._instance
+
+    def record_parse(
+        self,
+        doc: Document,
+        duration_ms: float,
+        anomalies: list[str] | None = None,
+    ) -> ParseEventMetric:
+        """Records telemetry for a parsed document."""
+        metric = ParseEventMetric(
+            file_name=doc.metadata.file_name,
+            file_type=doc.metadata.file_type,
+            page_count=doc.metadata.page_count or 1,
+            element_count=len(doc.content_tree),
+            duration_ms=round(duration_ms, 2),
+            has_scanned_pages=doc.metadata.has_scanned_pages,
+            anomalies=anomalies or [],
+        )
+        with self._lock:
+            self._events.append(metric)
+        return metric
+
+    def get_events(self) -> list[ParseEventMetric]:
+        with self._lock:
+            return list(self._events)
+
+    def get_summary(self) -> dict[str, Any]:
+        """Calculates aggregated performance statistics across all parsed documents."""
+        with self._lock:
+            events = list(self._events)
+
+        if not events:
+            return {
+                "total_documents": 0,
+                "total_pages": 0,
+                "total_elements": 0,
+                "avg_duration_ms": 0.0,
+                "format_breakdown": {},
+                "anomaly_count": 0,
+            }
+
+        total_docs = len(events)
+        total_pages = sum(e.page_count for e in events)
+        total_elements = sum(e.element_count for e in events)
+        avg_duration = round(sum(e.duration_ms for e in events) / total_docs, 2)
+
+        format_breakdown: dict[str, int] = {}
+        anomaly_count = 0
+        for e in events:
+            format_breakdown[e.file_type] = format_breakdown.get(e.file_type, 0) + 1
+            anomaly_count += len(e.anomalies)
+
+        return {
+            "total_documents": total_docs,
+            "total_pages": total_pages,
+            "total_elements": total_elements,
+            "avg_duration_ms": avg_duration,
+            "format_breakdown": format_breakdown,
+            "anomaly_count": anomaly_count,
+        }
+
+    def clear(self) -> None:
+        with self._lock:
+            self._events.clear()