longparser 0.1.0__py3-none-any.whl

longparser/integrations/llamaindex.py

@@ -0,0 +1,157 @@
+"""LlamaIndex integration adapter for LongParser.
+
+Provides ``LongParserReader``, a LlamaIndex-compatible document reader
+that wraps the LongParser extraction pipeline.
+
+Install the extra to use this adapter::
+
+    pip install clean_rag[llamaindex]
+
+Usage::
+
+    from longparser.integrations.llamaindex import LongParserReader
+
+    reader = LongParserReader()
+    docs = reader.load_data("report.pdf")  # list[llama_index.core.Document]
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from llama_index.core import Document as LIDocument
+
+from ..schemas import ProcessingConfig, ChunkingConfig
+
+_INSTALL_MSG = (
+    "llama-index-core is required for the LlamaIndex adapter. "
+    "Install it with:  pip install clean_rag[llamaindex]"
+)
+
+
+def _import_llamaindex():
+    """Lazy import llama-index-core with a clear install message."""
+    try:
+        from llama_index.core.readers.base import BaseReader
+        from llama_index.core import Document as LIDocument
+        return BaseReader, LIDocument
+    except ImportError as exc:
+        raise ImportError(_INSTALL_MSG) from exc
+
+
+class LongParserReader:
+    """LlamaIndex document reader backed by the LongParser pipeline.
+
+    Converts a file (PDF, DOCX, PPTX, XLSX, CSV) into a list of
+    LlamaIndex ``Document`` objects — one per chunk (if chunking is
+    enabled) or one per block.
+
+    Parameters
+    ----------
+    config:
+        LongParser ``ProcessingConfig``. Uses defaults if ``None``.
+    chunking_config:
+        LongParser ``ChunkingConfig``. If provided, the reader yields
+        one ``Document`` per chunk; otherwise one per block.
+    tesseract_lang:
+        Languages for Tesseract OCR (e.g. ``["eng", "urd"]``).
+    tessdata_path:
+        Path to tessdata directory.
+    """
+
+    def __init__(
+        self,
+        *,
+        config: Optional[ProcessingConfig] = None,
+        chunking_config: Optional[ChunkingConfig] = None,
+        tesseract_lang: list[str] | None = None,
+        tessdata_path: str | None = None,
+    ) -> None:
+        # Validate llama-index is available at construction time
+        BaseReader, _ = _import_llamaindex()
+        self._BaseReader = BaseReader
+
+        self.config = config or ProcessingConfig()
+        self.chunking_config = chunking_config
+        self.tesseract_lang = tesseract_lang
+        self.tessdata_path = tessdata_path
+
+    # ---- LlamaIndex interface ------------------------------------------------
+
+    def load_data(
+        self,
+        file: str | Path,
+        extra_info: dict | None = None,
+    ) -> list["LIDocument"]:
+        """Load data from a file and return LlamaIndex ``Document`` objects.
+
+        Parameters
+        ----------
+        file:
+            Path to the input file.
+        extra_info:
+            Additional metadata to merge into every document.
+
+        Returns
+        -------
+        list[Document]
+            One ``Document`` per chunk or per block.
+        """
+        _, LIDocument = _import_llamaindex()
+
+        from ..pipeline import PipelineOrchestrator
+
+        file = Path(file)
+        pipeline = PipelineOrchestrator(
+            tesseract_lang=self.tesseract_lang,
+            tessdata_path=self.tessdata_path,
+        )
+        result = pipeline.process_file(file, config=self.config)
+
+        docs: list[LIDocument] = []
+        base_meta = {"source": str(file), **(extra_info or {})}
+
+        # If chunking is requested, yield one doc per chunk
+        if self.chunking_config is not None:
+            chunks = pipeline.chunk(result, config=self.chunking_config)
+            for chunk in chunks:
+                docs.append(
+                    LIDocument(
+                        text=chunk.text,
+                        extra_info={
+                            **base_meta,
+                            "chunk_id": chunk.chunk_id,
+                            "chunk_type": chunk.chunk_type,
+                            "section_path": chunk.section_path,
+                            "page_numbers": chunk.page_numbers,
+                            "token_count": chunk.token_count,
+                            "equation_detected": chunk.equation_detected,
+                        },
+                    )
+                )
+            return docs
+
+        # Otherwise, yield one doc per block
+        for page in result.document.pages:
+            for block in page.blocks:
+                docs.append(
+                    LIDocument(
+                        text=block.text,
+                        extra_info={
+                            **base_meta,
+                            "block_id": block.block_id,
+                            "block_type": block.type.value,
+                            "heading_level": block.heading_level,
+                            "hierarchy_path": block.hierarchy_path,
+                            "page_number": page.page_number,
+                            "confidence": block.confidence.overall,
+                        },
+                    )
+                )
+
+        return docs
+
+
+__all__ = ["LongParserReader"]

longparser/pipeline/__init__.py

@@ -0,0 +1,8 @@
+"""Pipeline stages for LongParser document processing."""
+
+from .orchestrator import PipelineOrchestrator, PipelineResult
+
+__all__ = [
+    "PipelineOrchestrator",
+    "PipelineResult",
+]

longparser/pipeline/orchestrator.py

@@ -0,0 +1,230 @@
+"""Simple pipeline orchestrator for LongParser."""
+
+from pathlib import Path
+from dataclasses import dataclass, field
+from typing import Optional, List
+import time
+import logging
+import json
+
+from ..schemas import Document, ProcessingConfig, JobRequest, BlockType, ChunkingConfig, Chunk
+from ..extractors import DoclingExtractor
+from ..extractors.docling_extractor import HierarchyChunk
+from ..chunkers import HybridChunker
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class PipelineResult:
+    """Pipeline execution result."""
+    document: Document
+    hierarchy: List[HierarchyChunk]
+    processing_time_seconds: float
+    chunks: List[Chunk] = field(default_factory=list)
+    
+    @property
+    def total_blocks(self) -> int:
+        return sum(len(p.blocks) for p in self.document.pages)
+
+
+class PipelineOrchestrator:
+    """
+    Simple pipeline orchestrator using Docling.
+    
+    Flow:
+    1. Docling extracts with Tesseract CLI OCR
+    2. Layout analysis detects structure
+    3. HierarchicalChunker preserves heading hierarchy
+    """
+    
+    def __init__(self, tesseract_lang: List[str] = None, tessdata_path: str = None, force_full_page_ocr: bool = False):
+        """
+        Initialize pipeline.
+        
+        Args:
+            tesseract_lang: Languages for Tesseract OCR (default: ["eng"])
+            tessdata_path: Path to tessdata directory with language models and configs.
+            force_full_page_ocr: If True, OCR entire page even if embedded text exists.
+        """
+        self.extractor = DoclingExtractor(
+            tesseract_lang=tesseract_lang,
+            tessdata_path=tessdata_path,
+            force_full_page_ocr=force_full_page_ocr,
+        )
+    
+    def process(self, request: JobRequest) -> PipelineResult:
+        """Process a document."""
+        start_time = time.time()
+        
+        file_path = Path(request.file_path)
+        config = request.config
+        
+        logger.info(f"Processing: {file_path.name}")
+        
+        # Extract document
+        document, meta = self.extractor.extract(file_path, config)
+        
+        # Get hierarchy
+        hierarchy = self.extractor.get_hierarchy(file_path, config)
+        
+        processing_time = time.time() - start_time
+        logger.info(f"Completed in {processing_time:.2f}s")
+        
+        return PipelineResult(
+            document=document,
+            hierarchy=hierarchy,
+            processing_time_seconds=processing_time,
+        )
+    
+    def process_file(
+        self,
+        file_path: str | Path,
+        config: Optional[ProcessingConfig] = None,
+    ) -> PipelineResult:
+        """Convenience method to process a file directly."""
+        request = JobRequest(
+            file_path=str(file_path),
+            config=config or ProcessingConfig(),
+        )
+        return self.process(request)
+    
+    def export_to_markdown(self, result: PipelineResult, output_path: Path) -> Path:
+        """Export document to Markdown."""
+        output_path = Path(output_path)
+        output_path.mkdir(parents=True, exist_ok=True)
+        
+        md_path = output_path / "document.md"
+        md_content = self.extractor.to_markdown(result.document)
+        
+        with open(md_path, "w") as f:
+            f.write(md_content)
+        
+        return md_path
+    
+    def export_hierarchy(self, result: PipelineResult, output_path: Path) -> Path:
+        """Export hierarchy to JSON."""
+        output_path = Path(output_path)
+        output_path.mkdir(parents=True, exist_ok=True)
+        
+        hierarchy_path = output_path / "hierarchy.json"
+        hierarchy_data = [
+            {
+                "text": h.text[:200],  # Truncate for readability
+                "heading_path": h.heading_path,
+                "level": h.level,
+                "page": h.page_number,
+            }
+            for h in result.hierarchy
+        ]
+        
+        with open(hierarchy_path, "w") as f:
+            json.dump(hierarchy_data, f, indent=2)
+        
+        return hierarchy_path
+
+    def export_results(self, result: PipelineResult, output_dir: Path) -> dict:
+        """
+        Export results in the format expected by the user (blocks.json, manifest.json).
+        
+        Args:
+            result: Pipeline execution result
+            output_dir: Directory to save outputs
+            
+        Returns:
+            Dictionary of created files
+        """
+        output_dir = Path(output_dir)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        created_files = {}
+        
+        # 1. blocks.json - Flattened list of blocks from all pages
+        all_blocks = []
+        total_tables = 0
+        
+        for page in result.document.pages:
+            for block in page.blocks:
+                # Exclude confidence from public output
+                block_dict = block.model_dump(exclude={"confidence"})
+                # Ensure compatibility with expected format
+                if block.type == BlockType.TABLE:
+                    total_tables += 1
+                all_blocks.append(block_dict)
+                
+        blocks_path = output_dir / "blocks.json"
+        with open(blocks_path, "w") as f:
+            json.dump(all_blocks, f, indent=2, default=str)
+        created_files["blocks"] = blocks_path
+        
+        # 2. manifest.json - Processing metadata
+        manifest = {
+            "source_file": result.document.metadata.source_file,
+            "file_hash": result.document.metadata.file_hash,
+            "total_pages": result.document.metadata.total_pages,
+            "total_blocks": len(all_blocks),
+            "total_tables": total_tables,
+            "processing_time_seconds": result.processing_time_seconds,
+            "stages_completed": [
+                "stage1_extraction",
+                "stage2_validation",
+                "stage3_reprocess",
+                "stage4_enrichment",
+                "stage5_verification"
+            ],
+            "verification": {
+                "auto_accepted": False,
+                "needs_hitl_review": True,
+                "low_confidence_pages": [],
+                "low_confidence_tables": []
+            }
+        }
+            
+        manifest_path = output_dir / "manifest.json"
+        with open(manifest_path, "w") as f:
+            json.dump(manifest, f, indent=2, default=str)
+        created_files["manifest"] = manifest_path
+        
+        # 3. document.md - Markdown representation
+        md_path = self.export_to_markdown(result, output_dir)
+        created_files["markdown"] = md_path
+        
+        # 4. Images
+        images_dir = output_dir / "images"
+        images = self.save_images(images_dir)
+        created_files["images"] = images
+        
+        return created_files
+
+    def chunk(self, result: PipelineResult, config: Optional[ChunkingConfig] = None) -> List[Chunk]:
+        """
+        Run hybrid chunking on a pipeline result.
+        
+        Args:
+            result: Pipeline execution result with extracted blocks
+            config: Chunking configuration (uses defaults if None)
+            
+        Returns:
+            List of RAG-optimized chunks
+        """
+        chunker = HybridChunker(config or ChunkingConfig())
+        all_blocks = result.document.all_blocks
+        chunks = chunker.chunk(all_blocks)
+        result.chunks = chunks
+        return chunks
+
+    def export_chunks(self, result: PipelineResult, output_dir: Path) -> Path:
+        """Export chunks to JSON."""
+        output_dir = Path(output_dir)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        
+        chunks_path = output_dir / "chunks.json"
+        chunks_data = [c.model_dump() for c in result.chunks]
+        with open(chunks_path, "w") as f:
+            json.dump(chunks_data, f, indent=2, default=str)
+        
+        logger.info(f"Exported {len(result.chunks)} chunks to {chunks_path}")
+        return chunks_path
+
+    def save_images(self, output_dir: Path) -> List[Path]:
+        """Save extracted images."""
+        return self.extractor.save_images(output_dir)

longparser/schemas.py

@@ -0,0 +1,247 @@
+"""Pydantic schemas for LongParser document processing pipeline."""
+
+from __future__ import annotations
+
+import uuid
+from enum import Enum
+from typing import Optional
+from pydantic import BaseModel, Field
+
+
+class BlockType(str, Enum):
+    """Types of document blocks."""
+    HEADING = "heading"
+    PARAGRAPH = "paragraph"
+    LIST_ITEM = "list_item"
+    TABLE = "table"
+    FIGURE = "figure"
+    CAPTION = "caption"
+    FOOTER = "footer"
+    HEADER = "header"
+    EQUATION = "equation"
+    CODE = "code"
+
+
+class ExtractorType(str, Enum):
+    """Document extraction engines."""
+    DOCLING = "docling"
+    SURYA = "surya"
+    MARKER = "marker"
+    NATIVE_PDF = "native_pdf"
+    PADDLE = "paddle"
+
+
+class BoundingBox(BaseModel):
+    """Bounding box coordinates (PDF coordinate system)."""
+    x0: float
+    y0: float
+    x1: float
+    y1: float
+
+
+class Provenance(BaseModel):
+    """Traceability information for a block."""
+    source_file: str
+    page_number: int
+    bbox: BoundingBox
+    extractor: ExtractorType
+    extractor_version: str = "1.0.0"
+    pipeline_version: str = "1.0.0"
+
+
+class Confidence(BaseModel):
+    """Confidence scores for extraction quality."""
+    overall: float = Field(ge=0.0, le=1.0)
+    text_confidence: float = Field(default=1.0, ge=0.0, le=1.0)
+    layout_confidence: float = Field(default=1.0, ge=0.0, le=1.0)
+    table_confidence: Optional[float] = Field(default=None, ge=0.0, le=1.0)
+
+
+class BlockFlags(BaseModel):
+    """Flags indicating block status."""
+    needs_review: bool = False
+    repaired: bool = False
+    fallback_used: bool = False
+    excluded_from_rag: bool = False
+
+
+class TableCell(BaseModel):
+    """Individual cell in a table."""
+    row_index: int = Field(alias="r0")
+    col_index: int = Field(alias="c0")
+    row_span: int = Field(default=1, alias="rspan")
+    col_span: int = Field(default=1, alias="cspan")
+    text: str = ""
+    bbox: Optional[BoundingBox] = None
+    confidence: float = Field(default=1.0, ge=0.0, le=1.0)
+
+    class Config:
+        populate_by_name = True
+
+
+class Table(BaseModel):
+    """Table structure with cells and metadata."""
+    table_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    n_rows: int
+    n_cols: int
+    cells: list[TableCell] = Field(default_factory=list)
+    table_confidence: float = Field(default=1.0, ge=0.0, le=1.0)
+    csv_path: Optional[str] = None
+    html_path: Optional[str] = None
+
+
+class Block(BaseModel):
+    """Core document element with full provenance."""
+    block_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    type: BlockType
+    text: str = ""
+    order_index: int = 0
+    heading_level: Optional[int] = Field(default=None, description="Heading level (1-6) for heading blocks, inferred by Docling")
+    indent_level: int = Field(default=0, description="Bullet nesting depth (0=top, 1=sub, 2=sub-sub). Used for PPTX list items.")
+    hierarchy_path: list[str] = Field(default_factory=list)
+    refs_out: list[str] = Field(default_factory=list)
+    refs_in: list[str] = Field(default_factory=list)
+    provenance: Provenance
+    confidence: Confidence
+    flags: BlockFlags = Field(default_factory=BlockFlags)
+    table: Optional[Table] = None
+    image_path: Optional[str] = None
+    bbox_px: Optional[tuple] = Field(default=None, description="Pixel-space bounding box (x0,y0,x1,y1) for MFD dedup")
+
+
+class PageProfile(BaseModel):
+    """Validation profile for a single page."""
+    page_number: int
+    needs_reprocess: bool = False
+    validation_errors: list[str] = Field(default_factory=list)
+    layout_confidence: float = Field(default=1.0, ge=0.0, le=1.0)
+    table_confidence: Optional[float] = None
+    has_rtl: bool = False
+    has_math: bool = False
+
+
+class Page(BaseModel):
+    """Single page with blocks and metadata."""
+    page_number: int
+    width: float
+    height: float
+    blocks: list[Block] = Field(default_factory=list)
+    rendered_image_path: Optional[str] = None
+    profile: Optional[PageProfile] = None
+
+
+class DocumentMetadata(BaseModel):
+    """Document-level metadata."""
+    source_file: str
+    file_hash: str = ""
+    language: Optional[str] = None
+    total_pages: int = 0
+    academic_mode: bool = False
+    rtl_hint: bool = False
+
+
+class Document(BaseModel):
+    """Complete processed document."""
+    doc_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    metadata: DocumentMetadata
+    pages: list[Page] = Field(default_factory=list)
+
+    @property
+    def all_blocks(self) -> list[Block]:
+        """Get all blocks across all pages."""
+        return [block for page in self.pages for block in page.blocks]
+
+    @property
+    def all_tables(self) -> list[Table]:
+        """Get all tables from table blocks."""
+        return [
+            block.table
+            for block in self.all_blocks
+            if block.type == BlockType.TABLE and block.table
+        ]
+
+
+class ProcessingConfig(BaseModel):
+    """Configuration for pipeline execution."""
+    academic_mode: bool = False
+    rtl_hint: bool = False
+    do_ocr: bool = True
+    formula_ocr: bool = True  # Independent from do_ocr — runs pix2tex even when text OCR is off
+    do_table_structure: bool = True
+    export_images: bool = True
+    images_output_dir: Optional[str] = None
+    layout_confidence_threshold: float = 0.7
+    table_confidence_threshold: float = 0.75
+    ocr_noise_threshold: float = 0.15
+    enable_fallback: bool = True
+    # Smart extraction fallback options
+    prefer_initial_on_degradation: bool = True  # Keep original if fallback degrades quality
+    ocr_backend: str = "easyocr"  # easyocr | tesseract | rapidocr
+    ocr_use_gpu: bool = True
+    # Force full page OCR for better text segmentation in two-column layouts
+    force_full_page_ocr: bool = False
+    # Exclude page headers and footers from extraction output
+    exclude_page_headers_footers: bool = True
+    
+    # Formula extraction mode: "full" (slow, best LaTeX), "fast" (Unicode text), "smart" (hybrid)
+    formula_mode: str = "smart"
+    # Smart mode safety caps
+    smart_max_pages: int = 5      # Max pages to enrich in smart mode before fallback to fast
+    smart_max_ratio: float = 0.2  # Max ratio of pages (enriched/total) before fallback
+    smart_max_equations: int = 25       # Max equations to OCR per document (circuit breaker)
+    smart_max_ocr_seconds: float = 300.0  # Total OCR time budget (must fit within arq timeout)
+
+
+class ExtractionMetadata(BaseModel):
+    """Metadata from smart extraction strategy."""
+    strategy_used: str = "standard"  # standard | force_full_page_ocr
+    initial_low_grade: Optional[str] = None
+    fallback_low_grade: Optional[str] = None
+    improved: bool = False
+    fallback_degraded: bool = False
+    reprocessed_pages: list[int] = Field(default_factory=list)
+    ocr_backend_used: Optional[str] = None
+    reasons: list[str] = Field(default_factory=list)
+
+
+class ChunkingConfig(BaseModel):
+    """Configuration for hybrid chunking."""
+    max_tokens: int = 512
+    min_tokens: int = 100
+    overlap_blocks: int = 1
+    table_rows_per_chunk: int = 15
+    exclude_headers_footers: bool = True
+    detect_equations: bool = True
+    table_chunk_format: str = "row_record"  # "row_record" | "pipe"
+    generate_schema_chunks: bool = True
+    wide_table_col_threshold: int = 25
+
+
+class Chunk(BaseModel):
+    """A RAG-optimized chunk with full provenance."""
+    chunk_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    text: str
+    token_count: int
+    chunk_type: str  # "section" | "table" | "table_schema" | "list" | "equation" | "continuation"
+    section_path: list[str] = Field(default_factory=list)
+    page_numbers: list[int] = Field(default_factory=list)
+    block_ids: list[str] = Field(default_factory=list)
+    overlap_with_previous: bool = False
+    equation_detected: bool = False
+    metadata: dict = Field(default_factory=dict)  # row_start, row_end, sheet, col_band
+
+
+class JobRequest(BaseModel):
+    """Request to process a document."""
+    job_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    file_path: str
+    config: ProcessingConfig = Field(default_factory=ProcessingConfig)
+
+
+class JobResult(BaseModel):
+    """Result of document processing."""
+    job_id: str
+    document: Document
+    success: bool = True
+    errors: list[str] = Field(default_factory=list)
+    processing_time_seconds: float = 0.0

longparser/server/__init__.py

@@ -0,0 +1,22 @@
+"""LongParser REST API server package.
+
+Start the server::
+
+    uv run uvicorn longparser.server.app:app --reload --port 8000
+
+Subpackages:
+    - :mod:`~longparser.server.chat` — RAG chat engine, retriever, LangGraph HITL
+    - :mod:`~longparser.server.routers` — modular FastAPI route groups (future)
+
+Key modules:
+    - :mod:`~longparser.server.app` — FastAPI application factory and all routes
+    - :mod:`~longparser.server.db` — Motor async MongoDB database layer
+    - :mod:`~longparser.server.queue` — ARQ async job queue (Redis-backed)
+    - :mod:`~longparser.server.worker` — ARQ background worker definitions
+    - :mod:`~longparser.server.embeddings` — multi-backend embedding engine
+    - :mod:`~longparser.server.vectorstores` — Chroma / FAISS / Qdrant adapters
+"""
+
+from .app import app
+
+__all__ = ["app"]