preocr 0.1.0__py3-none-any.whl

preocr/__init__.py

@@ -0,0 +1,7 @@
+"""PreOCR - A fast, CPU-only library that detects whether files need OCR processing."""
+
+from .version import __version__
+from .detector import needs_ocr
+
+__all__ = ["needs_ocr", "__version__"]
+

preocr/constants.py

@@ -0,0 +1,17 @@
+"""Constants and configuration for preocr."""
+
+# Minimum text length to consider a file as having meaningful text
+MIN_TEXT_LENGTH = 50
+
+# Minimum text length for office documents to skip OCR
+MIN_OFFICE_TEXT_LENGTH = 100
+
+# File type categories
+CATEGORY_STRUCTURED = "structured"
+CATEGORY_UNSTRUCTURED = "unstructured"
+
+# Confidence thresholds
+HIGH_CONFIDENCE = 0.9
+MEDIUM_CONFIDENCE = 0.7
+LOW_CONFIDENCE = 0.5
+

preocr/decision.py

@@ -0,0 +1,129 @@
+"""Decision engine to determine if OCR is needed."""
+
+from typing import Dict, Tuple
+
+from .constants import (
+    CATEGORY_STRUCTURED,
+    CATEGORY_UNSTRUCTURED,
+    HIGH_CONFIDENCE,
+    LOW_CONFIDENCE,
+    MEDIUM_CONFIDENCE,
+    MIN_OFFICE_TEXT_LENGTH,
+    MIN_TEXT_LENGTH,
+)
+
+
+def decide(signals: Dict[str, any]) -> Tuple[bool, str, float, str]:
+    """
+    Decide if a file needs OCR based on collected signals.
+    
+    Args:
+        signals: Dictionary of signals from signals.collect_signals()
+        
+    Returns:
+        Tuple of:
+            - needs_ocr: Boolean indicating if OCR is needed
+            - reason: Human-readable reason for the decision
+            - confidence: Confidence score (0.0-1.0)
+            - category: "structured" or "unstructured"
+    """
+    mime = signals.get("mime", "")
+    text_length = signals.get("text_length", 0)
+    extension = signals.get("extension", "")
+    is_binary = signals.get("is_binary", True)
+    
+    # Rule 1: Plain text formats - NO OCR
+    if mime.startswith("text/"):
+        return (
+            False,
+            "text file with extractable content",
+            HIGH_CONFIDENCE,
+            CATEGORY_STRUCTURED,
+        )
+    
+    # Rule 2: Office documents with text - NO OCR
+    if "officedocument" in mime or extension in ["docx", "pptx", "xlsx"]:
+        if text_length >= MIN_OFFICE_TEXT_LENGTH:
+            return (
+                False,
+                f"office document with {text_length} characters of text",
+                HIGH_CONFIDENCE,
+                CATEGORY_STRUCTURED,
+            )
+        else:
+            return (
+                True,
+                f"office document with insufficient text ({text_length} chars)",
+                MEDIUM_CONFIDENCE,
+                CATEGORY_UNSTRUCTURED,
+            )
+    
+    # Rule 3: Images - YES OCR (always)
+    if mime.startswith("image/"):
+        return (
+            True,
+            "image file (no text extraction possible)",
+            HIGH_CONFIDENCE,
+            CATEGORY_UNSTRUCTURED,
+        )
+    
+    # Rule 4: PDFs
+    if mime == "application/pdf" or extension == "pdf":
+        if text_length >= MIN_TEXT_LENGTH:
+            return (
+                False,
+                f"digital PDF with {text_length} characters of extractable text",
+                HIGH_CONFIDENCE,
+                CATEGORY_STRUCTURED,
+            )
+        else:
+            return (
+                True,
+                f"PDF without extractable text ({text_length} chars) - likely scanned",
+                MEDIUM_CONFIDENCE,
+                CATEGORY_UNSTRUCTURED,
+            )
+    
+    # Rule 5: JSON/XML - NO OCR
+    if mime in ["application/json", "application/xml"] or extension in ["json", "xml"]:
+        return (
+            False,
+            "structured data file (JSON/XML)",
+            HIGH_CONFIDENCE,
+            CATEGORY_STRUCTURED,
+        )
+    
+    # Rule 6: HTML - NO OCR (text can be extracted)
+    if mime in ["text/html", "application/xhtml+xml"] or extension in ["html", "htm"]:
+        if text_length >= MIN_TEXT_LENGTH:
+            return (
+                False,
+                f"HTML file with {text_length} characters of text",
+                HIGH_CONFIDENCE,
+                CATEGORY_STRUCTURED,
+            )
+        else:
+            return (
+                True,
+                "HTML file with minimal content",
+                LOW_CONFIDENCE,
+                CATEGORY_UNSTRUCTURED,
+            )
+    
+    # Rule 7: Unknown binaries - YES OCR (conservative default)
+    if is_binary:
+        return (
+            True,
+            "unknown binary file type",
+            LOW_CONFIDENCE,
+            CATEGORY_UNSTRUCTURED,
+        )
+    
+    # Fallback: default to needing OCR
+    return (
+        True,
+        "unrecognized file type",
+        LOW_CONFIDENCE,
+        CATEGORY_UNSTRUCTURED,
+    )
+

preocr/detector.py

@@ -0,0 +1,100 @@
+"""Main API for OCR detection."""
+
+from pathlib import Path
+from typing import Any, Dict, Union
+
+from . import decision
+from . import filetype
+from . import image_probe
+from . import office_probe
+from . import pdf_probe
+from . import signals
+from . import text_probe
+
+
+def needs_ocr(file_path: Union[str, Path]) -> Dict[str, Any]:
+    """
+    Determine if a file needs OCR processing.
+    
+    This is the main API function. It analyzes the file type, extracts text
+    where possible, and makes an intelligent decision about whether OCR is needed.
+    
+    Args:
+        file_path: Path to the file to analyze (string or Path object)
+        
+    Returns:
+        Dictionary with keys:
+            - needs_ocr: Boolean indicating if OCR is needed
+            - file_type: Detected file type category (e.g., "image", "pdf", "office")
+            - category: "structured" (no OCR) or "unstructured" (needs OCR)
+            - confidence: Confidence score (0.0-1.0)
+            - reason: Human-readable reason for the decision
+            - signals: Dictionary of all collected signals (for debugging)
+            
+    Example:
+        >>> result = needs_ocr("document.pdf")
+        >>> if result["needs_ocr"]:
+        ...     run_ocr("document.pdf")
+    """
+    path = Path(file_path)
+    
+    if not path.exists():
+        raise FileNotFoundError(f"File not found: {file_path}")
+    
+    # Step 1: Detect file type
+    file_info = filetype.detect_file_type(str(path))
+    mime = file_info["mime"]
+    
+    # Step 2: Extract text based on file type
+    text_result = None
+    image_result = None
+    
+    if mime == "application/pdf":
+        # PDF text extraction
+        text_result = pdf_probe.extract_pdf_text(str(path))
+    elif "officedocument" in mime or file_info["extension"] in ["docx", "pptx", "xlsx"]:
+        # Office document text extraction
+        text_result = office_probe.extract_office_text(str(path), mime)
+    elif mime.startswith("text/") or mime in ["text/html", "application/xhtml+xml"]:
+        # Plain text or HTML extraction
+        text_result = text_probe.extract_text_from_file(str(path), mime)
+    elif mime.startswith("image/"):
+        # Image analysis (no text extraction)
+        image_result = image_probe.analyze_image(str(path))
+    
+    # Step 3: Collect all signals
+    collected_signals = signals.collect_signals(
+        str(path), file_info, text_result, image_result
+    )
+    
+    # Step 4: Make decision
+    needs_ocr_flag, reason, confidence, category = decision.decide(collected_signals)
+    
+    # Step 5: Determine file type category for user
+    file_type_category = _get_file_type_category(mime, file_info["extension"])
+    
+    return {
+        "needs_ocr": needs_ocr_flag,
+        "file_type": file_type_category,
+        "category": category,
+        "confidence": confidence,
+        "reason": reason,
+        "signals": collected_signals,
+    }
+
+
+def _get_file_type_category(mime: str, extension: str) -> str:
+    """Get a user-friendly file type category."""
+    if mime.startswith("image/"):
+        return "image"
+    elif mime == "application/pdf" or extension == "pdf":
+        return "pdf"
+    elif "officedocument" in mime or extension in ["docx", "pptx", "xlsx", "doc", "ppt", "xls"]:
+        return "office"
+    elif mime.startswith("text/") or extension in ["txt", "csv", "html", "htm"]:
+        return "text"
+    elif mime in ["application/json", "application/xml"] or extension in ["json", "xml"]:
+        return "structured"
+    else:
+        return "unknown"
+

preocr/filetype.py

@@ -0,0 +1,90 @@
+"""File type detection using MIME types and extensions."""
+
+import mimetypes
+from pathlib import Path
+from typing import Dict, Optional
+
+try:
+    import magic
+except ImportError:
+    magic = None
+
+
+def detect_file_type(file_path: str) -> Dict[str, str]:
+    """
+    Detect file type using MIME detection and extension fallback.
+    
+    Args:
+        file_path: Path to the file to analyze
+        
+    Returns:
+        Dictionary with keys:
+            - mime: MIME type string (e.g., "application/pdf")
+            - extension: File extension without dot (e.g., "pdf")
+            - is_binary: Boolean indicating if file is binary (True for non-text types)
+    """
+    path = Path(file_path)
+    extension = path.suffix.lower().lstrip(".")
+    
+    # Try python-magic first (more reliable)
+    mime_type = None
+    if magic:
+        try:
+            mime_type = magic.from_file(str(path), mime=True)
+        except (OSError, magic.MagicException):
+            # Fallback if magic fails
+            pass
+    
+    # Fallback to mimetypes module
+    if not mime_type:
+        mime_type, _ = mimetypes.guess_type(str(path))
+    
+    # Final fallback: use extension-based detection
+    if not mime_type:
+        mime_type = _guess_mime_from_extension(extension)
+    
+    # Default to application/octet-stream for unknown types
+    if not mime_type:
+        mime_type = "application/octet-stream"
+    
+    # Determine if binary (non-text types)
+    is_binary = not (
+        mime_type.startswith("text/") or
+        mime_type in ["application/json", "application/xml"]
+    )
+    
+    return {
+        "mime": mime_type,
+        "extension": extension,
+        "is_binary": is_binary,
+    }
+
+
+def _guess_mime_from_extension(extension: str) -> Optional[str]:
+    """Guess MIME type from file extension."""
+    extension_map = {
+        "pdf": "application/pdf",
+        "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "doc": "application/msword",
+        "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        "ppt": "application/vnd.ms-powerpoint",
+        "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "xls": "application/vnd.ms-excel",
+        "png": "image/png",
+        "jpg": "image/jpeg",
+        "jpeg": "image/jpeg",
+        "gif": "image/gif",
+        "tiff": "image/tiff",
+        "tif": "image/tiff",
+        "bmp": "image/bmp",
+        "webp": "image/webp",
+        "txt": "text/plain",
+        "csv": "text/csv",
+        "html": "text/html",
+        "htm": "text/html",
+        "json": "application/json",
+        "xml": "application/xml",
+        "eml": "message/rfc822",
+    }
+    return extension_map.get(extension.lower())
+

preocr/image_probe.py

@@ -0,0 +1,139 @@
+"""Image analysis and entropy calculation."""
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+if TYPE_CHECKING:
+    from PIL import Image as PILImage
+
+try:
+    from PIL import Image
+    import numpy as np
+except ImportError:
+    Image = None
+    np = None
+
+
+def is_image_file(mime_type: str) -> bool:
+    """
+    Check if MIME type represents an image.
+    
+    Args:
+        mime_type: MIME type string
+        
+    Returns:
+        True if MIME type is an image, False otherwise
+    """
+    return mime_type.startswith("image/")
+
+
+def analyze_image(file_path: str) -> Dict[str, any]:
+    """
+    Analyze image file and calculate entropy.
+    
+    Args:
+        file_path: Path to the image file
+        
+    Returns:
+        Dictionary with keys:
+            - entropy: Image entropy value (0-8, higher = more complex)
+            - width: Image width in pixels
+            - height: Image height in pixels
+            - mode: Image mode (RGB, L, etc.)
+            - is_image: Always True for images
+    """
+    if not Image:
+        return {
+            "entropy": None,
+            "width": None,
+            "height": None,
+            "mode": None,
+            "is_image": True,
+        }
+    
+    path = Path(file_path)
+    
+    try:
+        with Image.open(path) as img:
+            # Convert to grayscale for entropy calculation
+            gray_img = img.convert("L")
+            
+            # Calculate entropy
+            entropy = _calculate_entropy(gray_img)
+            
+            return {
+                "entropy": entropy,
+                "width": img.width,
+                "height": img.height,
+                "mode": img.mode,
+                "is_image": True,
+            }
+    except Exception:
+        return {
+            "entropy": None,
+            "width": None,
+            "height": None,
+            "mode": None,
+            "is_image": True,
+        }
+
+
+def _calculate_entropy(image: Any) -> float:
+    """
+    Calculate entropy of an image.
+    
+    Entropy measures the randomness/complexity of pixel values.
+    Low entropy (0-4): Simple images, likely scanned text
+    High entropy (4-8): Complex images, photos
+    
+    Args:
+        image: PIL Image object (should be grayscale)
+        
+    Returns:
+        Entropy value between 0 and 8
+    """
+    if not np:
+        # Fallback: simple histogram-based entropy
+        histogram = image.histogram()
+        histogram = [h for h in histogram if h > 0]  # Remove zeros
+        total_pixels = sum(histogram)
+        
+        if total_pixels == 0:
+            return 0.0
+        
+        entropy = 0.0
+        for count in histogram:
+            probability = count / total_pixels
+            if probability > 0:
+                entropy -= probability * (probability.bit_length() - 1)  # Approximate log2
+        return entropy
+    
+    # NumPy-based calculation (more accurate)
+    try:
+        img_array = np.array(image)
+        histogram, _ = np.histogram(img_array.flatten(), bins=256, range=(0, 256))
+        histogram = histogram[histogram > 0]  # Remove zeros
+        total_pixels = histogram.sum()
+        
+        if total_pixels == 0:
+            return 0.0
+        
+        probabilities = histogram / total_pixels
+        entropy = -np.sum(probabilities * np.log2(probabilities))
+        return float(entropy)
+    except Exception:
+        # Fallback to simple calculation
+        histogram = image.histogram()
+        histogram = [h for h in histogram if h > 0]
+        total_pixels = sum(histogram)
+        
+        if total_pixels == 0:
+            return 0.0
+        
+        entropy = 0.0
+        for count in histogram:
+            probability = count / total_pixels
+            if probability > 0:
+                entropy -= probability * (probability.bit_length() - 1)
+        return entropy
+

preocr/office_probe.py

@@ -0,0 +1,161 @@
+"""Office document text extraction (DOCX, PPTX, XLSX)."""
+
+from pathlib import Path
+from typing import Dict
+
+try:
+    from docx import Document
+except ImportError:
+    Document = None
+
+try:
+    from pptx import Presentation
+except ImportError:
+    Presentation = None
+
+try:
+    from openpyxl import load_workbook
+except ImportError:
+    load_workbook = None
+
+
+def extract_office_text(file_path: str, mime_type: str) -> Dict[str, any]:
+    """
+    Extract text from Office documents (DOCX, PPTX, XLSX).
+    
+    Args:
+        file_path: Path to the office document
+        mime_type: MIME type of the file
+        
+    Returns:
+        Dictionary with keys:
+            - text_length: Number of characters in extracted text
+            - text: Extracted text (may be truncated for large files)
+            - document_type: Type of document ("docx", "pptx", "xlsx")
+    """
+    path = Path(file_path)
+    
+    if "wordprocessingml" in mime_type or path.suffix.lower() == ".docx":
+        return _extract_docx(path)
+    elif "presentationml" in mime_type or path.suffix.lower() == ".pptx":
+        return _extract_pptx(path)
+    elif "spreadsheetml" in mime_type or path.suffix.lower() == ".xlsx":
+        return _extract_xlsx(path)
+    else:
+        return {
+            "text_length": 0,
+            "text": "",
+            "document_type": None,
+        }
+
+
+def _extract_docx(path: Path) -> Dict[str, any]:
+    """Extract text from DOCX file."""
+    if not Document:
+        return {
+            "text_length": 0,
+            "text": "",
+            "document_type": "docx",
+        }
+    
+    try:
+        doc = Document(path)
+        text_parts = []
+        
+        # Extract paragraphs
+        for paragraph in doc.paragraphs:
+            if paragraph.text.strip():
+                text_parts.append(paragraph.text)
+        
+        # Extract tables
+        for table in doc.tables:
+            for row in table.rows:
+                for cell in row.cells:
+                    if cell.text.strip():
+                        text_parts.append(cell.text)
+        
+        full_text = "\n".join(text_parts)
+        
+        return {
+            "text_length": len(full_text),
+            "text": full_text[:1000] if len(full_text) > 1000 else full_text,
+            "document_type": "docx",
+        }
+    except Exception:
+        return {
+            "text_length": 0,
+            "text": "",
+            "document_type": "docx",
+        }
+
+
+def _extract_pptx(path: Path) -> Dict[str, any]:
+    """Extract text from PPTX file."""
+    if not Presentation:
+        return {
+            "text_length": 0,
+            "text": "",
+            "document_type": "pptx",
+        }
+    
+    try:
+        prs = Presentation(path)
+        text_parts = []
+        
+        # Extract text from slides
+        for slide in prs.slides:
+            for shape in slide.shapes:
+                if hasattr(shape, "text") and shape.text.strip():
+                    text_parts.append(shape.text)
+        
+        full_text = "\n".join(text_parts)
+        
+        return {
+            "text_length": len(full_text),
+            "text": full_text[:1000] if len(full_text) > 1000 else full_text,
+            "document_type": "pptx",
+        }
+    except Exception:
+        return {
+            "text_length": 0,
+            "text": "",
+            "document_type": "pptx",
+        }
+
+
+def _extract_xlsx(path: Path) -> Dict[str, any]:
+    """Extract text from XLSX file."""
+    if not load_workbook:
+        return {
+            "text_length": 0,
+            "text": "",
+            "document_type": "xlsx",
+        }
+    
+    try:
+        wb = load_workbook(path, data_only=True)
+        text_parts = []
+        
+        # Extract text from all sheets
+        for sheet_name in wb.sheetnames:
+            sheet = wb[sheet_name]
+            for row in sheet.iter_rows(values_only=True):
+                row_text = " ".join(str(cell) for cell in row if cell is not None)
+                if row_text.strip():
+                    text_parts.append(row_text)
+        
+        wb.close()
+        full_text = "\n".join(text_parts)
+        
+        return {
+            "text_length": len(full_text),
+            "text": full_text[:1000] if len(full_text) > 1000 else full_text,
+            "document_type": "xlsx",
+        }
+    except Exception:
+        return {
+            "text_length": 0,
+            "text": "",
+            "document_type": "xlsx",
+        }
+