xfmr-zem 0.2.5__tar.gz → 0.2.7__tar.gz

{xfmr_zem-0.2.5 → xfmr_zem-0.2.7}/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.2.7] - 2026-02-03
+
+### Added
+- **Configurable OCR Parameters**: Added `scanned_threshold`, `zoom`, and `temp_dir` parameters to the OCR server for finer control over PDF processing.
+
+## [0.2.6] - 2026-02-03
+
+### Added
+- **PDF Extraction & Scanning**: Added support for processing multi-page PDFs in the OCR server. It automatically extracts digital text and falls back to OCR for scanned pages using `PyMuPDF`.
+
 ## [0.2.5] - 2026-02-03
 
 ### Added

{xfmr_zem-0.2.5 → xfmr_zem-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xfmr-zem
-Version: 0.2.5
+Version: 0.2.7
 Summary: Zem: Unified Data Pipeline Framework (ZenML + NeMo Curator + DataJuicer) for multi-domain processing
 Project-URL: Homepage, https://github.com/OAI-Labs/xfmr-zem
 Project-URL: Repository, https://github.com/OAI-Labs/xfmr-zem
@@ -51,6 +51,7 @@ Requires-Dist: paddlepaddle>=2.6.0; extra == 'ocr'
 Requires-Dist: pdfplumber>=0.11.0; extra == 'ocr'
 Requires-Dist: pillow>=10.0.0; extra == 'ocr'
 Requires-Dist: pyclipper; extra == 'ocr'
+Requires-Dist: pymupdf>=1.23.0; extra == 'ocr'
 Requires-Dist: pytesseract>=0.3.10; extra == 'ocr'
 Requires-Dist: ruamel-yaml>=0.17.0; extra == 'ocr'
 Requires-Dist: shapely; extra == 'ocr'

{xfmr_zem-0.2.5 → xfmr_zem-0.2.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "xfmr-zem"
-version = "0.2.5"
+version = "0.2.7"
 description = "Zem: Unified Data Pipeline Framework (ZenML + NeMo Curator + DataJuicer) for multi-domain processing"
 readme = "README.md"
 requires-python = ">=3.10,<3.13"
@@ -64,6 +64,7 @@ ocr = [
     "pyclipper",
     "einops",
     "pdfplumber>=0.11.0",
+    "pymupdf>=1.23.0",
     "ruamel.yaml>=0.17.0",
     "cachetools>=5.0.0",
 ]

{xfmr_zem-0.2.5 → xfmr_zem-0.2.7}/src/xfmr_zem/servers/ocr/parameters.yml

@@ -2,3 +2,6 @@
 extract_text:
   engine: "tesseract"
   model_id: null
+  scanned_threshold: 50
+  zoom: 2.0
+  temp_dir: "/tmp"

xfmr_zem-0.2.7/src/xfmr_zem/servers/ocr/server.py

@@ -0,0 +1,129 @@
+import os
+import pandas as pd
+from xfmr_zem.server import ZemServer
+from xfmr_zem.servers.ocr.engines import OCREngineFactory
+from loguru import logger
+from PIL import Image
+import io
+
+# Initialize ZemServer for OCR
+mcp = ZemServer("ocr")
+
+def extract_pdf_pages(
+    file_path: str, 
+    engine: str, 
+    ocr_engine, 
+    scanned_threshold: int = 50, 
+    zoom: float = 2.0, 
+    temp_dir: str = "/tmp"
+):
+    """Helper to process PDF pages with optional OCR for scanned content."""
+    import fitz  # PyMuPDF
+    
+    results = []
+    doc = fitz.open(file_path)
+    
+    # Ensure temp_dir exists
+    os.makedirs(temp_dir, exist_ok=True)
+    
+    for page_num in range(len(doc)):
+        page = doc[page_num]
+        text = page.get_text().strip()
+        
+        # Determine if we need to OCR (Strategy: text is too short or empty)
+        is_scanned = len(text) < scanned_threshold
+        
+        if is_scanned:
+            logger.info(f"Page {page_num + 1} appears scanned (text length: {len(text)}). Running OCR with {engine}...")
+            # Render page to image for OCR
+            pix = page.get_pixmap(matrix=fitz.Matrix(zoom, zoom))
+            img_data = pix.tobytes("png")
+            img = Image.open(io.BytesIO(img_data))
+            
+            # Temporary save for engine compatibility (engines expect path)
+            temp_path = os.path.join(temp_dir, f"ocr_page_{os.getpid()}_{page_num}.png")
+            img.save(temp_path)
+            
+            try:
+                ocr_result = ocr_engine.process(temp_path)
+                final_text = ocr_result["text"]
+                source = f"{engine}_ocr"
+            finally:
+                if os.path.exists(temp_path):
+                    os.remove(temp_path)
+        else:
+            final_text = text
+            source = "digital_pdf"
+            
+        results.append({
+            "text": final_text,
+            "page": page_num + 1,
+            "engine": source,
+            "metadata": {"file": file_path, "is_scanned": is_scanned}
+        })
+        
+    doc.close()
+    return results
+
+@mcp.tool()
+async def extract_text(
+    file_path: str, 
+    engine: str = "tesseract", 
+    model_id: str = None,
+    scanned_threshold: int = 50,
+    zoom: float = 2.0,
+    temp_dir: str = "/tmp"
+) -> pd.DataFrame:
+    """
+    Extracts text from an image or PDF using the specified OCR engine.
+    For PDFs, it will automatically handle scanned pages using the OCR engine.
+    
+    Args:
+        file_path: Path to the image or PDF file.
+        engine: The OCR engine to use ("tesseract", "paddle", "huggingface", "viet"). Defaults to "tesseract".
+        model_id: Optional model ID for the 'huggingface' engine.
+        scanned_threshold: Min characters required to skip OCR on PDF page. Defaults to 50.
+        zoom: Rendering zoom factor for scanned PDF pages. Defaults to 2.0.
+        temp_dir: Directory for temporary page images. Defaults to "/tmp".
+    """
+    logger.info(f"OCR Extraction: {file_path} using {engine} (scanned_threshold={scanned_threshold}, zoom={zoom})")
+    
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    try:
+        # Get engine from factory
+        ocr_engine = OCREngineFactory.get_engine(engine, model_id=model_id)
+        
+        # Handle PDF vs Image
+        if file_path.lower().endswith(".pdf"):
+            logger.info(f"Processing PDF file: {file_path}")
+            data = extract_pdf_pages(
+                file_path, 
+                engine, 
+                ocr_engine, 
+                scanned_threshold=scanned_threshold, 
+                zoom=zoom, 
+                temp_dir=temp_dir
+            )
+            df = pd.DataFrame(data)
+        else:
+            # Process image
+            result = ocr_engine.process(file_path)
+            df = pd.DataFrame([{
+                "text": result["text"],
+                "engine": result["engine"],
+                "metadata": result["metadata"]
+            }])
+        
+        logger.info(f"Successfully extracted text from {file_path}")
+        return df.to_dict(orient="records")
+        
+    except Exception as e:
+        logger.error(f"OCR Error with {engine}: {e}")
+        import traceback
+        logger.error(traceback.format_exc())
+        raise RuntimeError(f"OCR failed: {str(e)}")
+
+if __name__ == "__main__":
+    mcp.run()

{xfmr_zem-0.2.5 → xfmr_zem-0.2.7}/uv.lock

@@ -3683,6 +3683,21 @@ crypto = [
     { name = "cryptography" },
 ]
 
+[[package]]
+name = "pymupdf"
+version = "1.26.7"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/48/d6/09b28f027b510838559f7748807192149c419b30cb90e6d5f0cf916dc9dc/pymupdf-1.26.7.tar.gz", hash = "sha256:71add8bdc8eb1aaa207c69a13400693f06ad9b927bea976f5d5ab9df0bb489c3", size = 84327033, upload-time = "2025-12-11T21:48:50.694Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/94/35/cd74cea1787b2247702ef8522186bdef32e9cb30a099e6bb864627ef6045/pymupdf-1.26.7-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:07085718dfdae5ab83b05eb5eb397f863bcc538fe05135318a01ea353e7a1353", size = 23179369, upload-time = "2025-12-11T21:47:21.587Z" },
+    { url = "https://files.pythonhosted.org/packages/72/74/448b6172927c829c6a3fba80078d7b0a016ebbe2c9ee528821f5ea21677a/pymupdf-1.26.7-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:31aa9c8377ea1eea02934b92f4dcf79fb2abba0bf41f8a46d64c3e31546a3c02", size = 22470101, upload-time = "2025-12-11T21:47:37.105Z" },
+    { url = "https://files.pythonhosted.org/packages/65/e7/47af26f3ac76be7ac3dd4d6cc7ee105948a8355d774e5ca39857bf91c11c/pymupdf-1.26.7-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e419b609996434a14a80fa060adec72c434a1cca6a511ec54db9841bc5d51b3c", size = 23502486, upload-time = "2025-12-12T09:51:25.824Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/6b/3de1714d734ff949be1e90a22375d0598d3540b22ae73eb85c2d7d1f36a9/pymupdf-1.26.7-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:69dfc78f206a96e5b3ac22741263ebab945fdf51f0dbe7c5757c3511b23d9d72", size = 24115727, upload-time = "2025-12-11T21:47:51.274Z" },
+    { url = "https://files.pythonhosted.org/packages/62/9b/f86224847949577a523be2207315ae0fd3155b5d909cd66c274d095349a3/pymupdf-1.26.7-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1d5106f46e1ca0d64d46bd51892372a4f82076bdc14a9678d33d630702abca36", size = 24324386, upload-time = "2025-12-12T14:58:45.483Z" },
+    { url = "https://files.pythonhosted.org/packages/85/8e/a117d39092ca645fde8b903f4a941d9aa75b370a67b4f1f435f56393dc5a/pymupdf-1.26.7-cp310-abi3-win32.whl", hash = "sha256:7c9645b6f5452629c747690190350213d3e5bbdb6b2eca227d82702b327f6eee", size = 17203888, upload-time = "2025-12-12T13:59:57.613Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/c3/d0047678146c294469c33bae167c8ace337deafb736b0bf97b9bc481aa65/pymupdf-1.26.7-cp310-abi3-win_amd64.whl", hash = "sha256:425b1befe40d41b72eb0fe211711c7ae334db5eb60307e9dd09066ed060cceba", size = 18405952, upload-time = "2025-12-11T21:48:02.947Z" },
+]
+
 [[package]]
 name = "pymysql"
 version = "1.1.2"
@@ -5519,7 +5534,7 @@ wheels = [
 
 [[package]]
 name = "xfmr-zem"
-version = "0.2.5"
+version = "0.2.7"
 source = { editable = "." }
 dependencies = [
     { name = "click" },
@@ -5569,6 +5584,7 @@ ocr = [
     { name = "pdfplumber" },
     { name = "pillow" },
     { name = "pyclipper" },
+    { name = "pymupdf" },
     { name = "pytesseract" },
     { name = "ruamel-yaml" },
     { name = "shapely" },
@@ -5608,6 +5624,7 @@ requires-dist = [
     { name = "pyarrow", specifier = ">=15.0.0" },
     { name = "pyclipper", marker = "extra == 'ocr'" },
     { name = "pydantic", specifier = ">=2.0.0" },
+    { name = "pymupdf", marker = "extra == 'ocr'", specifier = ">=1.23.0" },
     { name = "pytesseract", marker = "extra == 'ocr'", specifier = ">=0.3.10" },
     { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" },
     { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0.0" },

xfmr_zem-0.2.5/src/xfmr_zem/servers/ocr/server.py

@@ -1,44 +0,0 @@
-import pandas as pd
-from xfmr_zem.server import ZemServer
-from xfmr_zem.servers.ocr.engines import OCREngineFactory
-from loguru import logger
-
-# Initialize ZemServer for OCR
-mcp = ZemServer("ocr")
-
-@mcp.tool()
-async def extract_text(file_path: str, engine: str = None, model_id: str = None) -> pd.DataFrame:
-    """
-    Extracts text from an image using the specified OCR engine.
-    
-    Args:
-        file_path: Path to the image file.
-        engine: The OCR engine to use ("tesseract", "paddle", "huggingface", "viet"). Defaults to "tesseract".
-        model_id: Optional model ID for the 'huggingface' engine (e.g., "Qwen/Qwen2-VL-2B-Instruct").
-    """
-    logger.info(f"OCR Extraction: {file_path} using {engine} (model: {model_id})")
-    
-    try:
-        # Get engine from factory (SOLID Strategy Pattern)
-        ocr_engine = OCREngineFactory.get_engine(engine, model_id=model_id)
-        
-        # Process image
-        result = ocr_engine.process(file_path)
-        
-        # Structure as a single-row DataFrame for Zem compatibility
-        # We wrap in a list to ensure pandas creates a row
-        df = pd.DataFrame([{
-            "text": result["text"],
-            "engine": result["engine"],
-            "metadata": result["metadata"]
-        }])
-        
-        logger.info(f"Successfully extracted text using {engine}")
-        return df.to_dict(orient="records")
-        
-    except Exception as e:
-        logger.error(f"OCR Error with {engine}: {e}")
-        raise RuntimeError(f"OCR failed: {str(e)}")
-
-if __name__ == "__main__":
-    mcp.run()