datafog 4.4.0__tar.gz → 4.4.0a2__tar.gz

{datafog-4.4.0 → datafog-4.4.0a2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datafog
-Version: 4.4.0
+Version: 4.4.0a2
 Summary: Lightning-fast PII detection and anonymization library with 190x performance advantage
 Author: Sid Mohan
 Author-email: sid@datafog.ai
@@ -35,45 +35,59 @@ Requires-Dist: torch<2.7,>=2.1.0; extra == "nlp-advanced"
 Requires-Dist: transformers>=4.20.0; extra == "nlp-advanced"
 Requires-Dist: huggingface-hub>=0.16.0; extra == "nlp-advanced"
 Provides-Extra: ocr
+Requires-Dist: numpy>=1.24.0; extra == "ocr"
 Requires-Dist: pytesseract>=0.3.0; extra == "ocr"
-Requires-Dist: Pillow>=10.0.0; extra == "ocr"
+Requires-Dist: Pillow>=12.2.0; extra == "ocr"
 Requires-Dist: sentencepiece>=0.2.0; extra == "ocr"
 Requires-Dist: protobuf>=4.0.0; extra == "ocr"
 Provides-Extra: distributed
 Requires-Dist: pandas>=2.0.0; extra == "distributed"
 Requires-Dist: numpy>=1.24.0; extra == "distributed"
+Requires-Dist: pyspark>=3.5.0; extra == "distributed"
 Provides-Extra: web
 Requires-Dist: fastapi>=0.100.0; extra == "web"
-Requires-Dist: aiohttp>=3.8.0; extra == "web"
-Requires-Dist: requests>=2.30.0; extra == "web"
+Requires-Dist: aiohttp>=3.13.4; extra == "web"
+Requires-Dist: certifi>=2025.4.26; extra == "web"
+Requires-Dist: requests>=2.33.0; extra == "web"
 Provides-Extra: cli
 Requires-Dist: typer>=0.12.0; extra == "cli"
 Requires-Dist: pydantic-settings>=2.0.0; extra == "cli"
 Provides-Extra: crypto
-Requires-Dist: cryptography>=40.0.0; extra == "crypto"
+Requires-Dist: cryptography>=46.0.7; extra == "crypto"
+Provides-Extra: test
+Requires-Dist: pytest>=9.0.3; extra == "test"
+Requires-Dist: pytest-asyncio>=1.3.0; extra == "test"
+Requires-Dist: pytest-cov>=7.1.0; extra == "test"
+Provides-Extra: docs
+Requires-Dist: sphinx>=7.2.6; extra == "docs"
+Provides-Extra: benchmark
+Requires-Dist: pytest-benchmark>=4.0.0; extra == "benchmark"
 Provides-Extra: dev
-Requires-Dist: pytest>=7.0.0; extra == "dev"
-Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
-Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
-Requires-Dist: sphinx>=7.0.0; extra == "dev"
+Requires-Dist: pytest>=9.0.3; extra == "dev"
+Requires-Dist: pytest-asyncio>=1.3.0; extra == "dev"
+Requires-Dist: pytest-cov>=7.1.0; extra == "dev"
+Requires-Dist: sphinx>=7.2.6; extra == "dev"
 Provides-Extra: all
 Requires-Dist: spacy<4.0,>=3.7.0; extra == "all"
 Requires-Dist: gliner>=0.2.5; extra == "all"
 Requires-Dist: torch<2.7,>=2.1.0; extra == "all"
 Requires-Dist: transformers>=4.20.0; extra == "all"
 Requires-Dist: huggingface-hub>=0.16.0; extra == "all"
+Requires-Dist: numpy>=1.24.0; extra == "all"
 Requires-Dist: pytesseract>=0.3.0; extra == "all"
-Requires-Dist: Pillow>=10.0.0; extra == "all"
+Requires-Dist: Pillow>=12.2.0; extra == "all"
 Requires-Dist: sentencepiece>=0.2.0; extra == "all"
 Requires-Dist: protobuf>=4.0.0; extra == "all"
 Requires-Dist: pandas>=2.0.0; extra == "all"
 Requires-Dist: numpy>=1.24.0; extra == "all"
+Requires-Dist: pyspark>=3.5.0; extra == "all"
 Requires-Dist: fastapi>=0.100.0; extra == "all"
-Requires-Dist: aiohttp>=3.8.0; extra == "all"
-Requires-Dist: requests>=2.30.0; extra == "all"
+Requires-Dist: aiohttp>=3.13.4; extra == "all"
+Requires-Dist: certifi>=2025.4.26; extra == "all"
+Requires-Dist: requests>=2.33.0; extra == "all"
 Requires-Dist: typer>=0.12.0; extra == "all"
 Requires-Dist: pydantic-settings>=2.0.0; extra == "all"
-Requires-Dist: cryptography>=40.0.0; extra == "all"
+Requires-Dist: cryptography>=46.0.7; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -251,5 +265,6 @@ cd datafog-python
 python -m venv .venv
 source .venv/bin/activate  # Windows: .venv\Scripts\activate
 pip install -e ".[all,dev]"
+pip install -r requirements-dev.txt
 pytest tests/
 ```

{datafog-4.4.0 → datafog-4.4.0a2}/README.md

@@ -162,5 +162,6 @@ cd datafog-python
 python -m venv .venv
 source .venv/bin/activate  # Windows: .venv\Scripts\activate
 pip install -e ".[all,dev]"
+pip install -r requirements-dev.txt
 pytest tests/
 ```

datafog-4.4.0a2/datafog/__about__.py

@@ -0,0 +1 @@
+__version__ = "4.4.0a2"

{datafog-4.4.0 → datafog-4.4.0a2}/datafog/client.py

@@ -181,7 +181,7 @@ def download_model(
     Download a model for specified engine.
 
     Examples:
-        spaCy: datafog download-model en_core_web_sm --engine spacy
+        spaCy: datafog download-model en_core_web_lg --engine spacy
         GLiNER: datafog download-model urchade/gliner_multi_pii-v1 --engine gliner
     """
     if engine == "spacy":

{datafog-4.4.0 → datafog-4.4.0a2}/datafog/engine.py

@@ -171,17 +171,13 @@ def _gliner_entities(text: str) -> list[Entity]:
 def _get_spacy_annotator():
     try:
         from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator
-    except ImportError:
-        return _UnavailableAnnotator(
-            "SpaCy engine requires the nlp extra. Install with: pip install datafog[nlp]"
-        )
+    except ImportError as exc:
+        return _UnavailableAnnotator(str(exc))
 
     try:
         return SpacyPIIAnnotator.create()
-    except ImportError:
-        return _UnavailableAnnotator(
-            "SpaCy engine requires the nlp extra. Install with: pip install datafog[nlp]"
-        )
+    except ImportError as exc:
+        return _UnavailableAnnotator(str(exc))
     except Exception as exc:
         return _UnavailableAnnotator(
             f"SpaCy engine initialization failed: {type(exc).__name__}: {exc}"
@@ -192,19 +188,13 @@ def _get_spacy_annotator():
 def _get_gliner_annotator():
     try:
         from .processing.text_processing.gliner_annotator import GLiNERAnnotator
-    except ImportError:
-        return _UnavailableAnnotator(
-            "GLiNER engine requires the nlp-advanced extra. "
-            "Install with: pip install datafog[nlp-advanced]"
-        )
+    except ImportError as exc:
+        return _UnavailableAnnotator(str(exc))
 
     try:
         annotator = GLiNERAnnotator.create()
-    except ImportError:
-        return _UnavailableAnnotator(
-            "GLiNER engine requires the nlp-advanced extra. "
-            "Install with: pip install datafog[nlp-advanced]"
-        )
+    except ImportError as exc:
+        return _UnavailableAnnotator(str(exc))
     except Exception as exc:
         return _UnavailableAnnotator(
             f"GLiNER engine initialization failed: {type(exc).__name__}: {exc}"

{datafog-4.4.0 → datafog-4.4.0a2}/datafog/models/spacy_nlp.py

@@ -9,10 +9,11 @@ from typing import List
 from uuid import uuid4
 
 import spacy
-from rich.progress import track
 
 from .annotator import AnnotationResult, AnnotatorRequest
 
+DEFAULT_SPACY_MODEL = "en_core_web_lg"
+
 
 class SpacyAnnotator:
     """
@@ -22,14 +23,18 @@ class SpacyAnnotator:
     Supports various NLP tasks including entity recognition and model management.
     """
 
-    def __init__(self, model_name: str = "en_core_web_lg"):
+    def __init__(self, model_name: str = DEFAULT_SPACY_MODEL):
         self.model_name = model_name
         self.nlp = None
 
     def load_model(self):
-        if not spacy.util.is_package(self.model_name):
-            spacy.cli.download(self.model_name)
-        self.nlp = spacy.load(self.model_name)
+        try:
+            self.nlp = spacy.load(self.model_name)
+        except OSError as exc:
+            raise ImportError(
+                f"spaCy model {self.model_name!r} is not installed. "
+                f"Download it explicitly with: datafog download-model {self.model_name} --engine spacy"
+            ) from exc
 
     def annotate_text(self, text: str, language: str = "en") -> List[AnnotationResult]:
         if not self.nlp:
@@ -47,7 +52,7 @@ class SpacyAnnotator:
         )
         doc = self.nlp(annotator_request.text)
         results = []
-        for ent in track(doc.ents, description="Processing entities"):
+        for ent in doc.ents:
             result = AnnotationResult(
                 start=ent.start_char,
                 end=ent.end_char,
@@ -72,6 +77,12 @@ class SpacyAnnotator:
         return spacy.util.get_installed_models()
 
     @staticmethod
-    def list_entities() -> List[str]:
-        nlp = spacy.load("en_core_web_lg")
+    def list_entities(model_name: str = DEFAULT_SPACY_MODEL) -> List[str]:
+        try:
+            nlp = spacy.load(model_name)
+        except OSError as exc:
+            raise ImportError(
+                f"spaCy model {model_name!r} is not installed. "
+                f"Download it explicitly with: datafog download-model {model_name} --engine spacy"
+            ) from exc
         return [ent for ent in nlp.pipe_labels["ner"]]

{datafog-4.4.0 → datafog-4.4.0a2}/datafog/processing/image_processing/donut_processor.py

@@ -6,14 +6,10 @@ for document understanding tasks, particularly OCR and information extraction
 from images of documents.
 """
 
-import importlib
-import importlib.util
 import json
 import logging
 import os
 import re
-import subprocess
-import sys
 from typing import TYPE_CHECKING, Any
 
 from .image_downloader import ImageDownloader
@@ -43,13 +39,12 @@ class DonutProcessor:
         self.model_path = model_path
         self.downloader = ImageDownloader()
 
-    def ensure_installed(self, package_name):
-        try:
-            importlib.import_module(package_name)
-        except ImportError:
-            subprocess.check_call(
-                [sys.executable, "-m", "pip", "install", package_name]
-            )
+    @staticmethod
+    def _missing_dependency_message(package_name: str) -> str:
+        return (
+            f"Donut OCR requires {package_name}. "
+            "Install with: pip install datafog[nlp-advanced,ocr]"
+        )
 
     def preprocess_image(self, image: "Image.Image") -> Any:
         import numpy as np
@@ -86,40 +81,40 @@ class DonutProcessor:
                 "PYTEST_DONUT=yes is set, running actual OCR in test environment"
             )
 
-        # Only import torch and transformers when actually needed and not in test environment
         try:
-            # Check if torch is available before trying to import it
-            try:
-                # Try to find the module without importing it
-                spec = importlib.util.find_spec("torch")
-                if spec is None:
-                    # If we're in a test that somehow bypassed the IN_TEST_ENV check,
-                    # still return a mock result instead of failing
-                    logging.warning("torch module not found, returning mock result")
-                    return json.dumps({"text": "Mock OCR text (torch not available)"})
-
-                # Ensure dependencies are installed
-                self.ensure_installed("torch")
-                self.ensure_installed("transformers")
-            except ImportError:
-                # If importlib.util is not available, fall back to direct try/except
-                pass
-
-            # Import dependencies only when needed
             try:
                 import torch
+            except ImportError as exc:
+                raise ImportError(self._missing_dependency_message("torch")) from exc
+
+            try:
                 from transformers import DonutProcessor as TransformersDonutProcessor
                 from transformers import VisionEncoderDecoderModel
             except ImportError as e:
-                logging.warning(f"Import error: {e}, returning mock result")
-                return json.dumps({"text": f"Mock OCR text (import error: {e})"})
+                raise ImportError(
+                    self._missing_dependency_message("transformers")
+                ) from e
 
             # Preprocess the image
             image_np = self.preprocess_image(image)
 
             # Initialize model components
-            processor = TransformersDonutProcessor.from_pretrained(self.model_path)
-            model = VisionEncoderDecoderModel.from_pretrained(self.model_path)
+            try:
+                processor = TransformersDonutProcessor.from_pretrained(
+                    self.model_path,
+                    local_files_only=True,
+                )
+                model = VisionEncoderDecoderModel.from_pretrained(
+                    self.model_path,
+                    local_files_only=True,
+                )
+            except OSError as exc:
+                raise RuntimeError(
+                    f"Donut model {self.model_path!r} is not available locally. "
+                    "Download it explicitly before using Donut OCR, or pass a local "
+                    "model path."
+                ) from exc
+
             device = "cuda" if torch.cuda.is_available() else "cpu"
             model.to(device)
             model.eval()
@@ -153,6 +148,8 @@ class DonutProcessor:
             result = processor.token2json(sequence)
             return json.dumps(result)
 
+        except (ImportError, RuntimeError):
+            raise
         except Exception as e:
             logging.error(f"Error in extract_text_from_image: {e}")
             # Return a placeholder in case of error

{datafog-4.4.0 → datafog-4.4.0a2}/datafog/processing/spark_processing/pyspark_udfs.py

@@ -2,17 +2,16 @@
 PySpark UDFs for PII annotation and related utilities.
 
 This module provides functions for PII (Personally Identifiable Information) annotation
-using SpaCy models in a PySpark environment. It includes utilities for installing
-dependencies, creating and broadcasting PII annotator UDFs, and performing PII annotation
-on text data.
+using SpaCy models in a PySpark environment. It includes utilities for validating
+dependencies, creating and broadcasting PII annotator UDFs, and performing PII
+annotation on text data.
 """
 
 import importlib
-import subprocess
-import sys
 
 PII_ANNOTATION_LABELS = ["DATE_TIME", "LOC", "NRP", "ORG", "PER"]
 MAXIMAL_STRING_SIZE = 1000000
+DEFAULT_SPACY_MODEL = "en_core_web_lg"
 
 
 def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]:
@@ -45,7 +44,7 @@ def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]:
 
 
 def broadcast_pii_annotator_udf(
-    spark_session=None, spacy_model: str = "en_core_web_lg"
+    spark_session=None, spacy_model: str = DEFAULT_SPACY_MODEL
 ):
     """Broadcast PII annotator across Spark cluster and create UDF"""
     ensure_installed("pyspark")
@@ -69,5 +68,14 @@ def broadcast_pii_annotator_udf(
 def ensure_installed(package_name):
     try:
         importlib.import_module(package_name)
-    except ImportError:
-        subprocess.check_call([sys.executable, "-m", "pip", "install", package_name])
+    except ImportError as exc:
+        if package_name == "pyspark":
+            extra = "distributed"
+        elif package_name == "spacy":
+            extra = "nlp"
+        else:
+            extra = "all"
+        raise ImportError(
+            f"{package_name} is required for Spark PII UDF support. "
+            f"Install with: pip install datafog[{extra}]"
+        ) from exc

{datafog-4.4.0 → datafog-4.4.0a2}/datafog/processing/text_processing/gliner_annotator.py

@@ -79,14 +79,18 @@ class GLiNERAnnotator(BaseModel):
 
         try:
             # Load the GLiNER model
-            model = GLiNER.from_pretrained(model_name)
+            model = GLiNER.from_pretrained(model_name, local_files_only=True)
             logging.info(f"Successfully loaded GLiNER model: {model_name}")
 
             return cls(model=model, entity_types=entity_types, model_name=model_name)
 
         except Exception as e:
             logging.error(f"Failed to load GLiNER model {model_name}: {str(e)}")
-            raise
+            raise RuntimeError(
+                f"GLiNER model {model_name!r} is not available locally. "
+                "Download it explicitly with: "
+                f"datafog download-model {model_name} --engine gliner"
+            ) from e
 
     def annotate(self, text: str) -> Dict[str, List[str]]:
         """

{datafog-4.4.0 → datafog-4.4.0a2}/datafog/processing/text_processing/spacy_pii_annotator.py

@@ -24,39 +24,34 @@ PII_ANNOTATION_LABELS = [
     "WORK_OF_ART",
 ]
 MAXIMAL_STRING_SIZE = 1000000
+DEFAULT_SPACY_MODEL = "en_core_web_lg"
 
 
 class SpacyPIIAnnotator(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     nlp: Any
+    model_name: str = DEFAULT_SPACY_MODEL
 
     @classmethod
-    def create(cls) -> "SpacyPIIAnnotator":
-        import spacy
-
+    def create(cls, model_name: str = DEFAULT_SPACY_MODEL) -> "SpacyPIIAnnotator":
         try:
-            nlp = spacy.load("en_core_web_lg")
-        except OSError:
-            import subprocess
-            import sys
+            import spacy
+        except ImportError as exc:
+            raise ImportError(
+                "SpaCy engine requires the nlp extra. "
+                "Install with: pip install datafog[nlp]"
+            ) from exc
 
-            interpreter_location = sys.executable
-            subprocess.run(
-                [
-                    interpreter_location,
-                    "-m",
-                    "pip",
-                    "install",
-                    "--no-deps",
-                    "--no-cache-dir",
-                    "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.7.1/en_core_web_lg-3.7.1-py3-none-any.whl",
-                ],
-                check=True,
-            )
-            nlp = spacy.load("en_core_web_lg")
+        try:
+            nlp = spacy.load(model_name)
+        except OSError as exc:
+            raise ImportError(
+                f"spaCy model {model_name!r} is not installed. "
+                f"Download it explicitly with: datafog download-model {model_name} --engine spacy"
+            ) from exc
 
-        return cls(nlp=nlp)
+        return cls(nlp=nlp, model_name=model_name)
 
     def annotate(self, text: str) -> Dict[str, List[str]]:
         try:

{datafog-4.4.0 → datafog-4.4.0a2}/datafog/services/spark_service.py

@@ -1,14 +1,12 @@
 """
 Spark service for data processing and analysis.
 
-Provides a wrapper around PySpark functionality, including session creation,
-JSON reading, and package management.
+Provides a wrapper around PySpark functionality, including session creation and
+JSON reading.
 """
 
 import importlib
 import os
-import subprocess
-import sys
 from typing import List
 
 
@@ -16,14 +14,13 @@ class SparkService:
     """
     Manages Spark operations and dependencies.
 
-    Initializes a Spark session, handles imports, and provides methods for
-    data reading and package installation.
+    Initializes a Spark session, handles imports, and provides methods for data
+    reading.
     """
 
     def __init__(self, master=None):
         self.master = master
 
-        # Ensure pyspark is installed first
         self.ensure_installed("pyspark")
 
         # Now import necessary modules after ensuring pyspark is installed
@@ -84,16 +81,8 @@ class SparkService:
     def ensure_installed(self, package_name):
         try:
             importlib.import_module(package_name)
-        except ImportError:
-            print(f"Installing {package_name}...")
-            try:
-                subprocess.check_call(
-                    [sys.executable, "-m", "pip", "install", package_name]
-                )
-                print(f"{package_name} installed successfully.")
-            except subprocess.CalledProcessError as e:
-                print(f"Failed to install {package_name}: {e}")
-                raise ImportError(
-                    f"Could not install {package_name}. "
-                    f"Please install it manually with 'pip install {package_name}'."
-                )
+        except ImportError as exc:
+            raise ImportError(
+                f"{package_name} is required for Spark support. "
+                "Install with: pip install datafog[distributed]"
+            ) from exc