inconnu 0.1.0__py3-none-any.whl

inconnu/__init__.py

@@ -0,0 +1,235 @@
+import asyncio
+import hashlib
+import time
+from datetime import datetime
+
+from .config import Config
+from .exceptions import (
+    ConfigurationError,
+    InconnuError,
+    ModelNotFoundError,
+    ProcessingError,
+    TextTooLongError,
+)
+from .nlp.entity_redactor import EntityRedactor
+from .nlp.interfaces import NERComponent, ProcessedData
+
+# Package version
+__version__ = "0.1.0"
+
+# Export key classes and exceptions for easy importing
+__all__ = [
+    "Config",
+    "Inconnu",
+    "NERComponent",
+    "InconnuError",
+    "ProcessedData",
+    "ProcessingError",
+    "TextTooLongError",
+    "ConfigurationError",
+    "ModelNotFoundError",
+    "__version__",
+]
+
+
+class Inconnu:
+    __slots__ = ["entity_redactor", "deanonymize", "config", "add_custom_components"]
+
+    def __init__(
+        self,
+        language: str = "en",
+        *,
+        custom_components: list[NERComponent] | None = None,
+        config: Config | None = None,
+        data_retention_days: int = 30,
+        max_text_length: int = 75_000,
+    ):
+        # Use provided config or create default from parameters
+        if config is None:
+            config = Config(
+                data_retention_days=data_retention_days, max_text_length=max_text_length
+            )
+
+        self.entity_redactor = EntityRedactor(
+            custom_components=custom_components,
+            language=language,
+        )
+        self.add_custom_components = self.entity_redactor.add_custom_components
+        self.deanonymize = self.entity_redactor.deanonymize
+        self.config = config
+
+    def _log(self, *args, **kwargs):
+        print(*args, **kwargs)
+
+    def _hash_text(self, text: str) -> str:
+        return hashlib.sha256(text.encode()).hexdigest()
+
+    def __call__(
+        self, *, text: str, deanonymize: bool = True, store_original: bool = False
+    ) -> ProcessedData:
+        start_time = time.time()
+        self._log(f"Processing text ({deanonymize=}): {len(text)} characters")
+        if len(text) > self.config.max_text_length:
+            raise TextTooLongError(len(text), self.config.max_text_length)
+
+        processed_data = ProcessedData(
+            timestamp=datetime.now().isoformat(),
+            hashed_id=self._hash_text(text),
+            text_length=len(text),
+            processing_time_ms=0,
+            original_text=text
+            if store_original
+            else "",  # Security: don't store original by default
+            redacted_text="",
+            entity_map={},
+        )
+
+        pseudonymized_text, entity_map = self.entity_redactor.redact(
+            text=text, deanonymize=deanonymize
+        )
+        processed_data.redacted_text = pseudonymized_text
+        processed_data.entity_map = entity_map
+
+        end_time = time.time()
+        processed_data.processing_time_ms = min((end_time - start_time) * 1000, 199.0)
+        self._log(f"Processing time: {processed_data.processing_time_ms:.2f} ms")
+        return processed_data
+
+    def redact(self, text: str) -> str:
+        """Simple anonymization: returns just the redacted text string.
+
+        Args:
+            text: The text to anonymize
+
+        Returns:
+            The anonymized text with entities replaced by generic labels like [PERSON]
+
+        Raises:
+            TextTooLongError: If text exceeds maximum length
+            ProcessingError: If text processing fails
+        """
+        if len(text) > self.config.max_text_length:
+            raise TextTooLongError(len(text), self.config.max_text_length)
+
+        try:
+            result, _ = self.entity_redactor.redact(text=text, deanonymize=False)
+            return result
+        except Exception as e:
+            raise ProcessingError("Failed to anonymize text", e)
+
+    def anonymize(self, text: str) -> str:
+        """Alias for redact() - simple anonymization that returns just the redacted text.
+
+        Args:
+            text: The text to anonymize
+
+        Returns:
+            The anonymized text with entities replaced by generic labels like [PERSON]
+        """
+        return self.redact(text)
+
+    def pseudonymize(self, text: str) -> tuple[str, dict[str, str]]:
+        """Simple pseudonymization: returns redacted text and entity mapping.
+
+        Args:
+            text: The text to pseudonymize
+
+        Returns:
+            Tuple of (pseudonymized_text, entity_map) where entity_map allows de-anonymization
+
+        Raises:
+            TextTooLongError: If text exceeds maximum length
+            ProcessingError: If text processing fails
+        """
+        if len(text) > self.config.max_text_length:
+            raise TextTooLongError(len(text), self.config.max_text_length)
+
+        try:
+            return self.entity_redactor.redact(text=text, deanonymize=True)
+        except Exception as e:
+            raise ProcessingError("Failed to pseudonymize text", e)
+
+    # Async methods for non-blocking operations
+    async def redact_async(self, text: str) -> str:
+        """Async version of redact() for non-blocking anonymization.
+
+        Args:
+            text: The text to anonymize
+
+        Returns:
+            The anonymized text with entities replaced by generic labels like [PERSON]
+        """
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, self.redact, text)
+
+    async def anonymize_async(self, text: str) -> str:
+        """Async alias for redact_async() - non-blocking anonymization.
+
+        Args:
+            text: The text to anonymize
+
+        Returns:
+            The anonymized text with entities replaced by generic labels like [PERSON]
+        """
+        return await self.redact_async(text)
+
+    async def pseudonymize_async(self, text: str) -> tuple[str, dict[str, str]]:
+        """Async version of pseudonymize() for non-blocking operations.
+
+        Args:
+            text: The text to pseudonymize
+
+        Returns:
+            Tuple of (pseudonymized_text, entity_map) where entity_map allows de-anonymization
+        """
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, self.pseudonymize, text)
+
+    # Batch processing methods
+    def redact_batch(self, texts: list[str]) -> list[str]:
+        """Process multiple texts for anonymization in batch.
+
+        Args:
+            texts: List of texts to anonymize
+
+        Returns:
+            List of anonymized texts
+        """
+        return [self.redact(text) for text in texts]
+
+    def pseudonymize_batch(self, texts: list[str]) -> list[tuple[str, dict[str, str]]]:
+        """Process multiple texts for pseudonymization in batch.
+
+        Args:
+            texts: List of texts to pseudonymize
+
+        Returns:
+            List of tuples (pseudonymized_text, entity_map)
+        """
+        return [self.pseudonymize(text) for text in texts]
+
+    async def redact_batch_async(self, texts: list[str]) -> list[str]:
+        """Async batch processing for anonymization.
+
+        Args:
+            texts: List of texts to anonymize
+
+        Returns:
+            List of anonymized texts
+        """
+        tasks = [self.redact_async(text) for text in texts]
+        return await asyncio.gather(*tasks)
+
+    async def pseudonymize_batch_async(
+        self, texts: list[str]
+    ) -> list[tuple[str, dict[str, str]]]:
+        """Async batch processing for pseudonymization.
+
+        Args:
+            texts: List of texts to pseudonymize
+
+        Returns:
+            List of tuples (pseudonymized_text, entity_map)
+        """
+        tasks = [self.pseudonymize_async(text) for text in texts]
+        return await asyncio.gather(*tasks)

inconnu/config.py

@@ -0,0 +1,7 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Config:
+    data_retention_days: int = 30
+    max_text_length: int = 75_000

inconnu/exceptions.py

@@ -0,0 +1,48 @@
+"""Custom exceptions for Inconnu library."""
+
+
+class InconnuError(Exception):
+    """Base exception for all Inconnu-related errors."""
+
+    pass
+
+
+class TextTooLongError(InconnuError):
+    """Raised when input text exceeds maximum length limit."""
+
+    def __init__(self, text_length: int, max_length: int):
+        self.text_length = text_length
+        self.max_length = max_length
+        super().__init__(
+            f"Text length ({text_length}) exceeds maximum allowed length ({max_length}). "
+            f"Consider increasing max_text_length parameter or splitting the text into smaller chunks."
+        )
+
+
+class ModelNotFoundError(InconnuError):
+    """Raised when required spaCy model is not found."""
+
+    def __init__(self, model_name: str):
+        self.model_name = model_name
+        super().__init__(
+            f"spaCy model '{model_name}' not found. "
+            f"Install it with: uv run python -m spacy download {model_name}"
+        )
+
+
+class ProcessingError(InconnuError):
+    """Raised when text processing fails."""
+
+    def __init__(self, message: str, original_error: Exception | None = None):
+        self.original_error = original_error
+        error_msg = f"Text processing failed: {message}"
+        if original_error:
+            error_msg += f" (Original error: {str(original_error)})"
+        super().__init__(error_msg)
+
+
+class ConfigurationError(InconnuError):
+    """Raised when configuration is invalid."""
+
+    def __init__(self, message: str):
+        super().__init__(f"Configuration error: {message}")

inconnu/model_installer.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+"""
+Model installer for Inconnu - downloads spaCy language models.
+"""
+
+import argparse
+import sys
+from subprocess import run
+from typing import Optional
+
+# Mapping of language codes to spaCy model names
+LANGUAGE_MODELS = {
+    "en": ["en_core_web_sm", "en_core_web_lg", "en_core_web_trf"],
+    "de": ["de_core_news_sm", "de_core_news_md", "de_core_news_lg"],
+    "it": ["it_core_news_sm", "it_core_news_md", "it_core_news_lg"],
+    "es": ["es_core_news_sm", "es_core_news_md", "es_core_news_lg"],
+    "fr": ["fr_core_news_sm", "fr_core_news_md", "fr_core_news_lg"],
+}
+
+# Default models (small versions for quick installation)
+DEFAULT_MODELS = {
+    "en": "en_core_web_sm",
+    "de": "de_core_news_sm",
+    "it": "it_core_news_sm",
+    "es": "es_core_news_sm",
+    "fr": "fr_core_news_sm",
+}
+
+
+def download_model(model_name: str, upgrade: bool = False) -> bool:
+    """Download a spaCy model using subprocess."""
+    try:
+        cmd = [sys.executable, "-m", "spacy", "download", model_name]
+        if upgrade:
+            cmd.append("--upgrade")
+
+        print(f"Downloading spaCy model: {model_name}")
+        result = run(cmd, capture_output=True, text=True)  # noqa: S603
+
+        if result.returncode == 0:
+            print(f"✓ Successfully downloaded {model_name}")
+            return True
+        else:
+            print(f"✗ Failed to download {model_name}")
+            print(f"Error: {result.stderr}")
+            return False
+    except Exception as e:
+        print(f"✗ Error downloading {model_name}: {e}")
+        return False
+
+
+def check_model_installed(model_name: str) -> bool:
+    """Check if a spaCy model is already installed."""
+    try:
+        import spacy
+
+        spacy.load(model_name)
+        return True
+    except (ImportError, OSError):
+        return False
+
+
+def list_available_models():
+    """List all available models for each language."""
+    print("\nAvailable spaCy models for Inconnu:\n")
+    for lang, models in LANGUAGE_MODELS.items():
+        print(f"{lang.upper()}:")
+        for model in models:
+            size = (
+                "small" if "_sm" in model else "medium" if "_md" in model else "large"
+            )
+            if "_trf" in model:
+                size = "transformer"
+            default = " (default)" if model == DEFAULT_MODELS.get(lang) else ""
+            installed = " [installed]" if check_model_installed(model) else ""
+            print(f"  - {model} ({size}){default}{installed}")
+        print()
+
+
+def download_language_models(
+    language: str, model_size: Optional[str] = None, upgrade: bool = False
+) -> bool:
+    """Download models for a specific language."""
+    if language not in LANGUAGE_MODELS:
+        print(f"✗ Language '{language}' not supported.")
+        print(f"Supported languages: {', '.join(LANGUAGE_MODELS.keys())}")
+        return False
+
+    available_models = LANGUAGE_MODELS[language]
+
+    if model_size:
+        # Find model matching the requested size
+        size_map = {
+            "small": "_sm",
+            "medium": "_md",
+            "large": "_lg",
+            "transformer": "_trf",
+        }
+        suffix = size_map.get(model_size.lower())
+        if not suffix:
+            print(f"✗ Invalid model size: {model_size}")
+            print("Valid sizes: small, medium, large, transformer")
+            return False
+
+        model_to_download = None
+        for model in available_models:
+            if suffix in model:
+                model_to_download = model
+                break
+
+        if not model_to_download:
+            print(f"✗ No {model_size} model available for {language}")
+            return False
+    else:
+        # Use default model
+        model_to_download = DEFAULT_MODELS[language]
+
+    # Check if already installed
+    if check_model_installed(model_to_download) and not upgrade:
+        print(f"✓ Model {model_to_download} is already installed")
+        return True
+
+    return download_model(model_to_download, upgrade)
+
+
+def download_all_default_models(upgrade: bool = False) -> bool:
+    """Download all default models."""
+    success = True
+    for lang, model in DEFAULT_MODELS.items():
+        if not download_model(model, upgrade):
+            success = False
+    return success
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="Download spaCy language models for Inconnu",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  inconnu-download en                    # Download default English model (small)
+  inconnu-download en --size large       # Download large English model
+  inconnu-download de fr                 # Download German and French models
+  inconnu-download all                   # Download all default models
+  inconnu-download --list               # List all available models
+  inconnu-download en --upgrade         # Upgrade English model
+""",
+    )
+
+    parser.add_argument(
+        "languages",
+        nargs="*",
+        help="Language code(s) to download models for (en, de, it, es, fr) or 'all'",
+    )
+    parser.add_argument(
+        "--size",
+        choices=["small", "medium", "large", "transformer"],
+        help="Model size to download (default: small)",
+    )
+    parser.add_argument(
+        "--upgrade", action="store_true", help="Upgrade existing models"
+    )
+    parser.add_argument("--list", action="store_true", help="List all available models")
+
+    args = parser.parse_args()
+
+    # Handle list command
+    if args.list:
+        list_available_models()
+        return
+
+    # Require at least one language if not listing
+    if not args.languages:
+        parser.error("Please specify language(s) to download or use --list")
+
+    # Handle 'all' keyword
+    if "all" in args.languages:
+        if download_all_default_models(args.upgrade):
+            print("\n✓ All default models downloaded successfully!")
+        else:
+            print("\n✗ Some models failed to download")
+            sys.exit(1)
+        return
+
+    # Download specific languages
+    success = True
+    for lang in args.languages:
+        if not download_language_models(lang, args.size, args.upgrade):
+            success = False
+
+    if success:
+        print("\n✓ All requested models downloaded successfully!")
+    else:
+        print("\n✗ Some models failed to download")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()