cognee 0.5.1.dev0__py3-none-any.whl → 0.5.2__py3-none-any.whl

cognee/tasks/translation/__init__.py

@@ -0,0 +1,96 @@
+"""
+Translation task for Cognee.
+
+This module provides multilingual content translation capabilities,
+allowing automatic detection and translation of non-English content
+to a target language while preserving original text and metadata.
+
+Main Components:
+- translate_content: Main task function for translating document chunks
+- translate_text: Convenience function for translating single texts
+- batch_translate_texts: Batch translation for multiple texts
+- detect_language: Language detection utility
+- TranslatedContent: DataPoint model for translated content
+- LanguageMetadata: DataPoint model for language information
+
+Supported Translation Providers:
+- LLM (default): Uses the configured LLM via existing infrastructure
+- Google Translate: Requires google-cloud-translate package
+- Azure Translator: Requires Azure Translator API key
+
+Example Usage:
+    ```python
+    from cognee.tasks.translation import translate_content, translate_text
+
+    # Translate document chunks in a pipeline
+    translated_chunks = await translate_content(
+        chunks,
+        target_language="en",
+        translation_provider="llm"
+    )
+
+    # Translate a single text
+    result = await translate_text("Bonjour le monde!")
+    print(result.translated_text)  # "Hello world!"
+    ```
+"""
+
+from .config import get_translation_config, TranslationConfig
+from .detect_language import (
+    detect_language,
+    detect_language_async,
+    LanguageDetectionResult,
+    get_language_name,
+)
+from .exceptions import (
+    TranslationError,
+    LanguageDetectionError,
+    TranslationProviderError,
+    UnsupportedLanguageError,
+    TranslationConfigError,
+)
+from .models import TranslatedContent, LanguageMetadata
+from .providers import (
+    TranslationProvider,
+    TranslationResult,
+    get_translation_provider,
+    LLMTranslationProvider,
+    GoogleTranslationProvider,
+    AzureTranslationProvider,
+)
+from .translate_content import (
+    translate_content,
+    translate_text,
+    batch_translate_texts,
+)
+
+__all__ = [
+    # Main task functions
+    "translate_content",
+    "translate_text",
+    "batch_translate_texts",
+    # Language detection
+    "detect_language",
+    "detect_language_async",
+    "LanguageDetectionResult",
+    "get_language_name",
+    # Models
+    "TranslatedContent",
+    "LanguageMetadata",
+    # Configuration
+    "get_translation_config",
+    "TranslationConfig",
+    # Providers
+    "TranslationProvider",
+    "TranslationResult",
+    "get_translation_provider",
+    "LLMTranslationProvider",
+    "GoogleTranslationProvider",
+    "AzureTranslationProvider",
+    # Exceptions
+    "TranslationError",
+    "LanguageDetectionError",
+    "TranslationProviderError",
+    "UnsupportedLanguageError",
+    "TranslationConfigError",
+]

cognee/tasks/translation/config.py

@@ -0,0 +1,110 @@
+from functools import lru_cache
+from typing import Literal, Optional
+
+from pydantic import AliasChoices, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+TranslationProviderType = Literal["llm", "google", "azure"]
+
+
+class TranslationConfig(BaseSettings):
+    """
+    Configuration settings for the translation task.
+
+    Environment variables can be used to configure these settings:
+    - TRANSLATION_PROVIDER: The translation service to use ("llm", "google", "azure")
+    - TARGET_LANGUAGE: Default target language (ISO 639-1 code, e.g., "en", "es", "fr")
+    - CONFIDENCE_THRESHOLD: Minimum confidence for language detection (0.0 to 1.0)
+    - GOOGLE_TRANSLATE_API_KEY: API key for Google Translate
+    - GOOGLE_PROJECT_ID: Google Cloud project ID
+    - AZURE_TRANSLATOR_KEY: API key for Azure Translator
+    - AZURE_TRANSLATOR_REGION: Region for Azure Translator
+    - AZURE_TRANSLATOR_ENDPOINT: Endpoint URL for Azure Translator
+    - TRANSLATION_BATCH_SIZE: Number of texts to translate per batch
+    - TRANSLATION_MAX_RETRIES: Maximum retry attempts on failure
+    - TRANSLATION_TIMEOUT_SECONDS: Request timeout in seconds
+    """
+
+    # Translation provider settings
+    translation_provider: TranslationProviderType = Field(
+        default="llm",
+        validation_alias=AliasChoices("TRANSLATION_PROVIDER", "translation_provider"),
+    )
+    target_language: str = Field(
+        default="en",
+        validation_alias=AliasChoices("TARGET_LANGUAGE", "target_language"),
+    )
+    confidence_threshold: float = Field(
+        default=0.8,
+        ge=0.0,
+        le=1.0,
+        validation_alias=AliasChoices("CONFIDENCE_THRESHOLD", "confidence_threshold"),
+    )
+
+    # Google Translate settings
+    google_translate_api_key: Optional[str] = Field(
+        default=None,
+        validation_alias=AliasChoices("GOOGLE_TRANSLATE_API_KEY", "google_translate_api_key"),
+    )
+    google_project_id: Optional[str] = Field(
+        default=None,
+        validation_alias=AliasChoices("GOOGLE_PROJECT_ID", "google_project_id"),
+    )
+
+    # Azure Translator settings
+    azure_translator_key: Optional[str] = Field(
+        default=None,
+        validation_alias=AliasChoices("AZURE_TRANSLATOR_KEY", "azure_translator_key"),
+    )
+    azure_translator_region: Optional[str] = Field(
+        default=None,
+        validation_alias=AliasChoices("AZURE_TRANSLATOR_REGION", "azure_translator_region"),
+    )
+    azure_translator_endpoint: str = Field(
+        default="https://api.cognitive.microsofttranslator.com",
+        validation_alias=AliasChoices("AZURE_TRANSLATOR_ENDPOINT", "azure_translator_endpoint"),
+    )
+
+    # LLM provider uses the existing LLM configuration
+
+    # Performance settings (with TRANSLATION_ prefix for env vars)
+    batch_size: int = Field(
+        default=10,
+        validation_alias=AliasChoices("TRANSLATION_BATCH_SIZE", "batch_size"),
+    )
+    max_retries: int = Field(
+        default=3,
+        validation_alias=AliasChoices("TRANSLATION_MAX_RETRIES", "max_retries"),
+    )
+    timeout_seconds: int = Field(
+        default=30,
+        validation_alias=AliasChoices("TRANSLATION_TIMEOUT_SECONDS", "timeout_seconds"),
+    )
+
+    # Language detection settings
+    min_text_length_for_detection: int = 10
+    skip_detection_for_short_text: bool = True
+
+    model_config = SettingsConfigDict(env_file=".env", extra="allow")
+
+    def to_dict(self) -> dict:
+        return {
+            "translation_provider": self.translation_provider,
+            "target_language": self.target_language,
+            "confidence_threshold": self.confidence_threshold,
+            "batch_size": self.batch_size,
+            "max_retries": self.max_retries,
+            "timeout_seconds": self.timeout_seconds,
+        }
+
+
+@lru_cache()
+def get_translation_config() -> TranslationConfig:
+    """Get the translation configuration singleton."""
+    return TranslationConfig()
+
+
+def clear_translation_config_cache():
+    """Clear the cached config for testing purposes."""
+    get_translation_config.cache_clear()

cognee/tasks/translation/detect_language.py

@@ -0,0 +1,190 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from cognee.shared.logging_utils import get_logger
+
+from .config import get_translation_config
+from .exceptions import LanguageDetectionError
+
+logger = get_logger(__name__)
+
+
+# ISO 639-1 language code to name mapping
+LANGUAGE_NAMES = {
+    "af": "Afrikaans",
+    "ar": "Arabic",
+    "bg": "Bulgarian",
+    "bn": "Bengali",
+    "ca": "Catalan",
+    "cs": "Czech",
+    "cy": "Welsh",
+    "da": "Danish",
+    "de": "German",
+    "el": "Greek",
+    "en": "English",
+    "es": "Spanish",
+    "et": "Estonian",
+    "fa": "Persian",
+    "fi": "Finnish",
+    "fr": "French",
+    "gu": "Gujarati",
+    "he": "Hebrew",
+    "hi": "Hindi",
+    "hr": "Croatian",
+    "hu": "Hungarian",
+    "id": "Indonesian",
+    "it": "Italian",
+    "ja": "Japanese",
+    "kn": "Kannada",
+    "ko": "Korean",
+    "lt": "Lithuanian",
+    "lv": "Latvian",
+    "mk": "Macedonian",
+    "ml": "Malayalam",
+    "mr": "Marathi",
+    "ne": "Nepali",
+    "nl": "Dutch",
+    "no": "Norwegian",
+    "pa": "Punjabi",
+    "pl": "Polish",
+    "pt": "Portuguese",
+    "ro": "Romanian",
+    "ru": "Russian",
+    "sk": "Slovak",
+    "sl": "Slovenian",
+    "so": "Somali",
+    "sq": "Albanian",
+    "sv": "Swedish",
+    "sw": "Swahili",
+    "ta": "Tamil",
+    "te": "Telugu",
+    "th": "Thai",
+    "tl": "Tagalog",
+    "tr": "Turkish",
+    "uk": "Ukrainian",
+    "ur": "Urdu",
+    "vi": "Vietnamese",
+    "zh-cn": "Chinese (Simplified)",
+    "zh-tw": "Chinese (Traditional)",
+}
+
+
+@dataclass
+class LanguageDetectionResult:
+    """Result of language detection."""
+
+    language_code: str
+    language_name: str
+    confidence: float
+    requires_translation: bool
+    character_count: int
+
+
+def get_language_name(language_code: str) -> str:
+    """Get the human-readable name for a language code."""
+    return LANGUAGE_NAMES.get(language_code.lower(), language_code)
+
+
+def detect_language(
+    text: str,
+    target_language: str = "en",
+    confidence_threshold: Optional[float] = None,
+) -> LanguageDetectionResult:
+    """
+    Detect the language of the given text.
+
+    Uses the langdetect library which is already a dependency of cognee.
+
+    Args:
+        text: The text to analyze
+        target_language: The target language for translation comparison
+        confidence_threshold: Minimum confidence to consider detection reliable
+
+    Returns:
+        LanguageDetectionResult with language info and translation requirement
+
+    Raises:
+        LanguageDetectionError: If language detection fails
+    """
+    config = get_translation_config()
+    threshold = confidence_threshold or config.confidence_threshold
+
+    # Handle empty or very short text
+    if not text or len(text.strip()) < config.min_text_length_for_detection:
+        if config.skip_detection_for_short_text:
+            return LanguageDetectionResult(
+                language_code="unknown",
+                language_name="Unknown",
+                confidence=0.0,
+                requires_translation=False,
+                character_count=len(text) if text else 0,
+            )
+        else:
+            raise LanguageDetectionError(
+                f"Text too short for reliable language detection: {len(text)} characters"
+            )
+
+    try:
+        from langdetect import detect_langs, LangDetectException
+    except ImportError:
+        raise LanguageDetectionError(
+            "langdetect is required for language detection. Install it with: pip install langdetect"
+        )
+
+    try:
+        # Get detection results with probabilities
+        detections = detect_langs(text)
+
+        if not detections:
+            raise LanguageDetectionError("No language detected")
+
+        # Get the most likely language
+        best_detection = detections[0]
+        language_code = best_detection.lang
+        confidence = best_detection.prob
+
+        # Check if translation is needed
+        requires_translation = (
+            language_code.lower() != target_language.lower() and confidence >= threshold
+        )
+
+        return LanguageDetectionResult(
+            language_code=language_code,
+            language_name=get_language_name(language_code),
+            confidence=confidence,
+            requires_translation=requires_translation,
+            character_count=len(text),
+        )
+
+    except LangDetectException as e:
+        logger.warning(f"Language detection failed: {e}")
+        raise LanguageDetectionError(f"Language detection failed: {e}", original_error=e)
+    except Exception as e:
+        logger.error(f"Unexpected error during language detection: {e}")
+        raise LanguageDetectionError(
+            f"Unexpected error during language detection: {e}", original_error=e
+        )
+
+
+async def detect_language_async(
+    text: str,
+    target_language: str = "en",
+    confidence_threshold: Optional[float] = None,
+) -> LanguageDetectionResult:
+    """
+    Async wrapper for language detection.
+
+    Args:
+        text: The text to analyze
+        target_language: The target language for translation comparison
+        confidence_threshold: Minimum confidence to consider detection reliable
+
+    Returns:
+        LanguageDetectionResult with language info and translation requirement
+    """
+    import asyncio
+
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(
+        None, detect_language, text, target_language, confidence_threshold
+    )

cognee/tasks/translation/exceptions.py

@@ -0,0 +1,62 @@
+class TranslationError(Exception):
+    """Base exception for translation errors."""
+
+    def __init__(self, message: str, original_error: Exception = None):
+        self.message = message
+        self.original_error = original_error
+        super().__init__(self.message)
+        if original_error:
+            self.__cause__ = original_error
+
+
+class LanguageDetectionError(TranslationError):
+    """Exception raised when language detection fails."""
+
+    def __init__(
+        self, message: str = "Failed to detect language", original_error: Exception = None
+    ):
+        super().__init__(message, original_error)
+
+
+class TranslationProviderError(TranslationError):
+    """Exception raised when the translation provider encounters an error."""
+
+    def __init__(
+        self,
+        provider: str,
+        message: str = "Translation provider error",
+        original_error: Exception = None,
+    ):
+        self.provider = provider
+        full_message = f"[{provider}] {message}"
+        super().__init__(full_message, original_error)
+
+
+class UnsupportedLanguageError(TranslationError):
+    """Exception raised when the language is not supported."""
+
+    def __init__(
+        self,
+        language: str,
+        provider: str = None,
+        message: str = None,
+        original_error: Exception = None,
+    ):
+        self.language = language
+        self.provider = provider
+        if message is None:
+            message = f"Language '{language}' is not supported"
+            if provider:
+                message += f" by {provider}"
+        super().__init__(message, original_error)
+
+
+class TranslationConfigError(TranslationError):
+    """Exception raised when translation configuration is invalid."""
+
+    def __init__(
+        self,
+        message: str = "Invalid translation configuration",
+        original_error: Exception = None,
+    ):
+        super().__init__(message, original_error)

cognee/tasks/translation/models.py

@@ -0,0 +1,72 @@
+from datetime import datetime, timezone
+from typing import Optional
+from uuid import UUID
+
+from cognee.infrastructure.engine import DataPoint
+from cognee.modules.chunking.models import DocumentChunk
+
+
+class TranslatedContent(DataPoint):
+    """
+    Represents translated content with quality metrics.
+
+    This class stores both the original and translated versions of content,
+    along with metadata about the translation process including source and
+    target languages, translation provider used, and confidence scores.
+
+    Instance variables include:
+
+    - original_chunk_id: UUID of the original document chunk
+    - original_text: The original text before translation
+    - translated_text: The translated text content
+    - source_language: Detected or specified source language code (e.g., "es", "fr", "de")
+    - target_language: Target language code for translation (default: "en")
+    - translation_provider: Name of the translation service used
+    - confidence_score: Translation quality/confidence score (0.0 to 1.0)
+    - translation_timestamp: When the translation was performed
+    - translated_from: Reference to the original DocumentChunk
+    """
+
+    original_chunk_id: UUID
+    original_text: str
+    translated_text: str
+    source_language: str
+    target_language: str = "en"
+    translation_provider: str
+    confidence_score: float
+    translation_timestamp: datetime = None
+    translated_from: Optional[DocumentChunk] = None
+
+    metadata: dict = {"index_fields": ["source_language", "translated_text"]}
+
+    def __init__(self, **data):
+        if data.get("translation_timestamp") is None:
+            data["translation_timestamp"] = datetime.now(timezone.utc)
+        super().__init__(**data)
+
+
+class LanguageMetadata(DataPoint):
+    """
+    Language information for content.
+
+    This class stores metadata about the detected language of content,
+    including confidence scores and whether translation is required.
+
+    Instance variables include:
+
+    - content_id: UUID of the associated content
+    - detected_language: ISO 639-1 language code (e.g., "en", "es", "fr")
+    - language_confidence: Confidence score for language detection (0.0 to 1.0)
+    - requires_translation: Whether the content needs translation
+    - character_count: Number of characters in the content
+    - language_name: Human-readable language name (e.g., "English", "Spanish")
+    """
+
+    content_id: UUID
+    detected_language: str
+    language_confidence: float
+    requires_translation: bool
+    character_count: int
+    language_name: Optional[str] = None
+
+    metadata: dict = {"index_fields": ["detected_language"]}

cognee/tasks/translation/providers/__init__.py

@@ -0,0 +1,44 @@
+from .base import TranslationProvider, TranslationResult
+from .llm_provider import LLMTranslationProvider
+from .google_provider import GoogleTranslationProvider
+from .azure_provider import AzureTranslationProvider
+
+__all__ = [
+    "TranslationProvider",
+    "TranslationResult",
+    "LLMTranslationProvider",
+    "GoogleTranslationProvider",
+    "AzureTranslationProvider",
+    "get_translation_provider",
+]
+
+
+def get_translation_provider(provider_name: str) -> TranslationProvider:
+    """
+    Factory function to get the appropriate translation provider.
+
+    Args:
+        provider_name: Name of the provider:
+            - "llm": Uses the configured LLM (OpenAI, Azure, Ollama, Anthropic, etc.)
+            - "google": Uses Google Cloud Translation API
+            - "azure": Uses Azure Translator API
+
+    Returns:
+        TranslationProvider instance
+
+    Raises:
+        ValueError: If the provider name is not recognized
+    """
+    providers = {
+        "llm": LLMTranslationProvider,
+        "google": GoogleTranslationProvider,
+        "azure": AzureTranslationProvider,
+    }
+
+    if provider_name.lower() not in providers:
+        raise ValueError(
+            f"Unknown translation provider: {provider_name}. "
+            f"Available providers: {list(providers.keys())}"
+        )
+
+    return providers[provider_name.lower()]()