django-cfg 1.3.11__py3-none-any.whl → 1.4.0__py3-none-any.whl

django_cfg/modules/django_llm/llm/stats/stats_manager.py

@@ -0,0 +1,107 @@
+"""
+Statistics manager for LLM client.
+
+Tracks requests, cache hits, tokens, costs, and usage patterns.
+"""
+
+import logging
+from typing import Dict, Any
+
+logger = logging.getLogger(__name__)
+
+
+class StatsManager:
+    """Manages LLM usage statistics."""
+
+    def __init__(self):
+        """Initialize statistics tracker."""
+        self.stats = {
+            'total_requests': 0,
+            'successful_requests': 0,
+            'failed_requests': 0,
+            'cache_hits': 0,
+            'cache_misses': 0,
+            'total_tokens_used': 0,
+            'total_cost_usd': 0.0,
+            'model_usage': {},
+            'provider_usage': {}
+        }
+
+    def record_request(self):
+        """Record a new request."""
+        self.stats['total_requests'] += 1
+
+    def record_cache_hit(self):
+        """Record cache hit."""
+        self.stats['cache_hits'] += 1
+
+    def record_cache_miss(self):
+        """Record cache miss."""
+        self.stats['cache_misses'] += 1
+
+    def record_success(self, tokens: int, cost: float, model: str, provider: str):
+        """
+        Record successful request.
+
+        Args:
+            tokens: Number of tokens used
+            cost: Cost in USD
+            model: Model used
+            provider: Provider used
+        """
+        self.stats['successful_requests'] += 1
+        self.stats['total_tokens_used'] += tokens
+        self.stats['total_cost_usd'] += cost
+        self.stats['model_usage'][model] = self.stats['model_usage'].get(model, 0) + 1
+        self.stats['provider_usage'][provider] = self.stats['provider_usage'].get(provider, 0) + 1
+
+    def record_failure(self):
+        """Record failed request."""
+        self.stats['failed_requests'] += 1
+
+    def get_stats(self) -> Dict[str, Any]:
+        """
+        Get copy of statistics.
+
+        Returns:
+            Dictionary with all statistics
+        """
+        return self.stats.copy()
+
+    def get_cache_hit_rate(self) -> float:
+        """
+        Calculate cache hit rate.
+
+        Returns:
+            Cache hit rate as percentage (0-100)
+        """
+        total_cache_requests = self.stats['cache_hits'] + self.stats['cache_misses']
+        if total_cache_requests == 0:
+            return 0.0
+        return (self.stats['cache_hits'] / total_cache_requests) * 100
+
+    def get_success_rate(self) -> float:
+        """
+        Calculate success rate.
+
+        Returns:
+            Success rate as percentage (0-100)
+        """
+        if self.stats['total_requests'] == 0:
+            return 0.0
+        return (self.stats['successful_requests'] / self.stats['total_requests']) * 100
+
+    def reset_stats(self):
+        """Reset all statistics to zero."""
+        self.stats = {
+            'total_requests': 0,
+            'successful_requests': 0,
+            'failed_requests': 0,
+            'cache_hits': 0,
+            'cache_misses': 0,
+            'total_tokens_used': 0,
+            'total_cost_usd': 0.0,
+            'model_usage': {},
+            'provider_usage': {}
+        }
+        logger.info("Statistics reset")

django_cfg/modules/django_llm/translator/detectors/__init__.py

@@ -0,0 +1,13 @@
+"""
+Language detection modules.
+
+Script-based and dictionary-based language detection.
+"""
+
+from .script_detector import ScriptDetector
+from .language_detector import LanguageDetector
+
+__all__ = [
+    'ScriptDetector',
+    'LanguageDetector',
+]

django_cfg/modules/django_llm/translator/detectors/language_detector.py

@@ -0,0 +1,90 @@
+"""
+Dictionary-based language detection.
+
+Detects language using common word dictionaries.
+"""
+
+from typing import Optional
+
+
+class LanguageDetector:
+    """Detect language using common word dictionaries."""
+
+    def __init__(self):
+        # Common words for each language
+        self.english_words = {
+            'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of',
+            'with', 'by', 'from', 'about', 'into', 'through', 'during', 'before',
+            'after', 'above', 'below', 'up', 'down', 'out', 'off', 'over', 'under',
+            'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where',
+            'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most',
+            'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same',
+            'so', 'than', 'too', 'very', 'can', 'will', 'just', 'should', 'now'
+        }
+
+        self.russian_words = {
+            'и', 'в', 'не', 'на', 'я', 'быть', 'он', 'с', 'что', 'а', 'по',
+            'это', 'она', 'этот', 'к', 'но', 'они', 'мы', 'как', 'из', 'у',
+            'который', 'то', 'за', 'свой', 'от', 'со', 'для', 'о', 'же', 'ты',
+            'все', 'если', 'люди', 'время', 'так', 'его', 'жизнь', 'может',
+            'год', 'только', 'над', 'еще', 'дом', 'после', 'большой', 'должен',
+            'хотеть', 'между'
+        }
+
+        self.spanish_words = {
+            'el', 'la', 'de', 'que', 'y', 'a', 'en', 'un', 'es', 'se', 'no',
+            'te', 'lo', 'le', 'da', 'su', 'por', 'son', 'con', 'para', 'al',
+            'del', 'los', 'las', 'una', 'como', 'pero', 'sus', 'ha', 'me', 'si',
+            'sin', 'sobre', 'este', 'ya', 'entre', 'cuando', 'todo', 'esta',
+            'ser', 'dos', 'también', 'fue', 'había', 'muy', 'hasta', 'desde',
+            'está'
+        }
+
+        self.portuguese_words = {
+            'o', 'a', 'de', 'que', 'e', 'do', 'da', 'em', 'um', 'para', 'é',
+            'com', 'não', 'uma', 'os', 'no', 'se', 'na', 'por', 'mais', 'as',
+            'dos', 'como', 'mas', 'foi', 'ao', 'ele', 'das', 'tem', 'à', 'seu',
+            'sua', 'ou', 'ser', 'quando', 'muito', 'há', 'nos', 'já', 'está',
+            'eu', 'também', 'só', 'pelo', 'pela', 'até', 'isso', 'ela', 'entre',
+            'era', 'depois', 'sem', 'mesmo', 'aos', 'ter', 'seus', 'suas',
+            'numa', 'pelos', 'pelas'
+        }
+
+    def detect_language(self, text: str) -> Optional[str]:
+        """
+        Detect language of text using common word dictionaries.
+
+        Args:
+            text: Text to analyze
+
+        Returns:
+            Language code or None if detection fails
+        """
+        if not text or len(text.strip()) < 3:
+            return None
+
+        text_lower = text.lower().strip()
+        words = set(text_lower.split())
+
+        # Count matches for each language
+        en_matches = len(words & self.english_words)
+        ru_matches = len(words & self.russian_words)
+        es_matches = len(words & self.spanish_words)
+        pt_matches = len(words & self.portuguese_words)
+
+        # Find the language with most matches
+        max_matches = max(en_matches, ru_matches, es_matches, pt_matches)
+
+        if max_matches == 0:
+            return 'en'  # Default to English if no matches
+
+        if en_matches == max_matches:
+            return 'en'
+        elif ru_matches == max_matches:
+            return 'ru'
+        elif es_matches == max_matches:
+            return 'es'
+        elif pt_matches == max_matches:
+            return 'pt'
+
+        return 'en'  # Default fallback

django_cfg/modules/django_llm/translator/detectors/script_detector.py

@@ -0,0 +1,153 @@
+"""
+Script-based language detection.
+
+Detects language based on character scripts (CJK, Cyrillic, etc.).
+"""
+
+import re
+from typing import Optional
+
+
+class ScriptDetector:
+    """Detect language based on character scripts."""
+
+    def __init__(self):
+        # CJK character ranges for detection
+        self.cjk_ranges = [
+            (0x4E00, 0x9FFF),   # CJK Unified Ideographs
+            (0x3400, 0x4DBF),   # CJK Extension A
+            (0x20000, 0x2A6DF), # CJK Extension B
+            (0x2A700, 0x2B73F), # CJK Extension C
+            (0x2B740, 0x2B81F), # CJK Extension D
+            (0x3040, 0x309F),   # Hiragana
+            (0x30A0, 0x30FF),   # Katakana
+            (0xAC00, 0xD7AF),   # Hangul Syllables
+        ]
+
+    def detect_language(self, text: str) -> str:
+        """
+        Detect language of text using character scripts.
+
+        Args:
+            text: Text to analyze
+
+        Returns:
+            Language code (ko, ja, zh, ru, or en)
+        """
+        if not text or not text.strip():
+            return 'unknown'
+
+        # Clean text for better detection
+        cleaned_text = self.clean_text(text)
+
+        if not cleaned_text:
+            return 'unknown'
+
+        # Check for CJK characters
+        if self.contains_cjk(text):
+            # Simple CJK detection
+            if self.contains_korean(text):
+                return 'ko'
+            elif self.contains_japanese(text):
+                return 'ja'
+            else:
+                return 'zh'  # Default to Chinese for other CJK
+
+        # Check for Cyrillic (Russian/Ukrainian/Belarusian)
+        if self.contains_cyrillic(text):
+            return 'ru'  # Default to Russian for Cyrillic
+
+        # Default to English for Latin script
+        return 'en'
+
+    def contains_cjk(self, text: str) -> bool:
+        """
+        Check if text contains CJK characters.
+
+        Args:
+            text: Text to check
+
+        Returns:
+            True if text contains CJK characters
+        """
+        for char in text:
+            char_code = ord(char)
+            for start, end in self.cjk_ranges:
+                if start <= char_code <= end:
+                    return True
+        return False
+
+    def contains_korean(self, text: str) -> bool:
+        """
+        Check if text contains Korean characters.
+
+        Args:
+            text: Text to check
+
+        Returns:
+            True if text contains Hangul syllables
+        """
+        for char in text:
+            char_code = ord(char)
+            if 0xAC00 <= char_code <= 0xD7AF:  # Hangul Syllables
+                return True
+        return False
+
+    def contains_japanese(self, text: str) -> bool:
+        """
+        Check if text contains Japanese characters.
+
+        Args:
+            text: Text to check
+
+        Returns:
+            True if text contains Hiragana or Katakana
+        """
+        for char in text:
+            char_code = ord(char)
+            if (0x3040 <= char_code <= 0x309F or  # Hiragana
+                0x30A0 <= char_code <= 0x30FF):   # Katakana
+                return True
+        return False
+
+    def contains_cyrillic(self, text: str) -> bool:
+        """
+        Check if text contains Cyrillic characters.
+
+        Args:
+            text: Text to check
+
+        Returns:
+            True if text contains Cyrillic characters
+        """
+        for char in text:
+            char_code = ord(char)
+            if 0x0400 <= char_code <= 0x04FF:  # Cyrillic
+                return True
+        return False
+
+    @staticmethod
+    def clean_text(text: str) -> str:
+        """
+        Clean text for better language detection.
+
+        Removes URLs, excessive whitespace, and numbers.
+
+        Args:
+            text: Text to clean
+
+        Returns:
+            Cleaned text
+        """
+        if not text:
+            return ""
+
+        # Remove excessive whitespace
+        text = ' '.join(text.split())
+
+        # Remove URLs and technical terms
+        text = re.sub(r'https?://\S+', '', text)
+        text = re.sub(r'www\.\S+', '', text)
+        text = re.sub(r'\b\d+\b', '', text)  # Remove standalone numbers
+
+        return text.strip()

django_cfg/modules/django_llm/translator/stats/__init__.py

@@ -0,0 +1,11 @@
+"""
+Statistics tracking for translation.
+
+Tracks usage, cache performance, and costs.
+"""
+
+from .stats_tracker import StatsTracker
+
+__all__ = [
+    'StatsTracker',
+]

django_cfg/modules/django_llm/translator/stats/stats_tracker.py

@@ -0,0 +1,85 @@
+"""
+Translation statistics tracking.
+
+Tracks translation usage, cache performance, and costs.
+"""
+
+from typing import Dict, Any
+
+
+class StatsTracker:
+    """Track translation statistics."""
+
+    def __init__(self):
+        self.stats = {
+            'total_translations': 0,
+            'cache_hits': 0,
+            'cache_misses': 0,
+            'total_tokens_used': 0,
+            'total_cost_usd': 0.0,
+            'language_pairs': {},
+            'successful_translations': 0,
+            'failed_translations': 0
+        }
+
+    def record_translation(
+        self,
+        source_language: str,
+        target_language: str,
+        response: Dict[str, Any]
+    ):
+        """
+        Record successful translation.
+
+        Args:
+            source_language: Source language code
+            target_language: Target language code
+            response: LLM response with tokens_used and cost_usd
+        """
+        self.stats['total_translations'] += 1
+        self.stats['successful_translations'] += 1
+
+        # Track tokens and cost
+        if response.get('tokens_used'):
+            self.stats['total_tokens_used'] += response['tokens_used']
+        if response.get('cost_usd'):
+            self.stats['total_cost_usd'] += response['cost_usd']
+
+        # Track language pairs
+        lang_pair = f"{source_language}-{target_language}"
+        self.stats['language_pairs'][lang_pair] = \
+            self.stats['language_pairs'].get(lang_pair, 0) + 1
+
+    def record_cache_hit(self):
+        """Record cache hit."""
+        self.stats['cache_hits'] += 1
+
+    def record_cache_miss(self):
+        """Record cache miss."""
+        self.stats['cache_misses'] += 1
+
+    def record_failure(self):
+        """Record translation failure."""
+        self.stats['failed_translations'] += 1
+
+    def get_stats(self) -> Dict[str, Any]:
+        """
+        Get translation statistics.
+
+        Returns:
+            Dictionary with statistics
+        """
+        return self.stats.copy()
+
+    def reset_stats(self):
+        """Reset all statistics to zero."""
+        self.stats = {
+            'total_translations': 0,
+            'cache_hits': 0,
+            'cache_misses': 0,
+            'total_tokens_used': 0,
+            'total_cost_usd': 0.0,
+            'language_pairs': {},
+            'successful_translations': 0,
+            'failed_translations': 0
+        }