tritopic 0.1.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

tritopic/multilingual/__init__.py

@@ -0,0 +1,38 @@
+"""
+TriTopic Multilingual Module
+
+Provides language detection, tokenization, and stopwords for 60+ languages.
+"""
+
+from .detection import (
+    detect_language,
+    detect_corpus_language,
+    normalize_language_code,
+    LANGUAGE_ALIASES,
+)
+
+from .tokenizers import (
+    TokenizerFactory,
+    tokenize_documents,
+)
+
+from .stopwords import (
+    get_stopwords,
+    get_available_languages,
+    STOPWORDS,
+)
+
+__all__ = [
+    # Detection
+    'detect_language',
+    'detect_corpus_language',
+    'normalize_language_code',
+    'LANGUAGE_ALIASES',
+    # Tokenizers
+    'TokenizerFactory',
+    'tokenize_documents',
+    # Stopwords
+    'get_stopwords',
+    'get_available_languages',
+    'STOPWORDS',
+]

tritopic/multilingual/detection.py

@@ -0,0 +1,208 @@
+"""
+Language Detection Module
+
+Automatic language detection for document collections.
+"""
+
+from typing import List, Dict, Tuple, Optional
+from collections import Counter
+import warnings
+
+
+def detect_language(text: str) -> Tuple[str, float]:
+    """
+    Detect the language of a single text.
+    
+    Parameters
+    ----------
+    text : str
+        The text to analyze
+        
+    Returns
+    -------
+    Tuple[str, float]
+        Language code and confidence score
+    """
+    # Try fasttext first (faster and more accurate)
+    try:
+        from fasttext_langdetect import detect
+        result = detect(text, low_memory=True)
+        return result['lang'], result['score']
+    except ImportError:
+        pass
+    
+    # Fall back to langdetect
+    try:
+        from langdetect import detect_langs
+        results = detect_langs(text)
+        if results:
+            return results[0].lang, results[0].prob
+    except ImportError:
+        warnings.warn(
+            "No language detection library found. "
+            "Install 'langdetect' or 'fasttext-langdetect' for automatic language detection."
+        )
+    except Exception:
+        pass
+    
+    return "en", 0.5  # Default to English with low confidence
+
+
+def detect_corpus_language(
+    documents: List[str],
+    sample_size: int = 100,
+    min_length: int = 20
+) -> Dict:
+    """
+    Detect the dominant language(s) in a document collection.
+    
+    Parameters
+    ----------
+    documents : List[str]
+        List of documents to analyze
+    sample_size : int
+        Number of documents to sample for detection
+    min_length : int
+        Minimum character length for a document to be considered
+        
+    Returns
+    -------
+    Dict
+        Dictionary with detection results:
+        - 'dominant_language': str - The most common language
+        - 'confidence': float - Confidence in the dominant language
+        - 'distribution': Dict[str, float] - Language distribution
+        - 'is_multilingual': bool - True if multiple languages detected
+        - 'detected_languages': List[str] - All detected languages
+    """
+    import random
+    
+    # Filter valid documents
+    valid_docs = [d for d in documents if isinstance(d, str) and len(d.strip()) >= min_length]
+    
+    if not valid_docs:
+        return {
+            'dominant_language': 'en',
+            'confidence': 0.5,
+            'distribution': {'en': 1.0},
+            'is_multilingual': False,
+            'detected_languages': ['en']
+        }
+    
+    # Sample documents
+    if len(valid_docs) > sample_size:
+        sample = random.sample(valid_docs, sample_size)
+    else:
+        sample = valid_docs
+    
+    # Detect language for each document
+    detections = []
+    for doc in sample:
+        try:
+            # Take first 500 chars for efficiency
+            lang, conf = detect_language(doc[:500])
+            detections.append((lang, conf))
+        except Exception:
+            continue
+    
+    if not detections:
+        return {
+            'dominant_language': 'en',
+            'confidence': 0.5,
+            'distribution': {'en': 1.0},
+            'is_multilingual': False,
+            'detected_languages': ['en']
+        }
+    
+    # Calculate distribution
+    lang_counts = Counter(lang for lang, _ in detections)
+    total = len(detections)
+    distribution = {lang: count / total for lang, count in lang_counts.items()}
+    
+    # Get dominant language
+    dominant_language = lang_counts.most_common(1)[0][0]
+    dominant_ratio = distribution[dominant_language]
+    
+    # Calculate average confidence for dominant language
+    dominant_confidences = [conf for lang, conf in detections if lang == dominant_language]
+    avg_confidence = sum(dominant_confidences) / len(dominant_confidences) if dominant_confidences else 0.5
+    
+    # Determine if corpus is multilingual
+    # If dominant language is less than 80% of corpus, consider it multilingual
+    is_multilingual = dominant_ratio < 0.8 and len(lang_counts) > 1
+    
+    return {
+        'dominant_language': dominant_language,
+        'confidence': avg_confidence * dominant_ratio,  # Combined confidence
+        'distribution': distribution,
+        'is_multilingual': is_multilingual,
+        'detected_languages': list(lang_counts.keys())
+    }
+
+
+# Language code normalization
+LANGUAGE_ALIASES = {
+    'german': 'de',
+    'deutsch': 'de',
+    'english': 'en',
+    'french': 'fr',
+    'français': 'fr',
+    'spanish': 'es',
+    'español': 'es',
+    'italian': 'it',
+    'italiano': 'it',
+    'portuguese': 'pt',
+    'português': 'pt',
+    'dutch': 'nl',
+    'chinese': 'zh',
+    'zh-cn': 'zh',
+    'zh-tw': 'zh',
+    'japanese': 'ja',
+    'korean': 'ko',
+    'russian': 'ru',
+    'arabic': 'ar',
+    'turkish': 'tr',
+    'polish': 'pl',
+    'swedish': 'sv',
+    'norwegian': 'no',
+    'danish': 'da',
+    'finnish': 'fi',
+    'greek': 'el',
+    'hebrew': 'he',
+    'thai': 'th',
+    'vietnamese': 'vi',
+    'indonesian': 'id',
+    'hindi': 'hi',
+    'czech': 'cs',
+    'hungarian': 'hu',
+    'romanian': 'ro',
+    'ukrainian': 'uk',
+}
+
+
+def normalize_language_code(language: str) -> str:
+    """
+    Normalize a language identifier to ISO 639-1 code.
+    
+    Parameters
+    ----------
+    language : str
+        Language name or code
+        
+    Returns
+    -------
+    str
+        ISO 639-1 language code
+    """
+    lang_lower = language.lower().strip()
+    
+    # Check if it's already a valid ISO code
+    if len(lang_lower) == 2:
+        return lang_lower
+    
+    # Check aliases
+    if lang_lower in LANGUAGE_ALIASES:
+        return LANGUAGE_ALIASES[lang_lower]
+    
+    # Return as-is if unknown
+    return lang_lower[:2] if len(lang_lower) > 2 else lang_lower