fastlangml 1.0.0__py3-none-any.whl

fastlangml/__init__.py

@@ -0,0 +1,164 @@
+"""FastLangID - Fast, accurate language detection with multiple backends.
+
+A production-first language detection toolkit optimized for short text that provides:
+- Stable, deterministic API
+- Unknown/abstain support (returns 'und' with reason when uncertain)
+- Top-k predictions with consistent confidence semantics
+- Multi-backend ensembling (fasttext, langdetect, lingua, pycld3, langid)
+- Batch + caching for throughput
+- Benchmark harness to compare backends
+
+Quick Start:
+    >>> from fastlangml import detect
+    >>> result = detect("Bonjour le monde")
+    >>> result.lang
+    'fr'
+    >>> result.confidence
+    0.95
+
+    >>> from fastlangml import detect
+    >>> result = detect("Hello world", top_k=3)
+    >>> result.candidates
+    [Candidate(lang='en', confidence=0.95), ...]
+
+Ensemble Usage:
+    >>> from fastlangml import ensemble
+    >>> result = ensemble(
+    ...     "Bonjour",
+    ...     backends=["fasttext", "lingua"],
+    ...     weights={"fasttext": 0.7, "lingua": 0.3},
+    ... )
+    >>> result.lang
+    'fr'
+
+Advanced Usage:
+    >>> from fastlangml import FastLangDetector
+    >>> detector = FastLangDetector()
+    >>> detector.add_hint("merci", "fr")
+    >>> result = detector.detect("Merci beaucoup!")
+    >>> result.lang
+    'fr'
+"""
+
+from __future__ import annotations
+
+from fastlangml.backends import (
+    Backend,
+    backend,
+    get_available_backends,
+    list_registered_backends,
+    register_backend,
+    unregister_backend,
+)
+from fastlangml.codeswitching import (
+    CodeSwitchDetector,
+    CodeSwitchResult,
+    detect_code_switching_pattern,
+)
+from fastlangml.context.conversation import ConversationContext, ConversationTurn
+from fastlangml.detector import (
+    DetectionConfig,
+    FastLangDetector,
+    FastLangDetectorBuilder,
+    detect,
+    detect_batch,
+    ensemble,
+)
+from fastlangml.ensemble.confusion import (
+    ConfusionResolver,
+    LanguageSimilarity,
+)
+from fastlangml.ensemble.voting import (
+    ConsensusVoting,
+    HardVoting,
+    SoftVoting,
+    VotingStrategy,
+    WeightedVoting,
+)
+from fastlangml.exceptions import (
+    BackendError,
+    BackendUnavailableError,
+    ConfigurationError,
+    DetectionError,
+    FastLangError,
+    HintError,
+    NoBackendsAvailableError,
+)
+from fastlangml.hints.dictionary import HintDictionary
+from fastlangml.hints.persistence import HintPersistence
+from fastlangml.normalize import (
+    compute_text_stats,
+    normalize_lang_tag,
+    normalize_text,
+)
+from fastlangml.preprocessing.script_filter import Script, detect_script
+from fastlangml.result import (
+    Candidate,
+    DetectionResult,
+    Reasons,
+)
+
+try:
+    from importlib.metadata import version as _get_version
+    __version__ = _get_version("fastlangml")
+except Exception:
+    __version__ = "0.1.0"  # Fallback for development
+
+__all__ = [
+    # Version
+    "__version__",
+    # Main API
+    "detect",
+    "detect_batch",
+    "ensemble",
+    # Result types
+    "DetectionResult",
+    "Candidate",
+    "Reasons",
+    # Detector
+    "FastLangDetector",
+    "FastLangDetectorBuilder",
+    "DetectionConfig",
+    # Context
+    "ConversationContext",
+    "ConversationTurn",
+    # Hints
+    "HintDictionary",
+    "HintPersistence",
+    # Script detection
+    "Script",
+    "detect_script",
+    # Normalization
+    "compute_text_stats",
+    "normalize_text",
+    "normalize_lang_tag",
+    # Utilities
+    "get_available_backends",
+    # Custom backend registration
+    "backend",  # Decorator (preferred)
+    "register_backend",
+    "unregister_backend",
+    "list_registered_backends",
+    "Backend",
+    # Voting strategies (for customization)
+    "VotingStrategy",
+    "HardVoting",
+    "SoftVoting",
+    "WeightedVoting",
+    "ConsensusVoting",
+    # Exceptions
+    "FastLangError",
+    "DetectionError",
+    "BackendUnavailableError",
+    "BackendError",
+    "ConfigurationError",
+    "HintError",
+    "NoBackendsAvailableError",
+    # Confusion resolution
+    "ConfusionResolver",
+    "LanguageSimilarity",
+    # Code-switching detection
+    "CodeSwitchDetector",
+    "CodeSwitchResult",
+    "detect_code_switching_pattern",
+]

fastlangml/backends/__init__.py

@@ -0,0 +1,283 @@
+"""Backend implementations for language detection.
+
+Supports built-in backends and custom backends via decorator registration.
+
+Example using decorator:
+    >>> from fastlangml import backend, Backend
+    >>> from fastlangml.backends.base import DetectionResult
+    >>>
+    >>> @backend("mybackend", reliability=4)
+    ... class MyBackend(Backend):
+    ...     @property
+    ...     def name(self) -> str:
+    ...         return "mybackend"
+    ...
+    ...     @property
+    ...     def is_available(self) -> bool:
+    ...         return True
+    ...
+    ...     def detect(self, text: str) -> DetectionResult:
+    ...         return DetectionResult("mybackend", "en", 0.9)
+    ...
+    ...     def supported_languages(self) -> set[str]:
+    ...         return {"en", "fr", "es"}
+    >>>
+    >>> # Now use it
+    >>> from fastlangml import detect
+    >>> detect("Hello", backends=["mybackend"]).lang
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING, TypeVar
+
+from fastlangml.backends.base import Backend, DetectionResult
+from fastlangml.exceptions import BackendUnavailableError
+
+if TYPE_CHECKING:
+    pass
+
+T = TypeVar("T", bound=type[Backend])
+
+
+# =============================================================================
+# Backend Registry
+# =============================================================================
+
+_CUSTOM_BACKEND_REGISTRY: dict[str, type[Backend]] = {}
+
+_BACKEND_RELIABILITY: dict[str, int] = {
+    "fasttext": 5,
+    "lingua": 4,
+    "pycld3": 3,
+    "langdetect": 2,
+}
+
+_BUILTIN_BACKEND_NAMES = frozenset({"fasttext", "langdetect", "lingua", "pycld3"})
+
+
+def backend(name: str, reliability: int = 3) -> Callable[[T], T]:
+    """Decorator to register a custom backend.
+
+    Use this decorator on a class extending Backend to register it
+    for use in FastLangID's ensemble detection.
+
+    Args:
+        name: Unique identifier for the backend.
+        reliability: Reliability score 1-5 (higher = more weight in voting).
+
+    Returns:
+        Decorator function that registers the class.
+
+    Example:
+        >>> from fastlangml import backend, Backend
+        >>> from fastlangml.backends.base import DetectionResult
+        >>>
+        >>> @backend("openai", reliability=5)
+        ... class OpenAIBackend(Backend):
+        ...     '''Custom backend using OpenAI API.'''
+        ...
+        ...     @property
+        ...     def name(self) -> str:
+        ...         return "openai"
+        ...
+        ...     @property
+        ...     def is_available(self) -> bool:
+        ...         try:
+        ...             import openai
+        ...             return True
+        ...         except ImportError:
+        ...             return False
+        ...
+        ...     def detect(self, text: str) -> DetectionResult:
+        ...         # Your OpenAI API call here
+        ...         lang, conf = self._detect_with_openai(text)
+        ...         return DetectionResult(self.name, lang, conf)
+        ...
+        ...     def supported_languages(self) -> set[str]:
+        ...         return {"en", "fr", "de", "es", "zh", "ja", "ko"}
+        >>>
+        >>> # Use in detection
+        >>> from fastlangml import FastLangDetector, DetectionConfig
+        >>> detector = FastLangDetector(
+        ...     config=DetectionConfig(backends=["openai", "fasttext"])
+        ... )
+    """
+    def decorator(cls: T) -> T:
+        register_backend(name, cls, reliability)
+        return cls
+    return decorator
+
+
+def register_backend(
+    name: str,
+    backend_class: type[Backend],
+    reliability: int = 3,
+) -> None:
+    """Register a custom backend (function form).
+
+    Prefer using the @backend decorator for cleaner code.
+
+    Args:
+        name: Unique identifier for the backend.
+        backend_class: Class extending Backend.
+        reliability: Reliability score 1-5.
+
+    Raises:
+        ValueError: If name conflicts with built-in or reliability out of range.
+        TypeError: If backend_class doesn't extend Backend.
+    """
+    if name in _BUILTIN_BACKEND_NAMES:
+        raise ValueError(
+            f"Cannot register '{name}': conflicts with built-in backend. "
+            f"Built-in: {sorted(_BUILTIN_BACKEND_NAMES)}"
+        )
+
+    if not isinstance(backend_class, type) or not issubclass(backend_class, Backend):
+        raise TypeError(
+            f"backend_class must extend Backend, got {type(backend_class)}"
+        )
+
+    if not isinstance(reliability, int) or not 1 <= reliability <= 5:
+        raise ValueError(f"reliability must be 1-5, got {reliability}")
+
+    _CUSTOM_BACKEND_REGISTRY[name] = backend_class
+    _BACKEND_RELIABILITY[name] = reliability
+
+
+def unregister_backend(name: str) -> bool:
+    """Remove a registered custom backend.
+
+    Args:
+        name: Backend name to remove.
+
+    Returns:
+        True if removed, False if not found.
+    """
+    if name in _BUILTIN_BACKEND_NAMES:
+        raise ValueError(f"Cannot unregister built-in backend: {name}")
+
+    removed = _CUSTOM_BACKEND_REGISTRY.pop(name, None) is not None
+    _BACKEND_RELIABILITY.pop(name, None)
+    return removed
+
+
+def list_registered_backends() -> list[str]:
+    """List custom registered backends."""
+    return list(_CUSTOM_BACKEND_REGISTRY.keys())
+
+
+def clear_registered_backends() -> None:
+    """Remove all custom backends (useful for testing)."""
+    for name in list(_CUSTOM_BACKEND_REGISTRY.keys()):
+        unregister_backend(name)
+
+
+# =============================================================================
+# Backend Factory
+# =============================================================================
+
+def _get_backend_class(name: str) -> type[Backend]:
+    """Get backend class by name (lazy import to avoid loading unavailable deps)."""
+    # Check custom registry first
+    if name in _CUSTOM_BACKEND_REGISTRY:
+        return _CUSTOM_BACKEND_REGISTRY[name]
+
+    # Built-in backends (lazy import)
+    if name == "fasttext":
+        from fastlangml.backends.fasttext_backend import FastTextBackend
+        return FastTextBackend
+    elif name == "langdetect":
+        from fastlangml.backends.langdetect_backend import LangdetectBackend
+        return LangdetectBackend
+    elif name == "lingua":
+        from fastlangml.backends.lingua_backend import LinguaBackend
+        return LinguaBackend
+    elif name == "pycld3":
+        from fastlangml.backends.pycld3_backend import PyCLD3Backend
+        return PyCLD3Backend
+    else:
+        available = sorted(_BUILTIN_BACKEND_NAMES | set(_CUSTOM_BACKEND_REGISTRY.keys()))
+        raise ValueError(f"Unknown backend: {name}. Available: {available}")
+
+
+# For backward compatibility
+BACKEND_NAMES = _BUILTIN_BACKEND_NAMES
+BACKEND_RELIABILITY = _BACKEND_RELIABILITY
+
+
+def create_backend(name: str, **kwargs: object) -> Backend:
+    """
+    Create a backend instance by name.
+
+    Args:
+        name: Backend name ("fasttext", "langdetect", "lingua", "pycld3", "langid")
+        **kwargs: Backend-specific configuration
+
+    Raises:
+        BackendUnavailableError: If backend dependencies not installed
+        ValueError: If backend name is unknown
+    """
+    backend_class = _get_backend_class(name)
+    backend = backend_class(**kwargs)  # type: ignore[arg-type]
+
+    if not backend.is_available:
+        install_cmd = f"pip install fastlang[{name}]"
+        raise BackendUnavailableError(
+            f"Backend '{name}' is not available. Install with: {install_cmd}",
+            backend_name=name,
+        )
+
+    return backend
+
+
+def get_available_backends() -> list[str]:
+    """Return list of available backend names (built-in + custom)."""
+    available = []
+
+    # Check built-in backends
+    for name in _BUILTIN_BACKEND_NAMES:
+        try:
+            backend_class = _get_backend_class(name)
+            backend_instance = backend_class()
+            if backend_instance.is_available:
+                available.append(name)
+        except Exception:
+            pass
+
+    # Check custom backends
+    for name, backend_class in _CUSTOM_BACKEND_REGISTRY.items():
+        try:
+            backend_instance = backend_class()
+            if backend_instance.is_available:
+                available.append(name)
+        except Exception:
+            pass
+
+    return sorted(available, key=lambda x: _BACKEND_RELIABILITY.get(x, 0), reverse=True)
+
+
+def get_backend_reliability(name: str) -> int:
+    """Get reliability score for a backend (higher = more reliable)."""
+    return BACKEND_RELIABILITY.get(name, 0)
+
+
+__all__ = [
+    # Base classes
+    "Backend",
+    "DetectionResult",
+    # Factory functions
+    "create_backend",
+    "get_available_backends",
+    "get_backend_reliability",
+    # Custom backend registration
+    "backend",  # Decorator (preferred)
+    "register_backend",
+    "unregister_backend",
+    "list_registered_backends",
+    "clear_registered_backends",
+    # Constants
+    "BACKEND_NAMES",
+    "BACKEND_RELIABILITY",
+]

fastlangml/backends/base.py

@@ -0,0 +1,73 @@
+"""Abstract base class for language detection backends."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+
+
+@dataclass
+class DetectionResult:
+    """Result from a single backend detection."""
+
+    backend_name: str
+    language: str
+    confidence: float
+    all_probabilities: dict[str, float] = field(default_factory=dict)
+    is_reliable: bool = True  # Backend-specific reliability flag
+
+    def __post_init__(self) -> None:
+        # Normalize language code to lowercase
+        self.language = self.language.lower() if self.language else "unknown"
+        # Ensure all_probabilities includes the top result
+        if self.language != "unknown" and self.language not in self.all_probabilities:
+            self.all_probabilities[self.language] = self.confidence
+
+
+class Backend(ABC):
+    """Abstract base class for language detection backends."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Backend identifier."""
+        pass
+
+    @property
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Check if backend dependencies are installed."""
+        pass
+
+    @abstractmethod
+    def detect(self, text: str) -> DetectionResult:
+        """
+        Detect language of text.
+
+        Args:
+            text: Preprocessed text to analyze
+
+        Returns:
+            DetectionResult with language code, confidence, and all probabilities
+        """
+        pass
+
+    @abstractmethod
+    def supported_languages(self) -> set[str]:
+        """Return set of ISO 639-1 codes this backend supports."""
+        pass
+
+    def detect_batch(self, texts: list[str]) -> list[DetectionResult]:
+        """
+        Detect language for multiple texts.
+
+        Default implementation calls detect() for each text.
+        Backends can override for optimized batch processing.
+
+        Args:
+            texts: List of texts to analyze
+
+        Returns:
+            List of DetectionResults
+        """
+        return [self.detect(text) for text in texts]

fastlangml/backends/fasttext_backend.py

@@ -0,0 +1,96 @@
+"""FastText-based language detection backend."""
+
+from __future__ import annotations
+
+from fastlangml.backends.base import Backend, DetectionResult
+
+
+class FastTextBackend(Backend):
+    """
+    FastText-based language detection backend.
+
+    Uses the fasttext-langdetect package which wraps Facebook's FastText
+    language identification model trained on Wikipedia data.
+
+    Supports 176 languages with high accuracy.
+    """
+
+    def __init__(self, low_memory: bool = False) -> None:
+        """
+        Args:
+            low_memory: Use low memory mode (slower but uses less RAM)
+        """
+        self._low_memory = low_memory
+
+    @property
+    def name(self) -> str:
+        return "fasttext"
+
+    @property
+    def is_available(self) -> bool:
+        try:
+            from ftlangdetect import detect  # noqa: F401
+
+            return True
+        except ImportError:
+            return False
+
+    def detect(self, text: str) -> DetectionResult:
+        from ftlangdetect import detect
+
+        # Clean text (fasttext expects single line)
+        clean_text = " ".join(text.split())
+
+        if not clean_text:
+            return DetectionResult(
+                backend_name=self.name,
+                language="unknown",
+                confidence=0.0,
+                all_probabilities={},
+                is_reliable=False,
+            )
+
+        try:
+            result = detect(text=clean_text, low_memory=self._low_memory)
+
+            lang = result.get("lang", "unknown")
+            score = float(result.get("score", 0.0))
+
+            return DetectionResult(
+                backend_name=self.name,
+                language=lang,
+                confidence=score,
+                all_probabilities={lang: score},
+                is_reliable=score > 0.5,
+            )
+        except Exception:
+            return DetectionResult(
+                backend_name=self.name,
+                language="unknown",
+                confidence=0.0,
+                all_probabilities={},
+                is_reliable=False,
+            )
+
+    def supported_languages(self) -> set[str]:
+        """FastText supports 176 languages."""
+        return {
+            "af", "als", "am", "an", "ar", "arz", "as", "ast", "av", "az",
+            "azb", "ba", "bar", "bcl", "be", "bg", "bh", "bn", "bo", "bpy",
+            "br", "bs", "bxr", "ca", "cbk", "ce", "ceb", "ckb", "co", "cs",
+            "cv", "cy", "da", "de", "diq", "dsb", "dty", "dv", "el", "eml",
+            "en", "eo", "es", "et", "eu", "fa", "fi", "fr", "frr", "fy",
+            "ga", "gd", "gl", "gn", "gom", "gu", "gv", "he", "hi", "hif",
+            "hr", "hsb", "ht", "hu", "hy", "ia", "id", "ie", "ilo", "io",
+            "is", "it", "ja", "jbo", "jv", "ka", "kk", "km", "kn", "ko",
+            "krc", "ku", "kv", "kw", "ky", "la", "lb", "lez", "li", "lmo",
+            "lo", "lrc", "lt", "lv", "mai", "mg", "mhr", "min", "mk", "ml",
+            "mn", "mr", "mrj", "ms", "mt", "mwl", "my", "myv", "mzn", "nah",
+            "nap", "nds", "ne", "new", "nl", "nn", "no", "oc", "or", "os",
+            "pa", "pam", "pfl", "pl", "pms", "pnb", "ps", "pt", "qu", "rm",
+            "ro", "ru", "rue", "sa", "sah", "sc", "scn", "sco", "sd", "sh",
+            "si", "sk", "sl", "so", "sq", "sr", "su", "sv", "sw", "ta",
+            "te", "tg", "th", "tk", "tl", "tr", "tt", "tyv", "ug", "uk",
+            "ur", "uz", "vec", "vep", "vi", "vls", "vo", "wa", "war", "wuu",
+            "xal", "xmf", "yi", "yo", "yue", "zh",
+        }