pygarble 0.1.0__py3-none-any.whl

pygarble/__init__.py

@@ -0,0 +1,7 @@
+__version__ = "0.1.0"
+__author__ = "Your Name"
+__email__ = "your.email@example.com"
+
+from .core import GarbleDetector, Strategy
+
+__all__ = ["GarbleDetector", "Strategy"]

pygarble/core.py

@@ -0,0 +1,92 @@
+import concurrent.futures
+from enum import Enum
+from typing import List, Union
+
+from .strategies import (
+    CharacterFrequencyStrategy,
+    EntropyBasedStrategy,
+    LanguageDetectionStrategy,
+    PatternMatchingStrategy,
+    StatisticalAnalysisStrategy,
+    WordLengthStrategy,
+)
+
+
+class Strategy(Enum):
+    CHARACTER_FREQUENCY = "character_frequency"
+    WORD_LENGTH = "word_length"
+    PATTERN_MATCHING = "pattern_matching"
+    STATISTICAL_ANALYSIS = "statistical_analysis"
+    ENTROPY_BASED = "entropy_based"
+    LANGUAGE_DETECTION = "language_detection"
+
+
+class GarbleDetector:
+    def __init__(
+        self,
+        strategy: Strategy,
+        threshold: float = 0.5,
+        threads: int = None,
+        **kwargs,
+    ):
+        self.strategy = strategy
+        self.threshold = threshold
+        self.threads = threads
+        self.kwargs = kwargs
+        self._strategy_instance = self._create_strategy_instance()
+
+    def _create_strategy_instance(self):
+        strategy_map = {
+            Strategy.CHARACTER_FREQUENCY: CharacterFrequencyStrategy,
+            Strategy.WORD_LENGTH: WordLengthStrategy,
+            Strategy.PATTERN_MATCHING: PatternMatchingStrategy,
+            Strategy.STATISTICAL_ANALYSIS: StatisticalAnalysisStrategy,
+            Strategy.ENTROPY_BASED: EntropyBasedStrategy,
+            Strategy.LANGUAGE_DETECTION: LanguageDetectionStrategy,
+        }
+
+        if self.strategy not in strategy_map:
+            raise NotImplementedError(
+                f"Strategy {self.strategy.value} is not implemented"
+            )
+
+        strategy_class = strategy_map[self.strategy]
+        return strategy_class(**self.kwargs)
+
+    def _process_text_proba(self, text: str) -> float:
+        return self._strategy_instance.predict_proba(text)
+
+    def _process_text_predict(self, text: str) -> bool:
+        proba = self._strategy_instance.predict_proba(text)
+        return proba >= self.threshold
+
+    def _process_batch_threaded(self, texts: List[str], process_func) -> List:
+        if self.threads is None or self.threads <= 1 or len(texts) < 10:
+            return [process_func(text) for text in texts]
+
+        max_workers = min(self.threads, len(texts))
+
+        with concurrent.futures.ThreadPoolExecutor(
+            max_workers=max_workers
+        ) as executor:
+            futures = [executor.submit(process_func, text) for text in texts]
+            return [future.result() for future in futures]
+
+    def predict(self, X: Union[str, List[str]]) -> Union[bool, List[bool]]:
+        if isinstance(X, str):
+            proba = self._strategy_instance.predict_proba(X)
+            return proba >= self.threshold
+        elif isinstance(X, list):
+            return self._process_batch_threaded(X, self._process_text_predict)
+        else:
+            raise ValueError("Input must be a string or list of strings")
+
+    def predict_proba(
+        self, X: Union[str, List[str]]
+    ) -> Union[float, List[float]]:
+        if isinstance(X, str):
+            return self._strategy_instance.predict_proba(X)
+        elif isinstance(X, list):
+            return self._process_batch_threaded(X, self._process_text_proba)
+        else:
+            raise ValueError("Input must be a string or list of strings")

pygarble/strategies/__init__.py

@@ -0,0 +1,17 @@
+from .base import BaseStrategy
+from .character_frequency import CharacterFrequencyStrategy
+from .entropy_based import EntropyBasedStrategy
+from .language_detection import LanguageDetectionStrategy
+from .pattern_matching import PatternMatchingStrategy
+from .statistical_analysis import StatisticalAnalysisStrategy
+from .word_length import WordLengthStrategy
+
+__all__ = [
+    "BaseStrategy",
+    "CharacterFrequencyStrategy",
+    "WordLengthStrategy",
+    "PatternMatchingStrategy",
+    "StatisticalAnalysisStrategy",
+    "EntropyBasedStrategy",
+    "LanguageDetectionStrategy",
+]

pygarble/strategies/base.py

@@ -0,0 +1,37 @@
+import re
+from abc import ABC, abstractmethod
+
+
+class BaseStrategy(ABC):
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+
+    def predict(self, text: str) -> bool:
+        if not text or not text.strip():
+            return False
+
+        if self._is_extremely_long_string(text):
+            return True
+
+        return self._predict_impl(text)
+
+    def predict_proba(self, text: str) -> float:
+        if not text or not text.strip():
+            return 0.0
+
+        if self._is_extremely_long_string(text):
+            return 1.0
+
+        return self._predict_proba_impl(text)
+
+    def _is_extremely_long_string(self, text: str) -> bool:
+        max_length = self.kwargs.get("max_string_length", 1000)
+        return len(text) > max_length and not re.search(r"\s", text)
+
+    @abstractmethod
+    def _predict_impl(self, text: str) -> bool:
+        pass
+
+    @abstractmethod
+    def _predict_proba_impl(self, text: str) -> float:
+        pass

pygarble/strategies/character_frequency.py

@@ -0,0 +1,33 @@
+from .base import BaseStrategy
+
+
+class CharacterFrequencyStrategy(BaseStrategy):
+    def _get_char_counts(self, text: str):
+        char_counts = {}
+        for char in text.lower():
+            if char.isalpha():
+                char_counts[char] = char_counts.get(char, 0) + 1
+        return char_counts
+
+    def _predict_impl(self, text: str) -> bool:
+        char_counts = self._get_char_counts(text)
+        total_chars = sum(char_counts.values())
+        if total_chars == 0:
+            return False
+
+        threshold = self.kwargs.get("frequency_threshold", 0.1)
+        for count in char_counts.values():
+            if count / total_chars > threshold:
+                return True
+        return False
+
+    def _predict_proba_impl(self, text: str) -> float:
+        char_counts = self._get_char_counts(text)
+        total_chars = sum(char_counts.values())
+        if total_chars == 0:
+            return 0.0
+
+        max_frequency = (
+            max(char_counts.values()) / total_chars if char_counts else 0.0
+        )
+        return min(max_frequency, 1.0)

pygarble/strategies/entropy_based.py

@@ -0,0 +1,47 @@
+import math
+
+from .base import BaseStrategy
+
+
+class EntropyBasedStrategy(BaseStrategy):
+    def _get_char_counts(self, text: str):
+        char_counts = {}
+        for char in text.lower():
+            if char.isalpha():
+                char_counts[char] = char_counts.get(char, 0) + 1
+        return char_counts
+
+    def _calculate_entropy(self, char_counts):
+        total_chars = sum(char_counts.values())
+        if total_chars == 0:
+            return 0.0
+
+        entropy = 0
+        for count in char_counts.values():
+            probability = count / total_chars
+            entropy -= probability * math.log2(probability)
+
+        return entropy
+
+    def _predict_impl(self, text: str) -> bool:
+        char_counts = self._get_char_counts(text)
+        total_chars = sum(char_counts.values())
+        if total_chars == 0:
+            return False
+
+        entropy = self._calculate_entropy(char_counts)
+        threshold = self.kwargs.get("entropy_threshold", 3.0)
+
+        return entropy < threshold
+
+    def _predict_proba_impl(self, text: str) -> float:
+        char_counts = self._get_char_counts(text)
+        total_chars = sum(char_counts.values())
+        if total_chars == 0:
+            return 0.0
+
+        entropy = self._calculate_entropy(char_counts)
+        max_entropy = math.log2(len(char_counts)) if char_counts else 1.0
+        normalized_entropy = entropy / max_entropy if max_entropy > 0 else 0.0
+
+        return 1.0 - normalized_entropy

pygarble/strategies/language_detection.py

@@ -0,0 +1,106 @@
+import os
+import urllib.request
+
+from .base import BaseStrategy
+
+
+class LanguageDetectionStrategy(BaseStrategy):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._model = None
+        self._model_path = self._get_model_path()
+
+    def _get_model_path(self):
+        custom_path = self.kwargs.get("model_path")
+        if custom_path:
+            return custom_path
+
+        model_dir = os.path.expanduser("~/.pygarble/models")
+        os.makedirs(model_dir, exist_ok=True)
+        return os.path.join(model_dir, "lid.176.bin")
+
+    def _download_model(self):
+        if os.path.exists(self._model_path):
+            return
+
+        custom_path = self.kwargs.get("model_path")
+        if custom_path:
+            raise FileNotFoundError(
+                f"Model file not found at custom path: {self._model_path}"
+            )
+
+        model_url = (
+            "https://dl.fbaipublicfiles.com/fasttext/supervised-models/"
+            "lid.176.bin"
+        )
+        print(
+            f"Downloading FastText language detection model to "
+            f"{self._model_path}..."
+        )
+        urllib.request.urlretrieve(model_url, self._model_path)
+        print("Model download completed.")
+
+    def _load_model(self):
+        if self._model is None:
+            try:
+                import fasttext
+                import numpy as np
+
+                self._download_model()
+                self._model = fasttext.load_model(self._model_path)
+
+                # Monkey patch to fix NumPy 2.0 compatibility issue
+                original_predict = self._model.predict
+
+                def patched_predict(text, k=1):
+                    labels, probs = original_predict(text, k)
+                    # Convert to list to avoid NumPy 2.0 copy=False issue
+                    if isinstance(probs, np.ndarray):
+                        probs = probs.tolist()
+                    return labels, probs
+
+                self._model.predict = patched_predict
+
+            except ImportError:
+                raise ImportError(
+                    "fasttext is required for LanguageDetectionStrategy. "
+                    "Install it with: pip install fasttext-wheel"
+                )
+
+    def _predict_impl(self, text: str) -> bool:
+        self._load_model()
+
+        predictions = self._model.predict(text, k=1)
+        labels, scores = predictions
+
+        if not labels or not scores:
+            return True
+
+        label = labels[0]
+        score = float(scores[0])
+
+        target_lang = self.kwargs.get("target_language", "en")
+
+        if label == f"__label__{target_lang}":
+            return score < 0.5
+        else:
+            return True
+
+    def _predict_proba_impl(self, text: str) -> float:
+        self._load_model()
+
+        predictions = self._model.predict(text, k=1)
+        labels, scores = predictions
+
+        if not labels or not scores:
+            return 1.0
+
+        label = labels[0]
+        score = float(scores[0])
+
+        target_lang = self.kwargs.get("target_language", "en")
+
+        if label == f"__label__{target_lang}":
+            return float(1.0 - score)
+        else:
+            return 1.0

pygarble/strategies/pattern_matching.py

@@ -0,0 +1,44 @@
+import re
+
+from .base import BaseStrategy
+
+
+class PatternMatchingStrategy(BaseStrategy):
+    DEFAULT_PATTERNS = {
+        "special_chars": r"[^a-zA-Z0-9\s]{3,}",
+        "repeated_chars": r"(.)\1{4,}",
+        "uppercase_sequence": r"[A-Z]{5,}",
+        "long_numbers": r"[0-9]{8,}",
+    }
+
+    def _get_patterns(self):
+        custom_patterns = self.kwargs.get("patterns", {})
+        override_defaults = self.kwargs.get("override_defaults", False)
+
+        if override_defaults:
+            patterns = custom_patterns
+        else:
+            patterns = {**self.DEFAULT_PATTERNS, **custom_patterns}
+
+        return patterns
+
+    def _predict_impl(self, text: str) -> bool:
+        patterns = self._get_patterns()
+
+        for pattern_name, pattern_regex in patterns.items():
+            if re.search(pattern_regex, text):
+                return True
+        return False
+
+    def _predict_proba_impl(self, text: str) -> float:
+        patterns = self._get_patterns()
+
+        if not patterns:
+            return 0.0
+
+        matches = 0
+        for pattern_name, pattern_regex in patterns.items():
+            if re.search(pattern_regex, text):
+                matches += 1
+
+        return min(matches / len(patterns), 1.0)

pygarble/strategies/statistical_analysis.py

@@ -0,0 +1,22 @@
+from .base import BaseStrategy
+
+
+class StatisticalAnalysisStrategy(BaseStrategy):
+    def _get_alpha_ratio(self, text: str) -> float:
+        alpha_chars = sum(1 for c in text if c.isalpha())
+        content_chars = sum(1 for c in text if not c.isspace())
+
+        if content_chars == 0:
+            return 0.0
+
+        return alpha_chars / content_chars
+
+    def _predict_impl(self, text: str) -> bool:
+        alpha_ratio = self._get_alpha_ratio(text)
+        threshold = self.kwargs.get("alpha_threshold", 0.5)
+
+        return alpha_ratio < threshold
+
+    def _predict_proba_impl(self, text: str) -> float:
+        alpha_ratio = self._get_alpha_ratio(text)
+        return 1.0 - alpha_ratio

pygarble/strategies/word_length.py

@@ -0,0 +1,29 @@
+from .base import BaseStrategy
+
+
+class WordLengthStrategy(BaseStrategy):
+    def _get_words(self, text: str):
+        return text.split()
+
+    def _predict_impl(self, text: str) -> bool:
+        words = self._get_words(text)
+        if not words:
+            return False
+
+        avg_length = sum(len(word) for word in words) / len(words)
+        max_length = self.kwargs.get("max_word_length", 20)
+
+        return avg_length > max_length
+
+    def _predict_proba_impl(self, text: str) -> float:
+        words = self._get_words(text)
+        if not words:
+            return 0.0
+
+        avg_length = sum(len(word) for word in words) / len(words)
+        max_length = self.kwargs.get("max_word_length", 20)
+
+        if avg_length <= max_length:
+            return 0.0
+
+        return min((avg_length - max_length) / max_length, 1.0)