fliprate 0.2.1__py3-none-any.whl

fliprate/__init__.py

@@ -0,0 +1,132 @@
+"""FlipRate: measure and reduce how often an NLP model changes its answer
+on messy romanized Hindi / Marathi / Tamil / Telugu and other Indian-language text.
+
+    import fliprate as fr
+
+    # Test robustness
+    report = fr.test(my_model, "data/test.csv")
+    print(report.summary())
+
+    # Fix it
+    safe_model = fr.protect(my_model)
+    report2 = fr.test(safe_model, "data/test.csv")
+
+    # Real-time gate
+    result = fr.gate(my_model, "mala product avadlaaa nahi")
+    print(result.decision)   # PASS / TRANSFORM / REVIEW
+
+    # Multi-language rules from Aksharantar (428K groups, 21 languages)
+    rules = fr.aksharantar_rules("tam", "tel")
+    report = fr.test(my_model, "data/test.csv", rules=rules)
+
+Author: Shiv Supane
+"""
+from __future__ import annotations
+
+# ── Core: most users only need these ──────────────────────────────────────────
+from .evaluation import (
+    Report,
+    iter_test,
+    load_dataset,
+    macro_f1,
+    paired_difference,
+    read_csv_text,
+    test,
+)
+from .gating import GateResult, gate, iter_gate
+from .models import Predictor, ProtectedModel, protect
+from .normalization import NormalizeResult, normalize
+from .perturbation import KINDS, Variant, perturb
+from .rules import DEFAULT_RULES, LANGUAGE_GROUPS, RuleSet, is_safe_variant
+
+# ── Model adapters ─────────────────────────────────────────────────────────────
+from .adapters import (
+    HFTextClassifier,
+    LLMClassifier,
+    from_sklearn,
+    llm_classifier,
+    twitter_preprocess,
+    with_retries,
+)
+
+# ── Experiment runner ──────────────────────────────────────────────────────────
+from .experiment import ExperimentResult, run_experiment
+
+# ── Multi-language dataset loaders ────────────────────────────────────────────
+from .dakshina import (
+    SUPPORTED_LANGUAGES as DAKSHINA_LANGUAGES,
+    dakshina_rules,
+    load_dakshina_groups,
+)
+from .aksharantar import (
+    SUPPORTED_LANGUAGES as AKSHARANTAR_LANGUAGES,
+    aksharantar_rules,
+    load_aksharantar_groups,
+)
+
+# ── Streaming helpers (for building custom event consumers) ───────────────────
+from .events import drain, make_event, to_sse
+
+__version__ = "0.2.1"
+__author__ = "Shiv Supane"
+
+__all__ = [
+    # ── Test & measure ──────────────────────────────────────────────────────
+    "test",
+    "iter_test",
+    "gate",
+    "iter_gate",
+    "Report",
+    "GateResult",
+    "paired_difference",
+
+    # ── Fix & protect ────────────────────────────────────────────────────────
+    "protect",
+    "normalize",
+    "perturb",
+    "is_safe_variant",
+
+    # ── Rules ────────────────────────────────────────────────────────────────
+    "RuleSet",
+    "DEFAULT_RULES",
+    "LANGUAGE_GROUPS",
+
+    # ── Multi-language datasets ──────────────────────────────────────────────
+    "dakshina_rules",
+    "load_dakshina_groups",
+    "DAKSHINA_LANGUAGES",
+    "aksharantar_rules",
+    "load_aksharantar_groups",
+    "AKSHARANTAR_LANGUAGES",
+
+    # ── Model adapters ────────────────────────────────────────────────────────
+    "HFTextClassifier",
+    "LLMClassifier",
+    "llm_classifier",
+    "from_sklearn",
+    "twitter_preprocess",
+    "with_retries",
+    "ProtectedModel",
+    "Predictor",
+
+    # ── Experiment ───────────────────────────────────────────────────────────
+    "run_experiment",
+    "ExperimentResult",
+
+    # ── Data loading ─────────────────────────────────────────────────────────
+    "load_dataset",
+    "read_csv_text",
+
+    # ── Types & helpers ──────────────────────────────────────────────────────
+    "NormalizeResult",
+    "Variant",
+    "KINDS",
+    "macro_f1",
+    "drain",
+    "make_event",
+    "to_sse",
+
+    # ── Version ──────────────────────────────────────────────────────────────
+    "__version__",
+    "__author__",
+]

fliprate/adapters.py

@@ -0,0 +1,143 @@
+"""Adapters that turn real models into FlipRate models (text -> label, with fast `predict_batch`).
+
+Heavy libraries (transformers, scikit-learn) are imported lazily, only when you use the adapter.
+"""
+from __future__ import annotations
+
+import re
+import time
+from typing import Any, Callable, Dict, List, Optional, Sequence
+
+
+def twitter_preprocess(text: str) -> str:
+    """The @user / http placeholders that the cardiffnlp Twitter models were trained with."""
+    out = []
+    for t in text.split(" "):
+        t = "@user" if t.startswith("@") and len(t) > 1 else t
+        t = "http" if t.startswith("http") else t
+        out.append(t)
+    return " ".join(out)
+
+
+class HFTextClassifier:
+    """Hugging Face text-classification model, batched.
+
+        clf = HFTextClassifier("cardiffnlp/twitter-xlm-roberta-base-sentiment",
+                               lowercase_labels=True, preprocess=twitter_preprocess)
+        fr.test(clf, rows)
+
+    `label_map` renames model labels to your dataset's labels. `lowercase_labels=True` maps
+    "Positive" -> "positive". `pipeline_factory` lets tests inject a fake pipeline.
+    """
+
+    def __init__(self, model_name: str, device: Any = None, batch_size: int = 32, max_length: int = 256,
+                 label_map: Optional[Dict[str, str]] = None, lowercase_labels: bool = False,
+                 preprocess: Optional[Callable[[str], str]] = None,
+                 pipeline_factory: Optional[Callable[[], Callable]] = None):
+        self.name = model_name
+        self.device, self.batch_size, self.max_length = device, batch_size, max_length
+        self.label_map, self.lowercase_labels, self.preprocess = label_map or {}, lowercase_labels, preprocess
+        self._factory = pipeline_factory
+        self._pipe: Optional[Callable] = None
+
+    def _get(self) -> Callable:
+        if self._pipe is None:
+            if self._factory:
+                self._pipe = self._factory()
+            else:
+                try:
+                    from transformers import pipeline
+                except ImportError as exc:
+                    raise ImportError("HFTextClassifier needs `pip install transformers torch`") from exc
+                kwargs = {"model": self.name, "tokenizer": self.name}
+                if self.device is not None:
+                    kwargs["device"] = self.device
+                self._pipe = pipeline("text-classification", **kwargs)
+        return self._pipe
+
+    def warmup(self) -> None:
+        self.predict_batch(["warmup"])
+
+    def predict_batch(self, texts: Sequence[str]) -> List[str]:
+        texts = [self.preprocess(t) if self.preprocess else t for t in texts]
+        outs = self._get()(texts, batch_size=self.batch_size, truncation=True, max_length=self.max_length)
+        labels = []
+        for o in outs:
+            lab = o["label"]
+            lab = self.label_map.get(lab, lab)
+            labels.append(lab.lower() if self.lowercase_labels else lab)
+        return labels
+
+    def __call__(self, text: str) -> str:
+        return self.predict_batch([text])[0]
+
+
+class SklearnAdapter:
+    """Wrap a fitted scikit-learn Pipeline (e.g. TfidfVectorizer + LogisticRegression)."""
+
+    def __init__(self, pipeline: Any, name: str = "sklearn"):
+        self.pipeline, self.name = pipeline, name
+
+    def predict_batch(self, texts: Sequence[str]) -> List[Any]:
+        out = self.pipeline.predict(list(texts))
+        return out.tolist() if hasattr(out, "tolist") else list(out)
+
+    def __call__(self, text: str):
+        return self.predict_batch([text])[0]
+
+
+def from_sklearn(pipeline: Any, name: str = "sklearn") -> SklearnAdapter:
+    return SklearnAdapter(pipeline, name)
+
+
+class LLMClassifier:
+    """Classify with any LLM through a function `complete(prompt) -> str` that you provide.
+
+    The reply is accepted only if it clearly names exactly one allowed label; otherwise it is recorded as
+    `invalid_label`. Use temperature 0 so results are repeatable. `n_invalid` counts unusable replies.
+    """
+
+    def __init__(self, complete: Callable[[str], str], labels: Sequence[str], instruction: Optional[str] = None,
+                 name: str = "llm", invalid_label: str = "__invalid__"):
+        self.complete, self.labels, self.name, self.invalid_label = complete, list(labels), name, invalid_label
+        self.instruction = instruction or "Classify the customer message into exactly one label."
+        self.n_invalid = 0
+
+    def _prompt(self, text: str) -> str:
+        return (f"{self.instruction}\nAllowed labels: {', '.join(self.labels)}\n"
+                f"Message: {text}\nAnswer with the label only.")
+
+    def parse(self, reply: str) -> str:
+        reply = (reply or "").strip().lower()
+        if reply in self.labels:
+            return reply
+        found = [l for l in self.labels if re.search(rf"(?<![a-z_]){re.escape(l.lower())}(?![a-z_])", reply)]
+        if len(found) == 1:
+            return found[0]
+        self.n_invalid += 1
+        return self.invalid_label
+
+    def __call__(self, text: str) -> str:
+        return self.parse(self.complete(self._prompt(text)))
+
+    def predict_batch(self, texts: Sequence[str]) -> List[str]:
+        return [self(t) for t in texts]
+
+
+def llm_classifier(complete: Callable[[str], str], labels: Sequence[str], **kw) -> LLMClassifier:
+    return LLMClassifier(complete, labels, **kw)
+
+
+def with_retries(fn: Callable, retries: int = 3, backoff: float = 0.5,
+                 exceptions: tuple = (Exception,)) -> Callable:
+    """Retry flaky calls (network/LLM APIs) with exponential backoff; the last error is re-raised."""
+    def wrapped(*args, **kwargs):
+        for attempt in range(retries + 1):
+            try:
+                return fn(*args, **kwargs)
+            except exceptions:
+                if attempt == retries:
+                    raise
+                time.sleep(backoff * (2 ** attempt))
+    wrapped.__name__ = getattr(fn, "__name__", "wrapped")
+    return wrapped

fliprate/aksharantar.py

@@ -0,0 +1,297 @@
+"""
+Loader for the Aksharantar dataset spelling variant groups.
+
+The Aksharantar dataset (AI4Bharat, 2022, CC BY 4.0) contains 24M+ native-script
+→ romanization word pairs for 21 Indian languages, sourced from IndicCorp,
+Samanantar, Dakshina, and other curated Indian language corpora.
+
+Reference
+---------
+Mittal et al. (2022) "Aksharantar: Towards Building Open Transliteration Tools
+for the Next Billion Users." https://arxiv.org/abs/2205.03018
+
+Dataset location
+----------------
+The loader looks for language directories (hin/, tam/, etc.) relative to the
+current working directory first, then next to the fliprate package source.
+Override with FLIPRATE_AKSHARANTAR_PATH environment variable.
+
+Usage
+-----
+    from fliprate.aksharantar import aksharantar_rules, load_aksharantar_groups
+
+    # Single language
+    rules = aksharantar_rules("ta")
+
+    # Multiple languages
+    rules = aksharantar_rules("ta", "kn", "te")
+
+    # All 21 languages
+    rules = aksharantar_rules()
+
+    # Just the groups (for manual inspection or further filtering)
+    groups = load_aksharantar_groups("hi")
+    print(groups[:5])
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from .rules import RuleSet, DEFAULT_RULES, word_key
+
+SUPPORTED_LANGUAGES = {
+    "hin": "Hindi",
+    "guj": "Gujarati",
+    "tam": "Tamil",
+    "tel": "Telugu",
+    "kan": "Kannada",
+    "mal": "Malayalam",
+    "mar": "Marathi",
+    "pan": "Punjabi",
+    "ben": "Bengali",
+    "urd": "Urdu",
+    "ori": "Odia",
+    "asm": "Assamese",
+    "kas": "Kashmiri",
+    "kok": "Konkani",
+    "mai": "Maithili",
+    "mni": "Manipuri",
+    "nep": "Nepali",
+    "san": "Sanskrit",
+    "brx": "Bodo",
+    "sid": "Sindhi",
+}
+
+_RUN_RE = re.compile(r"(.)\1+")
+_PURE_ALPHA = re.compile(r"^[a-z]{2,30}$")
+_NEGATION_KEYS = frozenset({
+    "nahi", "nhi", "nahin", "nahee", "nako", "nko", "mat", "maat",
+    "not", "no", "never",
+    "ila", "ilai", "beda", "ledu", "vadu", "kadu", "venda", "ala",
+    "noy", "nei", "nathi",
+})
+_SKIP = frozenset({
+    "na", "nai", "he", "to", "ho", "re", "ki", "ko", "po",
+    "la", "le", "se", "ka", "ke", "sa", "ne", "pe", "me",
+    "aa", "ee", "oo", "ha", "hi", "hu",
+})
+
+
+def _find_root() -> Optional[Path]:
+    env = os.environ.get("FLIPRATE_AKSHARANTAR_PATH")
+    if env:
+        p = Path(env)
+        if p.exists():
+            return p
+    candidates = [
+        # New consolidated folder first
+        Path.cwd() / "Aksharantar_All_Languages",
+        Path(__file__).parent.parent.parent / "Aksharantar_All_Languages",
+        Path(__file__).parent.parent.parent.parent / "Aksharantar_All_Languages",
+        # Legacy: language folders directly in cwd
+        Path.cwd(),
+        Path(__file__).parent.parent.parent,
+        Path(__file__).parent.parent.parent.parent,
+    ]
+    for c in candidates:
+        if (c / "hin").exists() and (c / "tam").exists():
+            return c
+    return None
+
+
+def _load_lang(root: Path, lang: str) -> Dict[str, Dict[str, int]]:
+    data: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
+    for split in ("train", "valid", "test"):
+        f = root / lang / f"{lang}_{split}.json"
+        if not f.exists():
+            continue
+        with open(f, encoding="utf-8") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    d = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                native = d.get("native word", "").strip()
+                roman = d.get("english word", "").strip().lower()
+                if native and roman:
+                    data[native][roman] += 1
+    return data
+
+
+def _extract(
+    lang_data: Dict[str, Dict[str, int]],
+    min_variants: int = 2,
+) -> List[Tuple[str, ...]]:
+    groups: List[Tuple[str, ...]] = []
+    registered: Dict[str, int] = {}
+
+    for native, roman_counts in lang_data.items():
+        clean = {r: c for r, c in roman_counts.items()
+                 if _PURE_ALPHA.match(r) and r not in _SKIP}
+        if len(clean) < min_variants:
+            continue
+
+        sorted_r = sorted(clean.items(), key=lambda x: (-x[1], x[0]))
+        members = [r for r, _ in sorted_r]
+
+        seen: Dict[str, bool] = {}
+        deduped = []
+        for r in members:
+            k = word_key(r)
+            if k not in seen:
+                seen[k] = True
+                deduped.append(r)
+
+        if len(deduped) < min_variants:
+            continue
+
+        group = tuple(deduped)
+        group_keys = {word_key(r) for r in group}
+
+        # Skip negation groups entirely (conservative)
+        if group_keys & _NEGATION_KEYS:
+            continue
+
+        # Cross-native-word key conflict — first-wins
+        gid = len(groups)
+        existing_gid = None
+        conflict = False
+        for k in group_keys:
+            existing = registered.get(k)
+            if existing is not None:
+                if existing_gid is None:
+                    existing_gid = existing
+                elif existing_gid != existing:
+                    conflict = True
+                    break
+        if conflict:
+            continue
+
+        for k in group_keys:
+            registered[k] = gid
+        groups.append(group)
+
+    return groups
+
+
+# Cache
+_cache: Dict[str, List[Tuple[str, ...]]] = {}
+
+
+def load_aksharantar_groups(
+    *languages: str,
+    min_variants: int = 2,
+    aksharantar_path: Optional[str] = None,
+) -> List[Tuple[str, ...]]:
+    """
+    Load Aksharantar spelling variant groups for the given languages.
+
+    Parameters
+    ----------
+    *languages : str
+        Language codes (e.g. "hin", "tam", "tel").
+        Pass no arguments to load all 21 supported languages.
+    min_variants : int
+        Minimum number of distinct romanizations per group (default 2).
+    aksharantar_path : str, optional
+        Override the auto-detected root directory.
+
+    Returns
+    -------
+    List of tuples — ready to pass to RuleSet() or dakshina_rules().
+
+    Raises
+    ------
+    FileNotFoundError
+        If the Aksharantar language directories cannot be found.
+    """
+    langs = list(languages) if languages else list(SUPPORTED_LANGUAGES.keys())
+    unknown = [l for l in langs if l not in SUPPORTED_LANGUAGES]
+    if unknown:
+        raise ValueError(f"Unknown language codes: {unknown}. "
+                         f"Supported: {list(SUPPORTED_LANGUAGES.keys())}")
+
+    root = Path(aksharantar_path) if aksharantar_path else _find_root()
+    if root is None:
+        raise FileNotFoundError(
+            "Aksharantar language directories not found. "
+            "Set FLIPRATE_AKSHARANTAR_PATH to the folder containing hin/, tam/, etc."
+        )
+
+    all_groups: List[Tuple[str, ...]] = []
+    for lang in langs:
+        cache_key = f"aksh:{lang}:{root}:{min_variants}"
+        if cache_key not in _cache:
+            lang_dir = root / lang
+            if not lang_dir.exists():
+                print(f"[fliprate.aksharantar] Warning: {lang_dir} not found, skipping",
+                      file=sys.stderr)
+                _cache[cache_key] = []
+                continue
+            data = _load_lang(root, lang)
+            _cache[cache_key] = _extract(data, min_variants)
+        all_groups.extend(_cache[cache_key])
+
+    return all_groups
+
+
+def aksharantar_rules(
+    *languages: str,
+    base: Optional[RuleSet] = None,
+    min_variants: int = 2,
+    aksharantar_path: Optional[str] = None,
+) -> RuleSet:
+    """
+    Build a RuleSet combining `base` with Aksharantar groups.
+
+    Parameters
+    ----------
+    *languages : str
+        Language codes. Pass none for all 21 languages.
+    base : RuleSet, optional
+        Base rules to combine with (default: DEFAULT_RULES).
+    min_variants : int
+        Minimum distinct romanizations per group (default 2).
+    aksharantar_path : str, optional
+        Override auto-detected root path.
+
+    Returns
+    -------
+    RuleSet
+
+    Examples
+    --------
+    >>> from fliprate.aksharantar import aksharantar_rules
+    >>> rules = aksharantar_rules("tam", "kn")
+    >>> fr.test(my_model, dataset, rules=rules)
+
+    Notes
+    -----
+    For the best coverage use both datasets:
+    >>> from fliprate.dakshina import dakshina_rules
+    >>> rules = dakshina_rules("ta").extended(
+    ...     groups=load_aksharantar_groups("tam")
+    ... )
+    """
+    groups = load_aksharantar_groups(
+        *languages,
+        min_variants=min_variants,
+        aksharantar_path=aksharantar_path,
+    )
+    b = base if base is not None else DEFAULT_RULES
+    combined = list(b.groups) + groups
+    return RuleSet(
+        groups=combined,
+        lexicon=list(b.lexicon),
+        negations=b.negations,
+        ambiguous=dict(b.ambiguous),
+    )