controlmt 0.1.0__py3-none-any.whl

controlmt/__init__.py

@@ -0,0 +1,25 @@
+"""ControlMT — Python SDK for the ControlMT v2.3 KN↔EN translator.
+
+Drop-in entry point for users who don't want to wire HuggingFace Transformers
+themselves. Picks the right device + dtype + quantization automatically;
+overridable when you need control.
+
+Quick start:
+    from controlmt import ControlMT
+    model = ControlMT.from_hf()                       # auto everything
+    print(model.translate("ನಾನು ಕನ್ನಡ ಮಾತನಾಡುತ್ತೇನೆ."))  # → "I speak Kannada."
+
+Explicit:
+    model = ControlMT.from_hf(device="cpu", quant="int8")     # int8 dynamic on CPU
+    model = ControlMT.from_hf(device="gpu", dtype="float16")  # fp16 on GPU
+    model = ControlMT.from_hf(device="auto")                  # GPU-with-CPU-fallback (default)
+
+Batched (user-defined size, defaults to 1 = no batching):
+    model.batch_translate(texts, batch_size=8)
+    model.batch_translate(texts, auto_batch=True)             # GPU-only auto-fit by VRAM
+"""
+
+from controlmt.client import ControlMT
+from controlmt._version import __version__
+
+__all__ = ["ControlMT", "__version__"]

controlmt/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

controlmt/batching.py

@@ -0,0 +1,47 @@
+"""Auto-batch sizing for GPU inference.
+
+ControlMT's `translate()` runs beam search sentence-by-sentence, but we
+ThreadPoolExecutor across N concurrent calls. N is bounded by free VRAM.
+
+Memory per concurrent translation at beam=2, max_len=256:
+    fp16: ~25 MB    (encoder + decoder state + beam tensors + kv-cache)
+    bf16: ~25 MB
+    fp32: ~50 MB
+"""
+
+from __future__ import annotations
+import warnings
+
+
+_MEM_PER_SENT_MB = {
+    "float16":  25,
+    "bfloat16": 25,
+    "float32":  50,
+    "int8-dynamic": 12,
+}
+_MAX_BATCH = 64       # ControlMT.translate() beam-search per-sentence — gain plateaus past ~16
+_TARGET_VRAM_FRAC = 0.8
+
+
+def auto_batch_size(device: str, dtype: str, quant: str = "none",
+                    free_vram_mb: float | None = None) -> int:
+    """Estimate batch size that fits in ~80% of free VRAM.
+
+    Returns 1 on CPU (no auto-batching there) with a warning.
+    On GPU returns 1..MAX_BATCH.
+    """
+    if device != "cuda":
+        warnings.warn(
+            "auto_batch=True ignored on CPU (no VRAM signal to size against). "
+            "Pass an explicit batch_size for batched CPU translation.",
+            RuntimeWarning, stacklevel=2)
+        return 1
+
+    if free_vram_mb is None:
+        import torch
+        free, _total = torch.cuda.mem_get_info()
+        free_vram_mb = free / 1024**2
+
+    per_sent_mb = _MEM_PER_SENT_MB.get(quant if quant != "none" else dtype, 50)
+    n = int((free_vram_mb * _TARGET_VRAM_FRAC) // per_sent_mb)
+    return max(1, min(_MAX_BATCH, n))

controlmt/client.py

@@ -0,0 +1,202 @@
+"""High-level client for ControlMT v2.3."""
+
+from __future__ import annotations
+import re
+import time
+import warnings
+from concurrent.futures import ThreadPoolExecutor
+from typing import Iterable, Sequence
+
+from controlmt.device import ResolvedConfig, detect_libraries, resolve
+from controlmt.batching import auto_batch_size
+
+
+DEFAULT_MODEL_ID = "anandkaman/controlmt-v2.3"
+
+# Heuristic: 'kn2en' if input is mostly Kannada chars, 'en2kn' otherwise.
+_KN_RE = re.compile(r"[ಀ-೿]")
+
+
+class ControlMT:
+    """High-level wrapper around the HuggingFace `model.translate()` API.
+
+    Usage:
+        from controlmt import ControlMT
+        model = ControlMT.from_hf()                              # auto
+        model.translate("ನಾನು ಕನ್ನಡ ಮಾತನಾಡುತ್ತೇನೆ.")
+        model.batch_translate(texts, batch_size=8)
+        model.batch_translate(texts, auto_batch=True)            # GPU only
+    """
+
+    # ──────────────────────────────────────────────────────────────
+    #  Construction
+    # ──────────────────────────────────────────────────────────────
+    def __init__(self, hf_model, tokenizer, config: ResolvedConfig, model_id: str):
+        self._model = hf_model
+        self._tokenizer = tokenizer
+        self.config = config
+        self.model_id = model_id
+
+    @classmethod
+    def from_hf(
+        cls,
+        model_id: str = DEFAULT_MODEL_ID,
+        *,
+        device: str = "auto",                    # "auto" | "gpu" | "cuda" | "cpu"
+        dtype: str | None = None,                # "float32" | "bfloat16" | "float16" | None
+        quant: str = "none",                     # "none" | "int8" (CPU-only)
+        revision: str | None = None,             # HF revision
+        verbose: bool = False,
+    ) -> "ControlMT":
+        """Load ControlMT from HuggingFace + auto-resolve config.
+
+        Defaults: GPU if available, else CPU. fp16 on GPU, bf16 on CPU. No quantization.
+        """
+        import torch
+        from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+        cfg = resolve(device=device, dtype=dtype, quant=quant)
+
+        if verbose:
+            libs = detect_libraries()
+            print(f"[controlmt] loading {model_id}  ({cfg.describe()})")
+            print(f"[controlmt] env  torch={libs.get('torch')}  transformers={libs.get('transformers')}  "
+                  f"cuda={libs.get('_cuda')}  device={libs.get('_cuda_device')}")
+
+        torch_dtype = {"float32": torch.float32, "bfloat16": torch.bfloat16,
+                       "float16": torch.float16}[cfg.dtype_str]
+
+        load_kw = {"trust_remote_code": True}
+        if revision: load_kw["revision"] = revision
+        if cfg.quant != "int8-dynamic" and cfg.dtype_str != "float32":
+            load_kw["dtype"] = torch_dtype   # let HF cast during load
+
+        tokenizer = AutoTokenizer.from_pretrained(model_id, **{k: v for k, v in load_kw.items() if k != "dtype"})
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_id, **load_kw)
+
+        # int8 dynamic quantization (CPU-only — checked in resolve())
+        if cfg.quant == "int8-dynamic":
+            model = torch.quantization.quantize_dynamic(model, {torch.nn.Linear}, dtype=torch.qint8)
+
+        model = model.to(torch.device(cfg.device)).eval()
+        return cls(hf_model=model, tokenizer=tokenizer, config=cfg, model_id=model_id)
+
+    # ──────────────────────────────────────────────────────────────
+    #  Inference
+    # ──────────────────────────────────────────────────────────────
+    def translate(
+        self,
+        text: str,
+        *,
+        direction: str | None = None,    # "kn2en" | "en2kn" | None=auto-detect
+        num_beams: int = 2,
+        anti_lm_alpha: float = 0.5,
+        max_length: int = 200,
+    ) -> str:
+        """Translate one sentence. Direction auto-detected from input script if not given."""
+        if not text or not text.strip():
+            return ""
+        if direction is None:
+            direction = self.detect_direction(text)
+        return self._model.translate(
+            text.strip(),
+            tokenizer=self._tokenizer,
+            direction=direction,
+            num_beams=num_beams,
+            anti_lm_alpha=anti_lm_alpha,
+            max_length=max_length,
+        )
+
+    def batch_translate(
+        self,
+        texts: Sequence[str],
+        *,
+        batch_size: int | None = None,         # None → 1 (safe), or int N
+        auto_batch: bool = False,              # GPU only — auto-fit by VRAM
+        direction: str | None = None,
+        num_beams: int = 2,
+        anti_lm_alpha: float = 0.5,
+        max_length: int = 200,
+    ) -> list[str]:
+        """Translate a batch. User must supply batch_size, or auto_batch=True on GPU.
+
+        Policy (matching DEPLOYMENT.md §11):
+          - batch_size=None and auto_batch=False  → batch_size=1 (one at a time, safe)
+          - batch_size=N                          → uses N concurrent translations
+          - auto_batch=True                       → GPU: probe free VRAM, pick N; CPU: warn + N=1
+
+        We use a ThreadPoolExecutor — each thread runs one model.translate() call.
+        For our model this is ~the same throughput as a true batched call (beam
+        search is per-sentence), with much simpler code.
+        """
+        if not texts:
+            return []
+
+        if auto_batch:
+            batch_size = auto_batch_size(
+                device=self.config.device,
+                dtype=self.config.dtype_str,
+                quant=self.config.quant,
+            )
+        if batch_size is None:
+            batch_size = 1
+        batch_size = max(1, int(batch_size))
+
+        # If batch_size=1, no need for threadpool overhead
+        if batch_size == 1:
+            return [self.translate(t, direction=direction, num_beams=num_beams,
+                                   anti_lm_alpha=anti_lm_alpha, max_length=max_length)
+                    for t in texts]
+
+        def _one(t: str) -> str:
+            return self.translate(t, direction=direction, num_beams=num_beams,
+                                  anti_lm_alpha=anti_lm_alpha, max_length=max_length)
+
+        with ThreadPoolExecutor(max_workers=batch_size) as ex:
+            return list(ex.map(_one, texts))
+
+    # ──────────────────────────────────────────────────────────────
+    #  Helpers
+    # ──────────────────────────────────────────────────────────────
+    @staticmethod
+    def detect_direction(text: str) -> str:
+        """Heuristic: > 30% Kannada characters → kn2en, else en2kn."""
+        if not text: return "en2kn"
+        kn_chars = sum(1 for c in text if _KN_RE.match(c))
+        total = sum(1 for c in text if not c.isspace() and c.isprintable())
+        return "kn2en" if total and kn_chars / total > 0.3 else "en2kn"
+
+    def warmup(self) -> float:
+        """JIT/compile kernels on a throwaway translation. Returns elapsed seconds."""
+        t0 = time.time()
+        self.translate("hello.", direction="en2kn", num_beams=1, max_length=20)
+        return time.time() - t0
+
+    def benchmark(self, num_beams: int = 2) -> dict:
+        """Run the 6-pair DEPLOYMENT.md verification suite on this loaded model.
+        Returns the same shape as scripts/verify_deployment.py's JSON output."""
+        TEST_PAIRS = [
+            ("kn2en", "ನಾನು ಕನ್ನಡ ಮಾತನಾಡುತ್ತೇನೆ."),
+            ("kn2en", "ಬೆಂಗಳೂರಿನಲ್ಲಿ ಮೆಟ್ರೋ ಬಹಳ ಅನುಕೂಲಕರವಾಗಿದೆ."),
+            ("kn2en", "ಆಪಲ್ ಹೊಸ ಐಫೋನ್ 17 ಬಿಡುಗಡೆ ಮಾಡಿದೆ."),
+            ("en2kn", "I speak Kannada."),
+            ("en2kn", "The new metro line opens next month."),
+            ("en2kn", "Please transfer money to my UPI ID."),
+        ]
+        self.warmup()
+        rows = []
+        for direction, src in TEST_PAIRS:
+            t0 = time.time()
+            out = self.translate(src, direction=direction, num_beams=num_beams)
+            rows.append({"direction": direction, "src": src, "out": out,
+                         "latency_s": round(time.time() - t0, 3)})
+        lats = sorted([r["latency_s"] for r in rows])
+        return {
+            "config": self.config.describe(),
+            "num_beams": num_beams,
+            "rows": rows,
+            "median_latency_s": lats[len(lats)//2],
+        }
+
+    def __repr__(self) -> str:
+        return f"<ControlMT model_id={self.model_id!r} {self.config.describe()}>"

controlmt/device.py

@@ -0,0 +1,129 @@
+"""Device + dtype + quantization auto-detection.
+
+Resolves the user's (device, dtype, quant) preferences into concrete torch
+objects. Always returns a usable config — falls back to CPU if GPU is
+requested but unavailable, with a warning.
+"""
+
+from __future__ import annotations
+import warnings
+from dataclasses import dataclass
+
+
+@dataclass
+class ResolvedConfig:
+    device: str         # "cuda" or "cpu"
+    dtype_str: str      # "float32" | "bfloat16" | "float16"
+    quant: str          # "none" | "int8-dynamic"
+    bf16_cpu: bool      # convenience flag — bf16 works on CPU iff torch supports it
+
+    def describe(self) -> str:
+        bits = [self.device, self.dtype_str]
+        if self.quant != "none":
+            bits.append(self.quant)
+        return " · ".join(bits)
+
+
+def resolve(
+    device: str = "auto",       # "auto" | "gpu" | "cuda" | "cpu"
+    dtype: str | None = None,   # "float32" | "bfloat16" | "float16" | None
+    quant: str = "none",        # "none" | "int8" / "int8-dynamic"
+) -> ResolvedConfig:
+    """Resolve the user's preferences into a concrete config.
+
+    Logic:
+      - device="auto" (default): GPU if available, else CPU with a quiet info note
+      - device="gpu" or "cuda": GPU required; fall back to CPU with a WARNING if absent
+      - device="cpu": forces CPU
+      - dtype=None: pick the best dtype for the resolved device
+            GPU: float16 (broadest compatibility — works on Volta+, Pascal too)
+            CPU: bfloat16 if torch supports it (~2.8× faster than fp32 in our tests),
+                 else float32
+      - quant: "int8" / "int8-dynamic" only valid on CPU. GPU + int8 = silently
+            falls back to fp16 with a warning (custom-arch incompat with bitsandbytes —
+            see DEPLOYMENT.md §9).
+    """
+    import torch
+
+    # ── device ────────────────────────────────────────────────────
+    device = device.lower()
+    if device in ("gpu", "cuda"):
+        if torch.cuda.is_available():
+            resolved_device = "cuda"
+        else:
+            warnings.warn("device='gpu' requested but CUDA unavailable; falling back to CPU.",
+                          RuntimeWarning, stacklevel=2)
+            resolved_device = "cpu"
+    elif device == "cpu":
+        resolved_device = "cpu"
+    elif device == "auto":
+        resolved_device = "cuda" if torch.cuda.is_available() else "cpu"
+    else:
+        raise ValueError(f"unknown device {device!r} — use 'auto', 'gpu', 'cuda', or 'cpu'")
+
+    # ── quant ─────────────────────────────────────────────────────
+    quant = (quant or "none").lower().replace("-dynamic", "")
+    if quant in ("int8", "int8_dynamic"):
+        quant = "int8-dynamic"
+    if quant not in ("none", "int8-dynamic"):
+        raise ValueError(f"unknown quant {quant!r} — use 'none' or 'int8' (CPU-only)")
+
+    if quant == "int8-dynamic" and resolved_device == "cuda":
+        warnings.warn(
+            "int8 dynamic quantization is CPU-only on this model "
+            "(see DEPLOYMENT.md §9). Falling back to fp16 on GPU.",
+            RuntimeWarning, stacklevel=2)
+        quant = "none"
+
+    # ── dtype ─────────────────────────────────────────────────────
+    bf16_cpu_ok = _cpu_supports_bf16(torch)
+    if dtype is None:
+        if quant == "int8-dynamic":
+            resolved_dtype = "float32"       # quantize_dynamic operates on fp32 weights
+        elif resolved_device == "cuda":
+            resolved_dtype = "float16"       # widest GPU compatibility
+        else:
+            resolved_dtype = "bfloat16" if bf16_cpu_ok else "float32"
+    else:
+        dtype = dtype.lower().replace("fp32", "float32").replace("fp16", "float16").replace("bf16", "bfloat16")
+        if dtype not in ("float32", "float16", "bfloat16"):
+            raise ValueError(f"unknown dtype {dtype!r} — use float32/float16/bfloat16")
+        resolved_dtype = dtype
+
+    return ResolvedConfig(
+        device=resolved_device,
+        dtype_str=resolved_dtype,
+        quant=quant,
+        bf16_cpu=bf16_cpu_ok,
+    )
+
+
+def _cpu_supports_bf16(torch_mod) -> bool:
+    """Quick runtime probe — try a 1-element bf16 add. Some old CPUs and some
+    libtorch builds segfault on bf16; this catches that path before model load."""
+    try:
+        x = torch_mod.zeros(1, dtype=torch_mod.bfloat16)
+        _ = x + x
+        return True
+    except Exception:
+        return False
+
+
+def detect_libraries() -> dict:
+    """Probe what's installed. Used for the SDK's status banner + auto-picking
+    code paths (e.g. if onnxruntime is present, future ONNX backend can be used)."""
+    out = {}
+    for name in ("torch", "transformers", "accelerate", "bitsandbytes", "onnxruntime", "safetensors"):
+        try:
+            mod = __import__(name)
+            out[name] = getattr(mod, "__version__", "?")
+        except ImportError:
+            out[name] = None
+    try:
+        import torch
+        out["_cuda"] = torch.cuda.is_available()
+        out["_cuda_device"] = torch.cuda.get_device_name(0) if out["_cuda"] else None
+    except Exception:
+        out["_cuda"] = False
+        out["_cuda_device"] = None
+    return out

controlmt-0.1.0.dist-info/METADATA

@@ -0,0 +1,135 @@
+Metadata-Version: 2.4
+Name: controlmt
+Version: 0.1.0
+Summary: Python SDK for the ControlMT v2.3 Kannada↔English translator
+Author: Anand Kaman
+License: Apache-2.0
+Project-URL: Homepage, https://huggingface.co/anandkaman/controlmt-v2.3
+Project-URL: Source, https://github.com/anandkaman/ControlMT
+Project-URL: Demo, https://huggingface.co/spaces/anandkaman/controlmt-demo
+Keywords: translation,kannada,machine-translation,nlp,indic,controlmt
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: torch<3,>=2.0
+Requires-Dist: transformers<5,>=4.40
+Requires-Dist: sentencepiece>=0.1.99
+Requires-Dist: safetensors>=0.4
+Requires-Dist: huggingface_hub>=0.27
+Provides-Extra: test
+Requires-Dist: pytest>=7; extra == "test"
+
+# controlmt
+
+Python SDK for **[ControlMT v2.3](https://huggingface.co/anandkaman/controlmt-v2.3)** —
+a compact 139M-parameter Kannada ↔ English translator.
+
+```bash
+pip install controlmt
+```
+
+## Quick start
+
+```python
+from controlmt import ControlMT
+
+model = ControlMT.from_hf()                                    # auto everything
+print(model.translate("ನಾನು ಕನ್ನಡ ಮಾತನಾಡುತ್ತೇನೆ."))
+# → "I speak Kannada."
+```
+
+That's it. The SDK auto-detects:
+- **Device** — GPU if CUDA is available, else CPU (overridable)
+- **Dtype** — fp16 on GPU, bf16 on CPU (overridable; bf16 only when supported)
+- **Direction** — Kannada → English if input is mostly Kannada chars, else reverse
+
+## Explicit control
+
+```python
+# Force CPU, even on a GPU box
+model = ControlMT.from_hf(device="cpu")
+
+# Force GPU; falls back to CPU with a warning if not present
+model = ControlMT.from_hf(device="gpu")
+
+# Pick an exact dtype
+model = ControlMT.from_hf(device="cpu", dtype="bf16")    # bfloat16
+model = ControlMT.from_hf(device="gpu", dtype="fp16")    # float16
+model = ControlMT.from_hf(dtype="fp32")                  # full precision
+
+# CPU int8 dynamic quantization (~2× faster than bf16 on CPU)
+model = ControlMT.from_hf(device="cpu", quant="int8")
+
+# Specific HF revision (e.g. a pre-quantized branch)
+model = ControlMT.from_hf(model_id="anandkaman/controlmt-v2.3-int8", quant="int8")
+
+# Loud: print the auto-pick decisions
+model = ControlMT.from_hf(verbose=True)
+```
+
+Inspect the resolved config:
+```python
+>>> model
+<ControlMT model_id='anandkaman/controlmt-v2.3' cuda · float16>
+>>> model.config
+ResolvedConfig(device='cuda', dtype_str='float16', quant='none', bf16_cpu=True)
+```
+
+## Batched translation
+
+By design: **you must specify `batch_size` to opt into batching.** Otherwise the SDK
+runs one sentence at a time — predictable memory, no surprises.
+
+```python
+texts = ["ನಾನು ಕನ್ನಡ.", "I speak English.", ...]
+
+# Default: one at a time (safe everywhere)
+outs = model.batch_translate(texts)
+
+# Explicit fixed batch size
+outs = model.batch_translate(texts, batch_size=8)
+
+# Auto-pick batch size from free VRAM (GPU only)
+outs = model.batch_translate(texts, auto_batch=True)
+```
+
+| Mode | CPU | GPU |
+|---|---|---|
+| `(no batch_size, no auto_batch)` | 1 sentence at a time | 1 sentence at a time |
+| `batch_size=N` | uses N | uses N |
+| `auto_batch=True` | ignored + warning → 1 | probes `torch.cuda.mem_get_info()`, picks N ≤ 64 |
+
+## Other endpoints
+
+```python
+# Heuristic direction detection (>30% KN chars → kn2en, else en2kn)
+ControlMT.detect_direction("ನಾನು ಕನ್ನಡ.")    # → "kn2en"
+
+# JIT/compile warmup — kills the 5–10s "first request" lag in production
+model.warmup()
+
+# Run the 6-pair DEPLOYMENT.md benchmark suite on YOUR hardware
+result = model.benchmark()
+# {'config': 'cuda · float16', 'num_beams': 2, 'median_latency_s': 0.19, 'rows': [...]}
+```
+
+## Architecture note
+
+ControlMT v2.3 is an **encoder-decoder seq2seq** model (T5/mBART family), not a
+decoder-only LM. That means:
+
+- ✅ Works: this SDK, raw Transformers, FastAPI, Docker, HF Inference Endpoints
+- ❌ Doesn't work without significant adapter work: vLLM, Ollama, llama.cpp/GGUF, HF TGI
+
+See [DEPLOYMENT.md](https://huggingface.co/anandkaman/controlmt-v2.3/blob/main/DEPLOYMENT.md) §9 for the full "not supported" table and why.
+
+## License
+
+Apache 2.0. Same as the underlying model weights.

controlmt-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+controlmt/__init__.py,sha256=gcO_c2HXEmaAEbB31WRYK7polvVQgbPr0rJJJcwN3DU,1096
+controlmt/_version.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
+controlmt/batching.py,sha256=w7-G6ayshjM7tuD1HIVDLO9Os-VggmzZmjGNw2WE05E,1511
+controlmt/client.py,sha256=1-K2mkdvY_GHY8lhcdolT8b2uRRl_VoK_PY4eHhQJT8,9221
+controlmt/device.py,sha256=k2EImENU5Vxtu1nKUCvK2I9rMha_KYqooSjjt7gbvD8,5435
+controlmt-0.1.0.dist-info/METADATA,sha256=AKtjzm9IcA-QGZePWOawV0ZSeoRX96GKAG5p5vQ-kIE,4639
+controlmt-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+controlmt-0.1.0.dist-info/top_level.txt,sha256=1fMNkQVgA081BBsoTHm_8vD7BpDL-kFl3t5VvdJZkbY,10
+controlmt-0.1.0.dist-info/RECORD,,

controlmt-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

controlmt-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+controlmt