bhashantar 0.1.0__py3-none-any.whl

bhashantar/__init__.py

@@ -0,0 +1,29 @@
+"""IndicTrans2 English-to-Indic translation on CPU, without transformers or fairseq."""
+
+from bhashantar.engine import (
+    MAX_BATCH_CHARS,
+    MAX_BATCH_SENTENCES,
+    MAX_SENTENCE_CHARS,
+    MAX_SOURCE_TOKENS,
+    InputTooLarge,
+    ModelNotAvailable,
+    Translator,
+    translate,
+)
+from bhashantar.languages import SOURCE, TARGETS, UnknownLanguage
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "MAX_BATCH_CHARS",
+    "MAX_BATCH_SENTENCES",
+    "MAX_SENTENCE_CHARS",
+    "MAX_SOURCE_TOKENS",
+    "SOURCE",
+    "TARGETS",
+    "InputTooLarge",
+    "ModelNotAvailable",
+    "Translator",
+    "UnknownLanguage",
+    "translate",
+]

bhashantar/cli.py

@@ -0,0 +1,70 @@
+"""bhashantar command line: translate lines, list targets, convert the checkpoint."""
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+from bhashantar import engine
+from bhashantar.languages import SOURCE, TARGETS, UnknownLanguage
+
+
+def _translate(args) -> int:
+    if args.text:
+        lines = args.text
+    else:
+        stream = open(args.input, encoding="utf-8") if args.input else sys.stdin
+        with stream:
+            lines = [line.rstrip("\n") for line in stream]
+    translator = engine.Translator(model_dir=args.model, threads=args.threads, beam_size=args.beam)
+    out = []
+    for start in range(0, len(lines), engine.MAX_BATCH_SENTENCES):
+        out += translator.translate(lines[start : start + engine.MAX_BATCH_SENTENCES], src=args.src, tgt=args.tgt)
+    sys.stdout.reconfigure(encoding="utf-8")
+    sys.stdout.write("\n".join(out) + "\n")
+    return 0
+
+
+def _convert(args) -> int:
+    from bhashantar import convert
+
+    checkpoint = Path(args.checkpoint) if args.checkpoint else convert.download_checkpoint()
+    out = convert.convert(checkpoint, Path(args.out) if args.out else engine.converted_model_dir())
+    print(out)
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="bhashantar", description="IndicTrans2 en-indic-dist-200M on CPU via CTranslate2 int8")
+    parser.add_argument("-v", "--verbose", action="store_true")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    t = sub.add_parser("translate", help="translate English lines (arguments, --input file, or stdin)")
+    t.add_argument("text", nargs="*")
+    t.add_argument("--tgt", required=True, help="target code, e.g. hin_Deva; see `bhashantar languages`")
+    t.add_argument("--src", default=SOURCE)
+    t.add_argument("-i", "--input", help="UTF-8 file, one sentence per line")
+    t.add_argument("--model", help="converted model directory (default: cache, then hub)")
+    t.add_argument("--threads", type=int, default=4)
+    t.add_argument("--beam", type=int, default=5)
+    t.set_defaults(run=_translate)
+
+    lang = sub.add_parser("languages", help="list target codes")
+    lang.set_defaults(run=lambda _args: print("\n".join(TARGETS)) or 0)
+
+    c = sub.add_parser("convert", help="convert the official checkpoint to CTranslate2 int8 (needs [convert] extra)")
+    c.add_argument("--checkpoint", help="local checkpoint directory (default: download the gated official repo)")
+    c.add_argument("--out", help=f"output directory (default: {engine.converted_model_dir()})")
+    c.set_defaults(run=_convert)
+
+    args = parser.parse_args(argv)
+    logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING, format="%(name)s: %(message)s")
+    try:
+        return args.run(args)
+    except (UnknownLanguage, engine.InputTooLarge, engine.ModelNotAvailable) as exc:
+        print(f"bhashantar: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bhashantar/convert.py

@@ -0,0 +1,176 @@
+"""Convert the Hugging Face IndicTrans2 checkpoint to a CTranslate2 model.
+
+CTranslate2 ships a converter for fairseq checkpoints but not for this
+checkpoint's custom `IndicTransForConditionalGeneration`, and the fairseq
+checkpoint needs fairseq installed. The architecture is the fairseq
+pre-norm transformer with `layernorm_embedding`, so this module fills the same
+`TransformerSpec` that `ctranslate2.converters.fairseq` fills, reading tensors
+by their Hugging Face names. Needs `pip install bhashantar[convert]`.
+"""
+
+import hashlib
+import json
+import logging
+import math
+import shutil
+from pathlib import Path
+
+import numpy as np
+
+log = logging.getLogger(__name__)
+
+SOURCE_REPO = "ai4bharat/indictrans2-en-indic-dist-200M"
+_ASSETS = ("model.SRC", "model.TGT", "LICENSE")
+
+
+def _load_tensors(checkpoint: Path) -> dict:
+    safetensors_file = checkpoint / "model.safetensors"
+    if safetensors_file.is_file():
+        from safetensors.numpy import load_file
+
+        return load_file(str(safetensors_file))
+    bin_file = checkpoint / "pytorch_model.bin"
+    if not bin_file.is_file():
+        raise FileNotFoundError(f"{checkpoint} has neither model.safetensors nor pytorch_model.bin")
+    try:
+        import torch
+    except ImportError as exc:
+        raise RuntimeError(f"{bin_file.name} is a torch pickle; install torch to read it, or use model.safetensors") from exc
+    state = torch.load(bin_file, map_location="cpu", weights_only=True, mmap=True)
+    return {name: tensor.float().numpy() for name, tensor in state.items()}
+
+
+def sinusoidal_positions(max_positions: int, dim: int, padding_idx: int) -> np.ndarray:
+    """Position table as the checkpoint computes it at runtime (it is not
+    stored). Rows start at padding_idx + 1, which is where the reference model
+    places the first token."""
+    rows = max_positions + padding_idx + 1
+    half = dim // 2
+    scale = math.log(10000) / (half - 1)
+    freqs = np.exp(np.arange(half, dtype=np.float32) * -scale)
+    angles = np.arange(rows, dtype=np.float32)[:, None] * freqs[None, :]
+    table = np.concatenate([np.sin(angles), np.cos(angles)], axis=1).astype(np.float32)
+    return table[padding_idx + 1 :]
+
+
+def _vocabulary(path: Path) -> list[str]:
+    token_to_id = json.loads(path.read_text(encoding="utf-8"))
+    tokens = [None] * len(token_to_id)
+    for token, index in token_to_id.items():
+        tokens[index] = token
+    if any(t is None for t in tokens):
+        raise ValueError(f"{path.name} ids are not contiguous from 0")
+    return tokens
+
+
+def build_spec(checkpoint: Path):
+    from ctranslate2.converters import utils
+    from ctranslate2.specs import common_spec, transformer_spec
+
+    config = json.loads((checkpoint / "config.json").read_text(encoding="utf-8"))
+    if config.get("activation_function") != "gelu":
+        raise ValueError(f"activation {config.get('activation_function')!r} not handled; expected 'gelu'")
+    if not (config["encoder_normalize_before"] and config["decoder_normalize_before"] and config["layernorm_embedding"]):
+        raise ValueError("expected pre-norm encoder and decoder with layernorm_embedding")
+
+    tensors = _load_tensors(checkpoint)
+    spec = transformer_spec.TransformerSpec.from_config(
+        (config["encoder_layers"], config["decoder_layers"]),
+        config["encoder_attention_heads"],
+        pre_norm=True,
+        activation=common_spec.Activation.GELU,
+        layernorm_embedding=True,
+    )
+
+    def take(name):
+        return tensors.pop(name)
+
+    def linear(target, prefix):
+        target.weight = take(f"{prefix}.weight")
+        target.bias = take(f"{prefix}.bias")
+
+    def norm(target, prefix):
+        target.gamma = take(f"{prefix}.weight")
+        target.beta = take(f"{prefix}.bias")
+
+    def attention(target, prefix, self_attention):
+        names = ("q_proj", "k_proj", "v_proj") if self_attention else ("k_proj", "v_proj")
+        parts = [common_spec.LinearSpec() for _ in names]
+        for part, name in zip(parts, names):
+            linear(part, f"{prefix}.{name}")
+        if self_attention:
+            utils.fuse_linear(target.linear[0], parts)
+        else:
+            linear(target.linear[0], f"{prefix}.q_proj")
+            utils.fuse_linear(target.linear[1], parts)
+        linear(target.linear[-1], f"{prefix}.out_proj")
+
+    def stack(target, prefix, dim_key, max_pos_key):
+        embeddings = target.embeddings[0] if isinstance(target.embeddings, list) else target.embeddings
+        embeddings.weight = take(f"{prefix}.embed_tokens.weight")
+        target.scale_embeddings = math.sqrt(config[dim_key]) if config["scale_embedding"] else 1.0
+        target.position_encodings.encodings = sinusoidal_positions(
+            config[max_pos_key], config[dim_key], config["pad_token_id"]
+        )
+        norm(target.layernorm_embedding, f"{prefix}.layernorm_embedding")
+        norm(target.layer_norm, f"{prefix}.layer_norm")
+        for i, layer in enumerate(target.layer):
+            p = f"{prefix}.layers.{i}"
+            attention(layer.self_attention, f"{p}.self_attn", self_attention=True)
+            norm(layer.self_attention.layer_norm, f"{p}.self_attn_layer_norm")
+            if hasattr(layer, "attention"):
+                attention(layer.attention, f"{p}.encoder_attn", self_attention=False)
+                norm(layer.attention.layer_norm, f"{p}.encoder_attn_layer_norm")
+            norm(layer.ffn.layer_norm, f"{p}.final_layer_norm")
+            linear(layer.ffn.linear_0, f"{p}.fc1")
+            linear(layer.ffn.linear_1, f"{p}.fc2")
+
+    stack(spec.encoder, "model.encoder", "encoder_embed_dim", "max_source_positions")
+    stack(spec.decoder, "model.decoder", "decoder_embed_dim", "max_target_positions")
+    spec.decoder.projection.weight = take("lm_head.weight")
+    leftovers = sorted(n for n in tensors if "embed_positions" not in n)
+    if leftovers:
+        raise ValueError(f"checkpoint tensors not mapped: {leftovers[:5]}")
+
+    spec.register_source_vocabulary(_vocabulary(checkpoint / "dict.SRC.json"))
+    spec.register_target_vocabulary(_vocabulary(checkpoint / "dict.TGT.json"))
+    spec.config.decoder_start_token = "</s>"
+    spec.config.add_source_eos = True
+    return spec
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as fh:
+        for block in iter(lambda: fh.read(1 << 20), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def convert(checkpoint: Path, out_dir: Path, quantization: str = "int8") -> Path:
+    checkpoint, out_dir = Path(checkpoint), Path(out_dir)
+    spec = build_spec(checkpoint)
+    spec.validate()
+    spec.optimize(quantization=quantization)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    spec.save(str(out_dir))
+    for name in _ASSETS:
+        if (checkpoint / name).is_file():
+            shutil.copy2(checkpoint / name, out_dir / name)
+    weights = next(p for p in (checkpoint / "model.safetensors", checkpoint / "pytorch_model.bin") if p.is_file())
+    provenance = {
+        "source_repo": SOURCE_REPO,
+        "source_weights": weights.name,
+        "source_sha256": _sha256(weights),
+        "quantization": quantization,
+    }
+    (out_dir / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n", encoding="utf-8")
+    log.info("wrote %s", out_dir)
+    return out_dir
+
+
+def download_checkpoint() -> Path:
+    from huggingface_hub import snapshot_download
+
+    files = ["config.json", "dict.SRC.json", "dict.TGT.json", "model.SRC", "model.TGT", "LICENSE", "model.safetensors"]
+    return Path(snapshot_download(SOURCE_REPO, allow_patterns=files))

bhashantar/engine.py

@@ -0,0 +1,187 @@
+"""CTranslate2 int8 inference for IndicTrans2 en-indic-dist-200M."""
+
+import logging
+import os
+from functools import lru_cache
+from pathlib import Path
+
+import ctranslate2
+import sentencepiece as spm
+
+from bhashantar import textprep
+from bhashantar.languages import SOURCE, check_direction
+
+log = logging.getLogger(__name__)
+
+HUB_REPO = "zaidwhy/indictrans2-en-indic-dist-200M-ct2-int8"
+MODEL_ENV = "BHASHANTAR_MODEL"
+
+MAX_SENTENCE_CHARS = 1000
+MAX_BATCH_SENTENCES = 512
+MAX_BATCH_CHARS = 50_000
+# The checkpoint was trained with 256 source positions: two language tags,
+# the pieces, and </s>. Longer input is refused, not truncated, because
+# truncation drops the end of the sentence without telling anyone.
+MAX_SOURCE_TOKENS = 256
+
+_SPECIAL = {"<s>", "<pad>", "</s>", "<unk>"}
+
+
+class InputTooLarge(ValueError):
+    """Input over one of the documented caps; the message names the cap."""
+
+
+class ModelNotAvailable(RuntimeError):
+    """No converted model on disk and none downloadable."""
+
+
+def converted_model_dir() -> Path:
+    hf_home = Path(os.environ.get("HF_HOME", Path.home() / ".cache" / "huggingface"))
+    return hf_home / "bhashantar" / "en-indic-dist-200M-ct2-int8"
+
+
+def resolve_model_dir(model_dir: str | os.PathLike | None = None) -> Path:
+    """Explicit path, then $BHASHANTAR_MODEL, then a local `bhashantar convert`
+    output, then the hub repo (cache first, network only on a cache miss)."""
+    if model_dir is None:
+        model_dir = os.environ.get(MODEL_ENV)
+    if model_dir is not None:
+        path = Path(model_dir)
+        if not (path / "model.bin").is_file():
+            raise ModelNotAvailable(f"{path} has no model.bin; point it at a converted CTranslate2 model directory")
+        return path
+    local = converted_model_dir()
+    if (local / "model.bin").is_file():
+        return local
+
+    from huggingface_hub import snapshot_download
+    from huggingface_hub.errors import HfHubHTTPError, LocalEntryNotFoundError
+
+    try:
+        return Path(snapshot_download(HUB_REPO, local_files_only=True))
+    except LocalEntryNotFoundError:
+        log.info("model not cached, downloading %s", HUB_REPO)
+    try:
+        return Path(snapshot_download(HUB_REPO))
+    except (HfHubHTTPError, LocalEntryNotFoundError, OSError) as exc:
+        raise ModelNotAvailable(
+            f"could not fetch {HUB_REPO} ({exc.__class__.__name__}). Either run "
+            f"`bhashantar convert` once (needs `pip install bhashantar[convert]` and a Hugging Face "
+            f"login with access to ai4bharat/indictrans2-en-indic-dist-200M), or set {MODEL_ENV} "
+            f"to a converted model directory"
+        ) from exc
+
+
+def _clean_up_spaces(text: str) -> str:
+    # transformers' clean_up_tokenization, which the reference decode applies.
+    for spaced, tight in (
+        (" .", "."),
+        (" ?", "?"),
+        (" !", "!"),
+        (" ,", ","),
+        (" ' ", "'"),
+        (" n't", "n't"),
+        (" 'm", "'m"),
+        (" 's", "'s"),
+        (" 've", "'ve"),
+        (" 're", "'re"),
+    ):
+        text = text.replace(spaced, tight)
+    return text
+
+
+class Translator:
+    def __init__(
+        self,
+        model_dir: str | os.PathLike | None = None,
+        threads: int = 4,
+        beam_size: int = 5,
+        max_batch_size: int = 16,
+    ):
+        # Loading waits for the first translate() so that bad codes and
+        # oversize input are refused before 200 MB of weights are read.
+        self._requested_dir = model_dir
+        self.threads = threads
+        self.beam_size = beam_size
+        self.max_batch_size = max_batch_size
+        self.model_dir = None
+        self._model = None
+
+    def _load(self) -> None:
+        self.model_dir = resolve_model_dir(self._requested_dir)
+        self._source_pieces = spm.SentencePieceProcessor(model_file=str(self.model_dir / "model.SRC"))
+        self._model = ctranslate2.Translator(
+            str(self.model_dir), device="cpu", compute_type="int8", intra_threads=self.threads
+        )
+        log.info("loaded %s", self.model_dir)
+
+    def _source_tokens(self, model_text: str) -> list[str]:
+        src_tag, tgt_tag, body = model_text.split(" ", 2)
+        return [src_tag, tgt_tag] + self._source_pieces.encode(body, out_type=str)
+
+    def translate(self, texts: list[str], src: str = SOURCE, tgt: str = "hin_Deva") -> list[str]:
+        check_direction(src, tgt)
+        if isinstance(texts, str):
+            raise TypeError("texts must be a list of strings; wrap a single sentence as [text]")
+        _check_batch(texts)
+        if self._model is None:
+            self._load()
+
+        prepared, sources, slots = [], [], []
+        for i, sentence in enumerate(texts):
+            if not sentence.strip():
+                continue
+            p = textprep.prepare(sentence, tgt)
+            tokens = self._source_tokens(p.model_text)
+            if len(tokens) + 1 > MAX_SOURCE_TOKENS:
+                raise InputTooLarge(
+                    f"texts[{i}] is {len(tokens) + 1} tokens after preprocessing; "
+                    f"MAX_SOURCE_TOKENS is {MAX_SOURCE_TOKENS}. Split the sentence."
+                )
+            prepared.append(p)
+            sources.append(tokens)
+            slots.append(i)
+
+        out = [""] * len(texts)
+        if not sources:
+            return out
+        results = self._model.translate_batch(
+            sources,
+            beam_size=self.beam_size,
+            max_batch_size=self.max_batch_size,
+            max_decoding_length=MAX_SOURCE_TOKENS,
+        )
+        for slot, p, result in zip(slots, prepared, results):
+            pieces = [t for t in result.hypotheses[0] if t not in _SPECIAL]
+            decoded = _clean_up_spaces("".join(pieces).replace("▁", " ").strip())
+            out[slot] = textprep.finish(decoded, tgt, p.placeholders)
+        return out
+
+
+def _check_batch(texts: list[str]) -> None:
+    if len(texts) > MAX_BATCH_SENTENCES:
+        raise InputTooLarge(f"{len(texts)} sentences in one call; MAX_BATCH_SENTENCES is {MAX_BATCH_SENTENCES}")
+    total = 0
+    for i, sentence in enumerate(texts):
+        if not isinstance(sentence, str):
+            raise TypeError(f"texts[{i}] is {type(sentence).__name__}, expected str")
+        if len(sentence) > MAX_SENTENCE_CHARS:
+            raise InputTooLarge(
+                f"texts[{i}] is {len(sentence)} characters; MAX_SENTENCE_CHARS is {MAX_SENTENCE_CHARS}"
+            )
+        total += len(sentence)
+    if total > MAX_BATCH_CHARS:
+        raise InputTooLarge(f"batch is {total} characters; MAX_BATCH_CHARS is {MAX_BATCH_CHARS}")
+
+
+@lru_cache(maxsize=1)
+def default_translator() -> Translator:
+    return Translator()
+
+
+def translate(texts: list[str], src: str = SOURCE, tgt: str = "hin_Deva") -> list[str]:
+    """Translate English sentences into one Indic language with the shared model.
+
+    The first call loads the model; later calls reuse it.
+    """
+    return default_translator().translate(texts, src=src, tgt=tgt)

bhashantar/indic_script.py

@@ -0,0 +1,98 @@
+"""Script conversion and detokenisation for Indic output.
+
+Vendored from indic_nlp_library (MIT, Copyright (c) 2013-present Anoop
+Kunchukuttan): `UnicodeIndicTransliterator.transliterate` and
+`trivial_detokenize_indic`, reduced to the paths IndicTrans2 inference uses.
+Behaviour is kept byte-for-byte, quirks included, because the parity number
+depends on it. The upstream package pulls pandas, morfessor and sphinx for
+code we do not call.
+
+IndicTrans2 decodes every Brahmic target in Devanagari; converting back to the
+target script is a fixed code-point offset inside the 0x00-0x6F "coordinated"
+range that all these Unicode blocks share.
+"""
+
+import regex as re
+
+SCRIPT_RANGES = {
+    "pa": 0x0A00,
+    "gu": 0x0A80,
+    "or": 0x0B00,
+    "ta": 0x0B80,
+    "te": 0x0C00,
+    "kn": 0x0C80,
+    "ml": 0x0D00,
+    "si": 0x0D80,
+    "hi": 0x0900,
+    "mr": 0x0900,
+    "kK": 0x0900,
+    "sa": 0x0900,
+    "ne": 0x0900,
+    "sd": 0x0900,
+    "bn": 0x0980,
+    "as": 0x0980,
+}
+_COORDINATED_LAST = 0x6F
+_DANDAS = ("।", "॥")
+
+
+def _tamil_offset(offset: int) -> int:
+    # Tamil has no voiced or aspirated plosives; fold them onto the unvoiced
+    # unaspirated letter of the same row (ja keeps its own letter).
+    if 0x15 <= offset <= 0x28 and offset != 0x1C and (offset - 0x15) % 5 not in (0, 4):
+        offset = 0x15 + 5 * ((offset - 0x15) // 5)
+    if offset in (0x2B, 0x2C, 0x2D):
+        offset = 0x2A
+    if offset == 0x36:
+        offset = 0x37
+    return offset
+
+
+def transliterate(text: str, from_lang: str, to_lang: str) -> str:
+    if from_lang not in SCRIPT_RANGES or to_lang not in SCRIPT_RANGES:
+        return text
+    base_from = SCRIPT_RANGES[from_lang]
+    base_to = SCRIPT_RANGES[to_lang]
+    out = []
+    for ch in text:
+        offset = ord(ch) - base_from
+        if 0 <= offset <= _COORDINATED_LAST and ch not in _DANDAS:
+            if to_lang == "ta":
+                offset = _tamil_offset(offset)
+            ch = chr(base_to + offset)
+        out.append(ch)
+    return "".join(out)
+
+
+_LEFT_ATTACH = re.compile(r"[ ]([!%)\]},.:;>?।॥])")
+_RIGHT_ATTACH = re.compile(r"([#$(\[{<@])[ ]")
+_BOTH_ATTACH = re.compile(r"[ ]([-/\\])[ ]")
+_NUMBER_RUN = re.compile(r"([0-9]+ [,.:/] )+[0-9]+")
+
+
+def detokenize(text: str) -> str:
+    parts = []
+    prev = 0
+    for m in _NUMBER_RUN.finditer(text):
+        # Upstream only collapses a number run when text precedes it; a run at
+        # position 0 is left spaced. Kept as is for parity.
+        if m.start() > prev:
+            parts.append(text[prev : m.start()])
+            parts.append(text[m.start() : m.end()].replace(" ", ""))
+            prev = m.end()
+    parts.append(text[prev:])
+    s = "".join(parts)
+    s = _BOTH_ATTACH.sub("\\1", s)
+    s = _LEFT_ATTACH.sub("\\1", s)
+    s = _RIGHT_ATTACH.sub("\\1", s)
+    for quote in "'\"`":
+        marked = []
+        count = 0
+        for ch in s:
+            if ch == quote:
+                marked.append("@RA" if count % 2 == 0 else "@LA")
+                count += 1
+            else:
+                marked.append(ch)
+        s = "".join(marked).replace("@RA ", quote).replace(" @LA", quote).replace("@RA", quote).replace("@LA", quote)
+    return s

bhashantar/languages.py

@@ -0,0 +1,80 @@
+"""Language codes the en-indic-dist-200M checkpoint was trained on.
+
+Codes are FLORES-200 style (`hin_Deva`). The checkpoint only translates out of
+English, so `eng_Latn` is the one source and the 25 Indic codes are targets.
+`ISO` maps each code to the indicnlp language key that the upstream
+post-processing used for script conversion and detokenisation; several codes
+map to "hi" on purpose, because their script needs no conversion.
+"""
+
+SOURCE = "eng_Latn"
+
+ISO = {
+    "asm_Beng": "as",
+    "ben_Beng": "bn",
+    "brx_Deva": "hi",
+    "doi_Deva": "hi",
+    "gom_Deva": "kK",
+    "guj_Gujr": "gu",
+    "hin_Deva": "hi",
+    "kan_Knda": "kn",
+    "kas_Arab": "ur",
+    "kas_Deva": "hi",
+    "mai_Deva": "hi",
+    "mal_Mlym": "ml",
+    "mar_Deva": "mr",
+    "mni_Beng": "bn",
+    "mni_Mtei": "hi",
+    "npi_Deva": "ne",
+    "ory_Orya": "or",
+    "pan_Guru": "pa",
+    "san_Deva": "hi",
+    "sat_Olck": "or",
+    "snd_Arab": "ur",
+    "snd_Deva": "hi",
+    "tam_Taml": "ta",
+    "tel_Telu": "te",
+    "urd_Arab": "ur",
+}
+
+TARGETS = tuple(sorted(ISO))
+
+# Unicode blocks per script suffix, used to tell whether output landed in the
+# script the caller asked for.
+SCRIPT_BLOCKS = {
+    "Deva": [(0x0900, 0x097F)],
+    "Beng": [(0x0980, 0x09FF)],
+    "Guru": [(0x0A00, 0x0A7F)],
+    "Gujr": [(0x0A80, 0x0AFF)],
+    "Orya": [(0x0B00, 0x0B7F)],
+    "Taml": [(0x0B80, 0x0BFF)],
+    "Telu": [(0x0C00, 0x0C7F)],
+    "Knda": [(0x0C80, 0x0CFF)],
+    "Mlym": [(0x0D00, 0x0D7F)],
+    "Arab": [(0x0600, 0x06FF), (0x0750, 0x077F), (0xFB50, 0xFDFF), (0xFE70, 0xFEFF)],
+    "Olck": [(0x1C50, 0x1C7F)],
+    "Mtei": [(0xABC0, 0xABFF), (0xAAE0, 0xAAFF)],
+}
+
+
+class UnknownLanguage(ValueError):
+    """A language code outside what this checkpoint supports."""
+
+
+def check_direction(src: str, tgt: str) -> None:
+    if src != SOURCE:
+        raise UnknownLanguage(
+            f"src={src!r} is not supported: this model translates from {SOURCE!r} only"
+        )
+    if tgt not in ISO:
+        raise UnknownLanguage(f"tgt={tgt!r} is not a supported target; valid codes: {', '.join(TARGETS)}")
+
+
+def script_share(text: str, code: str) -> float:
+    """Fraction of letters in `text` that fall inside the script of `code`."""
+    blocks = SCRIPT_BLOCKS[code.split("_", 1)[1]]
+    letters = [ch for ch in text if ch.isalpha() or 0x0900 <= ord(ch) <= 0x0DFF]
+    if not letters:
+        return 0.0
+    inside = sum(1 for ch in letters if any(lo <= ord(ch) <= hi for lo, hi in blocks))
+    return inside / len(letters)

bhashantar/textprep.py

@@ -0,0 +1,183 @@
+"""Text in and out of the model: English normalisation, entity placeholders,
+Indic script restoration.
+
+Vendored from IndicTransToolkit's IndicProcessor (MIT, Copyright (c) Varun
+Gumma), cut down to the English-source path this checkpoint uses. The upstream
+class keeps placeholder maps in a shared queue, so a failed batch leaves stale
+maps behind for the next one; here each sentence carries its own map.
+
+Regexes, their order and the "ID" failure spellings are upstream's. Changing
+any of them moves the parity number, so they are not tidied.
+"""
+
+from dataclasses import dataclass, field
+
+import regex as re
+from sacremoses import MosesPunctNormalizer, MosesTokenizer
+
+from bhashantar import indic_script
+from bhashantar.languages import ISO, SOURCE
+
+_DIGITS = {}
+for _ascii, _variants in enumerate(
+    [
+        "\u09e6\u0ae6\u0ce6\u0966\u0660\uabf0\u0b66\u0a66\u1c50\u06f0",
+        "\u09e7\u0ae7\u0967\u0ce7\u06f1\uabf1\u0b67\u0a67\u1c51\u0c67",
+        "\u09e8\u0ae8\u0968\u0ce8\u06f2\uabf2\u0b68\u0a68\u1c52\u0c68",
+        "\u09e9\u0ae9\u0969\u0ce9\u06f3\uabf3\u0b69\u0a69\u1c53\u0c69",
+        "\u09ea\u0aea\u096a\u0cea\u06f4\uabf4\u0b6a\u0a6a\u1c54\u0c6a",
+        "\u09eb\u0aeb\u096b\u0ceb\u06f5\uabf5\u0b6b\u0a6b\u1c55\u0c6b",
+        "\u09ec\u0aec\u096c\u0cec\u06f6\uabf6\u0b6c\u0a6c\u1c56\u0c6c",
+        "\u09ed\u0aed\u096d\u0ced\u06f7\uabf7\u0b6d\u0a6d\u1c57\u0c6d",
+        "\u09ee\u0aee\u096e\u0cee\u06f8\uabf8\u0b6e\u0a6e\u1c58\u0c6e",
+        "\u09ef\u0aef\u096f\u0cef\u06f9\uabf9\u0b6f\u0a6f\u1c59\u0c6f",
+    ]
+):
+    for _ch in _variants:
+        _DIGITS[ord(_ch)] = str(_ascii)
+
+_PUNC_REPLACEMENTS = [
+    (re.compile(r"\r"), ""),
+    (re.compile(r"\(\s*"), "("),
+    (re.compile(r"\s*\)"), ")"),
+    (re.compile(r"\s:\s?"), ":"),
+    (re.compile(r"\s;\s?"), ";"),
+    (re.compile(r"[`\u00b4\u2018\u201a\u2019]"), "'"),
+    (re.compile(r"[\u201e\u201c\u201d\u00ab\u00bb]"), '"'),
+    (re.compile("[\u2013\u2014]"), "-"),
+    (re.compile(r"\.\.\."), "..."),
+    (re.compile(r" %"), "%"),
+    (re.compile(r"n\u00ba "), "n\u00ba "),
+    (re.compile(r" \u00baC"), " \u00baC"),
+    (re.compile(r" [?!;]"), lambda m: m.group(0).strip()),
+    (re.compile(r", "), ", "),
+]
+_MULTISPACE = re.compile(r"[ ]{2,}")
+_BRACKET_PUNC = re.compile(r"\) ([\.!:?;,])")
+_DIGIT_SPACE_PERCENT = re.compile(r"(\d) %")
+_QUOTE_PUNC = re.compile(r"\"([,\.]+)")
+_DIGIT_SPACE_DIGIT = re.compile(r"(\d) (\d)")
+
+URL_PATTERN = re.compile(r"\b(?<![\w/.])(?:(?:https?|ftp)://)?(?:(?:[\w-]+\.)+(?!\.))(?:[\w/\-?#&=%.]+)+(?!\.\w+)\b")
+NUMERAL_PATTERN = re.compile(
+    r"(~?\d+\.?\d*\s?%?\s?-?\s?~?\d+\.?\d*\s?%|~?\d+%|\d+[-\/.,:']\d+[-\/.,:'+]\d+(?:\.\d+)?|\d+[-\/.:'+]\d+(?:\.\d+)?)"
+)
+EMAIL_PATTERN = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}")
+HANDLE_PATTERN = re.compile(r"[A-Za-z0-9]*[#|@]\w+")
+_PLACEHOLDER_ORDER = (EMAIL_PATTERN, URL_PATTERN, NUMERAL_PATTERN, HANDLE_PATTERN)
+
+# How the model sometimes mangles "<ID1>" in its output, per script.
+_ID_SPELLINGS = [
+    "\u0622\u06cc \u0688\u06cc ",
+    "\uabd1\uabe5\uabcf\uabd7\uabe4",
+    "\u0906\u0908\u0921\u0940",
+    "\u0906\u0908 . \u0921\u0940 . ",
+    "\u0906\u0908 . \u0921\u0940 .",
+    "\u0906\u0908. \u0921\u0940. ",
+    "\u0906\u0908. \u0921\u0940.",
+    "\u0906\u092f. \u0921\u0940. ",
+    "\u0906\u092f. \u0921\u0940.",
+    "\u0906\u092f . \u0921\u0940 . ",
+    # Upstream is missing a comma here, so these two spellings arrive fused.
+    "\u0906\u092f . \u0921\u0940 .\u0906\u0907 . \u0921\u0940 . ",
+    "\u0906\u0907 . \u0921\u0940 .",
+    "\u0906\u0907. \u0921\u0940. ",
+    "\u0906\u0907. \u0921\u0940.",
+    "\u0910\u091f\u093f",
+    "\u0622\u0626\u06cc \u0688\u06cc ",
+    "\u1c5f\u1c6d\u1c70\u1c64 \u1c7e",
+    "\u0906\u092f\u0921\u0940",
+    "\u0910\u0921\u093f",
+    "\u0906\u0907\u0921\u093f",
+    "\u1c5f\u1c6d\u1c70\u1c64",
+]
+
+_moses_norm = MosesPunctNormalizer()
+_moses_tok = MosesTokenizer(lang="en")
+
+
+@dataclass
+class PreparedSentence:
+    model_text: str
+    placeholders: dict = field(default_factory=dict)
+
+
+def _normalise_punctuation(text: str) -> str:
+    for pattern, replacement in _PUNC_REPLACEMENTS:
+        text = pattern.sub(replacement, text)
+    text = _MULTISPACE.sub(" ", text)
+    text = _BRACKET_PUNC.sub(r")\1", text)
+    text = _DIGIT_SPACE_PERCENT.sub(r"\1%", text)
+    text = _QUOTE_PUNC.sub(r'\1"', text)
+    text = _DIGIT_SPACE_DIGIT.sub(r"\1.\2", text)
+    return text.strip()
+
+
+def _spellings_for(serial: int) -> list[str]:
+    n = serial
+    spellings = []
+    for tag in ("ID", "id"):
+        spellings += [f"<{tag}{n}>", f"< {tag}{n} >", f"[{tag}{n}]", f"[ {tag}{n} ]", f"[{tag} {n}]"]
+        spellings += [f"<{tag}{n}]", f"< {tag}{n}]", f"<{tag}{n} ]"]
+    for c in _ID_SPELLINGS:
+        spellings += [
+            f"<{c}{n}>",
+            f"< {c}{n} >",
+            f"< {c} {n} >",
+            f"<{c} {n}]",
+            f"< {c} {n} ]",
+            f"[{c}{n}]",
+            f"[{c} {n}]",
+            f"[ {c}{n} ]",
+            f"[ {c} {n} ]",
+            f"{c} {n}",
+            f"{c}{n}",
+        ]
+    return spellings
+
+
+def _wrap_entities(text: str) -> tuple[str, dict]:
+    """Swap URLs, emails, numbers and handles for <IDn> so the model copies
+    them instead of translating them. Returns the text and the restore map."""
+    serial = 1
+    restore = {}
+    for pattern in _PLACEHOLDER_ORDER:
+        # Upstream iterates a set, so serial order follows string hashing and
+        # varies between processes; sorting makes it stable without changing
+        # which entities are wrapped.
+        for match in sorted(set(pattern.findall(text))):
+            if pattern is URL_PATTERN and len(match.replace(".", "")) < 4:
+                continue
+            if pattern is NUMERAL_PATTERN and len(match.replace(" ", "").replace(".", "").replace(":", "")) < 4:
+                continue
+            for spelling in _spellings_for(serial):
+                restore[spelling] = match
+            text = text.replace(match, f"<ID{serial}>")
+            serial += 1
+    text = re.sub(r"\s+", " ", text).replace(">/", ">").replace("]/", "]")
+    return text, restore
+
+
+def prepare(sentence: str, tgt: str) -> PreparedSentence:
+    text = _normalise_punctuation(sentence)
+    text = text.translate(_DIGITS)
+    text, restore = _wrap_entities(text)
+    text = _moses_norm.normalize(text.strip())
+    text = " ".join(_moses_tok.tokenize(text, escape=False)).strip()
+    return PreparedSentence(model_text=f"{SOURCE} {tgt} {text}", placeholders=restore)
+
+
+def finish(decoded: str, tgt: str, placeholders: dict) -> str:
+    lang, script = tgt.split("_", 1)
+    if script in ("Arab", "Aran"):
+        decoded = (
+            decoded.replace(" \u061f", "\u061f")
+            .replace(" \u06d4", "\u06d4")
+            .replace(" \u060c", "\u060c")
+            .replace("\u066e\u06ea", "\u0620")
+        )
+    if lang == "ory":
+        decoded = decoded.replace("\u0b2f\u0b3c", "\u0b5f")
+    for spelling, original in placeholders.items():
+        decoded = decoded.replace(spelling, original)
+    return indic_script.detokenize(indic_script.transliterate(decoded, "hi", ISO[tgt]))

bhashantar-0.1.0.dist-info/METADATA

@@ -0,0 +1,157 @@
+Metadata-Version: 2.5
+Name: bhashantar
+Version: 0.1.0
+Summary: IndicTrans2 English-to-Indic translation on CPU via CTranslate2 int8, no transformers or fairseq at runtime
+Author-email: Zaid Ali Syed <sidzaid72@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+License-File: NOTICE
+Keywords: ctranslate2,hindi,indic,indictrans2,machine-translation
+Classifier: Natural Language :: Hindi
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Requires-Dist: ctranslate2<5,>=4.4
+Requires-Dist: huggingface-hub>=0.24
+Requires-Dist: regex>=2023.0
+Requires-Dist: sacremoses>=0.1.1
+Requires-Dist: sentencepiece<0.3,>=0.2
+Provides-Extra: convert
+Requires-Dist: safetensors>=0.4; extra == 'convert'
+Provides-Extra: dev
+Requires-Dist: psutil>=5.9; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: sacrebleu>=2.4; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# bhashantar
+
+English to 25 Indic languages with AI4Bharat's IndicTrans2 (`en-indic-dist-200M`), on CPU, through CTranslate2 int8. No transformers, no torch, no fairseq and no IndicTransToolkit at runtime.
+
+```python
+from bhashantar import translate
+
+translate(["The weather in Mumbai is pleasant today."], tgt="hin_Deva")
+# ['मुंबई में आज मौसम सुखद है।']
+```
+
+```
+$ bhashantar translate --tgt hin_Deva "Send the report to ops@example.com by 15-03-2025."
+रिपोर्ट को 15-03-2025 द्वारा ops@example.com पर भेजें।
+```
+
+## Why this exists
+
+The official checkpoint loads through `trust_remote_code` files written against transformers 4.3x. On a fresh Python 3.12 venv with transformers 5.22.2 and IndicTransToolkit 1.1.1 (reproduced 2026-11-09):
+
+```
+from IndicTransToolkit.processor import IndicProcessor
+ImportError: cannot import name 'PreTrainedTokenizerBase' from 'transformers.tokenization_utils'
+
+AutoTokenizer.from_pretrained(..., trust_remote_code=True)
+ModuleNotFoundError: No module named 'transformers.onnx'
+
+AutoModelForSeq2SeqLM.from_pretrained(..., trust_remote_code=True)
+ModuleNotFoundError: No module named 'transformers.onnx'
+```
+
+This is [IndicTransToolkit#29](https://github.com/VarunGumma/IndicTransToolkit/issues/29). A fix for the remote code is open as discussion #15 on the model repo, unmerged at the time of writing. Pinning transformers 4.50.3 makes the stack load again; that pinned stack is the reference this package is measured against.
+
+## Data path
+
+```
+raw text
+  -> punctuation and digit normalisation, URLs/emails/numbers swapped for <IDn>   (textprep.prepare)
+  -> Moses tokenisation, "eng_Latn hin_Deva" tags, SentencePiece pieces           (engine)
+  -> CTranslate2 int8 beam search (beam 5), Devanagari-unified pieces out
+  -> detokenise pieces, restore <IDn>, Devanagari -> target script, reattach punctuation   (textprep.finish)
+```
+
+The model writes every Brahmic target in Devanagari; Tamil, Bengali and the rest come from a fixed Unicode offset afterwards. That step and the placeholder restore are where a port silently goes wrong, so both are vendored from IndicTransToolkit and indic_nlp_library (MIT, see `NOTICE`) with their regexes and quirks intact. Two deliberate changes: placeholder maps travel with each sentence instead of a shared queue, and placeholder numbering is sorted instead of following set order, which varied between processes.
+
+## Two ways this fails, and the guard for each
+
+**Silent quality loss from conversion or quantisation.** CTranslate2 has no converter for this checkpoint's custom class, so `bhashantar.convert` maps the Hugging Face tensors onto the same `TransformerSpec` the fairseq converter fills. A wrong position offset or a missed layer norm still produces fluent-looking text. The guard is `bench/parity.py`: chrF against human references for both this package and the upstream model, with the pass bar fixed before the first run.
+
+**Breakage on library upgrades.** Runtime dependencies are only what inference touches: `ctranslate2` (capped below 5, since `model.bin` is tied to the major), `sentencepiece`, `sacremoses`, `regex`, `huggingface_hub`. `.github/workflows/smoke.yml` installs the newest versions in range every week and translates.
+
+**The opinion:** parity is published as a number, not claimed. If the delta ever exceeds the bar, the table says so.
+
+## Parity
+
+FLORES-200 devtest, first 100 sentences, English source. The FLORES files come from the public `dl.fbaipublicfiles.com/nllb/flores200_dataset.tar.gz`; AI4Bharat's IN22 and the Hugging Face FLORES+ copy are both gated. Reference = upstream model, transformers 4.50.3 + IndicTransToolkit 1.1.1, fp32, beam 5. Pass bar, set before running: |delta| <= 1.0 chrF.
+
+| pair | chrF reference | chrF bhashantar | delta | identical outputs | pass |
+|---|---|---|---|---|---|
+| eng_Latn-hin_Deva | 65.69 | 65.17 | -0.52 | 61/100 | yes |
+| eng_Latn-ben_Beng | 54.52 | 54.62 | +0.10 | 54/100 | yes |
+| eng_Latn-tam_Taml | 61.33 | 61.59 | +0.27 | 56/100 | yes |
+
+Reproduce: `python bench/parity.py score <flores devtest dir> bench/results` (outputs and `parity.json` are committed in `bench/results`).
+
+"Identical outputs" counts sentences where both systems produce the same string; the rest differ in word choice (for example कनाडाई मधुमेह संघ against कैनेडियन डायबिटीज एसोसिएशन), the expected effect of int8 weights on beam search. Conversion error and quantisation error are not separated: an fp32 CTranslate2 conversion did not fit on the build machine's disk.
+
+Reference weights: the official repo is gated, so the reference used `Raghavan/indictrans2-en-indic-dist-200M`, whose `pytorch_model.bin` has the same Git LFS pointer as the official file (sha256 `52671a26...0fef0`, verified locally); tokenizer files match the official Git blob ids.
+
+## Speed and memory
+
+| | bhashantar (CT2 int8) | upstream (transformers fp32) |
+|---|---|---|
+| model load | 0.47 s | not timed |
+| ms per sentence, eng to tam_Taml, beam 5 | 323 (all 100 in one call) | 2,786 (batch 2, 80 sentences timed) |
+| peak RSS | 707 MB | 1,472 MB |
+
+The two latency figures are not like for like: the reference had to run at batch 2 because the build machine ran out of commit memory at batch 8, and it was resumed once mid-run, so only the last 80 sentences were timed. bhashantar on Hindi and Bengali: 328 and 308 ms per sentence.
+
+Measured on a laptop (Windows 11, 16 GB RAM, 4 threads, other jobs running), so treat them as relative, not absolute.
+
+## Limits
+
+- One direction: English to Indic. The `indic-en` and `indic-indic` checkpoints are not converted.
+- Caps, refused with the cap named in the error: 1,000 characters per sentence, 512 sentences and 50,000 characters per call, 256 subword tokens per sentence after preprocessing. Long input is refused, not truncated.
+- Santali (`sat_Olck`) output mixes in Oriya and Arabic words for some sentences. The upstream model does the same; the test marks it as a known model limit.
+- Sentence-level: split paragraphs yourself.
+
+## Model files
+
+`translate()` looks for a model in this order: the `model_dir` argument, `$BHASHANTAR_MODEL`, a local conversion at `$HF_HOME/bhashantar/en-indic-dist-200M-ct2-int8`, then the Hugging Face repo `zaidwhy/indictrans2-en-indic-dist-200M-ct2-int8` (cache first, so it works offline after the first download).
+
+That repo is not published yet. Until it is, convert once yourself (needs a Hugging Face login with the model's terms accepted):
+
+```
+pip install "bhashantar[convert]"
+hf auth login
+bhashantar convert
+```
+
+## Development
+
+```
+pip install -e ".[dev]"
+pytest                 # offline tests, no model
+pytest -m model        # end to end, needs the converted model
+```
+
+## Maintainer: publishing (not done yet)
+
+```
+# 1. weights, from a machine with the converted model
+hf auth login
+hf repo create zaidwhy/indictrans2-en-indic-dist-200M-ct2-int8 --repo-type model
+hf upload zaidwhy/indictrans2-en-indic-dist-200M-ct2-int8 ~/.cache/huggingface/bhashantar/en-indic-dist-200M-ct2-int8 .
+
+# 2. check that a clean install fetches it
+python -m venv /tmp/bt && /tmp/bt/bin/pip install . && /tmp/bt/bin/bhashantar translate --tgt tam_Taml "Good morning."
+
+# 3. package
+python -m pip install build twine
+python -m build
+twine check dist/*
+twine upload dist/*
+```
+
+The model directory carries AI4Bharat's MIT `LICENSE` and a `provenance.json` with the source weights' sha256; add a model card stating the parity numbers above before uploading.
+
+## License
+
+MIT. Vendored code and model weights keep their own MIT notices; see `NOTICE`.

bhashantar-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+bhashantar/__init__.py,sha256=QUPDOPNchCIe9ReXFNmtOlreGxX9CumP7fAW5BAplLg,627
+bhashantar/cli.py,sha256=dKlzR1YBnL5gFpPXOxfRHSF8pWUrmVMa7RY8vPMnbdw,2980
+bhashantar/convert.py,sha256=ZW_Bw-G3zTnOMex8xYuGzOz-SBGgZeTbu_egpsc9XGE,7579
+bhashantar/engine.py,sha256=6CVY8_YtZT4Byzvn4XzMmq0bnbAJsmonR_npd920Wd8,7142
+bhashantar/indic_script.py,sha256=jsU2Yg5gAF0AS5ZmeaE3_ovcja-Z7Nof4m5-HC2l8Vw,3180
+bhashantar/languages.py,sha256=QynSwLoSGNASx2ExlE1hf1rK-Te_yKlUMvqFjKjQI-0,2492
+bhashantar/textprep.py,sha256=sMVNqVsJUqYerUfsIHK5-co_D0HMbt44ziPYAzNDDwo,7098
+bhashantar-0.1.0.dist-info/METADATA,sha256=YGdSgNPRwmCSZRkdAOshWWNXxToEgEnsLIxqptgFud4,8673
+bhashantar-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+bhashantar-0.1.0.dist-info/entry_points.txt,sha256=Q5CQYoIc_r5RSHvEEgD48tyLVwJseiYS3i69dcuVpTg,51
+bhashantar-0.1.0.dist-info/licenses/LICENSE,sha256=B3roCCeZDO06exjk5EZUsN6DaDNEXHFd8B54pONKgXQ,1070
+bhashantar-0.1.0.dist-info/licenses/NOTICE,sha256=-xSkcwQsesBzl97EP7ZEa9NuXZOUtGcktol4-n4Dz8Q,1098
+bhashantar-0.1.0.dist-info/RECORD,,

bhashantar-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

bhashantar-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+bhashantar = bhashantar.cli:main

bhashantar-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Zaid Ali Syed
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bhashantar-0.1.0.dist-info/licenses/NOTICE

@@ -0,0 +1,25 @@
+bhashantar includes code adapted from the following MIT-licensed projects.
+
+src/bhashantar/textprep.py
+  Adapted from IndicTransToolkit (processor.pyx, IndicProcessor)
+  https://github.com/VarunGumma/IndicTransToolkit
+  MIT License, Copyright (c) Varun Gumma.
+
+src/bhashantar/indic_script.py
+  Adapted from indic_nlp_library (unicode_transliterate.py, indic_detokenize.py, langinfo.py)
+  https://github.com/anoopkunchukuttan/indic_nlp_library
+  MIT License, Copyright (c) 2013-present Anoop Kunchukuttan.
+
+src/bhashantar/convert.py
+  Weight mapping follows ctranslate2/converters/fairseq.py
+  https://github.com/OpenNMT/CTranslate2
+  MIT License, Copyright (c) 2019 SYSTRAN.
+
+Model weights (downloaded at runtime, not shipped in this package)
+  IndicTrans2 en-indic-dist-200M by AI4Bharat
+  https://huggingface.co/ai4bharat/indictrans2-en-indic-dist-200M
+  MIT License. The converted CTranslate2 model directory carries a copy of
+  that LICENSE file.
+
+Both MIT notices require that the copyright and permission notice be included
+in copies or substantial portions; the full MIT text is in LICENSE.