offline-translate 0.1.0__py3-none-any.whl

offline_translate/__init__.py

@@ -0,0 +1,12 @@
+"""Offline translation with Google Translate's on-device language packs.
+
+    from offline_translate import Translator
+    t = Translator("~/models")
+    t.download("he")
+    t.translate("Hello", target="he").text
+"""
+from .languages import NAMES, normalize
+from .translator import Language, TranslateError, Translation, Translator
+
+__version__ = "0.1.0"
+__all__ = ["Translator", "Translation", "Language", "TranslateError", "NAMES", "normalize", "__version__"]

offline_translate/catalog.py

@@ -0,0 +1,172 @@
+"""Google Translate offline package catalog.
+
+How the Android app finds packs (reverse engineered):
+  1. Manifest  https://dl.google.com/translate/offline/manifest/gtr/<channel>/manifest.pb
+     lists every package-profile version (+ sha1/size). The channel comes from a server flag ("prod").
+  2. Profile   https://dl.google.com/translate/offline/package_profile/<maj>/<min>/<patch>/translate_premium.pb.zlib
+     lists every language pack, its packages (nmt models, dictionary, translit) and their files.
+  3. Files     https://dl.google.com/translate/offline/nmt/<ver>/premium/<src>_<tgt>/0/<file>.zlib
+"""
+import struct
+import urllib.request
+import zlib
+from dataclasses import dataclass, field
+
+BASE = "https://dl.google.com/translate/offline"
+MANIFEST_URL = BASE + "/manifest/gtr/{channel}/manifest.pb"
+USER_AGENT = "Mozilla/5.0"
+
+PKG_NMT = 3
+# Files our inference engine knows how to run; packages with other layouts are skipped.
+NMT_FILES = {
+    "encoder_0.tflite", "decoder_init_0.tflite", "decoder_step_0.tflite",
+    "src_spm_model.model", "tgt_spm_model.model",
+}
+
+
+# --- minimal protobuf reader (Google's schema isn't public) ---
+def _varint(b, i):
+    r = s = 0
+    while True:
+        c = b[i]
+        i += 1
+        r |= (c & 0x7F) << s
+        s += 7
+        if c < 0x80:
+            return r, i
+
+
+def parse_pb(b):
+    """Parse a protobuf message into {field: [values]}; length-delimited values stay bytes."""
+    out, i = {}, 0
+    while i < len(b):
+        key, i = _varint(b, i)
+        f, t = key >> 3, key & 7
+        if t == 0:
+            v, i = _varint(b, i)
+        elif t == 2:
+            n, i = _varint(b, i)
+            v, i = b[i:i + n], i + n
+        elif t == 5:
+            v, i = struct.unpack("<f", b[i:i + 4])[0], i + 4
+        elif t == 1:
+            v, i = struct.unpack("<d", b[i:i + 8])[0], i + 8
+        else:
+            raise ValueError(f"unsupported protobuf wire type {t}")
+        out.setdefault(f, []).append(v)
+    return out
+
+
+def _str(m, f):
+    return m[f][0].decode()
+
+
+def _hash(m):
+    """{1: {1: algo, 2: hex}, 2: size} -> (hex, size)."""
+    return _str(parse_pb(m[1][0]), 2), m[2][0]
+
+
+def fetch(url, timeout=60):
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return r.read()
+
+
+def open_url(url, timeout=60):
+    return urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": USER_AGENT}), timeout=timeout)
+
+
+@dataclass(frozen=True)
+class ProfileRef:
+    version: tuple
+    url: str
+    sha1: str  # of the decompressed profile
+
+    @property
+    def version_str(self):
+        return ".".join(map(str, self.version))
+
+
+def parse_manifest(data):
+    """Returns the premium (NMT) package profiles listed in a manifest, oldest first."""
+    refs = []
+    groups = parse_pb(data).get(2, [])
+    for raw in (r for g in groups for r in parse_pb(g).get(2, [])):
+        m = parse_pb(raw)
+        url = _str(m, 2)
+        if "translate_premium" not in url:
+            continue
+        v = parse_pb(m[1][0])
+        version = tuple(v.get(k, [0])[0] for k in (1, 2, 3))
+        refs.append(ProfileRef(version, url, _hash(parse_pb(m[3][0]))[0]))
+    return sorted(refs, key=lambda r: r.version)
+
+
+@dataclass(frozen=True)
+class PackFile:
+    name: str       # file name on disk, e.g. encoder_0.tflite
+    url: str
+    sha1: str       # of the decompressed file
+    size: int       # decompressed bytes
+    download_size: int  # zlib-compressed bytes
+
+
+@dataclass(frozen=True)
+class Package:
+    name: str       # e.g. "en_iw" (direction src_tgt)
+    files: tuple
+
+    @property
+    def source(self):
+        return self.name.split("_", 1)[0]
+
+    @property
+    def target(self):
+        return self.name.split("_", 1)[1]
+
+
+@dataclass
+class Pack:
+    """A downloadable language: both directions to/from English."""
+    lang: str
+    packages: list = field(default_factory=list)
+    # False for older packs (nmt/8/1, WordPiece tokenizer) that the engine can't run yet.
+    supported: bool = True
+
+    @property
+    def download_size(self):
+        return sum(f.download_size for p in self.packages for f in p.files)
+
+    @property
+    def size(self):
+        return sum(f.size for p in self.packages for f in p.files)
+
+
+def parse_profile(data):
+    """Returns {lang_code: Pack} for every language with an NMT model (see Pack.supported)."""
+    top = parse_pb(data)
+    files = {}
+    for raw in top.get(3, []):
+        m = parse_pb(raw)
+        sha1, size = _hash(parse_pb(m[3][0]))
+        _, zsize = _hash(parse_pb(m[4][0]))
+        files[_str(m, 1)] = PackFile(_str(m, 5), _str(m, 2), sha1, size, zsize)
+
+    packages = {}
+    for raw in top.get(2, []):
+        m = parse_pb(raw)
+        if m[2][0] != PKG_NMT:
+            continue
+        pfiles = tuple(files[f.decode()] for f in m.get(4, []))
+        packages[_str(m, 1)] = Package(_str(m, 1).removesuffix("_premium_nmt"), pfiles)
+
+    packs = {}
+    for raw in top.get(1, []):
+        m = parse_pb(raw)
+        langs = [_str(parse_pb(x), 2) for x in parse_pb(m[3][0])[1]]
+        lang = next((l for l in langs if l != "en"), None)
+        pkgs = [packages[p.decode()] for p in m.get(2, []) if p.decode() in packages]
+        if lang and pkgs:
+            ok = all(NMT_FILES <= {f.name for f in p.files} for p in pkgs)
+            packs[lang] = Pack(lang, pkgs, ok)
+    return packs

offline_translate/engine.py

@@ -0,0 +1,118 @@
+"""Runs one direction of a Google Translate on-device NMT model (TFLite encoder + LSTM decoder)."""
+import os
+import re
+from pathlib import Path
+
+os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3")
+
+import numpy as np
+import sentencepiece as spm
+from ai_edge_litert.interpreter import Interpreter
+
+BOS, EOS = 1, 2
+STATE_KEYS = [f"beam_state:rnn_state_{i}_{x}:00" for i in (0, 1) for x in ("c", "m")]
+# The models are trained on single sentences; split on sentence-final punctuation + whitespace.
+SENT_RE = re.compile(r"(?<=[.!?…。！？])\s+")
+
+
+def split_sentences(text):
+    """Lines of sentences: line breaks are kept, each line is split into sentences."""
+    return [[s for s in SENT_RE.split(line.strip()) if s] for line in text.split("\n")]
+
+
+FINAL_PUNCT = ".!?…。！？:;\"'”’)»"
+
+
+def add_period_limit(processor_spec):
+    """max_sent_len of the pack's PreprocAddPeriod step, or None if the pack doesn't use it."""
+    m = re.search(rb"PreprocAddPeriod/[^:]*:max_sent_len=(\d+)", processor_spec)
+    return int(m.group(1)) if m else None
+
+
+def with_period(sentence, limit):
+    """Short sentences without final punctuation translate badly ("car" -> "tram"); the app adds
+    a period first. Returns (text to translate, whether a period was added)."""
+    if limit and len(sentence.split()) <= limit and sentence[-1:] not in FINAL_PUNCT:
+        return sentence + ".", True
+    return sentence, False
+
+
+def strip_period(translation):
+    return translation[:-1] if translation[-1:] in ".。" else translation
+
+
+class NmtModel:
+    def __init__(self, path):
+        path = Path(path)
+        self.src_sp = spm.SentencePieceProcessor(model_file=str(path / "src_spm_model.model"))
+        self.tgt_sp = spm.SentencePieceProcessor(model_file=str(path / "tgt_spm_model.model"))
+        runner = lambda name: Interpreter(model_path=str(path / f"{name}.tflite")).get_signature_runner()
+        self.encoder = runner("encoder_0")
+        self.dec_init = runner("decoder_init_0")
+        self.dec_step = runner("decoder_step_0")
+        spec = path / "processor_spec.pb"
+        self.period_limit = add_period_limit(spec.read_bytes()) if spec.exists() else None
+
+    def translate(self, text, beam=4):
+        return "\n".join(" ".join(self.translate_sentence(s, beam=beam) for s in line)
+                         for line in split_sentences(text))
+
+    def translate_sentence(self, text, beam=4):
+        text, added = with_period(text, self.period_limit)
+        out = self._decode(text, beam=beam)
+        return strip_period(out) if added else out
+
+    def _decode(self, text, beam=4, max_len=72, alpha=1.0):
+        # Source vocabularies are lowercase-only; the model restores casing on output.
+        ids = self.src_sp.encode(text.lower()) + [EOS]
+        src = np.array([ids], np.int32)
+        pad = np.zeros_like(src, np.float32)
+        encs = self.encoder(**{"feeds:src_ids:00": src, "feeds:src_paddings:00": pad})["fetches:src_encs:00"]
+        init = self.dec_init(**{
+            "feeds:num_hyps:00": np.array([1], np.int32),
+            "feeds:src_enc_paddings:00": pad,
+            "feeds:src_encs:00": encs,
+        })
+        vecs = init["atten_init:source_vecs:00"]
+        state = {k: np.zeros((1, 512), np.float32) for k in STATE_KEYS}
+        state["beam_state:atten_context:00"] = init["beam_state:atten_context:00"]
+
+        # Each live hypothesis: (tokens, cumulative log prob). Batch rows of `state` align with `hyps`.
+        hyps = [([], 0.0)]
+        done = []
+        max_len = min(max_len, 2 * len(ids) + 10)
+        for _ in range(max_len):
+            n = len(hyps)
+            last = np.array([[t[-1] if t else BOS] for t, _ in hyps], np.int32)
+            out = self.dec_step(**{
+                "atten_init:source_contexts:00": np.repeat(vecs, n, 0),
+                "atten_init:source_vecs:00": np.repeat(vecs, n, 0),
+                "atten_init:source_padding:00": np.repeat(pad, n, 0),
+                "feeds:step_ids:00": last,
+                **state,
+            })
+            logp = out["fetches:log_probs:00"]
+            scores = np.array([s for _, s in hyps])[:, None] + logp
+            flat = np.argsort(scores, axis=None)[::-1][: 2 * beam]
+            new_hyps, rows = [], []
+            for idx in flat:
+                row, tok = divmod(int(idx), logp.shape[1])
+                toks, sc = hyps[row][0] + [tok], float(scores[row, tok])
+                if tok == EOS:
+                    done.append((toks[:-1], sc / (((5 + len(toks)) / 6) ** alpha)))
+                elif len(new_hyps) < beam:
+                    new_hyps.append((toks, sc))
+                    rows.append(row)
+            if not new_hyps:
+                break
+            # Stop when the best finished hypothesis beats any live one's optimistic score.
+            if len(done) >= beam:
+                best_done = max(s for _, s in done)
+                best_live = max(s for _, s in new_hyps) / (((5 + max_len) / 6) ** alpha)
+                if best_done > best_live:
+                    break
+            hyps = new_hyps
+            state = {k: out[k][rows] for k in state}
+        if not done:
+            done = hyps
+        return self.tgt_sp.decode(max(done, key=lambda h: h[1])[0])

offline_translate/languages.py

@@ -0,0 +1,41 @@
+"""Language codes as used by Google Translate's offline catalog, with English names."""
+
+NAMES = {
+    "af": "Afrikaans", "am": "Amharic", "ar": "Arabic", "az": "Azerbaijani", "be": "Belarusian",
+    "bg": "Bulgarian", "bn": "Bengali", "bs": "Bosnian", "ca": "Catalan", "ceb": "Cebuano",
+    "co": "Corsican", "cs": "Czech", "cy": "Welsh", "da": "Danish", "de": "German", "el": "Greek",
+    "en": "English", "eo": "Esperanto", "es": "Spanish", "et": "Estonian", "eu": "Basque",
+    "fa": "Persian", "fi": "Finnish", "fr": "French", "fy": "Frisian", "ga": "Irish",
+    "gd": "Scottish Gaelic", "gl": "Galician", "gu": "Gujarati", "ha": "Hausa", "haw": "Hawaiian",
+    "hi": "Hindi", "hmn": "Hmong", "hr": "Croatian", "ht": "Haitian Creole", "hu": "Hungarian",
+    "hy": "Armenian", "id": "Indonesian", "ig": "Igbo", "is": "Icelandic", "it": "Italian",
+    "iw": "Hebrew", "ja": "Japanese", "jw": "Javanese", "ka": "Georgian", "kk": "Kazakh",
+    "km": "Khmer", "kn": "Kannada", "ko": "Korean", "ku": "Kurdish", "ky": "Kyrgyz", "la": "Latin",
+    "lb": "Luxembourgish", "lo": "Lao", "lt": "Lithuanian", "lv": "Latvian", "mg": "Malagasy",
+    "mi": "Maori", "mk": "Macedonian", "ml": "Malayalam", "mn": "Mongolian", "mr": "Marathi",
+    "ms": "Malay", "mt": "Maltese", "my": "Burmese", "ne": "Nepali", "nl": "Dutch",
+    "no": "Norwegian", "ny": "Chichewa", "or": "Odia", "pa": "Punjabi", "pl": "Polish",
+    "ps": "Pashto", "pt": "Portuguese", "ro": "Romanian", "ru": "Russian", "rw": "Kinyarwanda",
+    "sd": "Sindhi", "si": "Sinhala", "sk": "Slovak", "sl": "Slovenian", "sm": "Samoan",
+    "sn": "Shona", "so": "Somali", "sq": "Albanian", "st": "Sesotho", "su": "Sundanese",
+    "sv": "Swedish", "sw": "Swahili", "ta": "Tamil", "te": "Telugu", "tg": "Tajik", "th": "Thai",
+    "tk": "Turkmen", "tl": "Filipino", "tr": "Turkish", "tt": "Tatar", "ug": "Uyghur",
+    "uk": "Ukrainian", "ur": "Urdu", "uz": "Uzbek", "vi": "Vietnamese", "xh": "Xhosa",
+    "yi": "Yiddish", "yo": "Yoruba", "zh": "Chinese (Simplified)", "zh-Hant": "Chinese (Traditional)",
+    "zu": "Zulu",
+}
+
+# Google uses legacy codes for a few languages; accept the modern ones too.
+ALIASES = {"he": "iw", "jv": "jw", "fil": "tl", "nb": "no", "zh-cn": "zh", "zh-hans": "zh",
+           "zh-tw": "zh-Hant", "zh-hant": "zh-Hant"}
+
+
+def normalize(code):
+    """Map user input (code or English name, any case) to a catalog code."""
+    c = code.strip()
+    if c.lower() in ALIASES:
+        return ALIASES[c.lower()]
+    for k, name in NAMES.items():
+        if c.lower() in (k.lower(), name.lower()):
+            return k
+    return c

offline_translate/translator.py

@@ -0,0 +1,240 @@
+"""Public API: manage language packs in a folder and translate between installed languages."""
+import hashlib
+import json
+import shutil
+import zlib
+from dataclasses import dataclass
+from pathlib import Path
+
+from . import catalog as cat
+from .languages import NAMES, normalize
+
+PACK_META = "pack.json"
+
+
+class TranslateError(Exception):
+    pass
+
+
+@dataclass(frozen=True)
+class Language:
+    code: str
+    name: str
+    installed: bool
+    download_size: int     # bytes to download (0 for English: it needs no pack of its own)
+    size: int              # bytes on disk once installed
+    update_available: bool = False
+    builtin: bool = False  # English: always "installed", every pack is X<->English
+    supported: bool = True  # False: pack uses an older model format this engine can't run yet
+
+
+@dataclass(frozen=True)
+class Translation:
+    text: str
+    source: str
+    target: str
+    via: tuple  # language hops, e.g. ("iw", "en", "ar") when pivoting through English
+
+
+class Translator:
+    """
+    models_dir: folder holding the catalog cache and one sub-folder per model direction (e.g. en_iw/).
+    channel:    manifest channel the Android app reads ("prod").
+    """
+
+    def __init__(self, models_dir, channel="prod", beam=4):
+        self.dir = Path(models_dir).expanduser()
+        self.dir.mkdir(parents=True, exist_ok=True)
+        self.channel = channel
+        self.beam = beam
+        self._packs = None
+        self._models = {}
+
+    # ---------- catalog ----------
+    @property
+    def _catalog_dir(self):
+        return self.dir / "_catalog"
+
+    def update_catalog(self):
+        """Fetch the latest manifest + package profile from Google. Returns the profile version."""
+        refs = cat.parse_manifest(cat.fetch(cat.MANIFEST_URL.format(channel=self.channel)))
+        if not refs:
+            raise TranslateError("manifest lists no translation profiles")
+        ref = refs[-1]
+        data = zlib.decompress(cat.fetch(ref.url))
+        if hashlib.sha1(data).hexdigest() != ref.sha1:
+            raise TranslateError(f"checksum mismatch for {ref.url}")
+        self._catalog_dir.mkdir(exist_ok=True)
+        (self._catalog_dir / "profile.pb").write_bytes(data)
+        (self._catalog_dir / "profile.json").write_text(json.dumps({"version": ref.version_str, "url": ref.url}))
+        self._packs = cat.parse_profile(data)
+        return ref.version_str
+
+    def catalog_version(self):
+        meta = self._catalog_dir / "profile.json"
+        return json.loads(meta.read_text())["version"] if meta.exists() else None
+
+    def _catalog(self):
+        if self._packs is None:
+            path = self._catalog_dir / "profile.pb"
+            if path.exists():
+                self._packs = cat.parse_profile(path.read_bytes())
+            else:
+                self.update_catalog()
+        return self._packs
+
+    # ---------- languages ----------
+    def _pair_dir(self, pair):
+        return self.dir / pair
+
+    def _installed_pairs(self):
+        return {tuple(d.name.split("_", 1)) for d in self.dir.iterdir()
+                if d.is_dir() and (d / "encoder_0.tflite").exists()}
+
+    def _pack_meta(self, pair):
+        """sha1 of each installed file; computed once for packs installed by other means."""
+        d = self._pair_dir(pair)
+        path = d / PACK_META
+        if path.exists():
+            return json.loads(path.read_text())
+        meta = {f.name: hashlib.sha1(f.read_bytes()).hexdigest()
+                for f in sorted(d.iterdir()) if f.is_file() and f.name != PACK_META}
+        path.write_text(json.dumps(meta, indent=1))
+        return meta
+
+    def _package_state(self, pkg):
+        """(installed, update_available) for one direction."""
+        if not (self._pair_dir(pkg.name) / "encoder_0.tflite").exists():
+            return False, False
+        meta = self._pack_meta(pkg.name)
+        return True, any(meta.get(f.name) != f.sha1 for f in pkg.files)
+
+    def languages(self, installed_only=False):
+        """All languages in the catalog (plus English), sorted by name."""
+        out = [Language("en", NAMES["en"], True, 0, 0, builtin=True)]
+        for code, pack in self._catalog().items():
+            states = [self._package_state(p) for p in pack.packages]
+            installed = all(s[0] for s in states)
+            out.append(Language(code, NAMES.get(code, code), installed, pack.download_size, pack.size,
+                                update_available=installed and any(s[1] for s in states),
+                                supported=pack.supported))
+        out.sort(key=lambda l: l.name)
+        return [l for l in out if l.installed] if installed_only else out
+
+    def installed_languages(self):
+        """Languages usable for translation right now (works offline, without the catalog)."""
+        codes = {c for pair in self._installed_pairs() for c in pair}
+        return sorted(codes or {"en"}, key=lambda c: NAMES.get(c, c))
+
+    def _resolve(self, lang):
+        code = normalize(lang)
+        if code == "en":
+            raise TranslateError("English is built in; every language pack includes it")
+        if code not in self._catalog():
+            raise TranslateError(f"unknown language '{lang}'")
+        return code
+
+    def download(self, lang, progress=None, force=False):
+        """
+        Download (or update) a language pack. Already up-to-date directions are skipped unless force.
+        progress(done_bytes, total_bytes) is called while downloading (sizes are compressed bytes).
+        """
+        pack = self._catalog()[self._resolve(lang)]
+        if not pack.supported:
+            raise TranslateError(f"{NAMES.get(pack.lang, pack.lang)} uses an older model format that isn't supported yet")
+        todo = [p for p in pack.packages if force or self._package_state(p) != (True, False)]
+        total = sum(f.download_size for p in todo for f in p.files)
+        done = 0
+        for pkg in todo:
+            tmp = self.dir / f"{pkg.name}.partial"
+            tmp.mkdir(exist_ok=True)
+            meta = {}
+            for f in pkg.files:
+                dest = tmp / f.name
+                if dest.exists() and hashlib.sha1(dest.read_bytes()).hexdigest() == f.sha1:  # resume
+                    meta[f.name] = f.sha1
+                else:
+                    meta[f.name] = self._download_file(f, dest, lambda n: progress and progress(done + n, total))
+                done += f.download_size
+                if progress:
+                    progress(done, total)
+            (tmp / PACK_META).write_text(json.dumps(meta, indent=1))
+            final = self._pair_dir(pkg.name)
+            if final.exists():
+                shutil.rmtree(final)
+            tmp.rename(final)
+            self._models.pop(tuple(pkg.name.split("_", 1)), None)
+
+    @staticmethod
+    def _download_file(f, dest, on_bytes):
+        sha, dec, got = hashlib.sha1(), zlib.decompressobj(), 0
+        part = dest.with_suffix(dest.suffix + ".part")
+        with cat.open_url(f.url) as r, open(part, "wb") as out:
+            while chunk := r.read(1 << 16):
+                got += len(chunk)
+                on_bytes(got)
+                data = dec.decompress(chunk)
+                sha.update(data)
+                out.write(data)
+            tail = dec.flush()
+            sha.update(tail)
+            out.write(tail)
+        if sha.hexdigest() != f.sha1:
+            part.unlink()
+            raise TranslateError(f"checksum mismatch for {f.url}")
+        part.rename(dest)
+        return f.sha1
+
+    def remove(self, lang):
+        code = normalize(lang)
+        if code == "en":
+            raise TranslateError("English is built in and can't be removed")
+        removed = False
+        for pair in [(code, "en"), ("en", code)]:
+            d = self._pair_dir("_".join(pair))
+            for p in (d, d.with_name(d.name + ".partial")):
+                if p.exists():
+                    shutil.rmtree(p)
+                    removed = True
+            self._models.pop(pair, None)
+        if not removed:
+            raise TranslateError(f"'{lang}' is not installed")
+
+    # ---------- translation ----------
+    def detect(self, text):
+        """Guess the source among installed languages by vocabulary coverage of non-ASCII letters."""
+        letters = {c for c in text.lower() if c.isalpha() and not c.isascii()}
+        best, best_hits = "en", 0
+        if letters:
+            for src, tgt in sorted(self._installed_pairs()):
+                if src == "en":
+                    continue
+                vocab = (self._pair_dir(f"{src}_{tgt}") / "src_spm_model.vocab").read_text("utf-8")
+                hits = len(letters & set(vocab))
+                if hits > best_hits:
+                    best, best_hits = src, hits
+        return best
+
+    def route(self, source, target):
+        """Model hops for source->target: direct, or via English like the Android app."""
+        pairs = self._installed_pairs()
+        if source == target:
+            return []
+        if (source, target) in pairs:
+            return [(source, target)]
+        if (source, "en") in pairs and ("en", target) in pairs:
+            return [(source, "en"), ("en", target)]
+        missing = [l for l in (source, target) if l != "en" and not {(l, "en"), ("en", l)} <= pairs]
+        raise TranslateError(f"no model for {source} -> {target}; download: {', '.join(missing)}")
+
+    def translate(self, text, target, source=None):
+        source = normalize(source) if source else self.detect(text)
+        target = normalize(target)
+        hops = self.route(source, target)
+        for pair in hops:
+            if pair not in self._models:
+                from .engine import NmtModel  # deferred: TFLite import is the slow part
+                self._models[pair] = NmtModel(self._pair_dir("_".join(pair)))
+            text = self._models[pair].translate(text, beam=self.beam)
+        via = (source,) + tuple(t for _, t in hops) if hops else (source,)
+        return Translation(text, source, target, via)

offline_translate-0.1.0.dist-info/METADATA

@@ -0,0 +1,136 @@
+Metadata-Version: 2.5
+Name: offline-translate
+Version: 0.1.0
+Summary: Offline machine translation using Google Translate's on-device language packs
+Project-URL: Homepage, https://github.com/asimon655/offline-translate
+Project-URL: Issues, https://github.com/asimon655/offline-translate/issues
+Author: asimon655
+License-Expression: MIT
+License-File: LICENSE
+Keywords: google-translate,nmt,offline,tflite,translation
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: ai-edge-litert>=1.0
+Requires-Dist: numpy>=1.24
+Requires-Dist: sentencepiece>=0.2
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == 'test'
+Description-Content-Type: text/markdown
+
+# offline-translate
+
+Offline machine translation in Python using the same on-device neural translation models that the
+Google Translate Android app downloads for offline use.
+
+- ~100 languages, each pack translates to/from English (~50 MB per language)
+- Fully offline once a pack is downloaded; no API key, no account
+- Language pairs without English (e.g. Hebrew → French) pivot through English, like the app does
+- Runs on CPU via [LiteRT](https://ai.google.dev/edge/litert) (TensorFlow Lite); ~0.3 s per sentence
+
+```bash
+pip install offline-translate
+```
+
+Supported platforms are those with `ai-edge-litert` wheels: macOS (Apple Silicon), Linux (x86_64,
+aarch64) and Windows (x64), Python 3.10+.
+
+## Usage
+
+```python
+from offline_translate import Translator
+
+t = Translator("~/.local/share/offline-translate")   # where models are stored
+
+t.download("he")                          # Hebrew <-> English, ~55 MB
+t.translate("Good morning!", target="he").text
+# 'בוקר טוב!'
+
+t.translate("בוקר טוב", target="en")      # source is auto-detected among installed languages
+# Translation(text='Good morning', source='iw', target='en', via=('iw', 'en'))
+```
+
+### Managing languages
+
+```python
+for lang in t.languages():                # full catalog (fetched once from Google, then cached)
+    print(lang.code, lang.name, lang.installed, lang.download_size, lang.update_available)
+
+t.installed_languages()                   # ['en', 'iw'] — works offline, no catalog needed
+t.download("fr", progress=lambda done, total: print(f"{done}/{total}"))
+t.remove("fr")
+
+t.update_catalog()                        # check Google for a newer catalog
+[l for l in t.languages() if l.update_available]   # then t.download(code) to update
+```
+
+Language codes follow Google's catalog (`iw` for Hebrew, `zh-Hant`, `jw`, …). Common aliases
+(`he`, `zh-TW`, `jv`) and English names (`"Hebrew"`) are accepted everywhere.
+
+English is built in: there is no separate English pack, since every pack contains both
+directions to and from English.
+
+### API
+
+| | |
+|---|---|
+| `Translator(models_dir, channel="prod", beam=4)` | `models_dir` is created if missing. `beam=1` is faster, slightly worse. |
+| `translate(text, target, source=None) -> Translation` | `Translation(text, source, target, via)` |
+| `detect(text) -> str` | Guess among installed languages |
+| `languages(installed_only=False) -> list[Language]` | `Language(code, name, installed, download_size, size, update_available, builtin, supported)` |
+| `installed_languages() -> list[str]` | |
+| `download(lang, progress=None, force=False)` | Verifies SHA-1 of every file; resumes interrupted downloads |
+| `remove(lang)` | |
+| `update_catalog() -> str` | Returns the catalog version |
+| `TranslateError` | Raised for unknown/uninstalled languages, checksum failures, … |
+
+Network errors surface as `OSError`.
+
+## Command line
+
+The repository also has a small CLI built on the library (not part of the PyPI package):
+
+```bash
+./translate get he                 # download Hebrew
+./translate "Good morning"         # English -> Hebrew by default; source auto-detected
+./translate -s he -t fr "בוקר טוב"  # any pair, via English
+./translate list --installed
+```
+
+`tools/pull_from_phone.py` copies already-downloaded packs from a rooted Android phone over adb.
+
+## How it works
+
+The Android app reads a manifest at
+`https://dl.google.com/translate/offline/manifest/gtr/prod/manifest.pb`, which lists versions of
+a *package profile* — the catalog of language packs and their files. Each translation direction is
+a TFLite encoder, an LSTM decoder run step by step with beam search, and SentencePiece tokenizers.
+
+### Limitations
+
+- 15 languages (am, az, bs, hy, kk, ky, ml, mn, ne, pa, ps, si, so, tg, uz) still ship an older
+  model format with a WordPiece tokenizer that isn't supported yet; they're listed with
+  `supported=False`.
+- Quality matches the app's offline mode, which is below online Google Translate.
+
+## Development
+
+```bash
+pip install -e ".[test]"
+pytest                          # offline unit tests (fake Google server, fake model)
+pytest --network                # + checks against the real Google catalog
+pytest --models=~/models        # + end-to-end translation with installed Hebrew models
+```
+
+## Disclaimer
+
+This project is not affiliated with or endorsed by Google. It contains no Google code or models;
+language packs are downloaded from Google's public servers at runtime and are subject to Google's
+terms.

offline_translate-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+offline_translate/__init__.py,sha256=JUCBJLHohZBK5zC5uX5N9Vn8Nc07_SMKrHr9OXeT-p0,462
+offline_translate/catalog.py,sha256=acEQbX9EwmxkdTs3j_MzqsDJffNj7i0bqZ4gbPOqq6E,5401
+offline_translate/engine.py,sha256=_BQiYut2OlBLxr4hVXnKBlrHGf0IjN-L8o_DnVWXMbk,5220
+offline_translate/languages.py,sha256=R0JjzM1C5RI7-hW5cn6Td1v1b62N2kaU7e5MaS-XotE,2595
+offline_translate/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+offline_translate/translator.py,sha256=czx1MymuFuQ0jLpUZD0HYRIDoeiZFq-falvKgpttCgk,9980
+offline_translate-0.1.0.dist-info/METADATA,sha256=QpCeIIkNvx5SZLkAdSVmpDEtARamnuxnwx97U9z33KI,5542
+offline_translate-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+offline_translate-0.1.0.dist-info/licenses/LICENSE,sha256=lx7YwLwJwmgIHoe3Bd-oiG47NRYrCqpdjYDo-YzgWw0,1066
+offline_translate-0.1.0.dist-info/RECORD,,

offline_translate-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

offline_translate-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 asimon655
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.