secryst 0.1.0__tar.gz

secryst-0.1.0/LICENSE

@@ -0,0 +1,30 @@
+BSD 3-Clause License
+
+Copyright (c) 2026, Ribose Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

secryst-0.1.0/PKG-INFO

@@ -0,0 +1,64 @@
+Metadata-Version: 2.4
+Name: secryst
+Version: 0.1.0
+Summary: Python runtime for Interscript Model Format (IMF v1) — the phonological layer of Interscript
+Author: Interscript Project
+License: BSD-3-Clause
+Keywords: transliteration,diacritization,g2p,onnx,byt5
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: onnxruntime>=1.17
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: onnx>=1.16; extra == "dev"
+Dynamic: license-file
+
+# secryst — Python crystal
+
+**Secryst** is coined from *scrying* + *crystal*: gazing into an opaque
+script to reveal its hidden reading. This package is the Python crystal.
+
+The Python crystal — reference implementation of **IMF v1** model zips
+and the `models.yaml` index (the **interscript-ml** contract), the
+phonological layer of Interscript. The Ruby (`secryst` gem) and
+TypeScript (`npm i secryst`) crystals are diffed against this one on
+shared golden sets. This crystal owns golden generation and numerical
+adjudication for the family.
+
+Home repo: https://github.com/secryst/secryst-py (this copy in
+ml-models/runtime is the frozen origin; the package now lives there).
+
+```python
+from secryst import Model
+
+model = Model.load("khm-latn-1.0")        # id: index resolve -> download
+                                          # -> sha256-verify -> cache -> load
+model.translate("ភាសា")                   # -> "pheasaea"
+model.id                                   # "khm-latn-1.0"
+
+model = Model.load("khm-latn-1.0.zip")    # or: a local zip path directly
+```
+
+- Byte-level only: the canonical ByT5 table (byte `b` → id `b+3`,
+  trailing EOS) — no vocab files, no per-model tokenization code.
+- Greedy KV-cache decode when the zip ships `decoder-kv.onnx`
+  (default), plain full-recompute fallback otherwise.
+- Every `.onnx` member is sha256-verified against `metadata.yaml`
+  before the session is created; corrupt downloads fail loudly.
+- Dynamic fetch per the `models.yaml` contract (shared with the Ruby and
+  TypeScript runtimes): resolve id -> channel URL, download to temp,
+  verify whole-file sha256 against the index, atomically install into
+  `~/.cache/secryst/models/<id>/`. Overrides:
+  `SECRYST_INDEX` (URL or path), `SECRYST_CACHE`.
+
+Install: `pip install secryst` (or `pip install -e ".[dev]"` from the repo).
+
+Tests: `python -m pytest runtime/tests` — tiny-graph zips, no torch
+needed. The end-to-end golden test runs when `SECRYST_E2E_ZIP`
+points at a real zip (e.g. `models/khm-latn/khm-latn-1.0-fp32.zip`)
+and asserts byte-identical outputs against `golden/khm-latn-100.jsonl`.
+
+License: BSD-3-Clause.

secryst-0.1.0/README.md

@@ -0,0 +1,46 @@
+# secryst — Python crystal
+
+**Secryst** is coined from *scrying* + *crystal*: gazing into an opaque
+script to reveal its hidden reading. This package is the Python crystal.
+
+The Python crystal — reference implementation of **IMF v1** model zips
+and the `models.yaml` index (the **interscript-ml** contract), the
+phonological layer of Interscript. The Ruby (`secryst` gem) and
+TypeScript (`npm i secryst`) crystals are diffed against this one on
+shared golden sets. This crystal owns golden generation and numerical
+adjudication for the family.
+
+Home repo: https://github.com/secryst/secryst-py (this copy in
+ml-models/runtime is the frozen origin; the package now lives there).
+
+```python
+from secryst import Model
+
+model = Model.load("khm-latn-1.0")        # id: index resolve -> download
+                                          # -> sha256-verify -> cache -> load
+model.translate("ភាសា")                   # -> "pheasaea"
+model.id                                   # "khm-latn-1.0"
+
+model = Model.load("khm-latn-1.0.zip")    # or: a local zip path directly
+```
+
+- Byte-level only: the canonical ByT5 table (byte `b` → id `b+3`,
+  trailing EOS) — no vocab files, no per-model tokenization code.
+- Greedy KV-cache decode when the zip ships `decoder-kv.onnx`
+  (default), plain full-recompute fallback otherwise.
+- Every `.onnx` member is sha256-verified against `metadata.yaml`
+  before the session is created; corrupt downloads fail loudly.
+- Dynamic fetch per the `models.yaml` contract (shared with the Ruby and
+  TypeScript runtimes): resolve id -> channel URL, download to temp,
+  verify whole-file sha256 against the index, atomically install into
+  `~/.cache/secryst/models/<id>/`. Overrides:
+  `SECRYST_INDEX` (URL or path), `SECRYST_CACHE`.
+
+Install: `pip install secryst` (or `pip install -e ".[dev]"` from the repo).
+
+Tests: `python -m pytest runtime/tests` — tiny-graph zips, no torch
+needed. The end-to-end golden test runs when `SECRYST_E2E_ZIP`
+points at a real zip (e.g. `models/khm-latn/khm-latn-1.0-fp32.zip`)
+and asserts byte-identical outputs against `golden/khm-latn-100.jsonl`.
+
+License: BSD-3-Clause.

secryst-0.1.0/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "secryst"
+version = "0.1.0"
+description = "Python runtime for Interscript Model Format (IMF v1) — the phonological layer of Interscript"
+readme = "README.md"
+license = { text = "BSD-3-Clause" }
+requires-python = ">=3.10"
+authors = [{ name = "Interscript Project" }]
+keywords = ["transliteration", "diacritization", "g2p", "onnx", "byt5"]
+
+dependencies = [
+  "numpy>=1.26",
+  "pyyaml>=6.0",
+  "onnxruntime>=1.17",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0", "onnx>=1.16"]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+addopts = "-ra -q"

secryst-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

secryst-0.1.0/src/secryst/__init__.py

@@ -0,0 +1,33 @@
+"""interscript-ml — the Python runtime for Interscript Model Format (IMF v1).
+
+The reference implementation: the Ruby and TypeScript runtimes are
+diffed against this one on shared golden sets.
+
+    from secryst import Model
+    model = Model.load("khm-latn-1.0.zip")
+    model.translate("ភាសា")        # -> "pheasaea"
+
+Byte-level only: the tokenizer is the canonical ByT5 table (byte b ->
+token id b+3, trailing EOS), fixed and documented — no vocab files.
+"""
+
+from __future__ import annotations
+
+from secryst.loader import Manifest, ModelFormatError
+from secryst.model import Model
+from secryst.registry import RegistryError, resolve
+from secryst.tokens import BYTE_OFFSET, EOS_ID, PAD_ID, UNK_ID, decode, encode
+
+__all__ = [
+    "BYTE_OFFSET",
+    "EOS_ID",
+    "Manifest",
+    "Model",
+    "ModelFormatError",
+    "PAD_ID",
+    "RegistryError",
+    "UNK_ID",
+    "decode",
+    "encode",
+    "resolve",
+]

secryst-0.1.0/src/secryst/loader.py

@@ -0,0 +1,68 @@
+"""IMF v1 zip loading: sha256 verification + extraction."""
+
+from __future__ import annotations
+
+import hashlib
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+
+class ModelFormatError(ValueError):
+    """The zip is not a valid IMF v1 artifact (or fails integrity)."""
+
+
+@dataclass(frozen=True)
+class Manifest:
+    id: str
+    task: str
+    decoder: str
+    precision: str
+    opset: int
+    sha256: dict[str, str]
+
+
+def load_manifest(zip_path: Path | str) -> Manifest:
+    with zipfile.ZipFile(zip_path) as zf:
+        names = zf.namelist()
+        for required in ("metadata.yaml", "encoder.onnx", "decoder.onnx"):
+            if required not in names:
+                raise ModelFormatError(f"missing required file: {required}")
+        raw = yaml.safe_load(zf.read("metadata.yaml"))
+        if raw.get("format") != "imf-v1":
+            raise ModelFormatError(f"unsupported format: {raw.get('format')!r}")
+        if raw.get("tokenizer") != "bytes":
+            raise ModelFormatError(
+                f"tokenizer {raw.get('tokenizer')!r}: this runtime is byte-level only"
+            )
+        return Manifest(
+            id=raw["id"],
+            task=raw["task"],
+            decoder=raw.get("decoder", "plain"),
+            precision=raw.get("precision", "fp32"),
+            opset=int(raw.get("opset", 14)),
+            sha256=dict(raw.get("sha256", {})),
+        )
+
+
+def verify_and_read(zip_path: Path | str) -> dict[str, bytes]:
+    """Read .onnx members after verifying each sha256 against the
+    manifest — the corrupt-download failure mode fails loudly here."""
+    manifest = load_manifest(zip_path)
+    graphs: dict[str, bytes] = {}
+    with zipfile.ZipFile(zip_path) as zf:
+        for name in [n for n in zf.namelist() if n.endswith(".onnx")]:
+            member = zf.read(name)
+            recorded = manifest.sha256.get(name)
+            if recorded is None:
+                raise ModelFormatError(f"{name} is not covered by metadata sha256")
+            actual = hashlib.sha256(member).hexdigest()
+            if actual != recorded:
+                raise ModelFormatError(
+                    f"{name} sha256 mismatch: zip has {actual}, "
+                    f"metadata says {recorded}"
+                )
+            graphs[name] = member
+    return graphs

secryst-0.1.0/src/secryst/model.py

@@ -0,0 +1,127 @@
+"""Model.load(zip) + translate(text): greedy KV decode with plain fallback."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+
+from secryst.tokens import EOS_ID, PAD_ID, decode, encode
+from secryst.loader import load_manifest, verify_and_read
+
+
+class Model:
+    """A loaded, checksum-verified IMF v1 model.
+
+    >>> model = Model.load("khm-latn-1.0.zip")
+    >>> model.translate("ភាសា")
+    """
+
+    def __init__(self, zip_path: Path | str):
+        self.zip_path = Path(zip_path)
+        self.manifest = load_manifest(self.zip_path)
+        import onnxruntime as ort
+
+        options = ort.SessionOptions()
+        options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+        graphs = verify_and_read(self.zip_path)
+        self._encoder = ort.InferenceSession(
+            graphs["encoder.onnx"], options, providers=_providers()
+        )
+        decoder_name = (
+            "decoder-kv.onnx"
+            if self.manifest.decoder == "kv" and "decoder-kv.onnx" in graphs
+            else "decoder.onnx"
+        )
+        self._kv_session = decoder_name == "decoder-kv.onnx"
+        self._decoder = ort.InferenceSession(
+            graphs[decoder_name], options, providers=_providers()
+        )
+        self._pasts = {
+            meta.name: _zero_past(meta)
+            for meta in self._decoder.get_inputs()
+            if meta.name.startswith("past_")
+        }
+        self._output_names = [o.name for o in self._decoder.get_outputs()]
+
+    @classmethod
+    def load(cls, path_or_id: Path | str, index_url: str | None = None) -> "Model":
+        """Accepts a zip path OR a model id from models.yaml (dynamic
+        fetch: download -> verify -> cache)."""
+        candidate = str(path_or_id)
+        if candidate.endswith(".zip") or Path(candidate).exists():
+            return cls(candidate)
+        from secryst.registry import resolve
+
+        return cls(resolve(candidate, index_url))
+
+    @property
+    def id(self) -> str:
+        return self.manifest.id
+
+    def translate(self, text: str, max_len: int = 256) -> str:
+        token_ids = self.generate(text, max_len=max_len)
+        return decode(token_ids)
+
+    def generate(self, text: str, max_len: int = 256) -> list[int]:
+        ids = np.array([encode(text)], dtype=np.int64)
+        if ids.shape[1] == 1:  # only the trailing EOS: empty input
+            return []
+        hidden = self._encoder.run(None, {"input_ids": ids})[0]
+        if self._kv_session:
+            return self._greedy_kv(hidden, max_len)
+        return self._greedy_plain(hidden, max_len)
+
+    def _greedy_kv(self, hidden, max_len: int) -> list[int]:
+        pasts = dict(self._pasts)
+        current = np.array([[PAD_ID]], dtype=np.int64)
+        generated: list[int] = []
+        for _ in range(max_len):
+            outputs = self._decoder.run(
+                None,
+                {"input_ids": current, "encoder_hidden_states": hidden, **pasts},
+            )
+            results = dict(zip(self._output_names, outputs, strict=True))
+            token = int(np.argmax(results["logits"][0, -1]))
+            if token == EOS_ID:
+                break
+            generated.append(token)
+            pasts = {
+                name: results[name.replace("past_", "present_", 1)]
+                for name in pasts
+            }
+            current = np.array([[token]], dtype=np.int64)
+        return generated
+
+    def _greedy_plain(self, hidden, max_len: int) -> list[int]:
+        decoder_ids = np.array([[PAD_ID]], dtype=np.int64)
+        generated: list[int] = []
+        for _ in range(max_len):
+            logits = self._decoder.run(
+                None,
+                {"input_ids": decoder_ids, "encoder_hidden_states": hidden},
+            )[0]
+            token = int(np.argmax(logits[0, -1]))
+            if token == EOS_ID:
+                break
+            generated.append(token)
+            decoder_ids = np.concatenate(
+                [decoder_ids, np.array([[token]], dtype=np.int64)], axis=1
+            )
+        return generated
+
+
+def _providers() -> list[str]:
+    import onnxruntime as ort
+
+    available = ort.get_available_providers()
+    preferred = [p for p in ("CPUExecutionProvider",) if p in available]
+    return preferred or available
+
+
+def _zero_past(meta) -> object:
+    shape = meta.shape  # [batch, heads, past_seq, d_kv], dynamic dims are str
+    heads = shape[1] if isinstance(shape[1], int) else 4
+    d_kv = shape[3] if isinstance(shape[3], int) else 8
+    dtype = np.float16 if meta.type == "tensor(float16)" else np.float32
+    return np.zeros((1, heads, 0, d_kv), dtype=dtype)

secryst-0.1.0/src/secryst/registry.py

@@ -0,0 +1,159 @@
+"""Model index resolution + cached downloads (the dynamic-fetch layer).
+
+Implements the models.yaml contract shared by the Ruby and TypeScript
+runtimes: resolve an id, reuse a verified cache copy, or download +
+sha256-verify + atomically install into the cache.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import shutil
+import tempfile
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import urlparse
+
+import yaml
+
+DEFAULT_INDEX_URL = (
+    "https://raw.githubusercontent.com/interscript/interscript-ml/main/models.yaml"
+)
+ENV_INDEX = "SECRYST_INDEX"
+ENV_CACHE = "SECRYST_CACHE"
+
+
+class RegistryError(ValueError):
+    """The index cannot be fetched/parsed, or the id is unknown."""
+
+
+@dataclass(frozen=True)
+class Part:
+    url: str
+    sha256: str
+    size: int
+
+
+@dataclass(frozen=True)
+class IndexEntry:
+    id: str
+    filename: str
+    url: str
+    sha256: str
+    size: int
+    precision: str
+    task: str
+    parts: tuple[Part, ...] = ()
+
+
+def cache_dir() -> Path:
+    if os.environ.get(ENV_CACHE):
+        return Path(os.environ[ENV_CACHE])
+    return Path.home() / ".cache" / "secryst"
+
+
+def load_index(index_url: str | None = None) -> dict[str, IndexEntry]:
+    source = index_url or os.environ.get(ENV_INDEX) or DEFAULT_INDEX_URL
+    if source.startswith(("http://", "https://")):
+        with urllib.request.urlopen(source) as response:
+            text = response.read().decode("utf-8")
+    else:
+        text = Path(source).read_text(encoding="utf-8")
+    raw = yaml.safe_load(text)
+    if not isinstance(raw, dict) or raw.get("version") != 1:
+        raise RegistryError("index must be a mapping with version: 1")
+    entries: dict[str, IndexEntry] = {}
+    for model_id, spec in raw.get("models", {}).items():
+        parts = tuple(
+            Part(url=part["url"], sha256=part["sha256"], size=int(part.get("size", 0)))
+            for part in spec.get("parts", [])
+        )
+        entries[model_id] = IndexEntry(
+            id=model_id,
+            filename=spec["filename"],
+            url=spec.get("url", ""),
+            sha256=spec["sha256"],
+            size=int(spec.get("size", 0)),
+            precision=spec.get("precision", "fp32"),
+            task=spec.get("task", ""),
+            parts=parts,
+        )
+    return entries
+
+
+def _sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as fh:
+        while chunk := fh.read(1024 * 1024):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _open_channel(url: str):
+    if url.startswith("file://"):
+        return open(urlparse(url).path, "rb")
+    return urllib.request.urlopen(url)
+
+
+def _download_parts(entry: IndexEntry, downloaded: Path) -> None:
+    """Stream parts into `downloaded` in index order, verifying each part's
+    sha256 as it lands. Used when the artifact exceeds GitHub's 2 GiB
+    per-asset cap; the assembled file is checked against entry.sha256 by
+    the caller, so the cache contract is identical to single-file models."""
+    with downloaded.open("ab") as out:
+        for index, part in enumerate(entry.parts):
+            digest = hashlib.sha256()
+            with _open_channel(part.url) as remote:
+                while chunk := remote.read(1024 * 1024):
+                    out.write(chunk)
+                    digest.update(chunk)
+            actual = digest.hexdigest()
+            if actual != part.sha256:
+                raise RegistryError(
+                    f"part {index} of {entry.filename} sha256 mismatch: "
+                    f"got {actual}, index says {part.sha256}"
+                )
+
+
+def resolve(model_id: str, index_url: str | None = None) -> Path:
+    """Return a verified local zip path for `model_id`, downloading and
+    installing into the cache when needed. Never returns an unverified
+    file: cache hits are re-verified against the index sha256."""
+    entries = load_index(index_url)
+    if model_id not in entries:
+        raise RegistryError(
+            f"unknown model id {model_id!r} (known: {sorted(entries)})"
+        )
+    entry = entries[model_id]
+    target = cache_dir() / "models" / model_id / entry.filename
+    if target.is_file() and _sha256_file(target) == entry.sha256:
+        return target
+
+    target.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_name = tempfile.mkstemp(dir=target.parent, suffix=".part")
+    os.close(fd)
+    downloaded = Path(tmp_name)
+    if entry.parts:
+        _download_parts(entry, downloaded)
+    elif entry.url.startswith("file://"):
+        source = Path(urlparse(entry.url).path)
+        if not source.is_file():
+            raise RegistryError(f"channel file missing: {source}")
+        shutil.copyfile(source, downloaded)  # file:// is a mirror, not a move
+    else:
+        urllib.request.urlretrieve(entry.url, downloaded)
+    try:
+        actual = _sha256_file(downloaded)
+        if actual != entry.sha256:
+            raise RegistryError(
+                f"downloaded {entry.filename} sha256 mismatch: got {actual}, "
+                f"index says {entry.sha256}"
+            )
+        if downloaded != target:
+            os.replace(downloaded, target)
+    finally:
+        if downloaded != target and downloaded.exists():
+            downloaded.unlink()
+    return target

secryst-0.1.0/src/secryst/tokens.py

@@ -0,0 +1,30 @@
+"""The canonical ByT5 byte table (fixed, no vocab files).
+
+Stock google/byt5 tokenizers map UTF-8 byte b to id b+3 and append
+EOS(1); pad=0, unk=2. Ids are NOT raw byte values — feeding text.bytes
+directly produces silent garbage on real models.
+"""
+
+from __future__ import annotations
+
+BYTE_OFFSET = 3
+PAD_ID = 0
+EOS_ID = 1
+UNK_ID = 2
+
+
+def encode(text: str) -> list[int]:
+    """Canonical byte-level tokenization (byte+3 table, trailing EOS)."""
+    return [b + BYTE_OFFSET for b in text.encode("utf-8")] + [EOS_ID]
+
+
+def decode(token_ids: list[int]) -> str:
+    """Token ids -> text; stops at EOS, maps id-3 back to a byte."""
+    out = bytearray()
+    for token in token_ids:
+        if token == EOS_ID:
+            break
+        if token in (PAD_ID, UNK_ID):
+            continue
+        out.append((token - BYTE_OFFSET) % 256)
+    return out.decode("utf-8", errors="replace")

secryst-0.1.0/src/secryst.egg-info/PKG-INFO

@@ -0,0 +1,64 @@
+Metadata-Version: 2.4
+Name: secryst
+Version: 0.1.0
+Summary: Python runtime for Interscript Model Format (IMF v1) — the phonological layer of Interscript
+Author: Interscript Project
+License: BSD-3-Clause
+Keywords: transliteration,diacritization,g2p,onnx,byt5
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: onnxruntime>=1.17
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: onnx>=1.16; extra == "dev"
+Dynamic: license-file
+
+# secryst — Python crystal
+
+**Secryst** is coined from *scrying* + *crystal*: gazing into an opaque
+script to reveal its hidden reading. This package is the Python crystal.
+
+The Python crystal — reference implementation of **IMF v1** model zips
+and the `models.yaml` index (the **interscript-ml** contract), the
+phonological layer of Interscript. The Ruby (`secryst` gem) and
+TypeScript (`npm i secryst`) crystals are diffed against this one on
+shared golden sets. This crystal owns golden generation and numerical
+adjudication for the family.
+
+Home repo: https://github.com/secryst/secryst-py (this copy in
+ml-models/runtime is the frozen origin; the package now lives there).
+
+```python
+from secryst import Model
+
+model = Model.load("khm-latn-1.0")        # id: index resolve -> download
+                                          # -> sha256-verify -> cache -> load
+model.translate("ភាសា")                   # -> "pheasaea"
+model.id                                   # "khm-latn-1.0"
+
+model = Model.load("khm-latn-1.0.zip")    # or: a local zip path directly
+```
+
+- Byte-level only: the canonical ByT5 table (byte `b` → id `b+3`,
+  trailing EOS) — no vocab files, no per-model tokenization code.
+- Greedy KV-cache decode when the zip ships `decoder-kv.onnx`
+  (default), plain full-recompute fallback otherwise.
+- Every `.onnx` member is sha256-verified against `metadata.yaml`
+  before the session is created; corrupt downloads fail loudly.
+- Dynamic fetch per the `models.yaml` contract (shared with the Ruby and
+  TypeScript runtimes): resolve id -> channel URL, download to temp,
+  verify whole-file sha256 against the index, atomically install into
+  `~/.cache/secryst/models/<id>/`. Overrides:
+  `SECRYST_INDEX` (URL or path), `SECRYST_CACHE`.
+
+Install: `pip install secryst` (or `pip install -e ".[dev]"` from the repo).
+
+Tests: `python -m pytest runtime/tests` — tiny-graph zips, no torch
+needed. The end-to-end golden test runs when `SECRYST_E2E_ZIP`
+points at a real zip (e.g. `models/khm-latn/khm-latn-1.0-fp32.zip`)
+and asserts byte-identical outputs against `golden/khm-latn-100.jsonl`.
+
+License: BSD-3-Clause.

secryst-0.1.0/src/secryst.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+LICENSE
+README.md
+pyproject.toml
+src/secryst/__init__.py
+src/secryst/loader.py
+src/secryst/model.py
+src/secryst/registry.py
+src/secryst/tokens.py
+src/secryst.egg-info/PKG-INFO
+src/secryst.egg-info/SOURCES.txt
+src/secryst.egg-info/dependency_links.txt
+src/secryst.egg-info/requires.txt
+src/secryst.egg-info/top_level.txt
+tests/test_model.py
+tests/test_registry.py
+tests/tests_helpers.py

secryst-0.1.0/src/secryst.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

secryst-0.1.0/src/secryst.egg-info/requires.txt

@@ -0,0 +1,7 @@
+numpy>=1.26
+pyyaml>=6.0
+onnxruntime>=1.17
+
+[dev]
+pytest>=8.0
+onnx>=1.16

secryst-0.1.0/src/secryst.egg-info/top_level.txt

@@ -0,0 +1 @@
+secryst

secryst-0.1.0/tests/test_model.py

@@ -0,0 +1,159 @@
+"""Tests for the interscript-ml runtime.
+
+Tiny-graph zips built with the onnx package (no torch, no training
+repo). The end-to-end golden test runs only when a real zip is provided
+via SECRYST_E2E_ZIP.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import zipfile
+from pathlib import Path
+
+import pytest
+import yaml
+
+ort = pytest.importorskip("onnxruntime")
+onnx = pytest.importorskip("onnx")
+
+from secryst import Model, ModelFormatError, decode, encode  # noqa: E402
+from onnx import TensorProto, helper, numpy_helper  # noqa: E402
+
+import numpy as np  # noqa: E402
+
+
+def _graph(opset: int = 14) -> bytes:
+    graph = helper.make_graph(
+        nodes=[helper.make_node("Add", ["input_ids", "bias"], ["last_hidden_state"])],
+        name="tiny-enc",
+        inputs=[
+            helper.make_tensor_value_info("input_ids", TensorProto.INT64, ["batch", "seq"])
+        ],
+        outputs=[
+            helper.make_tensor_value_info(
+                "last_hidden_state", TensorProto.INT64, ["batch", "seq"]
+            )
+        ],
+        initializer=[numpy_helper.from_array(np.zeros(1, dtype=np.int64), "bias")],
+    )
+    model = helper.make_model(
+        graph, opset_imports=[helper.make_opsetid("", opset)], ir_version=7
+    )
+    return model.SerializeToString()
+
+
+def _decoder_graph() -> bytes:
+    graph = helper.make_graph(
+        nodes=[
+            helper.make_node("Add", ["input_ids", "bias"], ["logits"])
+        ],
+        name="tiny-dec",
+        inputs=[
+            helper.make_tensor_value_info("input_ids", TensorProto.INT64, ["batch", "seq"]),
+            helper.make_tensor_value_info(
+                "encoder_hidden_states", TensorProto.INT64, ["batch", "seq"]
+            ),
+        ],
+        outputs=[
+            helper.make_tensor_value_info("logits", TensorProto.INT64, ["batch", "seq"])
+        ],
+        initializer=[numpy_helper.from_array(np.zeros(1, dtype=np.int64), "bias")],
+    )
+    model = helper.make_model(
+        graph, opset_imports=[helper.make_opsetid("", 14)], ir_version=7
+    )
+    return model.SerializeToString()
+
+
+MANIFEST = {
+    "format": "imf-v1",
+    "id": "tiny-1.0",
+    "task": "translit",
+    "source_script": "Latn",
+    "target": "Latn",
+    "tokenizer": "bytes",
+    "opset": 14,
+    "decoder": "plain",
+    "precision": "fp32",
+    "license": "BSD-3-Clause",
+    "trained_from": "runtime test fixture",
+}
+
+
+def _tiny_zip(path: Path, tamper: bool = False, manifest: dict | None = None) -> Path:
+    encoder, decoder = _graph(), _decoder_graph()
+    sha = {
+        "encoder.onnx": hashlib.sha256(encoder).hexdigest(),
+        "decoder.onnx": hashlib.sha256(decoder).hexdigest(),
+    }
+    if tamper:
+        sha["encoder.onnx"] = "0" * 64
+    meta = dict(manifest if manifest is not None else MANIFEST)
+    meta["sha256"] = sha
+    with zipfile.ZipFile(path, "w") as zf:
+        zf.writestr("metadata.yaml", yaml.safe_dump(meta))
+        zf.writestr("encoder.onnx", encoder)
+        zf.writestr("decoder.onnx", decoder)
+        zf.writestr("README.md", "# tiny\n")
+    return path
+
+
+def test_token_table() -> None:
+    assert encode("rok") == [117, 114, 110, 1]
+    assert decode([117, 114, 110]) == "rok"
+    assert decode([117, 1, 114]) == "r"
+    assert decode([]) == ""
+
+
+def test_load_and_decode_tiny(tmp_path: Path) -> None:
+    z = _tiny_zip(tmp_path / "tiny.zip")
+    model = Model.load(z)
+    assert model.id == "tiny-1.0"
+    # tiny graphs are identity Adds: logits echo the decoder prefix, so
+    # greedy emits encode(PAD-prefix input)+... — deterministic, not
+    # meaningful; what matters is that the loop runs and decodes.
+    tokens = model.generate("he", max_len=4)
+    assert isinstance(tokens, list)
+    text = model.translate("he", max_len=4)
+    assert isinstance(text, str)
+
+
+def test_sha256_mismatch_rejected(tmp_path: Path) -> None:
+    z = _tiny_zip(tmp_path / "bad.zip", tamper=True)
+    with pytest.raises(ModelFormatError, match="sha256 mismatch"):
+        Model.load(z)
+
+
+def test_non_bytes_tokenizer_rejected(tmp_path: Path) -> None:
+    manifest = dict(MANIFEST, tokenizer="sentencepiece")
+    z = _tiny_zip(tmp_path / "spm.zip", manifest=manifest)
+    with pytest.raises(ModelFormatError, match="byte-level only"):
+        Model.load(z)
+
+
+def test_missing_graph_rejected(tmp_path: Path) -> None:
+    z = _tiny_zip(tmp_path / "m.zip")
+    truncated = tmp_path / "trunc.zip"
+    with zipfile.ZipFile(z) as src, zipfile.ZipFile(truncated, "w") as dst:
+        for name in src.namelist():
+            if name != "decoder.onnx":
+                dst.writestr(name, src.read(name))
+    with pytest.raises(ModelFormatError, match="decoder.onnx"):
+        Model.load(truncated)
+
+
+def test_golden_set_e2e() -> None:
+    """Run against a real zip: byte-identical outputs on the golden set."""
+    zip_path = os.environ.get("SECRYST_E2E_ZIP")
+    if not zip_path:
+        pytest.skip("set SECRYST_E2E_ZIP to a real IMF zip")
+    golden = Path(__file__).resolve().parent.parent.parent / "golden" / "khm-latn-100.jsonl"
+    if "khm" not in Path(zip_path).name:
+        pytest.skip("golden file is khm-latn specific")
+    model = Model.load(zip_path)
+    rows = [json.loads(line) for line in golden.read_text(encoding="utf-8").splitlines()]
+    for row in rows:
+        assert model.translate(row["input"], max_len=128) == row["output"], row["input"]

secryst-0.1.0/tests/test_registry.py

@@ -0,0 +1,170 @@
+"""Tests for the dynamic-fetch layer (models.yaml resolution + cache)."""
+
+from __future__ import annotations
+
+import hashlib
+import zipfile
+from pathlib import Path
+
+import pytest
+import yaml
+
+from secryst.registry import RegistryError, resolve
+from tests_helpers import build_tiny_zip
+
+import os  # noqa: E402
+
+
+def _index_file(tmp_path: Path, zip_path: Path, sha256: str | None = None) -> Path:
+    index = {
+        "version": 1,
+        "models": {
+            "tiny-1.0": {
+                "task": "translit",
+                "precision": "fp32",
+                "filename": zip_path.name,
+                "url": f"file://{zip_path}",
+                "sha256": sha256 or hashlib.sha256(zip_path.read_bytes()).hexdigest(),
+                "size": zip_path.stat().st_size,
+            }
+        },
+    }
+    path = tmp_path / "models.yaml"
+    path.write_text(yaml.safe_dump(index), encoding="utf-8")
+    return path
+
+
+def test_resolve_downloads_verifies_and_caches(tmp_path: Path) -> None:
+    zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
+    index = _index_file(tmp_path, zip_path)
+    cache = tmp_path / "cache"
+    os.environ["SECRYST_CACHE"] = str(cache)
+    try:
+        local = resolve("tiny-1.0", index_url=str(index))
+        assert local == cache / "models" / "tiny-1.0" / "tiny.zip"
+        assert local.is_file()
+        # second resolve is a verified cache hit (channel dir removed)
+        zip_path.unlink()
+        assert resolve("tiny-1.0", index_url=str(index)) == local
+    finally:
+        os.environ.pop("SECRYST_CACHE", None)
+
+
+def test_resolve_rejects_bad_download(tmp_path: Path) -> None:
+    zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
+    index = _index_file(tmp_path, zip_path, sha256="0" * 64)
+    os.environ["SECRYST_CACHE"] = str(tmp_path / "cache")
+    try:
+        with pytest.raises(RegistryError, match="sha256 mismatch"):
+            resolve("tiny-1.0", index_url=str(index))
+    finally:
+        os.environ.pop("SECRYST_CACHE", None)
+
+
+def test_resolve_unknown_id(tmp_path: Path) -> None:
+    index = tmp_path / "models.yaml"
+    index.write_text(yaml.safe_dump({"version": 1, "models": {}}), encoding="utf-8")
+    with pytest.raises(RegistryError, match="unknown model id"):
+        resolve("nope-1.0", index_url=str(index))
+
+
+def test_model_load_by_id(tmp_path: Path) -> None:
+    zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
+    index = _index_file(tmp_path, zip_path)
+    os.environ["SECRYST_CACHE"] = str(tmp_path / "cache")
+    try:
+        from secryst import Model
+
+        model = Model.load("tiny-1.0", index_url=str(index))
+        assert model.id == "tiny-1.0"
+        assert isinstance(model.translate("he", max_len=4), str)
+    finally:
+        os.environ.pop("SECRYST_CACHE", None)
+
+
+def test_resolve_parts_assembles_and_verifies(tmp_path: Path) -> None:
+    import hashlib
+
+    zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
+    blob = zip_path.read_bytes()
+    part_a, part_b = blob[: len(blob) // 2 + 3], blob[len(blob) // 2 + 3 :]
+    channel = tmp_path / "channel"
+    (channel / "tiny.zip.part-00").write_bytes(part_a)
+    (channel / "tiny.zip.part-01").write_bytes(part_b)
+    index = {
+        "version": 1,
+        "models": {
+            "tiny-1.0": {
+                "task": "translit",
+                "precision": "fp32",
+                "filename": "tiny.zip",
+                "sha256": hashlib.sha256(blob).hexdigest(),
+                "size": len(blob),
+                "parts": [
+                    {
+                        "url": f"file://{channel / 'tiny.zip.part-00'}",
+                        "sha256": hashlib.sha256(part_a).hexdigest(),
+                        "size": len(part_a),
+                    },
+                    {
+                        "url": f"file://{channel / 'tiny.zip.part-01'}",
+                        "sha256": hashlib.sha256(part_b).hexdigest(),
+                        "size": len(part_b),
+                    },
+                ],
+            },
+        },
+    }
+    index_path = tmp_path / "models.yaml"
+    index_path.write_text(yaml.safe_dump(index), encoding="utf-8")
+    cache = tmp_path / "cache"
+    os.environ["SECRYST_CACHE"] = str(cache)
+    try:
+        local = resolve("tiny-1.0", index_url=str(index_path))
+        assert local == cache / "models" / "tiny-1.0" / "tiny.zip"
+        assert local.read_bytes() == blob
+        zip_path.unlink()
+        (channel / "tiny.zip.part-00").unlink()
+        assert resolve("tiny-1.0", index_url=str(index_path)) == local
+    finally:
+        os.environ.pop("SECRYST_CACHE", None)
+
+
+def test_resolve_parts_rejects_corrupt_part(tmp_path: Path) -> None:
+    import hashlib
+
+    zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
+    blob = zip_path.read_bytes()
+    part_a, part_b = blob[:7], blob[7:]
+    channel = tmp_path / "channel"
+    (channel / "tiny.zip.part-00").write_bytes(part_a)
+    (channel / "tiny.zip.part-01").write_bytes(part_b)
+    index = {
+        "version": 1,
+        "models": {
+            "tiny-1.0": {
+                "filename": "tiny.zip",
+                "sha256": hashlib.sha256(blob).hexdigest(),
+                "parts": [
+                    {
+                        "url": f"file://{channel / 'tiny.zip.part-00'}",
+                        "sha256": "0" * 64,
+                        "size": len(part_a),
+                    },
+                    {
+                        "url": f"file://{channel / 'tiny.zip.part-01'}",
+                        "sha256": hashlib.sha256(part_b).hexdigest(),
+                        "size": len(part_b),
+                    },
+                ],
+            },
+        },
+    }
+    index_path = tmp_path / "models.yaml"
+    index_path.write_text(yaml.safe_dump(index), encoding="utf-8")
+    os.environ["SECRYST_CACHE"] = str(tmp_path / "cache")
+    try:
+        with pytest.raises(RegistryError, match="part 0"):
+            resolve("tiny-1.0", index_url=str(index_path))
+    finally:
+        os.environ.pop("SECRYST_CACHE", None)

secryst-0.1.0/tests/tests_helpers.py

@@ -0,0 +1,68 @@
+"""Shared tiny-graph zip builder for runtime tests."""
+
+from __future__ import annotations
+
+import hashlib
+import zipfile
+from pathlib import Path
+
+import numpy as np
+import yaml
+from onnx import TensorProto, helper, numpy_helper
+
+MANIFEST = {
+    "format": "imf-v1",
+    "id": "tiny-1.0",
+    "task": "translit",
+    "source_script": "Latn",
+    "target": "Latn",
+    "tokenizer": "bytes",
+    "opset": 14,
+    "decoder": "plain",
+    "precision": "fp32",
+    "license": "BSD-3-Clause",
+    "trained_from": "runtime test fixture",
+}
+
+
+def _add_graph(name: str, inputs: list[str], output: str) -> bytes:
+    graph = helper.make_graph(
+        nodes=[helper.make_node("Add", [inputs[0], "bias"], [output])],
+        name=name,
+        inputs=[
+            helper.make_tensor_value_info(n, TensorProto.INT64, ["batch", "seq"])
+            for n in inputs
+        ],
+        outputs=[
+            helper.make_tensor_value_info(output, TensorProto.INT64, ["batch", "seq"])
+        ],
+        initializer=[numpy_helper.from_array(np.zeros(1, dtype=np.int64), "bias")],
+    )
+    model = helper.make_model(
+        graph, opset_imports=[helper.make_opsetid("", 14)], ir_version=7
+    )
+    return model.SerializeToString()
+
+
+def build_tiny_zip(
+    path: Path, tamper: bool = False, manifest: dict | None = None
+) -> Path:
+    encoder = _add_graph("tiny-enc", ["input_ids"], "last_hidden_state")
+    decoder = _add_graph(
+        "tiny-dec", ["input_ids", "encoder_hidden_states"], "logits"
+    )
+    sha = {
+        "encoder.onnx": hashlib.sha256(encoder).hexdigest(),
+        "decoder.onnx": hashlib.sha256(decoder).hexdigest(),
+    }
+    if tamper:
+        sha["encoder.onnx"] = "0" * 64
+    meta = dict(manifest if manifest is not None else MANIFEST)
+    meta["sha256"] = sha
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with zipfile.ZipFile(path, "w") as zf:
+        zf.writestr("metadata.yaml", yaml.safe_dump(meta))
+        zf.writestr("encoder.onnx", encoder)
+        zf.writestr("decoder.onnx", decoder)
+        zf.writestr("README.md", "# tiny\n")
+    return path