interscript-ml 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- interscript_ml-0.1.1/PKG-INFO +28 -0
- interscript_ml-0.1.1/README.md +12 -0
- {interscript_ml-0.1.0 → interscript_ml-0.1.1}/pyproject.toml +1 -1
- interscript_ml-0.1.1/src/interscript_ml.egg-info/PKG-INFO +28 -0
- {interscript_ml-0.1.0 → interscript_ml-0.1.1}/src/interscript_ml.egg-info/SOURCES.txt +1 -4
- interscript_ml-0.1.0/PKG-INFO +0 -55
- interscript_ml-0.1.0/README.md +0 -39
- interscript_ml-0.1.0/src/interscript_ml.egg-info/PKG-INFO +0 -55
- interscript_ml-0.1.0/tests/test_model.py +0 -158
- interscript_ml-0.1.0/tests/test_registry.py +0 -167
- interscript_ml-0.1.0/tests/tests_helpers.py +0 -68
- {interscript_ml-0.1.0 → interscript_ml-0.1.1}/setup.cfg +0 -0
- {interscript_ml-0.1.0 → interscript_ml-0.1.1}/src/interscript_ml/__init__.py +0 -0
- {interscript_ml-0.1.0 → interscript_ml-0.1.1}/src/interscript_ml/loader.py +0 -0
- {interscript_ml-0.1.0 → interscript_ml-0.1.1}/src/interscript_ml/model.py +0 -0
- {interscript_ml-0.1.0 → interscript_ml-0.1.1}/src/interscript_ml/registry.py +0 -0
- {interscript_ml-0.1.0 → interscript_ml-0.1.1}/src/interscript_ml/tokens.py +0 -0
- {interscript_ml-0.1.0 → interscript_ml-0.1.1}/src/interscript_ml.egg-info/dependency_links.txt +0 -0
- {interscript_ml-0.1.0 → interscript_ml-0.1.1}/src/interscript_ml.egg-info/requires.txt +0 -0
- {interscript_ml-0.1.0 → interscript_ml-0.1.1}/src/interscript_ml.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: interscript-ml
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Python runtime for Interscript Model Format (IMF v1) — the phonological layer of Interscript
|
|
5
|
+
Author: Interscript Project
|
|
6
|
+
License: BSD-3-Clause
|
|
7
|
+
Keywords: transliteration,diacritization,g2p,onnx,byt5
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
Requires-Dist: numpy>=1.26
|
|
11
|
+
Requires-Dist: pyyaml>=6.0
|
|
12
|
+
Requires-Dist: onnxruntime>=1.17
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
15
|
+
Requires-Dist: onnx>=1.16; extra == "dev"
|
|
16
|
+
|
|
17
|
+
# interscript-ml
|
|
18
|
+
|
|
19
|
+
This package is renamed. The Python runtime for Interscript's neural
|
|
20
|
+
models publishes as **secryst**:
|
|
21
|
+
|
|
22
|
+
pip install secryst
|
|
23
|
+
|
|
24
|
+
from secryst import Model
|
|
25
|
+
model = Model.load("ara-diac-small-2.1-int8")
|
|
26
|
+
model.translate("...")
|
|
27
|
+
|
|
28
|
+
This 0.1.1 release exists only to carry this notice.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# interscript-ml
|
|
2
|
+
|
|
3
|
+
This package is renamed. The Python runtime for Interscript's neural
|
|
4
|
+
models publishes as **secryst**:
|
|
5
|
+
|
|
6
|
+
pip install secryst
|
|
7
|
+
|
|
8
|
+
from secryst import Model
|
|
9
|
+
model = Model.load("ara-diac-small-2.1-int8")
|
|
10
|
+
model.translate("...")
|
|
11
|
+
|
|
12
|
+
This 0.1.1 release exists only to carry this notice.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "interscript-ml"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.1"
|
|
8
8
|
description = "Python runtime for Interscript Model Format (IMF v1) — the phonological layer of Interscript"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "BSD-3-Clause" }
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: interscript-ml
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Python runtime for Interscript Model Format (IMF v1) — the phonological layer of Interscript
|
|
5
|
+
Author: Interscript Project
|
|
6
|
+
License: BSD-3-Clause
|
|
7
|
+
Keywords: transliteration,diacritization,g2p,onnx,byt5
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
Requires-Dist: numpy>=1.26
|
|
11
|
+
Requires-Dist: pyyaml>=6.0
|
|
12
|
+
Requires-Dist: onnxruntime>=1.17
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
15
|
+
Requires-Dist: onnx>=1.16; extra == "dev"
|
|
16
|
+
|
|
17
|
+
# interscript-ml
|
|
18
|
+
|
|
19
|
+
This package is renamed. The Python runtime for Interscript's neural
|
|
20
|
+
models publishes as **secryst**:
|
|
21
|
+
|
|
22
|
+
pip install secryst
|
|
23
|
+
|
|
24
|
+
from secryst import Model
|
|
25
|
+
model = Model.load("ara-diac-small-2.1-int8")
|
|
26
|
+
model.translate("...")
|
|
27
|
+
|
|
28
|
+
This 0.1.1 release exists only to carry this notice.
|
|
@@ -9,7 +9,4 @@ src/interscript_ml.egg-info/PKG-INFO
|
|
|
9
9
|
src/interscript_ml.egg-info/SOURCES.txt
|
|
10
10
|
src/interscript_ml.egg-info/dependency_links.txt
|
|
11
11
|
src/interscript_ml.egg-info/requires.txt
|
|
12
|
-
src/interscript_ml.egg-info/top_level.txt
|
|
13
|
-
tests/test_model.py
|
|
14
|
-
tests/test_registry.py
|
|
15
|
-
tests/tests_helpers.py
|
|
12
|
+
src/interscript_ml.egg-info/top_level.txt
|
interscript_ml-0.1.0/PKG-INFO
DELETED
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: interscript-ml
|
|
3
|
-
Version: 0.1.0
|
|
4
|
-
Summary: Python runtime for Interscript Model Format (IMF v1) — the phonological layer of Interscript
|
|
5
|
-
Author: Interscript Project
|
|
6
|
-
License: BSD-3-Clause
|
|
7
|
-
Keywords: transliteration,diacritization,g2p,onnx,byt5
|
|
8
|
-
Requires-Python: >=3.10
|
|
9
|
-
Description-Content-Type: text/markdown
|
|
10
|
-
Requires-Dist: numpy>=1.26
|
|
11
|
-
Requires-Dist: pyyaml>=6.0
|
|
12
|
-
Requires-Dist: onnxruntime>=1.17
|
|
13
|
-
Provides-Extra: dev
|
|
14
|
-
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
15
|
-
Requires-Dist: onnx>=1.16; extra == "dev"
|
|
16
|
-
|
|
17
|
-
# interscript-ml (Python runtime)
|
|
18
|
-
|
|
19
|
-
The reference Python runtime for **IMF v1** model zips — the phonological
|
|
20
|
-
layer of Interscript. The Ruby (secryst gem) and TypeScript
|
|
21
|
-
(@interscript/ml) runtimes are diffed against this one on shared golden
|
|
22
|
-
sets.
|
|
23
|
-
|
|
24
|
-
```python
|
|
25
|
-
from interscript_ml import Model
|
|
26
|
-
|
|
27
|
-
model = Model.load("khm-latn-1.0") # id: index resolve -> download
|
|
28
|
-
# -> sha256-verify -> cache -> load
|
|
29
|
-
model.translate("ភាសា") # -> "pheasaea"
|
|
30
|
-
model.id # "khm-latn-1.0"
|
|
31
|
-
|
|
32
|
-
model = Model.load("khm-latn-1.0.zip") # or: a local zip path directly
|
|
33
|
-
```
|
|
34
|
-
|
|
35
|
-
- Byte-level only: the canonical ByT5 table (byte `b` → id `b+3`,
|
|
36
|
-
trailing EOS) — no vocab files, no per-model tokenization code.
|
|
37
|
-
- Greedy KV-cache decode when the zip ships `decoder-kv.onnx`
|
|
38
|
-
(default), plain full-recompute fallback otherwise.
|
|
39
|
-
- Every `.onnx` member is sha256-verified against `metadata.yaml`
|
|
40
|
-
before the session is created; corrupt downloads fail loudly.
|
|
41
|
-
- Dynamic fetch per the `models.yaml` contract (shared with the Ruby and
|
|
42
|
-
TypeScript runtimes): resolve id -> channel URL, download to temp,
|
|
43
|
-
verify whole-file sha256 against the index, atomically install into
|
|
44
|
-
`~/.cache/interscript/models/<id>/`. Overrides:
|
|
45
|
-
`SECRYST_INDEX` (URL or path), `SECRYST_CACHE`.
|
|
46
|
-
|
|
47
|
-
Install: `pip install ./runtime` (from the interscript-ml checkout) or
|
|
48
|
-
`pip install -e "./runtime[dev]"` for development.
|
|
49
|
-
|
|
50
|
-
Tests: `python -m pytest runtime/tests` — tiny-graph zips, no torch
|
|
51
|
-
needed. The end-to-end golden test runs when `SECRYST_E2E_ZIP`
|
|
52
|
-
points at a real zip (e.g. `models/khm-latn/khm-latn-1.0-fp32.zip`)
|
|
53
|
-
and asserts byte-identical outputs against `golden/khm-latn-100.jsonl`.
|
|
54
|
-
|
|
55
|
-
License: BSD-3-Clause.
|
interscript_ml-0.1.0/README.md
DELETED
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
# interscript-ml (Python runtime)
|
|
2
|
-
|
|
3
|
-
The reference Python runtime for **IMF v1** model zips — the phonological
|
|
4
|
-
layer of Interscript. The Ruby (secryst gem) and TypeScript
|
|
5
|
-
(@interscript/ml) runtimes are diffed against this one on shared golden
|
|
6
|
-
sets.
|
|
7
|
-
|
|
8
|
-
```python
|
|
9
|
-
from interscript_ml import Model
|
|
10
|
-
|
|
11
|
-
model = Model.load("khm-latn-1.0") # id: index resolve -> download
|
|
12
|
-
# -> sha256-verify -> cache -> load
|
|
13
|
-
model.translate("ភាសា") # -> "pheasaea"
|
|
14
|
-
model.id # "khm-latn-1.0"
|
|
15
|
-
|
|
16
|
-
model = Model.load("khm-latn-1.0.zip") # or: a local zip path directly
|
|
17
|
-
```
|
|
18
|
-
|
|
19
|
-
- Byte-level only: the canonical ByT5 table (byte `b` → id `b+3`,
|
|
20
|
-
trailing EOS) — no vocab files, no per-model tokenization code.
|
|
21
|
-
- Greedy KV-cache decode when the zip ships `decoder-kv.onnx`
|
|
22
|
-
(default), plain full-recompute fallback otherwise.
|
|
23
|
-
- Every `.onnx` member is sha256-verified against `metadata.yaml`
|
|
24
|
-
before the session is created; corrupt downloads fail loudly.
|
|
25
|
-
- Dynamic fetch per the `models.yaml` contract (shared with the Ruby and
|
|
26
|
-
TypeScript runtimes): resolve id -> channel URL, download to temp,
|
|
27
|
-
verify whole-file sha256 against the index, atomically install into
|
|
28
|
-
`~/.cache/interscript/models/<id>/`. Overrides:
|
|
29
|
-
`SECRYST_INDEX` (URL or path), `SECRYST_CACHE`.
|
|
30
|
-
|
|
31
|
-
Install: `pip install ./runtime` (from the interscript-ml checkout) or
|
|
32
|
-
`pip install -e "./runtime[dev]"` for development.
|
|
33
|
-
|
|
34
|
-
Tests: `python -m pytest runtime/tests` — tiny-graph zips, no torch
|
|
35
|
-
needed. The end-to-end golden test runs when `SECRYST_E2E_ZIP`
|
|
36
|
-
points at a real zip (e.g. `models/khm-latn/khm-latn-1.0-fp32.zip`)
|
|
37
|
-
and asserts byte-identical outputs against `golden/khm-latn-100.jsonl`.
|
|
38
|
-
|
|
39
|
-
License: BSD-3-Clause.
|
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: interscript-ml
|
|
3
|
-
Version: 0.1.0
|
|
4
|
-
Summary: Python runtime for Interscript Model Format (IMF v1) — the phonological layer of Interscript
|
|
5
|
-
Author: Interscript Project
|
|
6
|
-
License: BSD-3-Clause
|
|
7
|
-
Keywords: transliteration,diacritization,g2p,onnx,byt5
|
|
8
|
-
Requires-Python: >=3.10
|
|
9
|
-
Description-Content-Type: text/markdown
|
|
10
|
-
Requires-Dist: numpy>=1.26
|
|
11
|
-
Requires-Dist: pyyaml>=6.0
|
|
12
|
-
Requires-Dist: onnxruntime>=1.17
|
|
13
|
-
Provides-Extra: dev
|
|
14
|
-
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
15
|
-
Requires-Dist: onnx>=1.16; extra == "dev"
|
|
16
|
-
|
|
17
|
-
# interscript-ml (Python runtime)
|
|
18
|
-
|
|
19
|
-
The reference Python runtime for **IMF v1** model zips — the phonological
|
|
20
|
-
layer of Interscript. The Ruby (secryst gem) and TypeScript
|
|
21
|
-
(@interscript/ml) runtimes are diffed against this one on shared golden
|
|
22
|
-
sets.
|
|
23
|
-
|
|
24
|
-
```python
|
|
25
|
-
from interscript_ml import Model
|
|
26
|
-
|
|
27
|
-
model = Model.load("khm-latn-1.0") # id: index resolve -> download
|
|
28
|
-
# -> sha256-verify -> cache -> load
|
|
29
|
-
model.translate("ភាសា") # -> "pheasaea"
|
|
30
|
-
model.id # "khm-latn-1.0"
|
|
31
|
-
|
|
32
|
-
model = Model.load("khm-latn-1.0.zip") # or: a local zip path directly
|
|
33
|
-
```
|
|
34
|
-
|
|
35
|
-
- Byte-level only: the canonical ByT5 table (byte `b` → id `b+3`,
|
|
36
|
-
trailing EOS) — no vocab files, no per-model tokenization code.
|
|
37
|
-
- Greedy KV-cache decode when the zip ships `decoder-kv.onnx`
|
|
38
|
-
(default), plain full-recompute fallback otherwise.
|
|
39
|
-
- Every `.onnx` member is sha256-verified against `metadata.yaml`
|
|
40
|
-
before the session is created; corrupt downloads fail loudly.
|
|
41
|
-
- Dynamic fetch per the `models.yaml` contract (shared with the Ruby and
|
|
42
|
-
TypeScript runtimes): resolve id -> channel URL, download to temp,
|
|
43
|
-
verify whole-file sha256 against the index, atomically install into
|
|
44
|
-
`~/.cache/interscript/models/<id>/`. Overrides:
|
|
45
|
-
`SECRYST_INDEX` (URL or path), `SECRYST_CACHE`.
|
|
46
|
-
|
|
47
|
-
Install: `pip install ./runtime` (from the interscript-ml checkout) or
|
|
48
|
-
`pip install -e "./runtime[dev]"` for development.
|
|
49
|
-
|
|
50
|
-
Tests: `python -m pytest runtime/tests` — tiny-graph zips, no torch
|
|
51
|
-
needed. The end-to-end golden test runs when `SECRYST_E2E_ZIP`
|
|
52
|
-
points at a real zip (e.g. `models/khm-latn/khm-latn-1.0-fp32.zip`)
|
|
53
|
-
and asserts byte-identical outputs against `golden/khm-latn-100.jsonl`.
|
|
54
|
-
|
|
55
|
-
License: BSD-3-Clause.
|
|
@@ -1,158 +0,0 @@
|
|
|
1
|
-
"""Tests for the interscript-ml runtime.
|
|
2
|
-
|
|
3
|
-
Tiny-graph zips built with the onnx package (no torch, no training
|
|
4
|
-
repo). The end-to-end golden test runs only when a real zip is provided
|
|
5
|
-
via SECRYST_E2E_ZIP.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
from __future__ import annotations
|
|
9
|
-
|
|
10
|
-
import hashlib
|
|
11
|
-
import json
|
|
12
|
-
import os
|
|
13
|
-
import zipfile
|
|
14
|
-
from pathlib import Path
|
|
15
|
-
|
|
16
|
-
import pytest
|
|
17
|
-
import yaml
|
|
18
|
-
|
|
19
|
-
ort = pytest.importorskip("onnxruntime")
|
|
20
|
-
onnx = pytest.importorskip("onnx")
|
|
21
|
-
|
|
22
|
-
import numpy as np # noqa: E402
|
|
23
|
-
from interscript_ml import Model, ModelFormatError, decode, encode # noqa: E402
|
|
24
|
-
from onnx import TensorProto, helper, numpy_helper # noqa: E402
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def _graph(opset: int = 14) -> bytes:
|
|
28
|
-
graph = helper.make_graph(
|
|
29
|
-
nodes=[helper.make_node("Add", ["input_ids", "bias"], ["last_hidden_state"])],
|
|
30
|
-
name="tiny-enc",
|
|
31
|
-
inputs=[
|
|
32
|
-
helper.make_tensor_value_info("input_ids", TensorProto.INT64, ["batch", "seq"])
|
|
33
|
-
],
|
|
34
|
-
outputs=[
|
|
35
|
-
helper.make_tensor_value_info(
|
|
36
|
-
"last_hidden_state", TensorProto.INT64, ["batch", "seq"]
|
|
37
|
-
)
|
|
38
|
-
],
|
|
39
|
-
initializer=[numpy_helper.from_array(np.zeros(1, dtype=np.int64), "bias")],
|
|
40
|
-
)
|
|
41
|
-
model = helper.make_model(
|
|
42
|
-
graph, opset_imports=[helper.make_opsetid("", opset)], ir_version=7
|
|
43
|
-
)
|
|
44
|
-
return model.SerializeToString()
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
def _decoder_graph() -> bytes:
|
|
48
|
-
graph = helper.make_graph(
|
|
49
|
-
nodes=[
|
|
50
|
-
helper.make_node("Add", ["input_ids", "bias"], ["logits"])
|
|
51
|
-
],
|
|
52
|
-
name="tiny-dec",
|
|
53
|
-
inputs=[
|
|
54
|
-
helper.make_tensor_value_info("input_ids", TensorProto.INT64, ["batch", "seq"]),
|
|
55
|
-
helper.make_tensor_value_info(
|
|
56
|
-
"encoder_hidden_states", TensorProto.INT64, ["batch", "seq"]
|
|
57
|
-
),
|
|
58
|
-
],
|
|
59
|
-
outputs=[
|
|
60
|
-
helper.make_tensor_value_info("logits", TensorProto.INT64, ["batch", "seq"])
|
|
61
|
-
],
|
|
62
|
-
initializer=[numpy_helper.from_array(np.zeros(1, dtype=np.int64), "bias")],
|
|
63
|
-
)
|
|
64
|
-
model = helper.make_model(
|
|
65
|
-
graph, opset_imports=[helper.make_opsetid("", 14)], ir_version=7
|
|
66
|
-
)
|
|
67
|
-
return model.SerializeToString()
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
MANIFEST = {
|
|
71
|
-
"format": "imf-v1",
|
|
72
|
-
"id": "tiny-1.0",
|
|
73
|
-
"task": "translit",
|
|
74
|
-
"source_script": "Latn",
|
|
75
|
-
"target": "Latn",
|
|
76
|
-
"tokenizer": "bytes",
|
|
77
|
-
"opset": 14,
|
|
78
|
-
"decoder": "plain",
|
|
79
|
-
"precision": "fp32",
|
|
80
|
-
"license": "BSD-3-Clause",
|
|
81
|
-
"trained_from": "runtime test fixture",
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
def _tiny_zip(path: Path, tamper: bool = False, manifest: dict | None = None) -> Path:
|
|
86
|
-
encoder, decoder = _graph(), _decoder_graph()
|
|
87
|
-
sha = {
|
|
88
|
-
"encoder.onnx": hashlib.sha256(encoder).hexdigest(),
|
|
89
|
-
"decoder.onnx": hashlib.sha256(decoder).hexdigest(),
|
|
90
|
-
}
|
|
91
|
-
if tamper:
|
|
92
|
-
sha["encoder.onnx"] = "0" * 64
|
|
93
|
-
meta = dict(manifest if manifest is not None else MANIFEST)
|
|
94
|
-
meta["sha256"] = sha
|
|
95
|
-
with zipfile.ZipFile(path, "w") as zf:
|
|
96
|
-
zf.writestr("metadata.yaml", yaml.safe_dump(meta))
|
|
97
|
-
zf.writestr("encoder.onnx", encoder)
|
|
98
|
-
zf.writestr("decoder.onnx", decoder)
|
|
99
|
-
zf.writestr("README.md", "# tiny\n")
|
|
100
|
-
return path
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
def test_token_table() -> None:
|
|
104
|
-
assert encode("rok") == [117, 114, 110, 1]
|
|
105
|
-
assert decode([117, 114, 110]) == "rok"
|
|
106
|
-
assert decode([117, 1, 114]) == "r"
|
|
107
|
-
assert decode([]) == ""
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
def test_load_and_decode_tiny(tmp_path: Path) -> None:
|
|
111
|
-
z = _tiny_zip(tmp_path / "tiny.zip")
|
|
112
|
-
model = Model.load(z)
|
|
113
|
-
assert model.id == "tiny-1.0"
|
|
114
|
-
# tiny graphs are identity Adds: logits echo the decoder prefix, so
|
|
115
|
-
# greedy emits encode(PAD-prefix input)+... — deterministic, not
|
|
116
|
-
# meaningful; what matters is that the loop runs and decodes.
|
|
117
|
-
tokens = model.generate("he", max_len=4)
|
|
118
|
-
assert isinstance(tokens, list)
|
|
119
|
-
text = model.translate("he", max_len=4)
|
|
120
|
-
assert isinstance(text, str)
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
def test_sha256_mismatch_rejected(tmp_path: Path) -> None:
|
|
124
|
-
z = _tiny_zip(tmp_path / "bad.zip", tamper=True)
|
|
125
|
-
with pytest.raises(ModelFormatError, match="sha256 mismatch"):
|
|
126
|
-
Model.load(z)
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
def test_non_bytes_tokenizer_rejected(tmp_path: Path) -> None:
|
|
130
|
-
manifest = dict(MANIFEST, tokenizer="sentencepiece")
|
|
131
|
-
z = _tiny_zip(tmp_path / "spm.zip", manifest=manifest)
|
|
132
|
-
with pytest.raises(ModelFormatError, match="byte-level only"):
|
|
133
|
-
Model.load(z)
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
def test_missing_graph_rejected(tmp_path: Path) -> None:
|
|
137
|
-
z = _tiny_zip(tmp_path / "m.zip")
|
|
138
|
-
truncated = tmp_path / "trunc.zip"
|
|
139
|
-
with zipfile.ZipFile(z) as src, zipfile.ZipFile(truncated, "w") as dst:
|
|
140
|
-
for name in src.namelist():
|
|
141
|
-
if name != "decoder.onnx":
|
|
142
|
-
dst.writestr(name, src.read(name))
|
|
143
|
-
with pytest.raises(ModelFormatError, match="decoder.onnx"):
|
|
144
|
-
Model.load(truncated)
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
def test_golden_set_e2e() -> None:
|
|
148
|
-
"""Run against a real zip: byte-identical outputs on the golden set."""
|
|
149
|
-
zip_path = os.environ.get("SECRYST_E2E_ZIP")
|
|
150
|
-
if not zip_path:
|
|
151
|
-
pytest.skip("set SECRYST_E2E_ZIP to a real IMF zip")
|
|
152
|
-
golden = Path(__file__).resolve().parent.parent.parent / "golden" / "khm-latn-100.jsonl"
|
|
153
|
-
if "khm" not in Path(zip_path).name:
|
|
154
|
-
pytest.skip("golden file is khm-latn specific")
|
|
155
|
-
model = Model.load(zip_path)
|
|
156
|
-
rows = [json.loads(line) for line in golden.read_text(encoding="utf-8").splitlines()]
|
|
157
|
-
for row in rows:
|
|
158
|
-
assert model.translate(row["input"], max_len=128) == row["output"], row["input"]
|
|
@@ -1,167 +0,0 @@
|
|
|
1
|
-
"""Tests for the dynamic-fetch layer (models.yaml resolution + cache)."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import hashlib
|
|
6
|
-
import os # noqa: E402
|
|
7
|
-
from pathlib import Path
|
|
8
|
-
|
|
9
|
-
import pytest
|
|
10
|
-
import yaml
|
|
11
|
-
from interscript_ml.registry import RegistryError, resolve
|
|
12
|
-
from tests_helpers import build_tiny_zip
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def _index_file(tmp_path: Path, zip_path: Path, sha256: str | None = None) -> Path:
|
|
16
|
-
index = {
|
|
17
|
-
"version": 1,
|
|
18
|
-
"models": {
|
|
19
|
-
"tiny-1.0": {
|
|
20
|
-
"task": "translit",
|
|
21
|
-
"precision": "fp32",
|
|
22
|
-
"filename": zip_path.name,
|
|
23
|
-
"url": f"file://{zip_path}",
|
|
24
|
-
"sha256": sha256 or hashlib.sha256(zip_path.read_bytes()).hexdigest(),
|
|
25
|
-
"size": zip_path.stat().st_size,
|
|
26
|
-
}
|
|
27
|
-
},
|
|
28
|
-
}
|
|
29
|
-
path = tmp_path / "models.yaml"
|
|
30
|
-
path.write_text(yaml.safe_dump(index), encoding="utf-8")
|
|
31
|
-
return path
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def test_resolve_downloads_verifies_and_caches(tmp_path: Path) -> None:
|
|
35
|
-
zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
|
|
36
|
-
index = _index_file(tmp_path, zip_path)
|
|
37
|
-
cache = tmp_path / "cache"
|
|
38
|
-
os.environ["SECRYST_CACHE"] = str(cache)
|
|
39
|
-
try:
|
|
40
|
-
local = resolve("tiny-1.0", index_url=str(index))
|
|
41
|
-
assert local == cache / "models" / "tiny-1.0" / "tiny.zip"
|
|
42
|
-
assert local.is_file()
|
|
43
|
-
# second resolve is a verified cache hit (channel dir removed)
|
|
44
|
-
zip_path.unlink()
|
|
45
|
-
assert resolve("tiny-1.0", index_url=str(index)) == local
|
|
46
|
-
finally:
|
|
47
|
-
os.environ.pop("SECRYST_CACHE", None)
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
def test_resolve_rejects_bad_download(tmp_path: Path) -> None:
|
|
51
|
-
zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
|
|
52
|
-
index = _index_file(tmp_path, zip_path, sha256="0" * 64)
|
|
53
|
-
os.environ["SECRYST_CACHE"] = str(tmp_path / "cache")
|
|
54
|
-
try:
|
|
55
|
-
with pytest.raises(RegistryError, match="sha256 mismatch"):
|
|
56
|
-
resolve("tiny-1.0", index_url=str(index))
|
|
57
|
-
finally:
|
|
58
|
-
os.environ.pop("SECRYST_CACHE", None)
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def test_resolve_unknown_id(tmp_path: Path) -> None:
|
|
62
|
-
index = tmp_path / "models.yaml"
|
|
63
|
-
index.write_text(yaml.safe_dump({"version": 1, "models": {}}), encoding="utf-8")
|
|
64
|
-
with pytest.raises(RegistryError, match="unknown model id"):
|
|
65
|
-
resolve("nope-1.0", index_url=str(index))
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
def test_model_load_by_id(tmp_path: Path) -> None:
|
|
69
|
-
zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
|
|
70
|
-
index = _index_file(tmp_path, zip_path)
|
|
71
|
-
os.environ["SECRYST_CACHE"] = str(tmp_path / "cache")
|
|
72
|
-
try:
|
|
73
|
-
from interscript_ml import Model
|
|
74
|
-
|
|
75
|
-
model = Model.load("tiny-1.0", index_url=str(index))
|
|
76
|
-
assert model.id == "tiny-1.0"
|
|
77
|
-
assert isinstance(model.translate("he", max_len=4), str)
|
|
78
|
-
finally:
|
|
79
|
-
os.environ.pop("SECRYST_CACHE", None)
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def test_resolve_parts_assembles_and_verifies(tmp_path: Path) -> None:
|
|
83
|
-
import hashlib
|
|
84
|
-
|
|
85
|
-
zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
|
|
86
|
-
blob = zip_path.read_bytes()
|
|
87
|
-
part_a, part_b = blob[: len(blob) // 2 + 3], blob[len(blob) // 2 + 3 :]
|
|
88
|
-
channel = tmp_path / "channel"
|
|
89
|
-
(channel / "tiny.zip.part-00").write_bytes(part_a)
|
|
90
|
-
(channel / "tiny.zip.part-01").write_bytes(part_b)
|
|
91
|
-
index = {
|
|
92
|
-
"version": 1,
|
|
93
|
-
"models": {
|
|
94
|
-
"tiny-1.0": {
|
|
95
|
-
"task": "translit",
|
|
96
|
-
"precision": "fp32",
|
|
97
|
-
"filename": "tiny.zip",
|
|
98
|
-
"sha256": hashlib.sha256(blob).hexdigest(),
|
|
99
|
-
"size": len(blob),
|
|
100
|
-
"parts": [
|
|
101
|
-
{
|
|
102
|
-
"url": f"file://{channel / 'tiny.zip.part-00'}",
|
|
103
|
-
"sha256": hashlib.sha256(part_a).hexdigest(),
|
|
104
|
-
"size": len(part_a),
|
|
105
|
-
},
|
|
106
|
-
{
|
|
107
|
-
"url": f"file://{channel / 'tiny.zip.part-01'}",
|
|
108
|
-
"sha256": hashlib.sha256(part_b).hexdigest(),
|
|
109
|
-
"size": len(part_b),
|
|
110
|
-
},
|
|
111
|
-
],
|
|
112
|
-
},
|
|
113
|
-
},
|
|
114
|
-
}
|
|
115
|
-
index_path = tmp_path / "models.yaml"
|
|
116
|
-
index_path.write_text(yaml.safe_dump(index), encoding="utf-8")
|
|
117
|
-
cache = tmp_path / "cache"
|
|
118
|
-
os.environ["SECRYST_CACHE"] = str(cache)
|
|
119
|
-
try:
|
|
120
|
-
local = resolve("tiny-1.0", index_url=str(index_path))
|
|
121
|
-
assert local == cache / "models" / "tiny-1.0" / "tiny.zip"
|
|
122
|
-
assert local.read_bytes() == blob
|
|
123
|
-
zip_path.unlink()
|
|
124
|
-
(channel / "tiny.zip.part-00").unlink()
|
|
125
|
-
assert resolve("tiny-1.0", index_url=str(index_path)) == local
|
|
126
|
-
finally:
|
|
127
|
-
os.environ.pop("SECRYST_CACHE", None)
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
def test_resolve_parts_rejects_corrupt_part(tmp_path: Path) -> None:
|
|
131
|
-
import hashlib
|
|
132
|
-
|
|
133
|
-
zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
|
|
134
|
-
blob = zip_path.read_bytes()
|
|
135
|
-
part_a, part_b = blob[:7], blob[7:]
|
|
136
|
-
channel = tmp_path / "channel"
|
|
137
|
-
(channel / "tiny.zip.part-00").write_bytes(part_a)
|
|
138
|
-
(channel / "tiny.zip.part-01").write_bytes(part_b)
|
|
139
|
-
index = {
|
|
140
|
-
"version": 1,
|
|
141
|
-
"models": {
|
|
142
|
-
"tiny-1.0": {
|
|
143
|
-
"filename": "tiny.zip",
|
|
144
|
-
"sha256": hashlib.sha256(blob).hexdigest(),
|
|
145
|
-
"parts": [
|
|
146
|
-
{
|
|
147
|
-
"url": f"file://{channel / 'tiny.zip.part-00'}",
|
|
148
|
-
"sha256": "0" * 64,
|
|
149
|
-
"size": len(part_a),
|
|
150
|
-
},
|
|
151
|
-
{
|
|
152
|
-
"url": f"file://{channel / 'tiny.zip.part-01'}",
|
|
153
|
-
"sha256": hashlib.sha256(part_b).hexdigest(),
|
|
154
|
-
"size": len(part_b),
|
|
155
|
-
},
|
|
156
|
-
],
|
|
157
|
-
},
|
|
158
|
-
},
|
|
159
|
-
}
|
|
160
|
-
index_path = tmp_path / "models.yaml"
|
|
161
|
-
index_path.write_text(yaml.safe_dump(index), encoding="utf-8")
|
|
162
|
-
os.environ["SECRYST_CACHE"] = str(tmp_path / "cache")
|
|
163
|
-
try:
|
|
164
|
-
with pytest.raises(RegistryError, match="part 0"):
|
|
165
|
-
resolve("tiny-1.0", index_url=str(index_path))
|
|
166
|
-
finally:
|
|
167
|
-
os.environ.pop("SECRYST_CACHE", None)
|
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
"""Shared tiny-graph zip builder for runtime tests."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import hashlib
|
|
6
|
-
import zipfile
|
|
7
|
-
from pathlib import Path
|
|
8
|
-
|
|
9
|
-
import numpy as np
|
|
10
|
-
import yaml
|
|
11
|
-
from onnx import TensorProto, helper, numpy_helper
|
|
12
|
-
|
|
13
|
-
MANIFEST = {
|
|
14
|
-
"format": "imf-v1",
|
|
15
|
-
"id": "tiny-1.0",
|
|
16
|
-
"task": "translit",
|
|
17
|
-
"source_script": "Latn",
|
|
18
|
-
"target": "Latn",
|
|
19
|
-
"tokenizer": "bytes",
|
|
20
|
-
"opset": 14,
|
|
21
|
-
"decoder": "plain",
|
|
22
|
-
"precision": "fp32",
|
|
23
|
-
"license": "BSD-3-Clause",
|
|
24
|
-
"trained_from": "runtime test fixture",
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def _add_graph(name: str, inputs: list[str], output: str) -> bytes:
|
|
29
|
-
graph = helper.make_graph(
|
|
30
|
-
nodes=[helper.make_node("Add", [inputs[0], "bias"], [output])],
|
|
31
|
-
name=name,
|
|
32
|
-
inputs=[
|
|
33
|
-
helper.make_tensor_value_info(n, TensorProto.INT64, ["batch", "seq"])
|
|
34
|
-
for n in inputs
|
|
35
|
-
],
|
|
36
|
-
outputs=[
|
|
37
|
-
helper.make_tensor_value_info(output, TensorProto.INT64, ["batch", "seq"])
|
|
38
|
-
],
|
|
39
|
-
initializer=[numpy_helper.from_array(np.zeros(1, dtype=np.int64), "bias")],
|
|
40
|
-
)
|
|
41
|
-
model = helper.make_model(
|
|
42
|
-
graph, opset_imports=[helper.make_opsetid("", 14)], ir_version=7
|
|
43
|
-
)
|
|
44
|
-
return model.SerializeToString()
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
def build_tiny_zip(
|
|
48
|
-
path: Path, tamper: bool = False, manifest: dict | None = None
|
|
49
|
-
) -> Path:
|
|
50
|
-
encoder = _add_graph("tiny-enc", ["input_ids"], "last_hidden_state")
|
|
51
|
-
decoder = _add_graph(
|
|
52
|
-
"tiny-dec", ["input_ids", "encoder_hidden_states"], "logits"
|
|
53
|
-
)
|
|
54
|
-
sha = {
|
|
55
|
-
"encoder.onnx": hashlib.sha256(encoder).hexdigest(),
|
|
56
|
-
"decoder.onnx": hashlib.sha256(decoder).hexdigest(),
|
|
57
|
-
}
|
|
58
|
-
if tamper:
|
|
59
|
-
sha["encoder.onnx"] = "0" * 64
|
|
60
|
-
meta = dict(manifest if manifest is not None else MANIFEST)
|
|
61
|
-
meta["sha256"] = sha
|
|
62
|
-
path.parent.mkdir(parents=True, exist_ok=True)
|
|
63
|
-
with zipfile.ZipFile(path, "w") as zf:
|
|
64
|
-
zf.writestr("metadata.yaml", yaml.safe_dump(meta))
|
|
65
|
-
zf.writestr("encoder.onnx", encoder)
|
|
66
|
-
zf.writestr("decoder.onnx", decoder)
|
|
67
|
-
zf.writestr("README.md", "# tiny\n")
|
|
68
|
-
return path
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{interscript_ml-0.1.0 → interscript_ml-0.1.1}/src/interscript_ml.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|