namz 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- namz-1.0.0/.gitignore +13 -0
- namz-1.0.0/LICENSE +21 -0
- namz-1.0.0/PKG-INFO +129 -0
- namz-1.0.0/README.md +98 -0
- namz-1.0.0/conformance/README.md +49 -0
- namz-1.0.0/conformance/generate.py +95 -0
- namz-1.0.0/conformance/invalid/bad-codec.namz +0 -0
- namz-1.0.0/conformance/invalid/bad-dtype.namz +0 -0
- namz-1.0.0/conformance/invalid/bad-magic.namz +0 -0
- namz-1.0.0/conformance/invalid/bad-version.namz +0 -0
- namz-1.0.0/conformance/invalid/empty.namz +0 -0
- namz-1.0.0/conformance/invalid/lying-metalen.namz +0 -0
- namz-1.0.0/conformance/invalid/magic-only.namz +1 -0
- namz-1.0.0/conformance/invalid/trailing-junk.namz +0 -0
- namz-1.0.0/conformance/invalid/truncated.namz +0 -0
- namz-1.0.0/conformance/manifest.json +121 -0
- namz-1.0.0/conformance/vectors/degenerate-array.nam +1 -0
- namz-1.0.0/conformance/vectors/degenerate-array.namz +0 -0
- namz-1.0.0/conformance/vectors/edge-floats.nam +1 -0
- namz-1.0.0/conformance/vectors/edge-floats.namz +0 -0
- namz-1.0.0/conformance/vectors/empty-weights.nam +1 -0
- namz-1.0.0/conformance/vectors/empty-weights.namz +0 -0
- namz-1.0.0/conformance/vectors/flat-noshuffle.nam +1 -0
- namz-1.0.0/conformance/vectors/flat-noshuffle.namz +0 -0
- namz-1.0.0/conformance/vectors/flat.nam +1 -0
- namz-1.0.0/conformance/vectors/flat.namz +0 -0
- namz-1.0.0/conformance/vectors/integer-weights.nam +1 -0
- namz-1.0.0/conformance/vectors/integer-weights.namz +0 -0
- namz-1.0.0/conformance/vectors/metadata.nam +1 -0
- namz-1.0.0/conformance/vectors/metadata.namz +0 -0
- namz-1.0.0/conformance/vectors/nested.nam +1 -0
- namz-1.0.0/conformance/vectors/nested.namz +0 -0
- namz-1.0.0/conformance/vectors/no-weights.nam +1 -0
- namz-1.0.0/conformance/vectors/no-weights.namz +0 -0
- namz-1.0.0/pyproject.toml +59 -0
- namz-1.0.0/src/namz/__init__.py +37 -0
- namz-1.0.0/src/namz/__main__.py +7 -0
- namz-1.0.0/src/namz/_codec.py +382 -0
- namz-1.0.0/src/namz/cli.py +192 -0
- namz-1.0.0/src/namz/py.typed +0 -0
- namz-1.0.0/tests/conftest.py +36 -0
- namz-1.0.0/tests/test_adversarial.py +220 -0
- namz-1.0.0/tests/test_conformance.py +91 -0
- namz-1.0.0/tests/test_differential.py +123 -0
- namz-1.0.0/tests/test_property.py +82 -0
- namz-1.0.0/tests/test_unit.py +143 -0
namz-1.0.0/.gitignore
ADDED
namz-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Oleh Tsymaienko <oleh@darwinscat.com> & Alisa Lafoks <alisa@darwinscat.com> (Darwin's Cat)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
namz-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: namz
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Lossless codec for NeuralAmpModeler .nam files — ~5.5x smaller, bit-exact float32, deterministic.
|
|
5
|
+
Project-URL: Homepage, https://github.com/darwinscat/namz
|
|
6
|
+
Project-URL: Repository, https://github.com/darwinscat/namz
|
|
7
|
+
Project-URL: Wire format, https://github.com/darwinscat/namz/blob/main/NAMZ-FORMAT.md
|
|
8
|
+
Author-email: Oleh Tsymaienko <oleh@darwinscat.com>, Alisa Lafoks <alisa@darwinscat.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: audio,codec,dsp,float32,guitar,nam,neural-amp-modeler
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Multimedia :: Sound/Audio
|
|
24
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.9
|
|
27
|
+
Requires-Dist: numpy>=1.21
|
|
28
|
+
Provides-Extra: test
|
|
29
|
+
Requires-Dist: pytest>=7; extra == 'test'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
<!-- SPDX-License-Identifier: MIT -->
|
|
33
|
+
# namz (Python)
|
|
34
|
+
|
|
35
|
+
Lossless codec for [NeuralAmpModeler](https://github.com/sdatkinson/neural-amp-modeler) `.nam` files.
|
|
36
|
+
|
|
37
|
+
A `.nam` is JSON whose bulk is one or more flat `"weights"` arrays written as ~20-character decimal
|
|
38
|
+
strings — which the NAM engine loads into `float32` **anyway**. `.namz` stores each weight as a 4-byte
|
|
39
|
+
float32 instead: **≈5.5× smaller**, **bit-exact** to what the engine computes, and **deterministic**
|
|
40
|
+
(no compression → byte-identical across runs, platforms, and languages). A small readable header exposes
|
|
41
|
+
tone/gear/device metadata without decoding the weights.
|
|
42
|
+
|
|
43
|
+
This package is a **native Python port** of the C++ reference and is **byte-for-byte compatible** with it
|
|
44
|
+
and with every other port — it is validated against the shared [conformance vectors](https://github.com/darwinscat/namz/tree/main/conformance)
|
|
45
|
+
(the cross-language TCK).
|
|
46
|
+
|
|
47
|
+
## Install
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
pip install namz
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Requires Python ≥ 3.9 and NumPy.
|
|
54
|
+
|
|
55
|
+
## Library
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
import namz
|
|
59
|
+
|
|
60
|
+
nam = b'{"architecture":"WaveNet","weights":[0.5,-0.5,0.25]}'
|
|
61
|
+
|
|
62
|
+
blob = namz.pack(nam) # -> .namz bytes
|
|
63
|
+
assert namz.is_namz(blob)
|
|
64
|
+
assert namz.unpack(blob) == nam # lossless to float32
|
|
65
|
+
|
|
66
|
+
# Stamp a readable metadata header (typed: "true"/"false" -> bool, digits -> int, else string):
|
|
67
|
+
opts = namz.PackOptions(metadata={"tone_type": "hi-gain", "boost": "true", "device": "tube:1,pnp:1"})
|
|
68
|
+
blob = namz.pack(nam, opts)
|
|
69
|
+
namz.read_meta(blob) # {'tone_type': 'hi-gain', 'boost': 'true', 'device': 'tube:1,pnp:1'} — no weight decode
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
### API
|
|
73
|
+
|
|
74
|
+
| function | description |
|
|
75
|
+
|---|---|
|
|
76
|
+
| `pack(nam_json, options=None) -> bytes` | `.nam` JSON (str/bytes) → `.namz`. `b""` on invalid JSON. |
|
|
77
|
+
| `unpack(data, max_json_bytes=256MiB) -> bytes` | `.namz` → `.nam` JSON. `b""` on corruption / over-cap. Never raises. |
|
|
78
|
+
| `read_meta(data) -> dict[str, str]` | display header without touching weights. `{}` for v1 / non-`.namz`. |
|
|
79
|
+
| `is_namz(data) -> bool` | cheap magic check. |
|
|
80
|
+
| `PackOptions(shuffle=True, metadata={})` | packing options. |
|
|
81
|
+
|
|
82
|
+
`unpack` is total: for **any** input bytes it returns `bytes` (empty on failure) and never raises.
|
|
83
|
+
|
|
84
|
+
## CLI
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
namz encode in.nam out.namz [--no-shuffle] [--set key=value ...]
|
|
88
|
+
namz decode in.namz out.nam
|
|
89
|
+
namz map in.namz [--json] # print the metadata header (no weight decode)
|
|
90
|
+
namz verify in.nam # pack->unpack round-trip check + ratio
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
(Also available as `python -m namz`.)
|
|
94
|
+
|
|
95
|
+
## Guarantees
|
|
96
|
+
|
|
97
|
+
- **Lossless to float32** — `unpack(pack(x))` is bit-exact at any nesting depth, including `-0.0`,
|
|
98
|
+
subnormals, and `FLT_MAX`.
|
|
99
|
+
- **Deterministic** — `pack(x)` is byte-identical across runs and platforms; the codec is idempotent.
|
|
100
|
+
- **Robust** — every malformed input (bad magic, unknown version/codec/dtype, truncation, trailing junk,
|
|
101
|
+
a lying `metaLen`/`numArrays`, an over-cap body) is rejected cleanly.
|
|
102
|
+
|
|
103
|
+
### Byte-match note
|
|
104
|
+
|
|
105
|
+
The skeleton is minified JSON with keys sorted ascending, matching the reference (nlohmann). It is
|
|
106
|
+
validated byte-for-byte against the reference over the whole NAM domain (a large differential fuzz —
|
|
107
|
+
config integers, simple decimals, non-ASCII, control chars, nesting). The weight payload itself is always
|
|
108
|
+
exact float32, never text. Two documented edges, both outside the NAM domain:
|
|
109
|
+
|
|
110
|
+
- Non-finite weights (`NaN`/`±Inf`) are out of contract and are **rejected**, exactly as the reference
|
|
111
|
+
rejects them.
|
|
112
|
+
- An integer literal outside `[int64_min, uint64_max]` in a *non-weight* field is coerced to a double, as
|
|
113
|
+
nlohmann does; this is byte-exact for double-representable magnitudes (e.g. `2**64`). For extreme
|
|
114
|
+
23+-digit integers nlohmann's own parser rounds imprecisely and may differ by one ULP — such values
|
|
115
|
+
never occur in NAM config.
|
|
116
|
+
|
|
117
|
+
## Development
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
pip install -e ".[test]"
|
|
121
|
+
pytest # runs the shared conformance vectors + property fuzz + adversarial suite
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Versioned independently of the C++ reference; tracks **wire format version 2**.
|
|
125
|
+
|
|
126
|
+
## License
|
|
127
|
+
|
|
128
|
+
MIT — see [LICENSE](LICENSE). © 2026 Oleh Tsymaienko <oleh@darwinscat.com> & Alisa Lafoks
|
|
129
|
+
<alisa@darwinscat.com> (Darwin's Cat).
|
namz-1.0.0/README.md
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
<!-- SPDX-License-Identifier: MIT -->
|
|
2
|
+
# namz (Python)
|
|
3
|
+
|
|
4
|
+
Lossless codec for [NeuralAmpModeler](https://github.com/sdatkinson/neural-amp-modeler) `.nam` files.
|
|
5
|
+
|
|
6
|
+
A `.nam` is JSON whose bulk is one or more flat `"weights"` arrays written as ~20-character decimal
|
|
7
|
+
strings — which the NAM engine loads into `float32` **anyway**. `.namz` stores each weight as a 4-byte
|
|
8
|
+
float32 instead: **≈5.5× smaller**, **bit-exact** to what the engine computes, and **deterministic**
|
|
9
|
+
(no compression → byte-identical across runs, platforms, and languages). A small readable header exposes
|
|
10
|
+
tone/gear/device metadata without decoding the weights.
|
|
11
|
+
|
|
12
|
+
This package is a **native Python port** of the C++ reference and is **byte-for-byte compatible** with it
|
|
13
|
+
and with every other port — it is validated against the shared [conformance vectors](https://github.com/darwinscat/namz/tree/main/conformance)
|
|
14
|
+
(the cross-language TCK).
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pip install namz
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Requires Python ≥ 3.9 and NumPy.
|
|
23
|
+
|
|
24
|
+
## Library
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
import namz
|
|
28
|
+
|
|
29
|
+
nam = b'{"architecture":"WaveNet","weights":[0.5,-0.5,0.25]}'
|
|
30
|
+
|
|
31
|
+
blob = namz.pack(nam) # -> .namz bytes
|
|
32
|
+
assert namz.is_namz(blob)
|
|
33
|
+
assert namz.unpack(blob) == nam # lossless to float32
|
|
34
|
+
|
|
35
|
+
# Stamp a readable metadata header (typed: "true"/"false" -> bool, digits -> int, else string):
|
|
36
|
+
opts = namz.PackOptions(metadata={"tone_type": "hi-gain", "boost": "true", "device": "tube:1,pnp:1"})
|
|
37
|
+
blob = namz.pack(nam, opts)
|
|
38
|
+
namz.read_meta(blob) # {'tone_type': 'hi-gain', 'boost': 'true', 'device': 'tube:1,pnp:1'} — no weight decode
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
### API
|
|
42
|
+
|
|
43
|
+
| function | description |
|
|
44
|
+
|---|---|
|
|
45
|
+
| `pack(nam_json, options=None) -> bytes` | `.nam` JSON (str/bytes) → `.namz`. `b""` on invalid JSON. |
|
|
46
|
+
| `unpack(data, max_json_bytes=256MiB) -> bytes` | `.namz` → `.nam` JSON. `b""` on corruption / over-cap. Never raises. |
|
|
47
|
+
| `read_meta(data) -> dict[str, str]` | display header without touching weights. `{}` for v1 / non-`.namz`. |
|
|
48
|
+
| `is_namz(data) -> bool` | cheap magic check. |
|
|
49
|
+
| `PackOptions(shuffle=True, metadata={})` | packing options. |
|
|
50
|
+
|
|
51
|
+
`unpack` is total: for **any** input bytes it returns `bytes` (empty on failure) and never raises.
|
|
52
|
+
|
|
53
|
+
## CLI
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
namz encode in.nam out.namz [--no-shuffle] [--set key=value ...]
|
|
57
|
+
namz decode in.namz out.nam
|
|
58
|
+
namz map in.namz [--json] # print the metadata header (no weight decode)
|
|
59
|
+
namz verify in.nam # pack->unpack round-trip check + ratio
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
(Also available as `python -m namz`.)
|
|
63
|
+
|
|
64
|
+
## Guarantees
|
|
65
|
+
|
|
66
|
+
- **Lossless to float32** — `unpack(pack(x))` is bit-exact at any nesting depth, including `-0.0`,
|
|
67
|
+
subnormals, and `FLT_MAX`.
|
|
68
|
+
- **Deterministic** — `pack(x)` is byte-identical across runs and platforms; the codec is idempotent.
|
|
69
|
+
- **Robust** — every malformed input (bad magic, unknown version/codec/dtype, truncation, trailing junk,
|
|
70
|
+
a lying `metaLen`/`numArrays`, an over-cap body) is rejected cleanly.
|
|
71
|
+
|
|
72
|
+
### Byte-match note
|
|
73
|
+
|
|
74
|
+
The skeleton is minified JSON with keys sorted ascending, matching the reference (nlohmann). It is
|
|
75
|
+
validated byte-for-byte against the reference over the whole NAM domain (a large differential fuzz —
|
|
76
|
+
config integers, simple decimals, non-ASCII, control chars, nesting). The weight payload itself is always
|
|
77
|
+
exact float32, never text. Two documented edges, both outside the NAM domain:
|
|
78
|
+
|
|
79
|
+
- Non-finite weights (`NaN`/`±Inf`) are out of contract and are **rejected**, exactly as the reference
|
|
80
|
+
rejects them.
|
|
81
|
+
- An integer literal outside `[int64_min, uint64_max]` in a *non-weight* field is coerced to a double, as
|
|
82
|
+
nlohmann does; this is byte-exact for double-representable magnitudes (e.g. `2**64`). For extreme
|
|
83
|
+
23+-digit integers nlohmann's own parser rounds imprecisely and may differ by one ULP — such values
|
|
84
|
+
never occur in NAM config.
|
|
85
|
+
|
|
86
|
+
## Development
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
pip install -e ".[test]"
|
|
90
|
+
pytest # runs the shared conformance vectors + property fuzz + adversarial suite
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Versioned independently of the C++ reference; tracks **wire format version 2**.
|
|
94
|
+
|
|
95
|
+
## License
|
|
96
|
+
|
|
97
|
+
MIT — see [LICENSE](LICENSE). © 2026 Oleh Tsymaienko <oleh@darwinscat.com> & Alisa Lafoks
|
|
98
|
+
<alisa@darwinscat.com> (Darwin's Cat).
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
<!-- SPDX-License-Identifier: MIT -->
|
|
2
|
+
# Conformance vectors — the cross-language TCK
|
|
3
|
+
|
|
4
|
+
These files are the **contract every language implementation of namz must satisfy.** They are how a
|
|
5
|
+
Python / JS / Rust / … port proves it is byte-for-byte compatible with the C++ reference — no port is
|
|
6
|
+
"the" binary; the vectors are the source of truth.
|
|
7
|
+
|
|
8
|
+
```
|
|
9
|
+
manifest.json machine-readable description of every case
|
|
10
|
+
vectors/<name>.nam a valid input
|
|
11
|
+
vectors/<name>.namz the reference output — your encode() must produce these exact bytes
|
|
12
|
+
invalid/<name>.namz malformed — your decode() must reject it (error/empty), never crash
|
|
13
|
+
generate.py regenerates everything from the reference CLI (see below)
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## What a port must check
|
|
17
|
+
|
|
18
|
+
For every entry in `manifest.json["valid"]`:
|
|
19
|
+
1. **encode** the `input` with the given `shuffle` flag and `set` metadata → must **byte-match** `output`.
|
|
20
|
+
2. **decode** the `output` → the weights must be **bit-exact** float32.
|
|
21
|
+
3. decode→encode must reproduce `output` (idempotence).
|
|
22
|
+
|
|
23
|
+
For every entry in `manifest.json["invalid"]`:
|
|
24
|
+
- **decode** the file → must be **rejected** (empty/error) and must not crash or hang.
|
|
25
|
+
|
|
26
|
+
Plus each port should add its own **local property fuzz** (generate N random finite float32, round-trip,
|
|
27
|
+
assert bit-exact) — that part isn't shared bytes, it's a per-language property.
|
|
28
|
+
|
|
29
|
+
## Byte-match gotchas (why an encode might differ)
|
|
30
|
+
|
|
31
|
+
- The skeleton is **minified JSON with object keys sorted ascending** (the reference uses nlohmann, which
|
|
32
|
+
sorts). Match it — Python: `json.dumps(obj, sort_keys=True, separators=(",", ":"))`; JS: `JSON.stringify`
|
|
33
|
+
does **not** sort, so you need a sorted+compact stringify.
|
|
34
|
+
- Integers are **little-endian**; the float payload is **IEEE-754 little-endian** (numpy:
|
|
35
|
+
`arr.astype("<f4").tobytes()`).
|
|
36
|
+
- The byte-plane **shuffle**: split each float's 4 bytes into 4 contiguous planes on encode; reverse on
|
|
37
|
+
decode (see `shuffleInto`/`unshuffleInto` in `include/namz.h`).
|
|
38
|
+
|
|
39
|
+
If a language genuinely cannot match key order, rely on **decode-bit-exact + idempotence** and document it
|
|
40
|
+
— never edit a golden file to fit a buggy port.
|
|
41
|
+
|
|
42
|
+
## Regenerating
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
cmake --build build # builds the reference CLI at build/namz
|
|
46
|
+
python3 conformance/generate.py build/namz
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
The reference C++ consumer is `tests/conformance.cpp` (run via `ctest -R namz_conformance`).
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
# Generate the namz conformance vectors (the cross-language TCK) using the reference CLI.
|
|
4
|
+
# Usage: python3 conformance/generate.py [path-to-namz-cli] (default: build/namz)
|
|
5
|
+
#
|
|
6
|
+
# Emits, relative to this file's directory:
|
|
7
|
+
# vectors/<name>.nam valid input
|
|
8
|
+
# vectors/<name>.namz reference output (from the CLI) — a port's encode must byte-match this
|
|
9
|
+
# invalid/<name>.namz malformed — a port's decode must reject it (error, never crash)
|
|
10
|
+
# manifest.json machine-readable description every port iterates
|
|
11
|
+
#
|
|
12
|
+
# Commit the outputs. Any language port validates against these files; if it diverges by one byte, it's wrong.
|
|
13
|
+
|
|
14
|
+
import json, os, struct, subprocess, sys, math
|
|
15
|
+
|
|
16
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
17
|
+
CLI = sys.argv[1] if len(sys.argv) > 1 else os.path.join(HERE, "..", "build", "namz")
|
|
18
|
+
VEC = os.path.join(HERE, "vectors")
|
|
19
|
+
INV = os.path.join(HERE, "invalid")
|
|
20
|
+
os.makedirs(VEC, exist_ok=True)
|
|
21
|
+
os.makedirs(INV, exist_ok=True)
|
|
22
|
+
|
|
23
|
+
def f32(x): # nearest float32, as a python float (so json emits a value that reparses to the same float32)
|
|
24
|
+
return struct.unpack("f", struct.pack("f", x))[0]
|
|
25
|
+
|
|
26
|
+
FLT_MAX = 3.4028234663852886e+38
|
|
27
|
+
SUBNORMAL = 1.401298464324817e-45 # denorm_min
|
|
28
|
+
MIN_NORMAL = 1.1754943508222875e-38
|
|
29
|
+
|
|
30
|
+
# ---- valid cases: (name, json-object, options) ; options = {"shuffle": bool, "set": {k: v}} -----------
|
|
31
|
+
valid = [
|
|
32
|
+
("flat", {"architecture": "WaveNet", "weights": [0.5, -0.5, 0.25]}, {"shuffle": True}),
|
|
33
|
+
("flat-noshuffle", {"architecture": "WaveNet", "weights": [0.5, -0.5, 0.25]}, {"shuffle": False}),
|
|
34
|
+
("nested", {"architecture": "SlimmableContainer", "sample_rate": 48000,
|
|
35
|
+
"config": {"submodels": [{"max_value": 0.5,
|
|
36
|
+
"model": {"architecture": "WaveNet", "weights": [0.1, -0.25, 12.5]}}]},
|
|
37
|
+
"weights": [1.0, -1.0]}, {"shuffle": True}),
|
|
38
|
+
("metadata", {"architecture": "WaveNet", "weights": [0.5]},
|
|
39
|
+
{"shuffle": True, "set": {"tone_type": "hi-gain", "boost": "true",
|
|
40
|
+
"device": "tube:1,pnp:1", "gear_make": "Two Notes"}}),
|
|
41
|
+
("no-weights", {"architecture": "LSTM", "sample_rate": 44100, "metadata": {"loudness": -18.0}}, {"shuffle": True}),
|
|
42
|
+
("empty-weights", {"architecture": "X", "weights": []}, {"shuffle": True}),
|
|
43
|
+
("edge-floats", {"architecture": "X", "weights": [0.0, -0.0, f32(SUBNORMAL), f32(MIN_NORMAL),
|
|
44
|
+
f32(FLT_MAX), -f32(FLT_MAX), f32(1.0/3.0)]}, {"shuffle": True}),
|
|
45
|
+
("integer-weights", {"architecture": "X", "weights": [0, 1, -1, 16777216, -16777216]}, {"shuffle": True}),
|
|
46
|
+
("degenerate-array", [1, 2, 3], {"shuffle": True}),
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
def run(args):
|
|
50
|
+
r = subprocess.run(args, capture_output=True, text=True)
|
|
51
|
+
if r.returncode != 0:
|
|
52
|
+
sys.exit("CLI failed: %s\n%s" % (" ".join(args), r.stderr))
|
|
53
|
+
|
|
54
|
+
manifest = {"format": "namz", "note": "Cross-language conformance vectors. Encode a valid input with the "
|
|
55
|
+
"given options and it must BYTE-MATCH the .namz; decode the .namz and the weights must be "
|
|
56
|
+
"bit-exact float32. Every invalid .namz must be rejected (empty/error, never crash).",
|
|
57
|
+
"valid": [], "invalid": []}
|
|
58
|
+
|
|
59
|
+
for name, obj, opts in valid:
|
|
60
|
+
nam = os.path.join(VEC, name + ".nam")
|
|
61
|
+
out = os.path.join(VEC, name + ".namz")
|
|
62
|
+
with open(nam, "w") as f:
|
|
63
|
+
json.dump(obj, f, separators=(",", ":"))
|
|
64
|
+
args = [CLI, "encode", nam, out]
|
|
65
|
+
if not opts.get("shuffle", True):
|
|
66
|
+
args.append("--no-shuffle")
|
|
67
|
+
for k, v in opts.get("set", {}).items():
|
|
68
|
+
args += ["--set", "%s=%s" % (k, v)]
|
|
69
|
+
run(args)
|
|
70
|
+
manifest["valid"].append({"name": name, "input": "vectors/%s.nam" % name,
|
|
71
|
+
"output": "vectors/%s.namz" % name,
|
|
72
|
+
"shuffle": opts.get("shuffle", True), "set": opts.get("set", {})})
|
|
73
|
+
|
|
74
|
+
# ---- invalid cases: derive from a valid blob so they're realistic ------------------------------------
|
|
75
|
+
base = open(os.path.join(VEC, "flat.namz"), "rb").read()
|
|
76
|
+
|
|
77
|
+
def write_invalid(name, blob, why):
|
|
78
|
+
open(os.path.join(INV, name + ".namz"), "wb").write(blob)
|
|
79
|
+
manifest["invalid"].append({"name": name, "file": "invalid/%s.namz" % name, "why": why})
|
|
80
|
+
|
|
81
|
+
write_invalid("bad-magic", b"XAMZ" + base[4:], "magic corrupted")
|
|
82
|
+
write_invalid("bad-version", base[:4] + bytes([9]) + base[5:], "formatVersion 9 (> 2) must be refused")
|
|
83
|
+
write_invalid("bad-codec", base[:5] + bytes([1]) + base[6:], "codec 1 (deflate, reserved) must be refused")
|
|
84
|
+
write_invalid("bad-dtype", base[:6] + bytes([1]) + base[7:], "dtype 1 (float16, reserved) must be refused")
|
|
85
|
+
write_invalid("truncated", base[:len(base) - 3], "body truncated — exact-length check must reject")
|
|
86
|
+
write_invalid("trailing-junk", base + b"junk", "extra bytes — exact-length check must reject")
|
|
87
|
+
write_invalid("lying-metalen", base[:8] + b"\xff\xff" + base[10:], "metaLen far bigger than the buffer")
|
|
88
|
+
write_invalid("empty", b"", "empty buffer")
|
|
89
|
+
write_invalid("magic-only", b"NAMZ", "header too short")
|
|
90
|
+
|
|
91
|
+
with open(os.path.join(HERE, "manifest.json"), "w") as f:
|
|
92
|
+
json.dump(manifest, f, indent=2)
|
|
93
|
+
f.write("\n")
|
|
94
|
+
|
|
95
|
+
print("generated %d valid + %d invalid vectors" % (len(manifest["valid"]), len(manifest["invalid"])))
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
File without changes
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
NAMZ
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
{
|
|
2
|
+
"format": "namz",
|
|
3
|
+
"note": "Cross-language conformance vectors. Encode a valid input with the given options and it must BYTE-MATCH the .namz; decode the .namz and the weights must be bit-exact float32. Every invalid .namz must be rejected (empty/error, never crash).",
|
|
4
|
+
"valid": [
|
|
5
|
+
{
|
|
6
|
+
"name": "flat",
|
|
7
|
+
"input": "vectors/flat.nam",
|
|
8
|
+
"output": "vectors/flat.namz",
|
|
9
|
+
"shuffle": true,
|
|
10
|
+
"set": {}
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"name": "flat-noshuffle",
|
|
14
|
+
"input": "vectors/flat-noshuffle.nam",
|
|
15
|
+
"output": "vectors/flat-noshuffle.namz",
|
|
16
|
+
"shuffle": false,
|
|
17
|
+
"set": {}
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"name": "nested",
|
|
21
|
+
"input": "vectors/nested.nam",
|
|
22
|
+
"output": "vectors/nested.namz",
|
|
23
|
+
"shuffle": true,
|
|
24
|
+
"set": {}
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"name": "metadata",
|
|
28
|
+
"input": "vectors/metadata.nam",
|
|
29
|
+
"output": "vectors/metadata.namz",
|
|
30
|
+
"shuffle": true,
|
|
31
|
+
"set": {
|
|
32
|
+
"tone_type": "hi-gain",
|
|
33
|
+
"boost": "true",
|
|
34
|
+
"device": "tube:1,pnp:1",
|
|
35
|
+
"gear_make": "Two Notes"
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"name": "no-weights",
|
|
40
|
+
"input": "vectors/no-weights.nam",
|
|
41
|
+
"output": "vectors/no-weights.namz",
|
|
42
|
+
"shuffle": true,
|
|
43
|
+
"set": {}
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"name": "empty-weights",
|
|
47
|
+
"input": "vectors/empty-weights.nam",
|
|
48
|
+
"output": "vectors/empty-weights.namz",
|
|
49
|
+
"shuffle": true,
|
|
50
|
+
"set": {}
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"name": "edge-floats",
|
|
54
|
+
"input": "vectors/edge-floats.nam",
|
|
55
|
+
"output": "vectors/edge-floats.namz",
|
|
56
|
+
"shuffle": true,
|
|
57
|
+
"set": {}
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"name": "integer-weights",
|
|
61
|
+
"input": "vectors/integer-weights.nam",
|
|
62
|
+
"output": "vectors/integer-weights.namz",
|
|
63
|
+
"shuffle": true,
|
|
64
|
+
"set": {}
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
"name": "degenerate-array",
|
|
68
|
+
"input": "vectors/degenerate-array.nam",
|
|
69
|
+
"output": "vectors/degenerate-array.namz",
|
|
70
|
+
"shuffle": true,
|
|
71
|
+
"set": {}
|
|
72
|
+
}
|
|
73
|
+
],
|
|
74
|
+
"invalid": [
|
|
75
|
+
{
|
|
76
|
+
"name": "bad-magic",
|
|
77
|
+
"file": "invalid/bad-magic.namz",
|
|
78
|
+
"why": "magic corrupted"
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"name": "bad-version",
|
|
82
|
+
"file": "invalid/bad-version.namz",
|
|
83
|
+
"why": "formatVersion 9 (> 2) must be refused"
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"name": "bad-codec",
|
|
87
|
+
"file": "invalid/bad-codec.namz",
|
|
88
|
+
"why": "codec 1 (deflate, reserved) must be refused"
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
"name": "bad-dtype",
|
|
92
|
+
"file": "invalid/bad-dtype.namz",
|
|
93
|
+
"why": "dtype 1 (float16, reserved) must be refused"
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
"name": "truncated",
|
|
97
|
+
"file": "invalid/truncated.namz",
|
|
98
|
+
"why": "body truncated \u2014 exact-length check must reject"
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"name": "trailing-junk",
|
|
102
|
+
"file": "invalid/trailing-junk.namz",
|
|
103
|
+
"why": "extra bytes \u2014 exact-length check must reject"
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
"name": "lying-metalen",
|
|
107
|
+
"file": "invalid/lying-metalen.namz",
|
|
108
|
+
"why": "metaLen far bigger than the buffer"
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"name": "empty",
|
|
112
|
+
"file": "invalid/empty.namz",
|
|
113
|
+
"why": "empty buffer"
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
"name": "magic-only",
|
|
117
|
+
"file": "invalid/magic-only.namz",
|
|
118
|
+
"why": "header too short"
|
|
119
|
+
}
|
|
120
|
+
]
|
|
121
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
[1,2,3]
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"architecture":"X","weights":[0.0,-0.0,1.401298464324817e-45,1.1754943508222875e-38,3.4028234663852886e+38,-3.4028234663852886e+38,0.3333333432674408]}
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"architecture":"X","weights":[]}
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"architecture":"WaveNet","weights":[0.5,-0.5,0.25]}
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"architecture":"WaveNet","weights":[0.5,-0.5,0.25]}
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"architecture":"X","weights":[0,1,-1,16777216,-16777216]}
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"architecture":"WaveNet","weights":[0.5]}
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"architecture":"SlimmableContainer","sample_rate":48000,"config":{"submodels":[{"max_value":0.5,"model":{"architecture":"WaveNet","weights":[0.1,-0.25,12.5]}}]},"weights":[1.0,-1.0]}
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"architecture":"LSTM","sample_rate":44100,"metadata":{"loudness":-18.0}}
|
|
Binary file
|