nat-tokenizer 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nat_tokenizer-0.1.0/.gitignore +4 -0
- nat_tokenizer-0.1.0/LICENSE +21 -0
- nat_tokenizer-0.1.0/PKG-INFO +147 -0
- nat_tokenizer-0.1.0/README.md +131 -0
- nat_tokenizer-0.1.0/pyproject.toml +20 -0
- nat_tokenizer-0.1.0/src/nat_tokenizer/__init__.py +4 -0
- nat_tokenizer-0.1.0/src/nat_tokenizer/model.py +241 -0
- nat_tokenizer-0.1.0/src/nat_tokenizer/tokenizer.py +145 -0
- nat_tokenizer-0.1.0/tests/test_tokenizer.py +78 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ahad Jawaid
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: nat-tokenizer
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: NAT action tokenizer: robot action chunks to time-ordered discrete tokens and back
|
|
5
|
+
Author: Ahad Jawaid
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Python: >=3.9
|
|
9
|
+
Requires-Dist: huggingface-hub>=0.20
|
|
10
|
+
Requires-Dist: numpy>=1.21
|
|
11
|
+
Requires-Dist: safetensors>=0.4
|
|
12
|
+
Requires-Dist: torch>=2
|
|
13
|
+
Provides-Extra: test
|
|
14
|
+
Requires-Dist: pytest>=7; extra == 'test'
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# NAT action tokenizer
|
|
18
|
+
|
|
19
|
+
Turns robot action chunks into discrete tokens for a vision-language-action model, and tokens back into actions.
|
|
20
|
+
Every 32-step chunk of up to 7 action columns becomes **14 tokens**, in time order, so a model can stream actions
|
|
21
|
+
before it has generated the whole chunk.
|
|
22
|
+
|
|
23
|
+
## Quick start
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
pip install nat-tokenizer
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
or from source: `pip install "git+https://github.com/ahadjawaid/nat.git"`.
|
|
30
|
+
|
|
31
|
+
The weights and config download from the Hugging Face repository
|
|
32
|
+
[ahad-j/nat-action-tokenizer](https://huggingface.co/ahad-j/nat-action-tokenizer) on first use.
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
import numpy as np
|
|
36
|
+
from nat_tokenizer import NATActionTokenizer
|
|
37
|
+
|
|
38
|
+
tokenizer = NATActionTokenizer.from_pretrained() # downloads ahad-j/nat-action-tokenizer
|
|
39
|
+
|
|
40
|
+
# 1. Fit the normalization once, on your dataset's actions (any shape ending in action_dim).
|
|
41
|
+
tokenizer.fit(dataset_actions)
|
|
42
|
+
|
|
43
|
+
# 2. Tokenize 32-step chunks: (batch, 32, action_dim) or one chunk (32, action_dim).
|
|
44
|
+
tokens = tokenizer(action_chunks) # list (one per chunk) of lists of ints in [0, 25600)
|
|
45
|
+
|
|
46
|
+
# 3. Decode back to actions in your original units.
|
|
47
|
+
actions = tokenizer.decode(tokens, action_dim=action_chunks.shape[-1]) # (batch, 32, action_dim)
|
|
48
|
+
|
|
49
|
+
# 4. Save the fitted tokenizer (keeps your min and max) and load it later the same way.
|
|
50
|
+
tokenizer.save_pretrained("my-nat-tokenizer")
|
|
51
|
+
tokenizer = NATActionTokenizer.from_pretrained("my-nat-tokenizer")
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Normalization (on by default)
|
|
55
|
+
|
|
56
|
+
The tokenizer was trained on actions in [-1, 1]. With `normalize=True` (the default) every column is mapped from
|
|
57
|
+
its min and max to [-1, 1] before encoding (values outside are clipped) and decoded actions are mapped back, so you
|
|
58
|
+
work in your own units throughout. Set the min and max with `fit(dataset_actions)`, or pass them directly:
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
tokenizer = NATActionTokenizer.from_pretrained(action_min=[...], action_max=[...])
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
If your actions are already in [-1, 1], pass `normalize=False` to both calls:
|
|
65
|
+
`tokenizer(chunks, normalize=False)` and `tokenizer.decode(tokens, action_dim=d, normalize=False)`.
|
|
66
|
+
A column whose min equals its max is encoded as 0 and decoded back to its constant value.
|
|
67
|
+
|
|
68
|
+
## Streaming (causal decoding)
|
|
69
|
+
|
|
70
|
+
Within a group the 14 tokens are time ordered: the first 4 decode the chunk's first 8 steps, the first 7 its first 16,
|
|
71
|
+
the first 11 its first 24, and all 14 the whole chunk. A prefix decodes exactly the same first steps as the full chunk,
|
|
72
|
+
so a model can execute actions while it is still generating:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
for steps in (8, 16, 24, 32):
|
|
76
|
+
prefix = [chunk[:tokenizer.prefix_token_count(steps)] for chunk in tokens]
|
|
77
|
+
first_steps = tokenizer.decode(prefix, action_dim=7) # (batch, steps, 7)
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Streaming works for actions of up to 7 columns (one group).
|
|
81
|
+
|
|
82
|
+
## Action dimensions
|
|
83
|
+
|
|
84
|
+
Columns are split into consecutive groups of up to 7 (8 columns = 7 + 1, 14 = 7 + 7, 21 = 7 + 7 + 7). Each group gets
|
|
85
|
+
its own 14 tokens and a chunk's tokens are the groups' tokens one after another: `tokenizer.tokens_per_chunk(action_dim)`
|
|
86
|
+
gives the count (14 per group). Chunks must be exactly 32 steps long.
|
|
87
|
+
|
|
88
|
+
## GPU and CUDA graphs
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
tokenizer = tokenizer.to("cuda") # eager on the GPU
|
|
92
|
+
tokenizer = tokenizer.to("cuda", cuda_graphs=True) # captures the encoder and decoder once per batch shape
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Inputs and outputs stay NumPy. GPU results match the CPU up to floating point (rarely a token on a quantization
|
|
96
|
+
boundary differs).
|
|
97
|
+
|
|
98
|
+
## API
|
|
99
|
+
|
|
100
|
+
| call | does |
|
|
101
|
+
|---|---|
|
|
102
|
+
| `NATActionTokenizer.from_pretrained(repo_or_folder, action_min=None, action_max=None)` | load (optionally with your min and max) |
|
|
103
|
+
| `tokenizer.fit(actions)` | per-column min and max of your actions (returns the tokenizer) |
|
|
104
|
+
| `tokenizer(chunks, normalize=True)` | `(batch, 32, d)` or `(32, d)` actions to a list of token lists |
|
|
105
|
+
| `tokenizer.decode(tokens, action_dim, normalize=True)` | token lists (whole chunks, or one group's prefix) to `(batch, steps, d)` actions |
|
|
106
|
+
| `tokenizer.prefix_token_count(steps)` | tokens that decode the first `steps` (8, 16, 24 or 32) |
|
|
107
|
+
| `tokenizer.tokens_per_chunk(action_dim)` | 14 per group of 7 columns |
|
|
108
|
+
| `tokenizer.to(device, cuda_graphs=False)` | run on a device, optionally with CUDA graphs |
|
|
109
|
+
| `tokenizer.save_pretrained(directory)` | save, with the fitted min and max |
|
|
110
|
+
| `tokenizer.vocab_size` | 25,600 (4 scales x 6,400 codes) |
|
|
111
|
+
|
|
112
|
+
## Model
|
|
113
|
+
|
|
114
|
+
4 latents per chunk, 4 FSQ scales of 6,400 codes (one coarse token per two latents, three finer tokens per latent),
|
|
115
|
+
trained on a large corpus of robot and human action data (every window once, batch 4,096). The repository holds only
|
|
116
|
+
the weights (`model.safetensors`) and a JSON config; the network is plain PyTorch in the `nat-tokenizer` package.
|
|
117
|
+
|
|
118
|
+
## Reconstruction on robot benchmarks, and speed
|
|
119
|
+
|
|
120
|
+
Random 32-step chunks of each benchmark's demonstrations, each normalized by its own min and max (MetaWorld's actions clipped to the [-1, 1] its simulator executes). Errors are mean absolute differences between the input and its reconstruction. Speed is in milliseconds per call on an RTX 4090 (shared with another job while measured) and 4 CPU threads, mean and SD over 20 calls.
|
|
121
|
+
|
|
122
|
+
| benchmark | chunks x shape | MAE in [-1, 1] units | MAE | max error |
|
|
123
|
+
|---|---|---|---|---|
|
|
124
|
+
| LIBERO-10 | 1000 x (32, 7) | 0.0304 | 0.0210 | 1.9762 |
|
|
125
|
+
| MetaWorld mt4 (clipped) | 1000 x (32, 4) | 0.0651 | 0.0651 | 1.7578 |
|
|
126
|
+
| RoboMimic mt4 | 1000 x (32, 7) | 0.0466 | 0.0404 | 1.5712 |
|
|
127
|
+
| RoboCasa mt4 | 1000 x (32, 12) | 0.0130 | 0.0107 | 0.9970 |
|
|
128
|
+
|
|
129
|
+
| device | batch of 32 x 7 chunks | encode ms | decode ms |
|
|
130
|
+
|---|---|---|---|
|
|
131
|
+
| cpu, eager | 1 | 11.57 ± 0.39 | 29.86 ± 4.06 |
|
|
132
|
+
| cpu, eager | 64 | 322.06 ± 1.76 | 263.23 ± 6.67 |
|
|
133
|
+
| cuda, eager | 1 | 6.54 ± 1.67 | 11.07 ± 1.18 |
|
|
134
|
+
| cuda, eager | 64 | 7.86 ± 2.41 | 12.67 ± 1.80 |
|
|
135
|
+
| cuda, CUDA graphs | 1 | 0.71 ± 0.57 | 2.44 ± 0.39 |
|
|
136
|
+
| cuda, CUDA graphs | 64 | 1.05 ± 0.35 | 5.18 ± 2.09 |
|
|
137
|
+
|
|
138
|
+
## Development
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
pip install -e ".[test]"
|
|
142
|
+
pytest
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## License
|
|
146
|
+
|
|
147
|
+
MIT, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# NAT action tokenizer
|
|
2
|
+
|
|
3
|
+
Turns robot action chunks into discrete tokens for a vision-language-action model, and tokens back into actions.
|
|
4
|
+
Every 32-step chunk of up to 7 action columns becomes **14 tokens**, in time order, so a model can stream actions
|
|
5
|
+
before it has generated the whole chunk.
|
|
6
|
+
|
|
7
|
+
## Quick start
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install nat-tokenizer
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
or from source: `pip install "git+https://github.com/ahadjawaid/nat.git"`.
|
|
14
|
+
|
|
15
|
+
The weights and config download from the Hugging Face repository
|
|
16
|
+
[ahad-j/nat-action-tokenizer](https://huggingface.co/ahad-j/nat-action-tokenizer) on first use.
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
import numpy as np
|
|
20
|
+
from nat_tokenizer import NATActionTokenizer
|
|
21
|
+
|
|
22
|
+
tokenizer = NATActionTokenizer.from_pretrained() # downloads ahad-j/nat-action-tokenizer
|
|
23
|
+
|
|
24
|
+
# 1. Fit the normalization once, on your dataset's actions (any shape ending in action_dim).
|
|
25
|
+
tokenizer.fit(dataset_actions)
|
|
26
|
+
|
|
27
|
+
# 2. Tokenize 32-step chunks: (batch, 32, action_dim) or one chunk (32, action_dim).
|
|
28
|
+
tokens = tokenizer(action_chunks) # list (one per chunk) of lists of ints in [0, 25600)
|
|
29
|
+
|
|
30
|
+
# 3. Decode back to actions in your original units.
|
|
31
|
+
actions = tokenizer.decode(tokens, action_dim=action_chunks.shape[-1]) # (batch, 32, action_dim)
|
|
32
|
+
|
|
33
|
+
# 4. Save the fitted tokenizer (keeps your min and max) and load it later the same way.
|
|
34
|
+
tokenizer.save_pretrained("my-nat-tokenizer")
|
|
35
|
+
tokenizer = NATActionTokenizer.from_pretrained("my-nat-tokenizer")
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Normalization (on by default)
|
|
39
|
+
|
|
40
|
+
The tokenizer was trained on actions in [-1, 1]. With `normalize=True` (the default) every column is mapped from
|
|
41
|
+
its min and max to [-1, 1] before encoding (values outside are clipped) and decoded actions are mapped back, so you
|
|
42
|
+
work in your own units throughout. Set the min and max with `fit(dataset_actions)`, or pass them directly:
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
tokenizer = NATActionTokenizer.from_pretrained(action_min=[...], action_max=[...])
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
If your actions are already in [-1, 1], pass `normalize=False` to both calls:
|
|
49
|
+
`tokenizer(chunks, normalize=False)` and `tokenizer.decode(tokens, action_dim=d, normalize=False)`.
|
|
50
|
+
A column whose min equals its max is encoded as 0 and decoded back to its constant value.
|
|
51
|
+
|
|
52
|
+
## Streaming (causal decoding)
|
|
53
|
+
|
|
54
|
+
Within a group the 14 tokens are time ordered: the first 4 decode the chunk's first 8 steps, the first 7 its first 16,
|
|
55
|
+
the first 11 its first 24, and all 14 the whole chunk. A prefix decodes exactly the same first steps as the full chunk,
|
|
56
|
+
so a model can execute actions while it is still generating:
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
for steps in (8, 16, 24, 32):
|
|
60
|
+
prefix = [chunk[:tokenizer.prefix_token_count(steps)] for chunk in tokens]
|
|
61
|
+
first_steps = tokenizer.decode(prefix, action_dim=7) # (batch, steps, 7)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Streaming works for actions of up to 7 columns (one group).
|
|
65
|
+
|
|
66
|
+
## Action dimensions
|
|
67
|
+
|
|
68
|
+
Columns are split into consecutive groups of up to 7 (8 columns = 7 + 1, 14 = 7 + 7, 21 = 7 + 7 + 7). Each group gets
|
|
69
|
+
its own 14 tokens and a chunk's tokens are the groups' tokens one after another: `tokenizer.tokens_per_chunk(action_dim)`
|
|
70
|
+
gives the count (14 per group). Chunks must be exactly 32 steps long.
|
|
71
|
+
|
|
72
|
+
## GPU and CUDA graphs
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
tokenizer = tokenizer.to("cuda") # eager on the GPU
|
|
76
|
+
tokenizer = tokenizer.to("cuda", cuda_graphs=True) # captures the encoder and decoder once per batch shape
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Inputs and outputs stay NumPy. GPU results match the CPU up to floating point (rarely a token on a quantization
|
|
80
|
+
boundary differs).
|
|
81
|
+
|
|
82
|
+
## API
|
|
83
|
+
|
|
84
|
+
| call | does |
|
|
85
|
+
|---|---|
|
|
86
|
+
| `NATActionTokenizer.from_pretrained(repo_or_folder, action_min=None, action_max=None)` | load (optionally with your min and max) |
|
|
87
|
+
| `tokenizer.fit(actions)` | per-column min and max of your actions (returns the tokenizer) |
|
|
88
|
+
| `tokenizer(chunks, normalize=True)` | `(batch, 32, d)` or `(32, d)` actions to a list of token lists |
|
|
89
|
+
| `tokenizer.decode(tokens, action_dim, normalize=True)` | token lists (whole chunks, or one group's prefix) to `(batch, steps, d)` actions |
|
|
90
|
+
| `tokenizer.prefix_token_count(steps)` | tokens that decode the first `steps` (8, 16, 24 or 32) |
|
|
91
|
+
| `tokenizer.tokens_per_chunk(action_dim)` | 14 per group of 7 columns |
|
|
92
|
+
| `tokenizer.to(device, cuda_graphs=False)` | run on a device, optionally with CUDA graphs |
|
|
93
|
+
| `tokenizer.save_pretrained(directory)` | save, with the fitted min and max |
|
|
94
|
+
| `tokenizer.vocab_size` | 25,600 (4 scales x 6,400 codes) |
|
|
95
|
+
|
|
96
|
+
## Model
|
|
97
|
+
|
|
98
|
+
4 latents per chunk, 4 FSQ scales of 6,400 codes (one coarse token per two latents, three finer tokens per latent),
|
|
99
|
+
trained on a large corpus of robot and human action data (every window once, batch 4,096). The repository holds only
|
|
100
|
+
the weights (`model.safetensors`) and a JSON config; the network is plain PyTorch in the `nat-tokenizer` package.
|
|
101
|
+
|
|
102
|
+
## Reconstruction on robot benchmarks, and speed
|
|
103
|
+
|
|
104
|
+
Random 32-step chunks of each benchmark's demonstrations, each normalized by its own min and max (MetaWorld's actions clipped to the [-1, 1] its simulator executes). Errors are mean absolute differences between the input and its reconstruction. Speed is in milliseconds per call on an RTX 4090 (shared with another job while measured) and 4 CPU threads, mean and SD over 20 calls.
|
|
105
|
+
|
|
106
|
+
| benchmark | chunks x shape | MAE in [-1, 1] units | MAE | max error |
|
|
107
|
+
|---|---|---|---|---|
|
|
108
|
+
| LIBERO-10 | 1000 x (32, 7) | 0.0304 | 0.0210 | 1.9762 |
|
|
109
|
+
| MetaWorld mt4 (clipped) | 1000 x (32, 4) | 0.0651 | 0.0651 | 1.7578 |
|
|
110
|
+
| RoboMimic mt4 | 1000 x (32, 7) | 0.0466 | 0.0404 | 1.5712 |
|
|
111
|
+
| RoboCasa mt4 | 1000 x (32, 12) | 0.0130 | 0.0107 | 0.9970 |
|
|
112
|
+
|
|
113
|
+
| device | batch of 32 x 7 chunks | encode ms | decode ms |
|
|
114
|
+
|---|---|---|---|
|
|
115
|
+
| cpu, eager | 1 | 11.57 ± 0.39 | 29.86 ± 4.06 |
|
|
116
|
+
| cpu, eager | 64 | 322.06 ± 1.76 | 263.23 ± 6.67 |
|
|
117
|
+
| cuda, eager | 1 | 6.54 ± 1.67 | 11.07 ± 1.18 |
|
|
118
|
+
| cuda, eager | 64 | 7.86 ± 2.41 | 12.67 ± 1.80 |
|
|
119
|
+
| cuda, CUDA graphs | 1 | 0.71 ± 0.57 | 2.44 ± 0.39 |
|
|
120
|
+
| cuda, CUDA graphs | 64 | 1.05 ± 0.35 | 5.18 ± 2.09 |
|
|
121
|
+
|
|
122
|
+
## Development
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
pip install -e ".[test]"
|
|
126
|
+
pytest
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
## License
|
|
130
|
+
|
|
131
|
+
MIT, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "nat-tokenizer"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "NAT action tokenizer: robot action chunks to time-ordered discrete tokens and back"
|
|
5
|
+
authors = [{ name = "Ahad Jawaid" }]
|
|
6
|
+
requires-python = ">=3.9"
|
|
7
|
+
dependencies = ["numpy>=1.21", "torch>=2", "safetensors>=0.4", "huggingface_hub>=0.20"]
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
license = "MIT"
|
|
10
|
+
license-files = ["LICENSE"]
|
|
11
|
+
|
|
12
|
+
[project.optional-dependencies]
|
|
13
|
+
test = ["pytest>=7"]
|
|
14
|
+
|
|
15
|
+
[build-system]
|
|
16
|
+
requires = ["hatchling"]
|
|
17
|
+
build-backend = "hatchling.build"
|
|
18
|
+
|
|
19
|
+
[tool.hatch.build.targets.wheel]
|
|
20
|
+
packages = ["src/nat_tokenizer"]
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
# Copyright (c) 2026 Ahad Jawaid
|
|
2
|
+
"""The NAT codec's inference network: a causal convolutional encoder, a multi-scale finite scalar quantizer whose codes
|
|
3
|
+
become time-ordered tokens, and a causal decoder that turns the tokens back into an action chunk."""
|
|
4
|
+
import math
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
from torch import nn
|
|
8
|
+
from torch.nn import functional
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class NATCodec(nn.Module):
|
|
12
|
+
"""Packed chunks of time_horizon steps x group_width columns, in [-1, 1], to tokens and back."""
|
|
13
|
+
|
|
14
|
+
def __init__(self, config):
|
|
15
|
+
super().__init__()
|
|
16
|
+
encoder, quantizer, decoder = config["encoder"], config["quantizer"], config["decoder"]
|
|
17
|
+
self.time_horizon, self.group_width = config["time_horizon"], config["group_width"]
|
|
18
|
+
values_per_latent = math.prod(encoder["downsampling_ratios"])
|
|
19
|
+
latents = self.time_horizon * self.group_width // values_per_latent
|
|
20
|
+
self.encoder = ActionEncoder(**encoder)
|
|
21
|
+
self.quantizer = MultiScaleFSQ(encoder["latent_dim"], latents=latents, **quantizer)
|
|
22
|
+
self.decoder = ActionDecoder(encoder["latent_dim"], values_per_latent, latents=latents, **decoder)
|
|
23
|
+
|
|
24
|
+
def encode(self, packed):
|
|
25
|
+
"""(batch, time_horizon, group_width) actions to (batch, tokens) token ids."""
|
|
26
|
+
latents = self.encoder(packed.reshape(len(packed), 1, -1)) # time-major: each latent covers whole steps
|
|
27
|
+
return self.quantizer.tokens(latents)
|
|
28
|
+
|
|
29
|
+
def decode(self, tokens):
|
|
30
|
+
"""(batch, tokens) token ids to (batch, time_horizon, group_width) actions."""
|
|
31
|
+
actions = self.decoder(self.quantizer.features(tokens))
|
|
32
|
+
return actions.reshape(len(tokens), self.time_horizon, self.group_width)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class ActionEncoder(nn.Module):
|
|
36
|
+
"""A causal SEANet encoder: a convolution, then per downsampling ratio a residual unit and a strided convolution,
|
|
37
|
+
an LSTM with a skip connection, and a convolution to the latent width."""
|
|
38
|
+
|
|
39
|
+
def __init__(self, channels, filters, downsampling_ratios, kernel_size, residual_kernel_size, last_kernel_size,
|
|
40
|
+
compress, lstm_layers, latent_dim):
|
|
41
|
+
super().__init__()
|
|
42
|
+
self.input_conv = CausalConv1d(channels, filters, kernel_size)
|
|
43
|
+
self.stages = nn.ModuleList()
|
|
44
|
+
for ratio in downsampling_ratios:
|
|
45
|
+
self.stages.append(EncoderStage(filters, filters // compress, residual_kernel_size, ratio))
|
|
46
|
+
filters *= 2
|
|
47
|
+
self.lstm = nn.LSTM(filters, filters, lstm_layers)
|
|
48
|
+
self.output_conv = CausalConv1d(filters, latent_dim, last_kernel_size)
|
|
49
|
+
|
|
50
|
+
def forward(self, actions):
|
|
51
|
+
hidden = self.input_conv(actions)
|
|
52
|
+
for stage in self.stages:
|
|
53
|
+
hidden = stage(hidden)
|
|
54
|
+
sequence = hidden.permute(2, 0, 1)
|
|
55
|
+
hidden = (self.lstm(sequence)[0] + sequence).permute(1, 2, 0)
|
|
56
|
+
return self.output_conv(functional.elu(hidden))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class EncoderStage(nn.Module):
|
|
60
|
+
def __init__(self, channels, hidden_channels, residual_kernel_size, ratio):
|
|
61
|
+
super().__init__()
|
|
62
|
+
self.residual = ResidualUnit(channels, hidden_channels, residual_kernel_size)
|
|
63
|
+
self.downsample = CausalConv1d(channels, 2 * channels, 2 * ratio, stride=ratio)
|
|
64
|
+
|
|
65
|
+
def forward(self, hidden):
|
|
66
|
+
return self.downsample(functional.elu(self.residual(hidden)))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class ResidualUnit(nn.Module):
|
|
70
|
+
def __init__(self, channels, hidden_channels, kernel_size):
|
|
71
|
+
super().__init__()
|
|
72
|
+
self.conv = CausalConv1d(channels, hidden_channels, kernel_size)
|
|
73
|
+
self.projection = CausalConv1d(hidden_channels, channels, 1)
|
|
74
|
+
self.shortcut = CausalConv1d(channels, channels, 1)
|
|
75
|
+
|
|
76
|
+
def forward(self, hidden):
|
|
77
|
+
return self.shortcut(hidden) + self.projection(functional.elu(self.conv(functional.elu(hidden))))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class CausalConv1d(nn.Conv1d):
|
|
81
|
+
"""A convolution that sees only the current and earlier steps, its input zero-padded on the left (and on the right
|
|
82
|
+
only to complete the last stride)."""
|
|
83
|
+
|
|
84
|
+
def forward(self, hidden):
|
|
85
|
+
kernel, stride = (self.kernel_size[0] - 1) * self.dilation[0] + 1, self.stride[0]
|
|
86
|
+
left = kernel - stride
|
|
87
|
+
frames = math.ceil((hidden.shape[-1] - kernel + left) / stride + 1)
|
|
88
|
+
right = (frames - 1) * stride + kernel - left - hidden.shape[-1]
|
|
89
|
+
return super().forward(functional.pad(hidden, (left, right)))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class MultiScaleFSQ(nn.Module):
|
|
93
|
+
"""Residual finite scalar quantization at several temporal scales: scale s pools scales[s] latents into one code
|
|
94
|
+
of prod(levels) values. A chunk's tokens are ordered by the first latent each code covers, coarse scales first,
|
|
95
|
+
and scale s's codes are offset by s x codebook_size, so a prefix of whole latents decodes on its own."""
|
|
96
|
+
|
|
97
|
+
def __init__(self, latent_dim, levels, scales, latents):
|
|
98
|
+
super().__init__()
|
|
99
|
+
self.scales, self.latents, self.codebook_size = scales, latents, math.prod(levels)
|
|
100
|
+
self.project_in = nn.ModuleList([nn.Linear(latent_dim, len(levels)) for _ in scales])
|
|
101
|
+
self.project_out = nn.ModuleList([nn.Linear(len(levels), latent_dim) for _ in scales])
|
|
102
|
+
self.register_buffer("levels", torch.tensor(levels, dtype=torch.int32), persistent=False)
|
|
103
|
+
self.register_buffer("basis", torch.cumprod(torch.tensor([1] + levels[:-1]), 0, dtype=torch.int32),
|
|
104
|
+
persistent=False)
|
|
105
|
+
self.token_order = sorted((position * span, scale, position) for scale, span in enumerate(scales)
|
|
106
|
+
for position in range(-(-latents // span)))
|
|
107
|
+
for scale in range(len(scales)):
|
|
108
|
+
self.register_buffer(f"scale_{scale}_columns", torch.tensor(
|
|
109
|
+
[index for index, (_, owner, _) in enumerate(self.token_order) if owner == scale]), persistent=False)
|
|
110
|
+
|
|
111
|
+
def tokens(self, latents):
|
|
112
|
+
residual, codes = latents, []
|
|
113
|
+
for scale, span in enumerate(self.scales):
|
|
114
|
+
pooled = functional.avg_pool1d(residual, span) if span > 1 else residual
|
|
115
|
+
indices, quantized = self.quantize(self.project_in[scale](pooled.transpose(1, 2)).float())
|
|
116
|
+
residual = residual - self.upsampled(self.project_out[scale](quantized).transpose(1, 2), span)
|
|
117
|
+
codes.append(indices)
|
|
118
|
+
return torch.stack([codes[scale][:, position] + scale * self.codebook_size
|
|
119
|
+
for _, scale, position in self.token_order], dim=1)
|
|
120
|
+
|
|
121
|
+
def features(self, tokens):
|
|
122
|
+
features = 0
|
|
123
|
+
for scale, span in enumerate(self.scales):
|
|
124
|
+
indices = tokens.index_select(1, getattr(self, f"scale_{scale}_columns")) - scale * self.codebook_size
|
|
125
|
+
codes = self.codes(indices.clamp(0, self.codebook_size - 1))
|
|
126
|
+
features = features + self.upsampled(self.project_out[scale](codes).transpose(1, 2), span)
|
|
127
|
+
return features
|
|
128
|
+
|
|
129
|
+
def quantize(self, projected):
|
|
130
|
+
"""Bounded, rounded values (codes in [-1, 1]) and their codebook indices."""
|
|
131
|
+
half_level = (self.levels - 1) * (1 + 1e-3) / 2
|
|
132
|
+
offset = torch.where(self.levels % 2 == 0, 0.5, 0.0)
|
|
133
|
+
bounded = (projected + (offset / half_level).atanh()).tanh() * half_level - offset
|
|
134
|
+
codes = (bounded + (bounded.round() - bounded)) / (self.levels // 2)
|
|
135
|
+
indices = ((codes * (self.levels // 2) + self.levels // 2) * self.basis).sum(-1).to(torch.int32)
|
|
136
|
+
return indices.long(), codes
|
|
137
|
+
|
|
138
|
+
def codes(self, indices):
|
|
139
|
+
digits = (indices.unsqueeze(-1) // self.basis) % self.levels
|
|
140
|
+
return (digits - self.levels // 2) / (self.levels // 2)
|
|
141
|
+
|
|
142
|
+
def upsampled(self, quantized, span):
|
|
143
|
+
return torch.repeat_interleave(quantized, span, dim=-1)[..., :self.latents]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class ActionDecoder(nn.Module):
|
|
147
|
+
"""A causal Vocos-style decoder: a convolution, residual and causal attention blocks, a conditioned layer norm,
|
|
148
|
+
ConvNeXt blocks and a linear head to each latent's values_per_latent time-major action values."""
|
|
149
|
+
|
|
150
|
+
def __init__(self, latent_dim, values_per_latent, latents, dim, intermediate_dim, num_blocks, condition_embeddings,
|
|
151
|
+
condition_id, attention_context, group_norm_groups):
|
|
152
|
+
super().__init__()
|
|
153
|
+
self.embed = CausalConv1d(latent_dim, dim, 7)
|
|
154
|
+
self.temporal_blocks = nn.Sequential(
|
|
155
|
+
ResnetBlock(dim, group_norm_groups), ResnetBlock(dim, group_norm_groups),
|
|
156
|
+
CausalSelfAttention(dim, group_norm_groups, attention_context, latents),
|
|
157
|
+
ResnetBlock(dim, group_norm_groups), ResnetBlock(dim, group_norm_groups))
|
|
158
|
+
self.temporal_norm = PerTimeGroupNorm(group_norm_groups, dim, eps=1e-6)
|
|
159
|
+
self.input_norm = ConditionedLayerNorm(condition_embeddings, dim)
|
|
160
|
+
self.blocks = nn.ModuleList([ConvNeXtBlock(dim, intermediate_dim, condition_embeddings)
|
|
161
|
+
for _ in range(num_blocks)])
|
|
162
|
+
self.output_norm = nn.LayerNorm(dim, eps=1e-6)
|
|
163
|
+
self.head = nn.Linear(dim, values_per_latent)
|
|
164
|
+
self.register_buffer("condition_id", torch.tensor([condition_id]), persistent=False)
|
|
165
|
+
|
|
166
|
+
def forward(self, features):
|
|
167
|
+
with torch.backends.cudnn.flags(enabled=torch.backends.cudnn.enabled, benchmark=torch.backends.cudnn.benchmark,
|
|
168
|
+
deterministic=torch.backends.cudnn.deterministic, allow_tf32=False):
|
|
169
|
+
hidden = self.temporal_norm(self.temporal_blocks(self.embed(features)))
|
|
170
|
+
hidden = self.input_norm(hidden.transpose(1, 2), self.condition_id).transpose(1, 2)
|
|
171
|
+
for block in self.blocks:
|
|
172
|
+
hidden = block(hidden, self.condition_id)
|
|
173
|
+
return self.head(self.output_norm(hidden.transpose(1, 2))).reshape(len(features), -1)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
class ResnetBlock(nn.Module):
|
|
177
|
+
def __init__(self, channels, groups):
|
|
178
|
+
super().__init__()
|
|
179
|
+
self.norm1, self.conv1 = PerTimeGroupNorm(groups, channels, eps=1e-6), CausalConv1d(channels, channels, 3)
|
|
180
|
+
self.norm2, self.conv2 = PerTimeGroupNorm(groups, channels, eps=1e-6), CausalConv1d(channels, channels, 3)
|
|
181
|
+
|
|
182
|
+
def forward(self, hidden):
|
|
183
|
+
return hidden + self.conv2(swish(self.norm2(self.conv1(swish(self.norm1(hidden))))))
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class CausalSelfAttention(nn.Module):
|
|
187
|
+
"""Attention over the current and up to context - 1 earlier latents."""
|
|
188
|
+
|
|
189
|
+
def __init__(self, channels, groups, context, latents):
|
|
190
|
+
super().__init__()
|
|
191
|
+
self.norm = PerTimeGroupNorm(groups, channels, eps=1e-6)
|
|
192
|
+
self.query, self.key, self.value, self.output = (nn.Conv1d(channels, channels, 1) for _ in range(4))
|
|
193
|
+
distances = torch.arange(latents)[:, None] - torch.arange(latents)[None, :]
|
|
194
|
+
self.register_buffer("allowed", (distances >= 0) & (distances < context), persistent=False)
|
|
195
|
+
|
|
196
|
+
def forward(self, hidden):
|
|
197
|
+
normalized = self.norm(hidden)
|
|
198
|
+
queries, keys, values = self.query(normalized), self.key(normalized), self.value(normalized)
|
|
199
|
+
weights = torch.bmm(queries.transpose(1, 2), keys) * queries.shape[1] ** -.5
|
|
200
|
+
allowed = self.allowed[:queries.shape[-1], :keys.shape[-1]]
|
|
201
|
+
weights = weights.masked_fill(~allowed, -torch.inf).softmax(-1)
|
|
202
|
+
return hidden + self.output(torch.bmm(values, weights.transpose(1, 2)))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class PerTimeGroupNorm(nn.GroupNorm):
|
|
206
|
+
"""Group normalization of each latent on its own, so no latent sees a later one."""
|
|
207
|
+
|
|
208
|
+
def forward(self, hidden):
|
|
209
|
+
batch, channels, length = hidden.shape
|
|
210
|
+
normalized = super().forward(hidden.transpose(1, 2).reshape(batch * length, channels, 1))
|
|
211
|
+
return normalized.reshape(batch, length, channels).transpose(1, 2)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
class ConditionedLayerNorm(nn.Module):
|
|
215
|
+
"""Layer normalization with a learned scale and shift per condition id."""
|
|
216
|
+
|
|
217
|
+
def __init__(self, conditions, channels):
|
|
218
|
+
super().__init__()
|
|
219
|
+
self.scale, self.shift = nn.Embedding(conditions, channels), nn.Embedding(conditions, channels)
|
|
220
|
+
|
|
221
|
+
def forward(self, hidden, condition_id):
|
|
222
|
+
normalized = functional.layer_norm(hidden, (hidden.shape[-1],), eps=1e-6)
|
|
223
|
+
return normalized * self.scale(condition_id) + self.shift(condition_id)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
class ConvNeXtBlock(nn.Module):
|
|
227
|
+
def __init__(self, channels, intermediate_channels, conditions):
|
|
228
|
+
super().__init__()
|
|
229
|
+
self.depthwise_conv = CausalConv1d(channels, channels, 7, groups=channels)
|
|
230
|
+
self.norm = ConditionedLayerNorm(conditions, channels)
|
|
231
|
+
self.expand, self.contract = nn.Linear(channels, intermediate_channels), nn.Linear(intermediate_channels, channels)
|
|
232
|
+
self.layer_scale = nn.Parameter(torch.ones(channels))
|
|
233
|
+
|
|
234
|
+
def forward(self, hidden, condition_id):
|
|
235
|
+
update = self.norm(self.depthwise_conv(hidden).transpose(1, 2), condition_id)
|
|
236
|
+
update = self.layer_scale * self.contract(functional.gelu(self.expand(update)))
|
|
237
|
+
return hidden + update.transpose(1, 2)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def swish(hidden):
|
|
241
|
+
return hidden * torch.sigmoid(hidden)
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# Copyright (c) 2026 Ahad Jawaid
|
|
2
|
+
"""NAT action tokenizer: 32-step action chunks to 14 time-ordered tokens per group of up to 7 action columns, and back.
|
|
3
|
+
|
|
4
|
+
from nat_tokenizer import NATActionTokenizer
|
|
5
|
+
tokenizer = NATActionTokenizer.from_pretrained()
|
|
6
|
+
tokenizer.fit(dataset_actions) # per-column min and max, mapped to [-1, 1]
|
|
7
|
+
tokens = tokenizer(action_chunks) # (batch, 32, action_dim) or (32, action_dim) -> list of token lists
|
|
8
|
+
actions = tokenizer.decode(tokens, action_dim=action_dim) # -> (batch, 32, action_dim), in the original units
|
|
9
|
+
|
|
10
|
+
With normalize=False the actions must already lie in [-1, 1]. Tokens are ids in [0, vocab_size). Each group's tokens are
|
|
11
|
+
time-ordered, so a prefix of prefix_token_count(steps) tokens decodes the chunk's first steps (one group only)."""
|
|
12
|
+
import json
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import torch
|
|
17
|
+
from safetensors.torch import load_file
|
|
18
|
+
|
|
19
|
+
from .model import NATCodec
|
|
20
|
+
|
|
21
|
+
CONFIG_NAME, WEIGHTS_NAME, REPOSITORY = "config.json", "model.safetensors", "ahad-j/nat-action-tokenizer"
|
|
22
|
+
SAVED_FILES = (WEIGHTS_NAME, "README.md", "LICENSE")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class NATActionTokenizer:
|
|
26
|
+
def __init__(self, folder, action_min=None, action_max=None, **config):
|
|
27
|
+
self.folder, self.device, self.cuda_graphs = Path(folder), torch.device("cpu"), None
|
|
28
|
+
self.model = NATCodec(config).eval().requires_grad_(False)
|
|
29
|
+
self.model.load_state_dict(load_file(str(self.folder / WEIGHTS_NAME)))
|
|
30
|
+
self.action_min = None if action_min is None else np.asarray(action_min, dtype=np.float32)
|
|
31
|
+
self.action_max = None if action_max is None else np.asarray(action_max, dtype=np.float32)
|
|
32
|
+
self.horizon, self.slots, self.vocab_size = config["time_horizon"], config["group_width"], config["vocab_size"]
|
|
33
|
+
self.prefix_tokens = {int(steps): tokens for steps, tokens in config["prefix_token_counts"].items()}
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def from_pretrained(cls, name_or_path=REPOSITORY, **keywords):
|
|
37
|
+
folder = Path(name_or_path)
|
|
38
|
+
if not folder.is_dir():
|
|
39
|
+
from huggingface_hub import snapshot_download
|
|
40
|
+
folder = Path(snapshot_download(str(name_or_path), token=keywords.get("token"), revision=keywords.get("revision"),
|
|
41
|
+
allow_patterns=[CONFIG_NAME, *SAVED_FILES]))
|
|
42
|
+
config = json.loads((folder / CONFIG_NAME).read_text())
|
|
43
|
+
config.update({name: value for name, value in keywords.items() if name in ("action_min", "action_max")})
|
|
44
|
+
return cls(folder, **config)
|
|
45
|
+
|
|
46
|
+
def save_pretrained(self, directory):
|
|
47
|
+
directory = Path(directory)
|
|
48
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
49
|
+
for name in SAVED_FILES:
|
|
50
|
+
if (self.folder / name).exists() and self.folder != directory:
|
|
51
|
+
(directory / name).write_bytes((self.folder / name).read_bytes())
|
|
52
|
+
config = json.loads((self.folder / CONFIG_NAME).read_text())
|
|
53
|
+
config.update(action_min=None if self.action_min is None else self.action_min.tolist(),
|
|
54
|
+
action_max=None if self.action_max is None else self.action_max.tolist())
|
|
55
|
+
(directory / CONFIG_NAME).write_text(json.dumps(config, indent=2))
|
|
56
|
+
|
|
57
|
+
def to(self, device, cuda_graphs=False):
|
|
58
|
+
"""Runs the encoder and decoder on device (e.g. "cuda"); inputs and outputs stay NumPy. With cuda_graphs each
|
|
59
|
+
is captured once per batch shape as a CUDA graph and replayed (fewer kernel launches, same results)."""
|
|
60
|
+
self.device = torch.device(device)
|
|
61
|
+
self.model = self.model.to(self.device)
|
|
62
|
+
self.cuda_graphs = {} if cuda_graphs else None
|
|
63
|
+
return self
|
|
64
|
+
|
|
65
|
+
def fit(self, actions):
|
|
66
|
+
"""Per-column min and max of a dataset's actions (any shape ending in action_dim), used to normalize."""
|
|
67
|
+
actions = np.asarray(actions, dtype=np.float32).reshape(-1, np.shape(actions)[-1])
|
|
68
|
+
self.action_min, self.action_max = actions.min(0), actions.max(0)
|
|
69
|
+
return self
|
|
70
|
+
|
|
71
|
+
def tokens_per_chunk(self, action_dim):
|
|
72
|
+
return len(self._groups(action_dim)) * self.prefix_tokens[self.horizon]
|
|
73
|
+
|
|
74
|
+
def prefix_token_count(self, steps):
|
|
75
|
+
"""Tokens that decode a single group's first steps (steps must end a latent: 8, 16, 24 or 32)."""
|
|
76
|
+
return self.prefix_tokens[steps]
|
|
77
|
+
|
|
78
|
+
def __call__(self, actions, normalize=True):
|
|
79
|
+
actions = np.asarray(actions, dtype=np.float32)
|
|
80
|
+
actions = actions[None] if actions.ndim == 2 else actions
|
|
81
|
+
if actions.ndim != 3 or actions.shape[1] != self.horizon:
|
|
82
|
+
raise ValueError(f"Expected action chunks of shape (batch, {self.horizon}, action_dim)")
|
|
83
|
+
if not np.isfinite(actions).all():
|
|
84
|
+
raise ValueError("Actions contain NaN or infinite values")
|
|
85
|
+
if normalize:
|
|
86
|
+
low, high = self._limits(actions)
|
|
87
|
+
actions = np.where(high > low, 2 * (actions - low) / np.where(high > low, high - low, 1) - 1, 0)
|
|
88
|
+
packed = self._packed(np.clip(actions, -1, 1))
|
|
89
|
+
with torch.no_grad():
|
|
90
|
+
tokens = self._run("encode", torch.from_numpy(packed).to(self.device))
|
|
91
|
+
return tokens.reshape(len(actions), -1).tolist()
|
|
92
|
+
|
|
93
|
+
def decode(self, tokens, action_dim, normalize=True):
|
|
94
|
+
"""Token lists (each a whole chunk, or for one group a prefix of whole latents) to actions."""
|
|
95
|
+
groups = self._groups(action_dim)
|
|
96
|
+
tokens = np.asarray(tokens, dtype=np.int64).reshape(len(tokens), -1)
|
|
97
|
+
steps = {count: steps for steps, count in self.prefix_tokens.items()}.get(tokens.shape[1])
|
|
98
|
+
if len(groups) == 1 and steps is not None and steps < self.horizon:
|
|
99
|
+
tokens = np.pad(tokens, ((0, 0), (0, self.prefix_tokens[self.horizon] - tokens.shape[1])))
|
|
100
|
+
elif tokens.shape[1] != self.tokens_per_chunk(action_dim):
|
|
101
|
+
raise ValueError(f"Expected {self.tokens_per_chunk(action_dim)} tokens per chunk for {action_dim} columns")
|
|
102
|
+
with torch.no_grad():
|
|
103
|
+
packed = self._run("decode", torch.from_numpy(tokens.reshape(-1, self.prefix_tokens[self.horizon])).to(self.device))
|
|
104
|
+
packed = np.clip(packed.cpu().numpy().reshape(len(tokens), len(groups), self.horizon, self.slots), -1, 1)
|
|
105
|
+
actions = np.concatenate([packed[:, index, :, :len(group)] for index, group in enumerate(groups)], axis=-1)
|
|
106
|
+
actions = actions[:, :steps or self.horizon]
|
|
107
|
+
if not normalize:
|
|
108
|
+
return actions
|
|
109
|
+
low, high = self._limits(actions)
|
|
110
|
+
return (actions + 1) / 2 * (high - low) + low
|
|
111
|
+
|
|
112
|
+
def _run(self, method, inputs):
|
|
113
|
+
if self.cuda_graphs is None:
|
|
114
|
+
return getattr(self.model, method)(inputs)
|
|
115
|
+
if (method, inputs.shape) not in self.cuda_graphs:
|
|
116
|
+
static, stream = inputs.clone(), torch.cuda.Stream()
|
|
117
|
+
stream.wait_stream(torch.cuda.current_stream())
|
|
118
|
+
with torch.cuda.stream(stream):
|
|
119
|
+
for _ in range(3):
|
|
120
|
+
getattr(self.model, method)(static)
|
|
121
|
+
torch.cuda.current_stream().wait_stream(stream)
|
|
122
|
+
graph = torch.cuda.CUDAGraph()
|
|
123
|
+
with torch.cuda.graph(graph):
|
|
124
|
+
output = getattr(self.model, method)(static)
|
|
125
|
+
self.cuda_graphs[method, inputs.shape] = static, graph, output
|
|
126
|
+
static, graph, output = self.cuda_graphs[method, inputs.shape]
|
|
127
|
+
static.copy_(inputs)
|
|
128
|
+
graph.replay()
|
|
129
|
+
return output.clone()
|
|
130
|
+
|
|
131
|
+
def _groups(self, action_dim):
|
|
132
|
+
return [range(start, min(start + self.slots, action_dim)) for start in range(0, action_dim, self.slots)]
|
|
133
|
+
|
|
134
|
+
def _packed(self, actions):
|
|
135
|
+
groups = self._groups(actions.shape[-1])
|
|
136
|
+
packed = np.zeros((len(actions), len(groups), self.horizon, self.slots), dtype=np.float32)
|
|
137
|
+
for index, group in enumerate(groups):
|
|
138
|
+
packed[:, index, :, :len(group)] = actions[..., group.start:group.stop]
|
|
139
|
+
return packed.reshape(-1, self.horizon, self.slots)
|
|
140
|
+
|
|
141
|
+
def _limits(self, actions):
|
|
142
|
+
if self.action_min is None or len(self.action_min) != actions.shape[-1]:
|
|
143
|
+
raise ValueError("normalize=True needs the actions' per-column min and max: call fit(dataset_actions), pass "
|
|
144
|
+
"action_min/action_max to from_pretrained, or pass normalize=False for actions in [-1, 1]")
|
|
145
|
+
return self.action_min, self.action_max
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# Copyright (c) 2026 Ahad Jawaid
|
|
2
|
+
import numpy as np
|
|
3
|
+
import pytest
|
|
4
|
+
import torch
|
|
5
|
+
|
|
6
|
+
from nat_tokenizer import NATActionTokenizer
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@pytest.fixture(scope="module")
|
|
10
|
+
def tokenizer():
|
|
11
|
+
return NATActionTokenizer.from_pretrained()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def chunks(action_dim, low=-1.0, high=1.0, seed=0):
|
|
15
|
+
walk = np.cumsum(np.random.default_rng(seed).normal(0, 0.08, (5, 32, action_dim)), axis=1)
|
|
16
|
+
return (low + (high - low) * (np.tanh(walk) + 1) / 2).astype(np.float32)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@pytest.mark.parametrize("action_dim", [1, 6, 7, 8, 14])
|
|
20
|
+
def test_round_trip_shapes_and_token_range(tokenizer, action_dim):
|
|
21
|
+
actions = chunks(action_dim)
|
|
22
|
+
tokens = tokenizer(actions, normalize=False)
|
|
23
|
+
assert len(tokens) == 5 and all(len(chunk) == tokenizer.tokens_per_chunk(action_dim) for chunk in tokens)
|
|
24
|
+
assert 0 <= min(map(min, tokens)) and max(map(max, tokens)) < tokenizer.vocab_size
|
|
25
|
+
decoded = tokenizer.decode(tokens, action_dim=action_dim, normalize=False)
|
|
26
|
+
assert decoded.shape == actions.shape and np.abs(decoded - actions).mean() < 0.1
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def test_normalization_round_trips_in_original_units(tokenizer):
|
|
30
|
+
raw = chunks(7, low=-3.0, high=7.5)
|
|
31
|
+
tokenizer.fit(raw)
|
|
32
|
+
decoded = tokenizer.decode(tokenizer(raw), action_dim=7)
|
|
33
|
+
assert decoded.min() >= raw.min() - 1e-5 and decoded.max() <= raw.max() + 1e-5
|
|
34
|
+
assert np.abs(decoded - raw).mean() < 0.1 * (raw.max() - raw.min())
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_constant_column_decodes_to_its_value(tokenizer):
|
|
38
|
+
raw = np.concatenate([chunks(6), np.full((5, 32, 1), 4.0, dtype=np.float32)], axis=-1)
|
|
39
|
+
tokenizer.fit(raw)
|
|
40
|
+
np.testing.assert_allclose(tokenizer.decode(tokenizer(raw), action_dim=7)[..., -1], 4.0)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@pytest.mark.parametrize("steps", [8, 16, 24])
|
|
44
|
+
def test_streamed_prefix_decodes_the_full_chunks_first_steps(tokenizer, steps):
|
|
45
|
+
tokens = tokenizer(chunks(7), normalize=False)
|
|
46
|
+
prefix = [chunk[:tokenizer.prefix_token_count(steps)] for chunk in tokens]
|
|
47
|
+
np.testing.assert_allclose(tokenizer.decode(prefix, action_dim=7, normalize=False),
|
|
48
|
+
tokenizer.decode(tokens, action_dim=7, normalize=False)[:, :steps], atol=1e-6)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_missing_statistics_and_bad_input_raise(tokenizer):
|
|
52
|
+
fresh = NATActionTokenizer.from_pretrained()
|
|
53
|
+
with pytest.raises(ValueError, match="fit"):
|
|
54
|
+
fresh(chunks(7))
|
|
55
|
+
bad = chunks(7)
|
|
56
|
+
bad[0, 0, 0] = np.nan
|
|
57
|
+
with pytest.raises(ValueError, match="NaN"):
|
|
58
|
+
fresh(bad, normalize=False)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_save_and_reload_keep_the_statistics(tokenizer, tmp_path):
|
|
62
|
+
raw = chunks(7, low=0.0, high=2.0)
|
|
63
|
+
tokenizer.fit(raw).save_pretrained(tmp_path)
|
|
64
|
+
assert NATActionTokenizer.from_pretrained(tmp_path)(raw) == tokenizer(raw)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@pytest.mark.skipif(not torch.cuda.is_available(), reason="no GPU")
|
|
68
|
+
def test_gpu_and_cuda_graphs_agree_with_the_cpu(tokenizer):
|
|
69
|
+
actions = chunks(8)
|
|
70
|
+
cpu_tokens = np.asarray(tokenizer(actions, normalize=False))
|
|
71
|
+
for cuda_graphs in (False, True):
|
|
72
|
+
gpu = NATActionTokenizer.from_pretrained().to("cuda", cuda_graphs=cuda_graphs)
|
|
73
|
+
assert np.mean(np.asarray(gpu(actions, normalize=False)) == cpu_tokens) > 0.99
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_the_downloaded_repository_holds_only_data(tokenizer):
|
|
77
|
+
assert sorted(path.name for path in tokenizer.folder.iterdir()) == ["LICENSE", "README.md", "config.json",
|
|
78
|
+
"model.safetensors"]
|