nat-tokenizer 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,4 @@
1
+ # Copyright (c) 2026 Ahad Jawaid
2
+ from .tokenizer import NATActionTokenizer
3
+
4
+ __all__ = ["NATActionTokenizer"]
nat_tokenizer/model.py ADDED
@@ -0,0 +1,241 @@
1
+ # Copyright (c) 2026 Ahad Jawaid
2
+ """The NAT codec's inference network: a causal convolutional encoder, a multi-scale finite scalar quantizer whose codes
3
+ become time-ordered tokens, and a causal decoder that turns the tokens back into an action chunk."""
4
+ import math
5
+
6
+ import torch
7
+ from torch import nn
8
+ from torch.nn import functional
9
+
10
+
11
+ class NATCodec(nn.Module):
12
+ """Packed chunks of time_horizon steps x group_width columns, in [-1, 1], to tokens and back."""
13
+
14
+ def __init__(self, config):
15
+ super().__init__()
16
+ encoder, quantizer, decoder = config["encoder"], config["quantizer"], config["decoder"]
17
+ self.time_horizon, self.group_width = config["time_horizon"], config["group_width"]
18
+ values_per_latent = math.prod(encoder["downsampling_ratios"])
19
+ latents = self.time_horizon * self.group_width // values_per_latent
20
+ self.encoder = ActionEncoder(**encoder)
21
+ self.quantizer = MultiScaleFSQ(encoder["latent_dim"], latents=latents, **quantizer)
22
+ self.decoder = ActionDecoder(encoder["latent_dim"], values_per_latent, latents=latents, **decoder)
23
+
24
+ def encode(self, packed):
25
+ """(batch, time_horizon, group_width) actions to (batch, tokens) token ids."""
26
+ latents = self.encoder(packed.reshape(len(packed), 1, -1)) # time-major: each latent covers whole steps
27
+ return self.quantizer.tokens(latents)
28
+
29
+ def decode(self, tokens):
30
+ """(batch, tokens) token ids to (batch, time_horizon, group_width) actions."""
31
+ actions = self.decoder(self.quantizer.features(tokens))
32
+ return actions.reshape(len(tokens), self.time_horizon, self.group_width)
33
+
34
+
35
+ class ActionEncoder(nn.Module):
36
+ """A causal SEANet encoder: a convolution, then per downsampling ratio a residual unit and a strided convolution,
37
+ an LSTM with a skip connection, and a convolution to the latent width."""
38
+
39
+ def __init__(self, channels, filters, downsampling_ratios, kernel_size, residual_kernel_size, last_kernel_size,
40
+ compress, lstm_layers, latent_dim):
41
+ super().__init__()
42
+ self.input_conv = CausalConv1d(channels, filters, kernel_size)
43
+ self.stages = nn.ModuleList()
44
+ for ratio in downsampling_ratios:
45
+ self.stages.append(EncoderStage(filters, filters // compress, residual_kernel_size, ratio))
46
+ filters *= 2
47
+ self.lstm = nn.LSTM(filters, filters, lstm_layers)
48
+ self.output_conv = CausalConv1d(filters, latent_dim, last_kernel_size)
49
+
50
+ def forward(self, actions):
51
+ hidden = self.input_conv(actions)
52
+ for stage in self.stages:
53
+ hidden = stage(hidden)
54
+ sequence = hidden.permute(2, 0, 1)
55
+ hidden = (self.lstm(sequence)[0] + sequence).permute(1, 2, 0)
56
+ return self.output_conv(functional.elu(hidden))
57
+
58
+
59
+ class EncoderStage(nn.Module):
60
+ def __init__(self, channels, hidden_channels, residual_kernel_size, ratio):
61
+ super().__init__()
62
+ self.residual = ResidualUnit(channels, hidden_channels, residual_kernel_size)
63
+ self.downsample = CausalConv1d(channels, 2 * channels, 2 * ratio, stride=ratio)
64
+
65
+ def forward(self, hidden):
66
+ return self.downsample(functional.elu(self.residual(hidden)))
67
+
68
+
69
+ class ResidualUnit(nn.Module):
70
+ def __init__(self, channels, hidden_channels, kernel_size):
71
+ super().__init__()
72
+ self.conv = CausalConv1d(channels, hidden_channels, kernel_size)
73
+ self.projection = CausalConv1d(hidden_channels, channels, 1)
74
+ self.shortcut = CausalConv1d(channels, channels, 1)
75
+
76
+ def forward(self, hidden):
77
+ return self.shortcut(hidden) + self.projection(functional.elu(self.conv(functional.elu(hidden))))
78
+
79
+
80
+ class CausalConv1d(nn.Conv1d):
81
+ """A convolution that sees only the current and earlier steps, its input zero-padded on the left (and on the right
82
+ only to complete the last stride)."""
83
+
84
+ def forward(self, hidden):
85
+ kernel, stride = (self.kernel_size[0] - 1) * self.dilation[0] + 1, self.stride[0]
86
+ left = kernel - stride
87
+ frames = math.ceil((hidden.shape[-1] - kernel + left) / stride + 1)
88
+ right = (frames - 1) * stride + kernel - left - hidden.shape[-1]
89
+ return super().forward(functional.pad(hidden, (left, right)))
90
+
91
+
92
+ class MultiScaleFSQ(nn.Module):
93
+ """Residual finite scalar quantization at several temporal scales: scale s pools scales[s] latents into one code
94
+ of prod(levels) values. A chunk's tokens are ordered by the first latent each code covers, coarse scales first,
95
+ and scale s's codes are offset by s x codebook_size, so a prefix of whole latents decodes on its own."""
96
+
97
+ def __init__(self, latent_dim, levels, scales, latents):
98
+ super().__init__()
99
+ self.scales, self.latents, self.codebook_size = scales, latents, math.prod(levels)
100
+ self.project_in = nn.ModuleList([nn.Linear(latent_dim, len(levels)) for _ in scales])
101
+ self.project_out = nn.ModuleList([nn.Linear(len(levels), latent_dim) for _ in scales])
102
+ self.register_buffer("levels", torch.tensor(levels, dtype=torch.int32), persistent=False)
103
+ self.register_buffer("basis", torch.cumprod(torch.tensor([1] + levels[:-1]), 0, dtype=torch.int32),
104
+ persistent=False)
105
+ self.token_order = sorted((position * span, scale, position) for scale, span in enumerate(scales)
106
+ for position in range(-(-latents // span)))
107
+ for scale in range(len(scales)):
108
+ self.register_buffer(f"scale_{scale}_columns", torch.tensor(
109
+ [index for index, (_, owner, _) in enumerate(self.token_order) if owner == scale]), persistent=False)
110
+
111
+ def tokens(self, latents):
112
+ residual, codes = latents, []
113
+ for scale, span in enumerate(self.scales):
114
+ pooled = functional.avg_pool1d(residual, span) if span > 1 else residual
115
+ indices, quantized = self.quantize(self.project_in[scale](pooled.transpose(1, 2)).float())
116
+ residual = residual - self.upsampled(self.project_out[scale](quantized).transpose(1, 2), span)
117
+ codes.append(indices)
118
+ return torch.stack([codes[scale][:, position] + scale * self.codebook_size
119
+ for _, scale, position in self.token_order], dim=1)
120
+
121
+ def features(self, tokens):
122
+ features = 0
123
+ for scale, span in enumerate(self.scales):
124
+ indices = tokens.index_select(1, getattr(self, f"scale_{scale}_columns")) - scale * self.codebook_size
125
+ codes = self.codes(indices.clamp(0, self.codebook_size - 1))
126
+ features = features + self.upsampled(self.project_out[scale](codes).transpose(1, 2), span)
127
+ return features
128
+
129
+ def quantize(self, projected):
130
+ """Bounded, rounded values (codes in [-1, 1]) and their codebook indices."""
131
+ half_level = (self.levels - 1) * (1 + 1e-3) / 2
132
+ offset = torch.where(self.levels % 2 == 0, 0.5, 0.0)
133
+ bounded = (projected + (offset / half_level).atanh()).tanh() * half_level - offset
134
+ codes = (bounded + (bounded.round() - bounded)) / (self.levels // 2)
135
+ indices = ((codes * (self.levels // 2) + self.levels // 2) * self.basis).sum(-1).to(torch.int32)
136
+ return indices.long(), codes
137
+
138
+ def codes(self, indices):
139
+ digits = (indices.unsqueeze(-1) // self.basis) % self.levels
140
+ return (digits - self.levels // 2) / (self.levels // 2)
141
+
142
+ def upsampled(self, quantized, span):
143
+ return torch.repeat_interleave(quantized, span, dim=-1)[..., :self.latents]
144
+
145
+
146
+ class ActionDecoder(nn.Module):
147
+ """A causal Vocos-style decoder: a convolution, residual and causal attention blocks, a conditioned layer norm,
148
+ ConvNeXt blocks and a linear head to each latent's values_per_latent time-major action values."""
149
+
150
+ def __init__(self, latent_dim, values_per_latent, latents, dim, intermediate_dim, num_blocks, condition_embeddings,
151
+ condition_id, attention_context, group_norm_groups):
152
+ super().__init__()
153
+ self.embed = CausalConv1d(latent_dim, dim, 7)
154
+ self.temporal_blocks = nn.Sequential(
155
+ ResnetBlock(dim, group_norm_groups), ResnetBlock(dim, group_norm_groups),
156
+ CausalSelfAttention(dim, group_norm_groups, attention_context, latents),
157
+ ResnetBlock(dim, group_norm_groups), ResnetBlock(dim, group_norm_groups))
158
+ self.temporal_norm = PerTimeGroupNorm(group_norm_groups, dim, eps=1e-6)
159
+ self.input_norm = ConditionedLayerNorm(condition_embeddings, dim)
160
+ self.blocks = nn.ModuleList([ConvNeXtBlock(dim, intermediate_dim, condition_embeddings)
161
+ for _ in range(num_blocks)])
162
+ self.output_norm = nn.LayerNorm(dim, eps=1e-6)
163
+ self.head = nn.Linear(dim, values_per_latent)
164
+ self.register_buffer("condition_id", torch.tensor([condition_id]), persistent=False)
165
+
166
+ def forward(self, features):
167
+ with torch.backends.cudnn.flags(enabled=torch.backends.cudnn.enabled, benchmark=torch.backends.cudnn.benchmark,
168
+ deterministic=torch.backends.cudnn.deterministic, allow_tf32=False):
169
+ hidden = self.temporal_norm(self.temporal_blocks(self.embed(features)))
170
+ hidden = self.input_norm(hidden.transpose(1, 2), self.condition_id).transpose(1, 2)
171
+ for block in self.blocks:
172
+ hidden = block(hidden, self.condition_id)
173
+ return self.head(self.output_norm(hidden.transpose(1, 2))).reshape(len(features), -1)
174
+
175
+
176
+ class ResnetBlock(nn.Module):
177
+ def __init__(self, channels, groups):
178
+ super().__init__()
179
+ self.norm1, self.conv1 = PerTimeGroupNorm(groups, channels, eps=1e-6), CausalConv1d(channels, channels, 3)
180
+ self.norm2, self.conv2 = PerTimeGroupNorm(groups, channels, eps=1e-6), CausalConv1d(channels, channels, 3)
181
+
182
+ def forward(self, hidden):
183
+ return hidden + self.conv2(swish(self.norm2(self.conv1(swish(self.norm1(hidden))))))
184
+
185
+
186
+ class CausalSelfAttention(nn.Module):
187
+ """Attention over the current and up to context - 1 earlier latents."""
188
+
189
+ def __init__(self, channels, groups, context, latents):
190
+ super().__init__()
191
+ self.norm = PerTimeGroupNorm(groups, channels, eps=1e-6)
192
+ self.query, self.key, self.value, self.output = (nn.Conv1d(channels, channels, 1) for _ in range(4))
193
+ distances = torch.arange(latents)[:, None] - torch.arange(latents)[None, :]
194
+ self.register_buffer("allowed", (distances >= 0) & (distances < context), persistent=False)
195
+
196
+ def forward(self, hidden):
197
+ normalized = self.norm(hidden)
198
+ queries, keys, values = self.query(normalized), self.key(normalized), self.value(normalized)
199
+ weights = torch.bmm(queries.transpose(1, 2), keys) * queries.shape[1] ** -.5
200
+ allowed = self.allowed[:queries.shape[-1], :keys.shape[-1]]
201
+ weights = weights.masked_fill(~allowed, -torch.inf).softmax(-1)
202
+ return hidden + self.output(torch.bmm(values, weights.transpose(1, 2)))
203
+
204
+
205
+ class PerTimeGroupNorm(nn.GroupNorm):
206
+ """Group normalization of each latent on its own, so no latent sees a later one."""
207
+
208
+ def forward(self, hidden):
209
+ batch, channels, length = hidden.shape
210
+ normalized = super().forward(hidden.transpose(1, 2).reshape(batch * length, channels, 1))
211
+ return normalized.reshape(batch, length, channels).transpose(1, 2)
212
+
213
+
214
+ class ConditionedLayerNorm(nn.Module):
215
+ """Layer normalization with a learned scale and shift per condition id."""
216
+
217
+ def __init__(self, conditions, channels):
218
+ super().__init__()
219
+ self.scale, self.shift = nn.Embedding(conditions, channels), nn.Embedding(conditions, channels)
220
+
221
+ def forward(self, hidden, condition_id):
222
+ normalized = functional.layer_norm(hidden, (hidden.shape[-1],), eps=1e-6)
223
+ return normalized * self.scale(condition_id) + self.shift(condition_id)
224
+
225
+
226
+ class ConvNeXtBlock(nn.Module):
227
+ def __init__(self, channels, intermediate_channels, conditions):
228
+ super().__init__()
229
+ self.depthwise_conv = CausalConv1d(channels, channels, 7, groups=channels)
230
+ self.norm = ConditionedLayerNorm(conditions, channels)
231
+ self.expand, self.contract = nn.Linear(channels, intermediate_channels), nn.Linear(intermediate_channels, channels)
232
+ self.layer_scale = nn.Parameter(torch.ones(channels))
233
+
234
+ def forward(self, hidden, condition_id):
235
+ update = self.norm(self.depthwise_conv(hidden).transpose(1, 2), condition_id)
236
+ update = self.layer_scale * self.contract(functional.gelu(self.expand(update)))
237
+ return hidden + update.transpose(1, 2)
238
+
239
+
240
+ def swish(hidden):
241
+ return hidden * torch.sigmoid(hidden)
@@ -0,0 +1,145 @@
1
+ # Copyright (c) 2026 Ahad Jawaid
2
+ """NAT action tokenizer: 32-step action chunks to 14 time-ordered tokens per group of up to 7 action columns, and back.
3
+
4
+ from nat_tokenizer import NATActionTokenizer
5
+ tokenizer = NATActionTokenizer.from_pretrained()
6
+ tokenizer.fit(dataset_actions) # per-column min and max, mapped to [-1, 1]
7
+ tokens = tokenizer(action_chunks) # (batch, 32, action_dim) or (32, action_dim) -> list of token lists
8
+ actions = tokenizer.decode(tokens, action_dim=action_dim) # -> (batch, 32, action_dim), in the original units
9
+
10
+ With normalize=False the actions must already lie in [-1, 1]. Tokens are ids in [0, vocab_size). Each group's tokens are
11
+ time-ordered, so a prefix of prefix_token_count(steps) tokens decodes the chunk's first steps (one group only)."""
12
+ import json
13
+ from pathlib import Path
14
+
15
+ import numpy as np
16
+ import torch
17
+ from safetensors.torch import load_file
18
+
19
+ from .model import NATCodec
20
+
21
+ CONFIG_NAME, WEIGHTS_NAME, REPOSITORY = "config.json", "model.safetensors", "ahad-j/nat-action-tokenizer"
22
+ SAVED_FILES = (WEIGHTS_NAME, "README.md", "LICENSE")
23
+
24
+
25
+ class NATActionTokenizer:
26
+ def __init__(self, folder, action_min=None, action_max=None, **config):
27
+ self.folder, self.device, self.cuda_graphs = Path(folder), torch.device("cpu"), None
28
+ self.model = NATCodec(config).eval().requires_grad_(False)
29
+ self.model.load_state_dict(load_file(str(self.folder / WEIGHTS_NAME)))
30
+ self.action_min = None if action_min is None else np.asarray(action_min, dtype=np.float32)
31
+ self.action_max = None if action_max is None else np.asarray(action_max, dtype=np.float32)
32
+ self.horizon, self.slots, self.vocab_size = config["time_horizon"], config["group_width"], config["vocab_size"]
33
+ self.prefix_tokens = {int(steps): tokens for steps, tokens in config["prefix_token_counts"].items()}
34
+
35
+ @classmethod
36
+ def from_pretrained(cls, name_or_path=REPOSITORY, **keywords):
37
+ folder = Path(name_or_path)
38
+ if not folder.is_dir():
39
+ from huggingface_hub import snapshot_download
40
+ folder = Path(snapshot_download(str(name_or_path), token=keywords.get("token"), revision=keywords.get("revision"),
41
+ allow_patterns=[CONFIG_NAME, *SAVED_FILES]))
42
+ config = json.loads((folder / CONFIG_NAME).read_text())
43
+ config.update({name: value for name, value in keywords.items() if name in ("action_min", "action_max")})
44
+ return cls(folder, **config)
45
+
46
+ def save_pretrained(self, directory):
47
+ directory = Path(directory)
48
+ directory.mkdir(parents=True, exist_ok=True)
49
+ for name in SAVED_FILES:
50
+ if (self.folder / name).exists() and self.folder != directory:
51
+ (directory / name).write_bytes((self.folder / name).read_bytes())
52
+ config = json.loads((self.folder / CONFIG_NAME).read_text())
53
+ config.update(action_min=None if self.action_min is None else self.action_min.tolist(),
54
+ action_max=None if self.action_max is None else self.action_max.tolist())
55
+ (directory / CONFIG_NAME).write_text(json.dumps(config, indent=2))
56
+
57
+ def to(self, device, cuda_graphs=False):
58
+ """Runs the encoder and decoder on device (e.g. "cuda"); inputs and outputs stay NumPy. With cuda_graphs each
59
+ is captured once per batch shape as a CUDA graph and replayed (fewer kernel launches, same results)."""
60
+ self.device = torch.device(device)
61
+ self.model = self.model.to(self.device)
62
+ self.cuda_graphs = {} if cuda_graphs else None
63
+ return self
64
+
65
+ def fit(self, actions):
66
+ """Per-column min and max of a dataset's actions (any shape ending in action_dim), used to normalize."""
67
+ actions = np.asarray(actions, dtype=np.float32).reshape(-1, np.shape(actions)[-1])
68
+ self.action_min, self.action_max = actions.min(0), actions.max(0)
69
+ return self
70
+
71
+ def tokens_per_chunk(self, action_dim):
72
+ return len(self._groups(action_dim)) * self.prefix_tokens[self.horizon]
73
+
74
+ def prefix_token_count(self, steps):
75
+ """Tokens that decode a single group's first steps (steps must end a latent: 8, 16, 24 or 32)."""
76
+ return self.prefix_tokens[steps]
77
+
78
+ def __call__(self, actions, normalize=True):
79
+ actions = np.asarray(actions, dtype=np.float32)
80
+ actions = actions[None] if actions.ndim == 2 else actions
81
+ if actions.ndim != 3 or actions.shape[1] != self.horizon:
82
+ raise ValueError(f"Expected action chunks of shape (batch, {self.horizon}, action_dim)")
83
+ if not np.isfinite(actions).all():
84
+ raise ValueError("Actions contain NaN or infinite values")
85
+ if normalize:
86
+ low, high = self._limits(actions)
87
+ actions = np.where(high > low, 2 * (actions - low) / np.where(high > low, high - low, 1) - 1, 0)
88
+ packed = self._packed(np.clip(actions, -1, 1))
89
+ with torch.no_grad():
90
+ tokens = self._run("encode", torch.from_numpy(packed).to(self.device))
91
+ return tokens.reshape(len(actions), -1).tolist()
92
+
93
+ def decode(self, tokens, action_dim, normalize=True):
94
+ """Token lists (each a whole chunk, or for one group a prefix of whole latents) to actions."""
95
+ groups = self._groups(action_dim)
96
+ tokens = np.asarray(tokens, dtype=np.int64).reshape(len(tokens), -1)
97
+ steps = {count: steps for steps, count in self.prefix_tokens.items()}.get(tokens.shape[1])
98
+ if len(groups) == 1 and steps is not None and steps < self.horizon:
99
+ tokens = np.pad(tokens, ((0, 0), (0, self.prefix_tokens[self.horizon] - tokens.shape[1])))
100
+ elif tokens.shape[1] != self.tokens_per_chunk(action_dim):
101
+ raise ValueError(f"Expected {self.tokens_per_chunk(action_dim)} tokens per chunk for {action_dim} columns")
102
+ with torch.no_grad():
103
+ packed = self._run("decode", torch.from_numpy(tokens.reshape(-1, self.prefix_tokens[self.horizon])).to(self.device))
104
+ packed = np.clip(packed.cpu().numpy().reshape(len(tokens), len(groups), self.horizon, self.slots), -1, 1)
105
+ actions = np.concatenate([packed[:, index, :, :len(group)] for index, group in enumerate(groups)], axis=-1)
106
+ actions = actions[:, :steps or self.horizon]
107
+ if not normalize:
108
+ return actions
109
+ low, high = self._limits(actions)
110
+ return (actions + 1) / 2 * (high - low) + low
111
+
112
+ def _run(self, method, inputs):
113
+ if self.cuda_graphs is None:
114
+ return getattr(self.model, method)(inputs)
115
+ if (method, inputs.shape) not in self.cuda_graphs:
116
+ static, stream = inputs.clone(), torch.cuda.Stream()
117
+ stream.wait_stream(torch.cuda.current_stream())
118
+ with torch.cuda.stream(stream):
119
+ for _ in range(3):
120
+ getattr(self.model, method)(static)
121
+ torch.cuda.current_stream().wait_stream(stream)
122
+ graph = torch.cuda.CUDAGraph()
123
+ with torch.cuda.graph(graph):
124
+ output = getattr(self.model, method)(static)
125
+ self.cuda_graphs[method, inputs.shape] = static, graph, output
126
+ static, graph, output = self.cuda_graphs[method, inputs.shape]
127
+ static.copy_(inputs)
128
+ graph.replay()
129
+ return output.clone()
130
+
131
+ def _groups(self, action_dim):
132
+ return [range(start, min(start + self.slots, action_dim)) for start in range(0, action_dim, self.slots)]
133
+
134
+ def _packed(self, actions):
135
+ groups = self._groups(actions.shape[-1])
136
+ packed = np.zeros((len(actions), len(groups), self.horizon, self.slots), dtype=np.float32)
137
+ for index, group in enumerate(groups):
138
+ packed[:, index, :, :len(group)] = actions[..., group.start:group.stop]
139
+ return packed.reshape(-1, self.horizon, self.slots)
140
+
141
+ def _limits(self, actions):
142
+ if self.action_min is None or len(self.action_min) != actions.shape[-1]:
143
+ raise ValueError("normalize=True needs the actions' per-column min and max: call fit(dataset_actions), pass "
144
+ "action_min/action_max to from_pretrained, or pass normalize=False for actions in [-1, 1]")
145
+ return self.action_min, self.action_max
@@ -0,0 +1,147 @@
1
+ Metadata-Version: 2.5
2
+ Name: nat-tokenizer
3
+ Version: 0.1.0
4
+ Summary: NAT action tokenizer: robot action chunks to time-ordered discrete tokens and back
5
+ Author: Ahad Jawaid
6
+ License-Expression: MIT
7
+ License-File: LICENSE
8
+ Requires-Python: >=3.9
9
+ Requires-Dist: huggingface-hub>=0.20
10
+ Requires-Dist: numpy>=1.21
11
+ Requires-Dist: safetensors>=0.4
12
+ Requires-Dist: torch>=2
13
+ Provides-Extra: test
14
+ Requires-Dist: pytest>=7; extra == 'test'
15
+ Description-Content-Type: text/markdown
16
+
17
+ # NAT action tokenizer
18
+
19
+ Turns robot action chunks into discrete tokens for a vision-language-action model, and tokens back into actions.
20
+ Every 32-step chunk of up to 7 action columns becomes **14 tokens**, in time order, so a model can stream actions
21
+ before it has generated the whole chunk.
22
+
23
+ ## Quick start
24
+
25
+ ```bash
26
+ pip install nat-tokenizer
27
+ ```
28
+
29
+ or from source: `pip install "git+https://github.com/ahadjawaid/nat.git"`.
30
+
31
+ The weights and config download from the Hugging Face repository
32
+ [ahad-j/nat-action-tokenizer](https://huggingface.co/ahad-j/nat-action-tokenizer) on first use.
33
+
34
+ ```python
35
+ import numpy as np
36
+ from nat_tokenizer import NATActionTokenizer
37
+
38
+ tokenizer = NATActionTokenizer.from_pretrained() # downloads ahad-j/nat-action-tokenizer
39
+
40
+ # 1. Fit the normalization once, on your dataset's actions (any shape ending in action_dim).
41
+ tokenizer.fit(dataset_actions)
42
+
43
+ # 2. Tokenize 32-step chunks: (batch, 32, action_dim) or one chunk (32, action_dim).
44
+ tokens = tokenizer(action_chunks) # list (one per chunk) of lists of ints in [0, 25600)
45
+
46
+ # 3. Decode back to actions in your original units.
47
+ actions = tokenizer.decode(tokens, action_dim=action_chunks.shape[-1]) # (batch, 32, action_dim)
48
+
49
+ # 4. Save the fitted tokenizer (keeps your min and max) and load it later the same way.
50
+ tokenizer.save_pretrained("my-nat-tokenizer")
51
+ tokenizer = NATActionTokenizer.from_pretrained("my-nat-tokenizer")
52
+ ```
53
+
54
+ ## Normalization (on by default)
55
+
56
+ The tokenizer was trained on actions in [-1, 1]. With `normalize=True` (the default) every column is mapped from
57
+ its min and max to [-1, 1] before encoding (values outside are clipped) and decoded actions are mapped back, so you
58
+ work in your own units throughout. Set the min and max with `fit(dataset_actions)`, or pass them directly:
59
+
60
+ ```python
61
+ tokenizer = NATActionTokenizer.from_pretrained(action_min=[...], action_max=[...])
62
+ ```
63
+
64
+ If your actions are already in [-1, 1], pass `normalize=False` to both calls:
65
+ `tokenizer(chunks, normalize=False)` and `tokenizer.decode(tokens, action_dim=d, normalize=False)`.
66
+ A column whose min equals its max is encoded as 0 and decoded back to its constant value.
67
+
68
+ ## Streaming (causal decoding)
69
+
70
+ Within a group the 14 tokens are time ordered: the first 4 decode the chunk's first 8 steps, the first 7 its first 16,
71
+ the first 11 its first 24, and all 14 the whole chunk. A prefix decodes exactly the same first steps as the full chunk,
72
+ so a model can execute actions while it is still generating:
73
+
74
+ ```python
75
+ for steps in (8, 16, 24, 32):
76
+ prefix = [chunk[:tokenizer.prefix_token_count(steps)] for chunk in tokens]
77
+ first_steps = tokenizer.decode(prefix, action_dim=7) # (batch, steps, 7)
78
+ ```
79
+
80
+ Streaming works for actions of up to 7 columns (one group).
81
+
82
+ ## Action dimensions
83
+
84
+ Columns are split into consecutive groups of up to 7 (8 columns = 7 + 1, 14 = 7 + 7, 21 = 7 + 7 + 7). Each group gets
85
+ its own 14 tokens and a chunk's tokens are the groups' tokens one after another: `tokenizer.tokens_per_chunk(action_dim)`
86
+ gives the count (14 per group). Chunks must be exactly 32 steps long.
87
+
88
+ ## GPU and CUDA graphs
89
+
90
+ ```python
91
+ tokenizer = tokenizer.to("cuda") # eager on the GPU
92
+ tokenizer = tokenizer.to("cuda", cuda_graphs=True) # captures the encoder and decoder once per batch shape
93
+ ```
94
+
95
+ Inputs and outputs stay NumPy. GPU results match the CPU up to floating point (rarely a token on a quantization
96
+ boundary differs).
97
+
98
+ ## API
99
+
100
+ | call | does |
101
+ |---|---|
102
+ | `NATActionTokenizer.from_pretrained(repo_or_folder, action_min=None, action_max=None)` | load (optionally with your min and max) |
103
+ | `tokenizer.fit(actions)` | per-column min and max of your actions (returns the tokenizer) |
104
+ | `tokenizer(chunks, normalize=True)` | `(batch, 32, d)` or `(32, d)` actions to a list of token lists |
105
+ | `tokenizer.decode(tokens, action_dim, normalize=True)` | token lists (whole chunks, or one group's prefix) to `(batch, steps, d)` actions |
106
+ | `tokenizer.prefix_token_count(steps)` | tokens that decode the first `steps` (8, 16, 24 or 32) |
107
+ | `tokenizer.tokens_per_chunk(action_dim)` | 14 per group of 7 columns |
108
+ | `tokenizer.to(device, cuda_graphs=False)` | run on a device, optionally with CUDA graphs |
109
+ | `tokenizer.save_pretrained(directory)` | save, with the fitted min and max |
110
+ | `tokenizer.vocab_size` | 25,600 (4 scales x 6,400 codes) |
111
+
112
+ ## Model
113
+
114
+ 4 latents per chunk, 4 FSQ scales of 6,400 codes (one coarse token per two latents, three finer tokens per latent),
115
+ trained on a large corpus of robot and human action data (every window once, batch 4,096). The repository holds only
116
+ the weights (`model.safetensors`) and a JSON config; the network is plain PyTorch in the `nat-tokenizer` package.
117
+
118
+ ## Reconstruction on robot benchmarks, and speed
119
+
120
+ Random 32-step chunks of each benchmark's demonstrations, each normalized by its own min and max (MetaWorld's actions clipped to the [-1, 1] its simulator executes). Errors are mean absolute differences between the input and its reconstruction. Speed is in milliseconds per call on an RTX 4090 (shared with another job while measured) and 4 CPU threads, mean and SD over 20 calls.
121
+
122
+ | benchmark | chunks x shape | MAE in [-1, 1] units | MAE | max error |
123
+ |---|---|---|---|---|
124
+ | LIBERO-10 | 1000 x (32, 7) | 0.0304 | 0.0210 | 1.9762 |
125
+ | MetaWorld mt4 (clipped) | 1000 x (32, 4) | 0.0651 | 0.0651 | 1.7578 |
126
+ | RoboMimic mt4 | 1000 x (32, 7) | 0.0466 | 0.0404 | 1.5712 |
127
+ | RoboCasa mt4 | 1000 x (32, 12) | 0.0130 | 0.0107 | 0.9970 |
128
+
129
+ | device | batch of 32 x 7 chunks | encode ms | decode ms |
130
+ |---|---|---|---|
131
+ | cpu, eager | 1 | 11.57 ± 0.39 | 29.86 ± 4.06 |
132
+ | cpu, eager | 64 | 322.06 ± 1.76 | 263.23 ± 6.67 |
133
+ | cuda, eager | 1 | 6.54 ± 1.67 | 11.07 ± 1.18 |
134
+ | cuda, eager | 64 | 7.86 ± 2.41 | 12.67 ± 1.80 |
135
+ | cuda, CUDA graphs | 1 | 0.71 ± 0.57 | 2.44 ± 0.39 |
136
+ | cuda, CUDA graphs | 64 | 1.05 ± 0.35 | 5.18 ± 2.09 |
137
+
138
+ ## Development
139
+
140
+ ```bash
141
+ pip install -e ".[test]"
142
+ pytest
143
+ ```
144
+
145
+ ## License
146
+
147
+ MIT, see [LICENSE](LICENSE).
@@ -0,0 +1,7 @@
1
+ nat_tokenizer/__init__.py,sha256=QP6XeMH6d8MIQ28q-2KyVN3BJ95R6nNGWDGCrLNxr4Q,109
2
+ nat_tokenizer/model.py,sha256=aGchjcdDUrm0yA4a_wAkvTjE4WgKTgyIjVoq1SpqCVU,12689
3
+ nat_tokenizer/tokenizer.py,sha256=xzJ7nwM9tUEqPVYhXB_jd8IDE3NaVs4ZGZuchRI6LbE,8275
4
+ nat_tokenizer-0.1.0.dist-info/METADATA,sha256=NkDrJYNEde9zTFFmuTpp57l9hfz5c9GpDP49rqUw2lA,6463
5
+ nat_tokenizer-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
6
+ nat_tokenizer-0.1.0.dist-info/licenses/LICENSE,sha256=RlWosyKDlCLhT3_qpJqtOW4bAEcZgV9PgT3RQ8S6FAM,1068
7
+ nat_tokenizer-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ahad Jawaid
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.