instmodel 1.3.1__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {instmodel-1.3.1 → instmodel-1.4.0}/PKG-INFO +32 -2
- instmodel-1.4.0/README.md +66 -0
- {instmodel-1.3.1 → instmodel-1.4.0}/instmodel/instruction_model.py +90 -6
- instmodel-1.4.0/instmodel/sequence/__init__.py +31 -0
- instmodel-1.4.0/instmodel/sequence/_spec.py +305 -0
- instmodel-1.4.0/instmodel/sequence/reference.py +70 -0
- instmodel-1.4.0/instmodel/sequence/tf.py +331 -0
- instmodel-1.4.0/instmodel/sequence/torch.py +283 -0
- {instmodel-1.3.1 → instmodel-1.4.0}/instmodel.egg-info/PKG-INFO +32 -2
- {instmodel-1.3.1 → instmodel-1.4.0}/instmodel.egg-info/SOURCES.txt +9 -1
- {instmodel-1.3.1 → instmodel-1.4.0}/instmodel.egg-info/requires.txt +1 -1
- {instmodel-1.3.1 → instmodel-1.4.0}/pyproject.toml +3 -3
- instmodel-1.4.0/tests/test_sequence.py +168 -0
- instmodel-1.4.0/tests/test_sequence_tf.py +269 -0
- instmodel-1.4.0/tests/test_sequence_torch.py +205 -0
- instmodel-1.3.1/README.md +0 -36
- {instmodel-1.3.1 → instmodel-1.4.0}/LICENSE +0 -0
- {instmodel-1.3.1 → instmodel-1.4.0}/instmodel/__init__.py +0 -0
- {instmodel-1.3.1 → instmodel-1.4.0}/instmodel/_core.py +0 -0
- {instmodel-1.3.1 → instmodel-1.4.0}/instmodel/tf.py +0 -0
- {instmodel-1.3.1 → instmodel-1.4.0}/instmodel/torch.py +0 -0
- {instmodel-1.3.1 → instmodel-1.4.0}/instmodel/training_utils.py +0 -0
- {instmodel-1.3.1 → instmodel-1.4.0}/instmodel.egg-info/dependency_links.txt +0 -0
- {instmodel-1.3.1 → instmodel-1.4.0}/instmodel.egg-info/top_level.txt +0 -0
- {instmodel-1.3.1 → instmodel-1.4.0}/setup.cfg +0 -0
- {instmodel-1.3.1 → instmodel-1.4.0}/tests/test_backends.py +0 -0
- {instmodel-1.3.1 → instmodel-1.4.0}/tests/test_instruction_model.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: instmodel
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: Train TF/PyTorch models and export them to a JSON instruction format with NumPy inference
|
|
5
5
|
Author-email: Joao Ferreira <joaoprcf@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -17,7 +17,7 @@ Provides-Extra: tensorflow
|
|
|
17
17
|
Requires-Dist: tensorflow>=2.20; extra == "tensorflow"
|
|
18
18
|
Requires-Dist: protobuf>=6.0; extra == "tensorflow"
|
|
19
19
|
Provides-Extra: pytorch
|
|
20
|
-
Requires-Dist: torch>=2.
|
|
20
|
+
Requires-Dist: torch>=2.5; extra == "pytorch"
|
|
21
21
|
Dynamic: license-file
|
|
22
22
|
|
|
23
23
|
# instmodel
|
|
@@ -53,6 +53,36 @@ exported = model.create_instruction_model() # JSON-serialisable dict
|
|
|
53
53
|
print(instruction_model_inference(exported, np.random.rand(3, 4))[-1])
|
|
54
54
|
```
|
|
55
55
|
|
|
56
|
+
## Sequence models
|
|
57
|
+
|
|
58
|
+
`instmodel.sequence` trains small causal transformers and exports them as an instruction model
|
|
59
|
+
that processes one token per run. The attention caches are explicit state owned by the caller,
|
|
60
|
+
so any runtime can decode token by token (NumPy here, or `SequenceModel` in the Rust port).
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
import numpy as np
|
|
64
|
+
from instmodel import sequence
|
|
65
|
+
from instmodel.sequence import torch as backend # same API in instmodel.sequence.tf
|
|
66
|
+
|
|
67
|
+
config = sequence.TransformerConfig(vocab_size=16, width=16, heads=4, kv_heads=2)
|
|
68
|
+
model = backend.Transformer(config, members=2) # two copies trained in one batch
|
|
69
|
+
|
|
70
|
+
start = np.random.randint(0, 16, (4096, 1))
|
|
71
|
+
tokens = (start + np.arange(13)) % 16 # rows [N, time + 1]; learn "next = token + 1"
|
|
72
|
+
rows = dict(tokens=backend.torch.as_tensor(tokens), groups=backend.torch.zeros(4096, 12, dtype=int))
|
|
73
|
+
trainer = backend.Trainer(model, rows, batch=64, lr=3e-3, ema=0.99)
|
|
74
|
+
for step in range(400):
|
|
75
|
+
trainer.step(backend.schedule(step / 400, 3e-3))
|
|
76
|
+
|
|
77
|
+
exported = sequence.export_step_model(config, trainer.averaged().member_parameters(0))
|
|
78
|
+
session = sequence.SequenceSession(exported) # keeps the key/value cache between steps
|
|
79
|
+
print(session.step([3]).argmax(-1)) # -> [4]
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
`TransformerConfig` also covers grouped-query attention, looped (weight-shared) passes and
|
|
83
|
+
`SegmentFeatures`; `sequence_loss` takes per-position groups, allowed-token masks and soft targets.
|
|
84
|
+
`add_validation_sequences` stores reference logits in the file for other runtimes to replay.
|
|
85
|
+
|
|
56
86
|
## License
|
|
57
87
|
|
|
58
88
|
[MIT](LICENSE)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# instmodel
|
|
2
|
+
|
|
3
|
+
This repo is meant to be used with coding agents, not read manually.
|
|
4
|
+
|
|
5
|
+
Build and train small neural networks with PyTorch or TensorFlow/Keras through one shared graph API,
|
|
6
|
+
then export them to a compact JSON "instruction model" that runs anywhere with plain NumPy (or the
|
|
7
|
+
Rust/C++ inference ports) — no training framework needed at inference time.
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install instmodel # NumPy inference only
|
|
13
|
+
pip install instmodel[pytorch] # + PyTorch training backend (instmodel.torch)
|
|
14
|
+
pip install instmodel[tensorflow] # + TensorFlow/Keras training backend (instmodel.tf)
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Example
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
import numpy as np
|
|
21
|
+
from instmodel.torch import Dense, InputBuffer, ModelGraph # same API in instmodel.tf
|
|
22
|
+
from instmodel.instruction_model import instruction_model_inference
|
|
23
|
+
|
|
24
|
+
x = InputBuffer(4)
|
|
25
|
+
y = Dense(1, activation="sigmoid")(Dense(8, activation="relu")(x))
|
|
26
|
+
model = ModelGraph(x, y)
|
|
27
|
+
model.compile(optimizer="adam", loss="binary_crossentropy")
|
|
28
|
+
model.fit(np.random.rand(64, 4), np.random.randint(0, 2, (64, 1)), epochs=5, verbose=0)
|
|
29
|
+
|
|
30
|
+
exported = model.create_instruction_model() # JSON-serialisable dict
|
|
31
|
+
print(instruction_model_inference(exported, np.random.rand(3, 4))[-1])
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Sequence models
|
|
35
|
+
|
|
36
|
+
`instmodel.sequence` trains small causal transformers and exports them as an instruction model
|
|
37
|
+
that processes one token per run. The attention caches are explicit state owned by the caller,
|
|
38
|
+
so any runtime can decode token by token (NumPy here, or `SequenceModel` in the Rust port).
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
import numpy as np
|
|
42
|
+
from instmodel import sequence
|
|
43
|
+
from instmodel.sequence import torch as backend # same API in instmodel.sequence.tf
|
|
44
|
+
|
|
45
|
+
config = sequence.TransformerConfig(vocab_size=16, width=16, heads=4, kv_heads=2)
|
|
46
|
+
model = backend.Transformer(config, members=2) # two copies trained in one batch
|
|
47
|
+
|
|
48
|
+
start = np.random.randint(0, 16, (4096, 1))
|
|
49
|
+
tokens = (start + np.arange(13)) % 16 # rows [N, time + 1]; learn "next = token + 1"
|
|
50
|
+
rows = dict(tokens=backend.torch.as_tensor(tokens), groups=backend.torch.zeros(4096, 12, dtype=int))
|
|
51
|
+
trainer = backend.Trainer(model, rows, batch=64, lr=3e-3, ema=0.99)
|
|
52
|
+
for step in range(400):
|
|
53
|
+
trainer.step(backend.schedule(step / 400, 3e-3))
|
|
54
|
+
|
|
55
|
+
exported = sequence.export_step_model(config, trainer.averaged().member_parameters(0))
|
|
56
|
+
session = sequence.SequenceSession(exported) # keeps the key/value cache between steps
|
|
57
|
+
print(session.step([3]).argmax(-1)) # -> [4]
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
`TransformerConfig` also covers grouped-query attention, looped (weight-shared) passes and
|
|
61
|
+
`SegmentFeatures`; `sequence_loss` takes per-position groups, allowed-token masks and soft targets.
|
|
62
|
+
`add_validation_sequences` stores reference logits in the file for other runtimes to replay.
|
|
63
|
+
|
|
64
|
+
## License
|
|
65
|
+
|
|
66
|
+
[MIT](LICENSE)
|
|
@@ -99,7 +99,11 @@ def _validate_references(model):
|
|
|
99
99
|
"parameters": len(model["parameters"]),
|
|
100
100
|
"parameters_min": len(model["parameters"]),
|
|
101
101
|
"parameters_max": len(model["parameters"]),
|
|
102
|
+
"gain": len(model["parameters"]),
|
|
103
|
+
"bias": len(model["parameters"]),
|
|
102
104
|
"map": len(model.get("maps", [])),
|
|
105
|
+
"slot": len(model.get("sequence", {}).get("kv_slots", [])),
|
|
106
|
+
"scanner": model.get("sequence", {}).get("scanners", 0),
|
|
103
107
|
}
|
|
104
108
|
for instr in model["instructions"]:
|
|
105
109
|
inputs = instr.get("input")
|
|
@@ -123,6 +127,48 @@ def _validate_references(model):
|
|
|
123
127
|
_check_span(
|
|
124
128
|
instr["internal_output_index"], instr["size"], sizes[instr["output"]], "MAP_TRANSFORM"
|
|
125
129
|
)
|
|
130
|
+
elif instr["type"] == "EMBEDDING":
|
|
131
|
+
_check_index(instr.get("internal_input_index", 0), sizes[instr["input"]], "EMBEDDING column")
|
|
132
|
+
elif instr["type"] == "SEGMENT_FEATURES":
|
|
133
|
+
if sizes[instr["output"]] != 2:
|
|
134
|
+
raise ValueError("SEGMENT_FEATURES writes an (ordinal, index) buffer of size 2")
|
|
135
|
+
elif instr["type"] == "CAUSAL_ATTENTION":
|
|
136
|
+
slot = model["sequence"]["kv_slots"][instr["slot"]]
|
|
137
|
+
heads, shared, head = instr["heads"], instr["kv_heads"], slot["head_dim"]
|
|
138
|
+
if (heads, shared) != (slot["heads"], slot["kv_heads"]) or shared < 1 or heads % shared:
|
|
139
|
+
raise ValueError("CAUSAL_ATTENTION heads differ from its key/value slot")
|
|
140
|
+
if sizes[instr["input"]] != (heads + 2 * shared) * head or sizes[instr["output"]] != heads * head:
|
|
141
|
+
raise ValueError("CAUSAL_ATTENTION buffers do not match its heads and head_dim")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _segment_features(instruction, tokens, scanner):
|
|
145
|
+
"""Advances one feature scanner by a token per row and returns ``[ordinal, index]`` columns."""
|
|
146
|
+
member = np.isin(tokens, instruction["members"])
|
|
147
|
+
run = member | (np.isin(tokens, instruction["openers"]) & ~scanner["prev_member"])
|
|
148
|
+
start = run & ~scanner["prev_run"]
|
|
149
|
+
scanner["ordinal"] = scanner["ordinal"] + start
|
|
150
|
+
scanner["index"] = np.where(start, 1, np.where(run, scanner["index"] + 1, 0))
|
|
151
|
+
scanner["prev_member"], scanner["prev_run"] = member, run
|
|
152
|
+
limit = instruction["rows"] - 1
|
|
153
|
+
return np.minimum(np.stack((scanner["ordinal"], scanner["index"]), -1), limit).astype(np.float32)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _causal_attention(instruction, x, slot, position):
|
|
157
|
+
"""Appends this token's keys and values to ``slot`` and attends over the cached prefix."""
|
|
158
|
+
heads, shared = instruction["heads"], instruction["kv_heads"]
|
|
159
|
+
head = x.shape[1] // (heads + 2 * shared)
|
|
160
|
+
rows = x.shape[0]
|
|
161
|
+
if position >= slot["keys"].shape[1]:
|
|
162
|
+
raise ValueError("sequence exceeds the model's max_positions")
|
|
163
|
+
query = x[:, : heads * head].reshape(rows, shared, heads // shared, head)
|
|
164
|
+
slot["keys"][:, position] = x[:, heads * head : (heads + shared) * head].reshape(rows, shared, head)
|
|
165
|
+
slot["values"][:, position] = x[:, (heads + shared) * head :].reshape(rows, shared, head)
|
|
166
|
+
keys, values = slot["keys"][:, : position + 1], slot["values"][:, : position + 1]
|
|
167
|
+
# Query head h reads key/value head h // (heads // kv_heads).
|
|
168
|
+
scores = np.einsum("rgqd,rtgd->rgqt", query, keys) * np.float32(head ** -0.5)
|
|
169
|
+
scores = np.exp(scores - scores.max(-1, keepdims=True))
|
|
170
|
+
scores = scores / scores.sum(-1, keepdims=True)
|
|
171
|
+
return np.einsum("rgqt,rtgd->rgqd", scores, values).reshape(rows, heads * head).astype(np.float32)
|
|
126
172
|
|
|
127
173
|
|
|
128
174
|
def _output_buffer(buffers, buffer_sizes, index, rows):
|
|
@@ -135,15 +181,20 @@ _BUFFER_REDUCERS = {"ADD_ELEMENTWISE_BUFFERS": np.sum, "MULTIPLY_ELEMENTWISE_BUF
|
|
|
135
181
|
_HEAD_COMBINERS = {"ADD_BUFFER_HEADS": np.add, "MULTIPLY_BUFFER_HEADS": np.multiply}
|
|
136
182
|
|
|
137
183
|
|
|
138
|
-
def
|
|
184
|
+
def model_arrays(model):
|
|
185
|
+
"""Validates references and returns the model's (weights, bias, parameters) as float32 arrays."""
|
|
186
|
+
_validate_references(model)
|
|
187
|
+
return tuple([np.array(v, dtype=np.float32) for v in model[key]] for key in ("weights", "bias", "parameters"))
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def instruction_model_inference(model, input_data, state=None, arrays=None):
|
|
139
191
|
"""Runs the model on a 2D array (split across the leading buffers) or a list of per-buffer arrays.
|
|
140
192
|
|
|
141
|
-
Returns every buffer; the last one is the model output.
|
|
193
|
+
Returns every buffer; the last one is the model output. ``state`` carries
|
|
194
|
+
the caches of the stateful instructions (see ``instmodel.sequence``), with
|
|
195
|
+
one row per sequence; ``arrays`` reuses the result of ``model_arrays``.
|
|
142
196
|
"""
|
|
143
|
-
|
|
144
|
-
weights = [np.array(w, dtype=np.float32) for w in model["weights"]]
|
|
145
|
-
bias = [np.array(b, dtype=np.float32) for b in model["bias"]]
|
|
146
|
-
parameters = [np.array(p, dtype=np.float32) for p in model["parameters"]]
|
|
197
|
+
weights, bias, parameters = arrays if arrays is not None else model_arrays(model)
|
|
147
198
|
buffer_sizes = model["buffer_sizes"]
|
|
148
199
|
buffers = [None] * len(buffer_sizes)
|
|
149
200
|
|
|
@@ -196,6 +247,35 @@ def instruction_model_inference(model, input_data):
|
|
|
196
247
|
np.sum(buffers[src], axis=-1, keepdims=True), dtype=np.float32
|
|
197
248
|
)
|
|
198
249
|
continue
|
|
250
|
+
if kind == "EMBEDDING":
|
|
251
|
+
table = weights[instruction["weights"]]
|
|
252
|
+
ids = np.rint(buffers[src][:, instruction.get("internal_input_index", 0)]).astype(np.int64)
|
|
253
|
+
if ids.min() < 0 or ids.max() >= len(table):
|
|
254
|
+
raise ValueError(f"EMBEDDING id outside [0, {len(table)})")
|
|
255
|
+
buffers[instruction["output"]] = table[ids]
|
|
256
|
+
continue
|
|
257
|
+
if kind == "LAYER_NORM":
|
|
258
|
+
x = buffers[src]
|
|
259
|
+
centered = x - x.mean(-1, keepdims=True)
|
|
260
|
+
scale = np.sqrt((centered * centered).mean(-1, keepdims=True) + np.float32(instruction["epsilon"]))
|
|
261
|
+
buffers[instruction["output"]] = np.asarray(
|
|
262
|
+
centered / scale * parameters[instruction["gain"]] + parameters[instruction["bias"]],
|
|
263
|
+
dtype=np.float32,
|
|
264
|
+
)
|
|
265
|
+
continue
|
|
266
|
+
if kind in ("SEGMENT_FEATURES", "CAUSAL_ATTENTION"):
|
|
267
|
+
if state is None:
|
|
268
|
+
raise ValueError(f"{kind} needs sequence state; step the model with instmodel.sequence")
|
|
269
|
+
if kind == "SEGMENT_FEATURES":
|
|
270
|
+
tokens = np.rint(buffers[src][:, 0]).astype(np.int64)
|
|
271
|
+
buffers[instruction["output"]] = _segment_features(
|
|
272
|
+
instruction, tokens, state["scanners"][instruction["scanner"]]
|
|
273
|
+
)
|
|
274
|
+
else:
|
|
275
|
+
buffers[instruction["output"]] = _causal_attention(
|
|
276
|
+
instruction, buffers[src], state["kv"][instruction["slot"]], state["position"]
|
|
277
|
+
)
|
|
278
|
+
continue
|
|
199
279
|
|
|
200
280
|
x = buffers[src]
|
|
201
281
|
if kind == "COPY_MASKED":
|
|
@@ -256,6 +336,8 @@ def validate_instruction_model(model):
|
|
|
256
336
|
|
|
257
337
|
def concatenate_instruction_models(models, start_from_first=False):
|
|
258
338
|
"""Chains models: each model's first buffer is the previous model's last buffer."""
|
|
339
|
+
if any("sequence" in model for model in models):
|
|
340
|
+
raise ValueError("sequence models carry state and cannot be concatenated")
|
|
259
341
|
concatenated_buffer_sizes = models[0]["buffer_sizes"][:]
|
|
260
342
|
for model in models[1:]:
|
|
261
343
|
if concatenated_buffer_sizes[-1] != model["buffer_sizes"][0]:
|
|
@@ -284,6 +366,8 @@ def concatenate_instruction_models(models, start_from_first=False):
|
|
|
284
366
|
"parameters": len(new_model["parameters"]),
|
|
285
367
|
"parameters_min": len(new_model["parameters"]),
|
|
286
368
|
"parameters_max": len(new_model["parameters"]),
|
|
369
|
+
"gain": len(new_model["parameters"]),
|
|
370
|
+
"bias": len(new_model["parameters"]),
|
|
287
371
|
"map": len(new_model["maps"]),
|
|
288
372
|
"output": offset,
|
|
289
373
|
"key": offset,
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Causal transformers: training on PyTorch or TensorFlow, token-by-token inference from JSON."""
|
|
2
|
+
|
|
3
|
+
import importlib
|
|
4
|
+
|
|
5
|
+
from ._spec import ( # noqa: F401
|
|
6
|
+
FORMAT_VERSION,
|
|
7
|
+
SegmentFeatures,
|
|
8
|
+
TransformerConfig,
|
|
9
|
+
check_parameters,
|
|
10
|
+
export_step_model,
|
|
11
|
+
init_parameters,
|
|
12
|
+
parameter_count,
|
|
13
|
+
parameter_shapes,
|
|
14
|
+
segment_features,
|
|
15
|
+
)
|
|
16
|
+
from .reference import ( # noqa: F401
|
|
17
|
+
SequenceSession,
|
|
18
|
+
add_validation_sequences,
|
|
19
|
+
new_state,
|
|
20
|
+
validate_sequence_model,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def __getattr__(name):
|
|
25
|
+
# Training backends are optional and heavy; import them on first access.
|
|
26
|
+
if name in ("tf", "torch"):
|
|
27
|
+
try:
|
|
28
|
+
return importlib.import_module(f".{name}", __name__)
|
|
29
|
+
except ImportError as e:
|
|
30
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r} ({e})") from e
|
|
31
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
"""Causal transformer definition shared by the training backends and the step-program export.
|
|
2
|
+
|
|
3
|
+
A model is a dict of named float32 arrays (see ``parameter_shapes``). Training
|
|
4
|
+
backends hold the same arrays with a leading members axis and run them over
|
|
5
|
+
whole sequences. ``export_step_model`` turns one member into a format-v2
|
|
6
|
+
instruction model: the instruction list processes one token, and the state
|
|
7
|
+
declared under ``"sequence"`` carries the attention caches and feature scanner
|
|
8
|
+
from one token to the next.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from dataclasses import asdict, dataclass
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
FORMAT_VERSION = 2
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class SegmentFeatures:
|
|
20
|
+
"""Per-token ``(ordinal, index)`` of runs of tokens, each with a learned embedding table.
|
|
21
|
+
|
|
22
|
+
A token belongs to a run when it is in ``members``, or in ``openers`` and
|
|
23
|
+
the previous token is not a member (a sign before a digit). ``ordinal``
|
|
24
|
+
counts the runs started so far; ``index`` is the one-based place inside
|
|
25
|
+
the current run and 0 outside a run. Both clamp to ``rows - 1``.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
members: tuple
|
|
29
|
+
openers: tuple = ()
|
|
30
|
+
rows: int = 16
|
|
31
|
+
|
|
32
|
+
def __post_init__(self):
|
|
33
|
+
object.__setattr__(self, "members", tuple(int(t) for t in self.members))
|
|
34
|
+
object.__setattr__(self, "openers", tuple(int(t) for t in self.openers))
|
|
35
|
+
if self.rows < 2 or not self.members:
|
|
36
|
+
raise ValueError("segment features need members and at least two rows")
|
|
37
|
+
if set(self.members) & set(self.openers):
|
|
38
|
+
raise ValueError("a token cannot be both a member and an opener")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class TransformerConfig:
|
|
43
|
+
"""Pre-norm causal transformer.
|
|
44
|
+
|
|
45
|
+
``heads`` query heads share ``kv_heads`` key/value heads (grouped-query
|
|
46
|
+
attention; query head ``h`` uses key/value head ``h // (heads // kv_heads)``).
|
|
47
|
+
With ``loops`` above one the stack of ``layers`` blocks runs several times
|
|
48
|
+
with shared weights; ``inject`` re-adds the input embedding before each
|
|
49
|
+
later pass, ``pass_embed`` adds a learned vector per pass, ``pass_norms``
|
|
50
|
+
gives every pass its own layer-norm parameters, and ``lora`` is the rank
|
|
51
|
+
of a per-pass correction to each block's query/key/value projection.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
vocab_size: int
|
|
55
|
+
width: int
|
|
56
|
+
heads: int
|
|
57
|
+
layers: int = 2
|
|
58
|
+
ff_width: int = 0
|
|
59
|
+
kv_heads: int = 0
|
|
60
|
+
loops: int = 1
|
|
61
|
+
inject: bool = False
|
|
62
|
+
pass_embed: bool = False
|
|
63
|
+
pass_norms: bool = False
|
|
64
|
+
lora: int = 0
|
|
65
|
+
tied: bool = True
|
|
66
|
+
features: SegmentFeatures | None = None
|
|
67
|
+
max_positions: int = 192
|
|
68
|
+
epsilon: float = 1e-5
|
|
69
|
+
|
|
70
|
+
def __post_init__(self):
|
|
71
|
+
if isinstance(self.features, dict):
|
|
72
|
+
object.__setattr__(self, "features", SegmentFeatures(**self.features))
|
|
73
|
+
object.__setattr__(self, "ff_width", self.ff_width or self.width)
|
|
74
|
+
object.__setattr__(self, "kv_heads", self.kv_heads or self.heads)
|
|
75
|
+
if self.vocab_size < 2 or self.width < 1 or self.layers < 1 or self.loops < 1:
|
|
76
|
+
raise ValueError("vocab_size, width, layers and loops must be positive")
|
|
77
|
+
if self.width % self.heads or self.heads % self.kv_heads:
|
|
78
|
+
raise ValueError("width must divide into heads, and heads into kv_heads")
|
|
79
|
+
if self.lora < 0 or self.max_positions < 1 or self.ff_width < 1:
|
|
80
|
+
raise ValueError("lora, max_positions and ff_width must be valid sizes")
|
|
81
|
+
if self.features is not None and max(self.features.members + self.features.openers) >= self.vocab_size:
|
|
82
|
+
raise ValueError("segment feature tokens must be vocabulary ids")
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def head_dim(self):
|
|
86
|
+
return self.width // self.heads
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def qkv_size(self):
|
|
90
|
+
return self.width + 2 * self.kv_heads * self.head_dim
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def norm_passes(self):
|
|
94
|
+
return self.loops if self.pass_norms else 1
|
|
95
|
+
|
|
96
|
+
def to_dict(self):
|
|
97
|
+
return asdict(self)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def parameter_shapes(config):
|
|
101
|
+
"""Name -> shape of every learned array of one member; linear maps are ``[input, output]``."""
|
|
102
|
+
c, d = config, config.width
|
|
103
|
+
shapes = {"embedding": (c.vocab_size, d)}
|
|
104
|
+
if c.features is not None:
|
|
105
|
+
shapes["ordinal"] = (c.features.rows, d)
|
|
106
|
+
shapes["index"] = (c.features.rows, d)
|
|
107
|
+
if c.pass_embed:
|
|
108
|
+
shapes["passes"] = (c.loops, d)
|
|
109
|
+
for layer in range(c.layers):
|
|
110
|
+
prefix = f"blocks.{layer}."
|
|
111
|
+
shapes[prefix + "norm1"] = (c.norm_passes, 2, d) # Row 0 is the gain, row 1 the bias.
|
|
112
|
+
shapes[prefix + "qkv"] = (d, c.qkv_size)
|
|
113
|
+
shapes[prefix + "qkv_bias"] = (c.qkv_size,)
|
|
114
|
+
if c.lora:
|
|
115
|
+
shapes[prefix + "lora_a"] = (c.loops, d, c.lora)
|
|
116
|
+
shapes[prefix + "lora_b"] = (c.loops, c.lora, c.qkv_size)
|
|
117
|
+
shapes[prefix + "out"] = (d, d)
|
|
118
|
+
shapes[prefix + "out_bias"] = (d,)
|
|
119
|
+
shapes[prefix + "norm2"] = (c.norm_passes, 2, d)
|
|
120
|
+
shapes[prefix + "up"] = (d, c.ff_width)
|
|
121
|
+
shapes[prefix + "up_bias"] = (c.ff_width,)
|
|
122
|
+
shapes[prefix + "down"] = (c.ff_width, d)
|
|
123
|
+
shapes[prefix + "down_bias"] = (d,)
|
|
124
|
+
shapes["norm"] = (2, d)
|
|
125
|
+
if not c.tied:
|
|
126
|
+
shapes["readout"] = (d, c.vocab_size)
|
|
127
|
+
shapes["readout_bias"] = (c.vocab_size,)
|
|
128
|
+
return shapes
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def parameter_count(config):
|
|
132
|
+
return sum(int(np.prod(shape)) for shape in parameter_shapes(config).values())
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def init_parameters(config, seed=0, members=None):
|
|
136
|
+
"""Initial float32 parameters; with ``members`` every array gains a leading members axis.
|
|
137
|
+
|
|
138
|
+
Tables and LoRA inputs are normal with scale ``width ** -0.5``; linear maps
|
|
139
|
+
are uniform in ``±1/sqrt(input)``; norms start as identity; biases, pass
|
|
140
|
+
vectors and LoRA outputs start at zero, so every pass starts shared.
|
|
141
|
+
"""
|
|
142
|
+
rng = np.random.default_rng(seed)
|
|
143
|
+
lead = () if members is None else (members,)
|
|
144
|
+
d, out = config.width, {}
|
|
145
|
+
for name, shape in parameter_shapes(config).items():
|
|
146
|
+
leaf = name.rsplit(".", 1)[-1]
|
|
147
|
+
full = lead + shape
|
|
148
|
+
if leaf in ("embedding", "ordinal", "index", "lora_a"):
|
|
149
|
+
value = rng.standard_normal(full) * d ** -0.5
|
|
150
|
+
elif leaf in ("norm1", "norm2", "norm"):
|
|
151
|
+
value = np.zeros(full)
|
|
152
|
+
value[..., 0, :] = 1
|
|
153
|
+
elif leaf in ("qkv", "out", "up", "down", "readout"):
|
|
154
|
+
bound = shape[0] ** -0.5
|
|
155
|
+
value = rng.uniform(-bound, bound, full)
|
|
156
|
+
else:
|
|
157
|
+
value = np.zeros(full)
|
|
158
|
+
out[name] = value.astype(np.float32)
|
|
159
|
+
return out
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def check_parameters(config, parameters):
|
|
163
|
+
shapes = parameter_shapes(config)
|
|
164
|
+
if set(parameters) != set(shapes):
|
|
165
|
+
raise ValueError(f"parameter names differ: {sorted(set(parameters) ^ set(shapes))}")
|
|
166
|
+
for name, shape in shapes.items():
|
|
167
|
+
if tuple(np.shape(parameters[name])) != shape:
|
|
168
|
+
raise ValueError(f"{name} has shape {np.shape(parameters[name])}, expected {shape}")
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def segment_features(tokens, features):
|
|
172
|
+
"""NumPy ``(ordinal, index)`` for token arrays shaped ``[..., time]``."""
|
|
173
|
+
tokens = np.asarray(tokens)
|
|
174
|
+
|
|
175
|
+
def shifted(mask):
|
|
176
|
+
return np.concatenate((np.zeros_like(mask[..., :1]), mask[..., :-1]), -1)
|
|
177
|
+
|
|
178
|
+
member = np.isin(tokens, features.members)
|
|
179
|
+
run = member | (np.isin(tokens, features.openers) & ~shifted(member))
|
|
180
|
+
start = run & ~shifted(run)
|
|
181
|
+
place = np.broadcast_to(np.arange(tokens.shape[-1]), tokens.shape)
|
|
182
|
+
last = np.maximum.accumulate(np.where(start, place, -1), -1)
|
|
183
|
+
index = np.where(run, place - last + 1, 0)
|
|
184
|
+
ordinal = np.cumsum(start, -1)
|
|
185
|
+
limit = features.rows - 1
|
|
186
|
+
return np.minimum(ordinal, limit), np.minimum(index, limit)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
class _Program:
|
|
190
|
+
"""Accumulates buffers, instructions and deduplicated weight and parameter entries."""
|
|
191
|
+
|
|
192
|
+
def __init__(self):
|
|
193
|
+
self.model = dict(buffer_sizes=[], instructions=[], maps=[], weights=[], bias=[], parameters=[])
|
|
194
|
+
self._entries = {}
|
|
195
|
+
|
|
196
|
+
def buffer(self, size):
|
|
197
|
+
self.model["buffer_sizes"].append(int(size))
|
|
198
|
+
return len(self.model["buffer_sizes"]) - 1
|
|
199
|
+
|
|
200
|
+
def emit(self, **instruction):
|
|
201
|
+
self.model["instructions"].append(instruction)
|
|
202
|
+
return instruction.get("output")
|
|
203
|
+
|
|
204
|
+
def weights(self, key, matrix, bias=None):
|
|
205
|
+
"""Index of an ``[output][input]`` matrix, stored once per ``key``."""
|
|
206
|
+
if key not in self._entries:
|
|
207
|
+
matrix = np.asarray(matrix, dtype=np.float32)
|
|
208
|
+
bias = np.zeros(len(matrix), dtype=np.float32) if bias is None else np.asarray(bias, dtype=np.float32)
|
|
209
|
+
self.model["weights"].append(matrix.tolist())
|
|
210
|
+
self.model["bias"].append(bias.tolist())
|
|
211
|
+
self._entries[key] = len(self.model["weights"]) - 1
|
|
212
|
+
return self._entries[key]
|
|
213
|
+
|
|
214
|
+
def parameters(self, key, vector):
|
|
215
|
+
key = ("parameters", key)
|
|
216
|
+
if key not in self._entries:
|
|
217
|
+
self.model["parameters"].append(np.asarray(vector, dtype=np.float32).tolist())
|
|
218
|
+
self._entries[key] = len(self.model["parameters"]) - 1
|
|
219
|
+
return self._entries[key]
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def export_step_model(config, parameters, vocabulary=None, metadata=None):
|
|
223
|
+
"""Format-v2 instruction model that processes one token of one member.
|
|
224
|
+
|
|
225
|
+
Buffer 0 holds the token id and the last buffer the next-token logits.
|
|
226
|
+
Looped passes are unrolled; their instructions reference the same weight
|
|
227
|
+
entries, and every attention instruction owns one key/value cache slot.
|
|
228
|
+
"""
|
|
229
|
+
check_parameters(config, parameters)
|
|
230
|
+
c, p, d = config, parameters, config.width
|
|
231
|
+
program = _Program()
|
|
232
|
+
token = program.buffer(1)
|
|
233
|
+
|
|
234
|
+
def linear(name, source, size, bias=None, activation=None, key=None):
|
|
235
|
+
index = program.weights(key or name, np.asarray(p[name]).T, None if bias is None else p[bias])
|
|
236
|
+
instruction = dict(type="DOT", input=source, output=program.buffer(size), weights=index)
|
|
237
|
+
if activation:
|
|
238
|
+
instruction["activation"] = activation
|
|
239
|
+
return program.emit(**instruction)
|
|
240
|
+
|
|
241
|
+
def norm(name, values, source):
|
|
242
|
+
gain = program.parameters(name + ".gain", values[0])
|
|
243
|
+
bias = program.parameters(name + ".bias", values[1])
|
|
244
|
+
return program.emit(type="LAYER_NORM", input=source, output=program.buffer(d), gain=gain, bias=bias,
|
|
245
|
+
epsilon=c.epsilon)
|
|
246
|
+
|
|
247
|
+
def add(*sources):
|
|
248
|
+
return program.emit(type="ADD_ELEMENTWISE_BUFFERS", input=list(sources), output=program.buffer(d))
|
|
249
|
+
|
|
250
|
+
# The token table doubles as the tied readout matrix, whose bias is the readout bias.
|
|
251
|
+
table = program.weights("embedding", p["embedding"], p["readout_bias"] if c.tied else None)
|
|
252
|
+
embedded = [program.emit(type="EMBEDDING", input=token, output=program.buffer(d), weights=table)]
|
|
253
|
+
if c.features is not None:
|
|
254
|
+
scanned = program.emit(type="SEGMENT_FEATURES", input=token, output=program.buffer(2),
|
|
255
|
+
members=list(c.features.members), openers=list(c.features.openers),
|
|
256
|
+
rows=c.features.rows, scanner=0)
|
|
257
|
+
for column, name in enumerate(("ordinal", "index")):
|
|
258
|
+
embedded.append(program.emit(type="EMBEDDING", input=scanned, internal_input_index=column,
|
|
259
|
+
output=program.buffer(d), weights=program.weights(name, p[name])))
|
|
260
|
+
source = add(*embedded) if len(embedded) > 1 else embedded[0]
|
|
261
|
+
|
|
262
|
+
x, slots = source, []
|
|
263
|
+
for step in range(c.loops):
|
|
264
|
+
if step and c.inject:
|
|
265
|
+
x = add(x, source)
|
|
266
|
+
if c.pass_embed:
|
|
267
|
+
if x == source: # The embedding stays intact for later injection.
|
|
268
|
+
x = program.emit(type="COPY", input=source, output=program.buffer(d), internal_index=0)
|
|
269
|
+
program.emit(type="ADD_ELEMENTWISE", input=x,
|
|
270
|
+
parameters=program.parameters(f"passes.{step}", p["passes"][step]))
|
|
271
|
+
row = step if c.pass_norms else 0
|
|
272
|
+
for layer in range(c.layers):
|
|
273
|
+
prefix = f"blocks.{layer}."
|
|
274
|
+
hidden = norm(f"{prefix}norm1.{row}", p[prefix + "norm1"][row], x)
|
|
275
|
+
qkv = linear(prefix + "qkv", hidden, c.qkv_size, prefix + "qkv_bias")
|
|
276
|
+
if c.lora:
|
|
277
|
+
low = program.emit(type="DOT", input=hidden, output=program.buffer(c.lora),
|
|
278
|
+
weights=program.weights(f"{prefix}lora_a.{step}", p[prefix + "lora_a"][step].T))
|
|
279
|
+
delta = program.emit(type="DOT", input=low, output=program.buffer(c.qkv_size),
|
|
280
|
+
weights=program.weights(f"{prefix}lora_b.{step}", p[prefix + "lora_b"][step].T))
|
|
281
|
+
qkv = program.emit(type="ADD_ELEMENTWISE_BUFFERS", input=[qkv, delta],
|
|
282
|
+
output=program.buffer(c.qkv_size))
|
|
283
|
+
attended = program.emit(type="CAUSAL_ATTENTION", input=qkv, output=program.buffer(d),
|
|
284
|
+
heads=c.heads, kv_heads=c.kv_heads, slot=len(slots))
|
|
285
|
+
slots.append(dict(heads=c.heads, kv_heads=c.kv_heads, head_dim=c.head_dim))
|
|
286
|
+
x = add(x, linear(prefix + "out", attended, d, prefix + "out_bias"))
|
|
287
|
+
hidden = norm(f"{prefix}norm2.{row}", p[prefix + "norm2"][row], x)
|
|
288
|
+
raised = linear(prefix + "up", hidden, c.ff_width, prefix + "up_bias", activation="GELU")
|
|
289
|
+
x = add(x, linear(prefix + "down", raised, d, prefix + "down_bias"))
|
|
290
|
+
final = norm("norm", p["norm"], x)
|
|
291
|
+
if c.tied:
|
|
292
|
+
program.emit(type="DOT", input=final, output=program.buffer(c.vocab_size), weights=table)
|
|
293
|
+
else:
|
|
294
|
+
linear("readout", final, c.vocab_size, "readout_bias")
|
|
295
|
+
|
|
296
|
+
model = program.model
|
|
297
|
+
model["format_version"] = FORMAT_VERSION
|
|
298
|
+
model["features"] = ["token"]
|
|
299
|
+
model["sequence"] = dict(
|
|
300
|
+
vocabulary=list(vocabulary) if vocabulary is not None else [str(i) for i in range(c.vocab_size)],
|
|
301
|
+
max_positions=c.max_positions, kv_slots=slots, scanners=int(c.features is not None),
|
|
302
|
+
config=c.to_dict(), metadata=dict(metadata or {}))
|
|
303
|
+
if len(model["sequence"]["vocabulary"]) != c.vocab_size:
|
|
304
|
+
raise ValueError("vocabulary length differs from vocab_size")
|
|
305
|
+
return model
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""NumPy reference for sequence models: token-by-token steps with explicit state.
|
|
2
|
+
|
|
3
|
+
This is the ground truth the training backends and the Rust runtime are
|
|
4
|
+
checked against. It runs the exported instruction list once per token on
|
|
5
|
+
float32 arrays, for several sequences at a time.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
from ..instruction_model import instruction_model_inference, model_arrays
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def new_state(model, sequences=1):
|
|
14
|
+
"""Empty state for ``sequences`` parallel sequences: attention caches, scanners and position."""
|
|
15
|
+
info = model["sequence"]
|
|
16
|
+
capacity = info["max_positions"]
|
|
17
|
+
|
|
18
|
+
def cache(slot):
|
|
19
|
+
return np.zeros((sequences, capacity, slot["kv_heads"], slot["head_dim"]), dtype=np.float32)
|
|
20
|
+
|
|
21
|
+
def scanner():
|
|
22
|
+
return dict(prev_member=np.zeros(sequences, dtype=bool), prev_run=np.zeros(sequences, dtype=bool),
|
|
23
|
+
index=np.zeros(sequences, dtype=np.int64), ordinal=np.zeros(sequences, dtype=np.int64))
|
|
24
|
+
|
|
25
|
+
return dict(position=0, kv=[dict(keys=cache(slot), values=cache(slot)) for slot in info["kv_slots"]],
|
|
26
|
+
scanners=[scanner() for _ in range(info["scanners"])])
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SequenceSession:
|
|
30
|
+
"""Steps a format-v2 sequence model; ``state`` may be copied, kept and restored by the caller."""
|
|
31
|
+
|
|
32
|
+
def __init__(self, model, sequences=1):
|
|
33
|
+
if model.get("format_version") != 2 or "sequence" not in model:
|
|
34
|
+
raise ValueError("not a format-v2 sequence model")
|
|
35
|
+
self.model, self.sequences = model, sequences
|
|
36
|
+
self._arrays = model_arrays(model)
|
|
37
|
+
self.state = new_state(model, sequences)
|
|
38
|
+
|
|
39
|
+
def step(self, tokens):
|
|
40
|
+
"""Feeds one token id per sequence and returns next-token logits ``[sequences, vocabulary]``."""
|
|
41
|
+
tokens = np.asarray(tokens, dtype=np.float32).reshape(self.sequences, 1)
|
|
42
|
+
logits = instruction_model_inference(self.model, [tokens], self.state, self._arrays)[-1]
|
|
43
|
+
self.state["position"] += 1
|
|
44
|
+
return logits
|
|
45
|
+
|
|
46
|
+
def run(self, tokens):
|
|
47
|
+
"""Logits ``[sequences, time, vocabulary]`` for whole token arrays ``[sequences, time]``."""
|
|
48
|
+
tokens = np.asarray(tokens).reshape(self.sequences, -1)
|
|
49
|
+
return np.stack([self.step(tokens[:, t]) for t in range(tokens.shape[1])], 1)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def add_validation_sequences(model, sequences):
|
|
53
|
+
"""Stores token sequences with the reference logits, for runtimes to replay on load."""
|
|
54
|
+
model["validation_sequences"] = [
|
|
55
|
+
dict(tokens=[int(t) for t in tokens], logits=SequenceSession(model).run(tokens)[0].tolist())
|
|
56
|
+
for tokens in sequences
|
|
57
|
+
]
|
|
58
|
+
return model
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def validate_sequence_model(model, atol=1e-4):
|
|
62
|
+
"""Replays ``validation_sequences``; returns the largest logit error or raises on a mismatch."""
|
|
63
|
+
worst = 0.0
|
|
64
|
+
for case in model.get("validation_sequences", []):
|
|
65
|
+
expected = np.asarray(case["logits"], dtype=np.float32)
|
|
66
|
+
actual = SequenceSession(model).run(case["tokens"])[0]
|
|
67
|
+
worst = max(worst, float(np.abs(actual - expected).max()))
|
|
68
|
+
if worst > atol or (actual.argmax(-1) != expected.argmax(-1)).any():
|
|
69
|
+
raise AssertionError(f"sequence model differs from its validation logits (max error {worst})")
|
|
70
|
+
return worst
|