instmodel 1.3.1__tar.gz → 1.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {instmodel-1.3.1 → instmodel-1.4.0}/PKG-INFO +32 -2
  2. instmodel-1.4.0/README.md +66 -0
  3. {instmodel-1.3.1 → instmodel-1.4.0}/instmodel/instruction_model.py +90 -6
  4. instmodel-1.4.0/instmodel/sequence/__init__.py +31 -0
  5. instmodel-1.4.0/instmodel/sequence/_spec.py +305 -0
  6. instmodel-1.4.0/instmodel/sequence/reference.py +70 -0
  7. instmodel-1.4.0/instmodel/sequence/tf.py +331 -0
  8. instmodel-1.4.0/instmodel/sequence/torch.py +283 -0
  9. {instmodel-1.3.1 → instmodel-1.4.0}/instmodel.egg-info/PKG-INFO +32 -2
  10. {instmodel-1.3.1 → instmodel-1.4.0}/instmodel.egg-info/SOURCES.txt +9 -1
  11. {instmodel-1.3.1 → instmodel-1.4.0}/instmodel.egg-info/requires.txt +1 -1
  12. {instmodel-1.3.1 → instmodel-1.4.0}/pyproject.toml +3 -3
  13. instmodel-1.4.0/tests/test_sequence.py +168 -0
  14. instmodel-1.4.0/tests/test_sequence_tf.py +269 -0
  15. instmodel-1.4.0/tests/test_sequence_torch.py +205 -0
  16. instmodel-1.3.1/README.md +0 -36
  17. {instmodel-1.3.1 → instmodel-1.4.0}/LICENSE +0 -0
  18. {instmodel-1.3.1 → instmodel-1.4.0}/instmodel/__init__.py +0 -0
  19. {instmodel-1.3.1 → instmodel-1.4.0}/instmodel/_core.py +0 -0
  20. {instmodel-1.3.1 → instmodel-1.4.0}/instmodel/tf.py +0 -0
  21. {instmodel-1.3.1 → instmodel-1.4.0}/instmodel/torch.py +0 -0
  22. {instmodel-1.3.1 → instmodel-1.4.0}/instmodel/training_utils.py +0 -0
  23. {instmodel-1.3.1 → instmodel-1.4.0}/instmodel.egg-info/dependency_links.txt +0 -0
  24. {instmodel-1.3.1 → instmodel-1.4.0}/instmodel.egg-info/top_level.txt +0 -0
  25. {instmodel-1.3.1 → instmodel-1.4.0}/setup.cfg +0 -0
  26. {instmodel-1.3.1 → instmodel-1.4.0}/tests/test_backends.py +0 -0
  27. {instmodel-1.3.1 → instmodel-1.4.0}/tests/test_instruction_model.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: instmodel
3
- Version: 1.3.1
3
+ Version: 1.4.0
4
4
  Summary: Train TF/PyTorch models and export them to a JSON instruction format with NumPy inference
5
5
  Author-email: Joao Ferreira <joaoprcf@gmail.com>
6
6
  License-Expression: MIT
@@ -17,7 +17,7 @@ Provides-Extra: tensorflow
17
17
  Requires-Dist: tensorflow>=2.20; extra == "tensorflow"
18
18
  Requires-Dist: protobuf>=6.0; extra == "tensorflow"
19
19
  Provides-Extra: pytorch
20
- Requires-Dist: torch>=2.0; extra == "pytorch"
20
+ Requires-Dist: torch>=2.5; extra == "pytorch"
21
21
  Dynamic: license-file
22
22
 
23
23
  # instmodel
@@ -53,6 +53,36 @@ exported = model.create_instruction_model() # JSON-serialisable dict
53
53
  print(instruction_model_inference(exported, np.random.rand(3, 4))[-1])
54
54
  ```
55
55
 
56
+ ## Sequence models
57
+
58
+ `instmodel.sequence` trains small causal transformers and exports them as an instruction model
59
+ that processes one token per run. The attention caches are explicit state owned by the caller,
60
+ so any runtime can decode token by token (NumPy here, or `SequenceModel` in the Rust port).
61
+
62
+ ```python
63
+ import numpy as np
64
+ from instmodel import sequence
65
+ from instmodel.sequence import torch as backend # same API in instmodel.sequence.tf
66
+
67
+ config = sequence.TransformerConfig(vocab_size=16, width=16, heads=4, kv_heads=2)
68
+ model = backend.Transformer(config, members=2) # two copies trained in one batch
69
+
70
+ start = np.random.randint(0, 16, (4096, 1))
71
+ tokens = (start + np.arange(13)) % 16 # rows [N, time + 1]; learn "next = token + 1"
72
+ rows = dict(tokens=backend.torch.as_tensor(tokens), groups=backend.torch.zeros(4096, 12, dtype=int))
73
+ trainer = backend.Trainer(model, rows, batch=64, lr=3e-3, ema=0.99)
74
+ for step in range(400):
75
+ trainer.step(backend.schedule(step / 400, 3e-3))
76
+
77
+ exported = sequence.export_step_model(config, trainer.averaged().member_parameters(0))
78
+ session = sequence.SequenceSession(exported) # keeps the key/value cache between steps
79
+ print(session.step([3]).argmax(-1)) # -> [4]
80
+ ```
81
+
82
+ `TransformerConfig` also covers grouped-query attention, looped (weight-shared) passes and
83
+ `SegmentFeatures`; `sequence_loss` takes per-position groups, allowed-token masks and soft targets.
84
+ `add_validation_sequences` stores reference logits in the file for other runtimes to replay.
85
+
56
86
  ## License
57
87
 
58
88
  [MIT](LICENSE)
@@ -0,0 +1,66 @@
1
+ # instmodel
2
+
3
+ This repo is meant to be used with coding agents, not read manually.
4
+
5
+ Build and train small neural networks with PyTorch or TensorFlow/Keras through one shared graph API,
6
+ then export them to a compact JSON "instruction model" that runs anywhere with plain NumPy (or the
7
+ Rust/C++ inference ports) — no training framework needed at inference time.
8
+
9
+ ## Install
10
+
11
+ ```bash
12
+ pip install instmodel # NumPy inference only
13
+ pip install instmodel[pytorch] # + PyTorch training backend (instmodel.torch)
14
+ pip install instmodel[tensorflow] # + TensorFlow/Keras training backend (instmodel.tf)
15
+ ```
16
+
17
+ ## Example
18
+
19
+ ```python
20
+ import numpy as np
21
+ from instmodel.torch import Dense, InputBuffer, ModelGraph # same API in instmodel.tf
22
+ from instmodel.instruction_model import instruction_model_inference
23
+
24
+ x = InputBuffer(4)
25
+ y = Dense(1, activation="sigmoid")(Dense(8, activation="relu")(x))
26
+ model = ModelGraph(x, y)
27
+ model.compile(optimizer="adam", loss="binary_crossentropy")
28
+ model.fit(np.random.rand(64, 4), np.random.randint(0, 2, (64, 1)), epochs=5, verbose=0)
29
+
30
+ exported = model.create_instruction_model() # JSON-serialisable dict
31
+ print(instruction_model_inference(exported, np.random.rand(3, 4))[-1])
32
+ ```
33
+
34
+ ## Sequence models
35
+
36
+ `instmodel.sequence` trains small causal transformers and exports them as an instruction model
37
+ that processes one token per run. The attention caches are explicit state owned by the caller,
38
+ so any runtime can decode token by token (NumPy here, or `SequenceModel` in the Rust port).
39
+
40
+ ```python
41
+ import numpy as np
42
+ from instmodel import sequence
43
+ from instmodel.sequence import torch as backend # same API in instmodel.sequence.tf
44
+
45
+ config = sequence.TransformerConfig(vocab_size=16, width=16, heads=4, kv_heads=2)
46
+ model = backend.Transformer(config, members=2) # two copies trained in one batch
47
+
48
+ start = np.random.randint(0, 16, (4096, 1))
49
+ tokens = (start + np.arange(13)) % 16 # rows [N, time + 1]; learn "next = token + 1"
50
+ rows = dict(tokens=backend.torch.as_tensor(tokens), groups=backend.torch.zeros(4096, 12, dtype=int))
51
+ trainer = backend.Trainer(model, rows, batch=64, lr=3e-3, ema=0.99)
52
+ for step in range(400):
53
+ trainer.step(backend.schedule(step / 400, 3e-3))
54
+
55
+ exported = sequence.export_step_model(config, trainer.averaged().member_parameters(0))
56
+ session = sequence.SequenceSession(exported) # keeps the key/value cache between steps
57
+ print(session.step([3]).argmax(-1)) # -> [4]
58
+ ```
59
+
60
+ `TransformerConfig` also covers grouped-query attention, looped (weight-shared) passes and
61
+ `SegmentFeatures`; `sequence_loss` takes per-position groups, allowed-token masks and soft targets.
62
+ `add_validation_sequences` stores reference logits in the file for other runtimes to replay.
63
+
64
+ ## License
65
+
66
+ [MIT](LICENSE)
@@ -99,7 +99,11 @@ def _validate_references(model):
99
99
  "parameters": len(model["parameters"]),
100
100
  "parameters_min": len(model["parameters"]),
101
101
  "parameters_max": len(model["parameters"]),
102
+ "gain": len(model["parameters"]),
103
+ "bias": len(model["parameters"]),
102
104
  "map": len(model.get("maps", [])),
105
+ "slot": len(model.get("sequence", {}).get("kv_slots", [])),
106
+ "scanner": model.get("sequence", {}).get("scanners", 0),
103
107
  }
104
108
  for instr in model["instructions"]:
105
109
  inputs = instr.get("input")
@@ -123,6 +127,48 @@ def _validate_references(model):
123
127
  _check_span(
124
128
  instr["internal_output_index"], instr["size"], sizes[instr["output"]], "MAP_TRANSFORM"
125
129
  )
130
+ elif instr["type"] == "EMBEDDING":
131
+ _check_index(instr.get("internal_input_index", 0), sizes[instr["input"]], "EMBEDDING column")
132
+ elif instr["type"] == "SEGMENT_FEATURES":
133
+ if sizes[instr["output"]] != 2:
134
+ raise ValueError("SEGMENT_FEATURES writes an (ordinal, index) buffer of size 2")
135
+ elif instr["type"] == "CAUSAL_ATTENTION":
136
+ slot = model["sequence"]["kv_slots"][instr["slot"]]
137
+ heads, shared, head = instr["heads"], instr["kv_heads"], slot["head_dim"]
138
+ if (heads, shared) != (slot["heads"], slot["kv_heads"]) or shared < 1 or heads % shared:
139
+ raise ValueError("CAUSAL_ATTENTION heads differ from its key/value slot")
140
+ if sizes[instr["input"]] != (heads + 2 * shared) * head or sizes[instr["output"]] != heads * head:
141
+ raise ValueError("CAUSAL_ATTENTION buffers do not match its heads and head_dim")
142
+
143
+
144
+ def _segment_features(instruction, tokens, scanner):
145
+ """Advances one feature scanner by a token per row and returns ``[ordinal, index]`` columns."""
146
+ member = np.isin(tokens, instruction["members"])
147
+ run = member | (np.isin(tokens, instruction["openers"]) & ~scanner["prev_member"])
148
+ start = run & ~scanner["prev_run"]
149
+ scanner["ordinal"] = scanner["ordinal"] + start
150
+ scanner["index"] = np.where(start, 1, np.where(run, scanner["index"] + 1, 0))
151
+ scanner["prev_member"], scanner["prev_run"] = member, run
152
+ limit = instruction["rows"] - 1
153
+ return np.minimum(np.stack((scanner["ordinal"], scanner["index"]), -1), limit).astype(np.float32)
154
+
155
+
156
+ def _causal_attention(instruction, x, slot, position):
157
+ """Appends this token's keys and values to ``slot`` and attends over the cached prefix."""
158
+ heads, shared = instruction["heads"], instruction["kv_heads"]
159
+ head = x.shape[1] // (heads + 2 * shared)
160
+ rows = x.shape[0]
161
+ if position >= slot["keys"].shape[1]:
162
+ raise ValueError("sequence exceeds the model's max_positions")
163
+ query = x[:, : heads * head].reshape(rows, shared, heads // shared, head)
164
+ slot["keys"][:, position] = x[:, heads * head : (heads + shared) * head].reshape(rows, shared, head)
165
+ slot["values"][:, position] = x[:, (heads + shared) * head :].reshape(rows, shared, head)
166
+ keys, values = slot["keys"][:, : position + 1], slot["values"][:, : position + 1]
167
+ # Query head h reads key/value head h // (heads // kv_heads).
168
+ scores = np.einsum("rgqd,rtgd->rgqt", query, keys) * np.float32(head ** -0.5)
169
+ scores = np.exp(scores - scores.max(-1, keepdims=True))
170
+ scores = scores / scores.sum(-1, keepdims=True)
171
+ return np.einsum("rgqt,rtgd->rgqd", scores, values).reshape(rows, heads * head).astype(np.float32)
126
172
 
127
173
 
128
174
  def _output_buffer(buffers, buffer_sizes, index, rows):
@@ -135,15 +181,20 @@ _BUFFER_REDUCERS = {"ADD_ELEMENTWISE_BUFFERS": np.sum, "MULTIPLY_ELEMENTWISE_BUF
135
181
  _HEAD_COMBINERS = {"ADD_BUFFER_HEADS": np.add, "MULTIPLY_BUFFER_HEADS": np.multiply}
136
182
 
137
183
 
138
- def instruction_model_inference(model, input_data):
184
+ def model_arrays(model):
185
+ """Validates references and returns the model's (weights, bias, parameters) as float32 arrays."""
186
+ _validate_references(model)
187
+ return tuple([np.array(v, dtype=np.float32) for v in model[key]] for key in ("weights", "bias", "parameters"))
188
+
189
+
190
+ def instruction_model_inference(model, input_data, state=None, arrays=None):
139
191
  """Runs the model on a 2D array (split across the leading buffers) or a list of per-buffer arrays.
140
192
 
141
- Returns every buffer; the last one is the model output.
193
+ Returns every buffer; the last one is the model output. ``state`` carries
194
+ the caches of the stateful instructions (see ``instmodel.sequence``), with
195
+ one row per sequence; ``arrays`` reuses the result of ``model_arrays``.
142
196
  """
143
- _validate_references(model)
144
- weights = [np.array(w, dtype=np.float32) for w in model["weights"]]
145
- bias = [np.array(b, dtype=np.float32) for b in model["bias"]]
146
- parameters = [np.array(p, dtype=np.float32) for p in model["parameters"]]
197
+ weights, bias, parameters = arrays if arrays is not None else model_arrays(model)
147
198
  buffer_sizes = model["buffer_sizes"]
148
199
  buffers = [None] * len(buffer_sizes)
149
200
 
@@ -196,6 +247,35 @@ def instruction_model_inference(model, input_data):
196
247
  np.sum(buffers[src], axis=-1, keepdims=True), dtype=np.float32
197
248
  )
198
249
  continue
250
+ if kind == "EMBEDDING":
251
+ table = weights[instruction["weights"]]
252
+ ids = np.rint(buffers[src][:, instruction.get("internal_input_index", 0)]).astype(np.int64)
253
+ if ids.min() < 0 or ids.max() >= len(table):
254
+ raise ValueError(f"EMBEDDING id outside [0, {len(table)})")
255
+ buffers[instruction["output"]] = table[ids]
256
+ continue
257
+ if kind == "LAYER_NORM":
258
+ x = buffers[src]
259
+ centered = x - x.mean(-1, keepdims=True)
260
+ scale = np.sqrt((centered * centered).mean(-1, keepdims=True) + np.float32(instruction["epsilon"]))
261
+ buffers[instruction["output"]] = np.asarray(
262
+ centered / scale * parameters[instruction["gain"]] + parameters[instruction["bias"]],
263
+ dtype=np.float32,
264
+ )
265
+ continue
266
+ if kind in ("SEGMENT_FEATURES", "CAUSAL_ATTENTION"):
267
+ if state is None:
268
+ raise ValueError(f"{kind} needs sequence state; step the model with instmodel.sequence")
269
+ if kind == "SEGMENT_FEATURES":
270
+ tokens = np.rint(buffers[src][:, 0]).astype(np.int64)
271
+ buffers[instruction["output"]] = _segment_features(
272
+ instruction, tokens, state["scanners"][instruction["scanner"]]
273
+ )
274
+ else:
275
+ buffers[instruction["output"]] = _causal_attention(
276
+ instruction, buffers[src], state["kv"][instruction["slot"]], state["position"]
277
+ )
278
+ continue
199
279
 
200
280
  x = buffers[src]
201
281
  if kind == "COPY_MASKED":
@@ -256,6 +336,8 @@ def validate_instruction_model(model):
256
336
 
257
337
  def concatenate_instruction_models(models, start_from_first=False):
258
338
  """Chains models: each model's first buffer is the previous model's last buffer."""
339
+ if any("sequence" in model for model in models):
340
+ raise ValueError("sequence models carry state and cannot be concatenated")
259
341
  concatenated_buffer_sizes = models[0]["buffer_sizes"][:]
260
342
  for model in models[1:]:
261
343
  if concatenated_buffer_sizes[-1] != model["buffer_sizes"][0]:
@@ -284,6 +366,8 @@ def concatenate_instruction_models(models, start_from_first=False):
284
366
  "parameters": len(new_model["parameters"]),
285
367
  "parameters_min": len(new_model["parameters"]),
286
368
  "parameters_max": len(new_model["parameters"]),
369
+ "gain": len(new_model["parameters"]),
370
+ "bias": len(new_model["parameters"]),
287
371
  "map": len(new_model["maps"]),
288
372
  "output": offset,
289
373
  "key": offset,
@@ -0,0 +1,31 @@
1
+ """Causal transformers: training on PyTorch or TensorFlow, token-by-token inference from JSON."""
2
+
3
+ import importlib
4
+
5
+ from ._spec import ( # noqa: F401
6
+ FORMAT_VERSION,
7
+ SegmentFeatures,
8
+ TransformerConfig,
9
+ check_parameters,
10
+ export_step_model,
11
+ init_parameters,
12
+ parameter_count,
13
+ parameter_shapes,
14
+ segment_features,
15
+ )
16
+ from .reference import ( # noqa: F401
17
+ SequenceSession,
18
+ add_validation_sequences,
19
+ new_state,
20
+ validate_sequence_model,
21
+ )
22
+
23
+
24
+ def __getattr__(name):
25
+ # Training backends are optional and heavy; import them on first access.
26
+ if name in ("tf", "torch"):
27
+ try:
28
+ return importlib.import_module(f".{name}", __name__)
29
+ except ImportError as e:
30
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r} ({e})") from e
31
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,305 @@
1
+ """Causal transformer definition shared by the training backends and the step-program export.
2
+
3
+ A model is a dict of named float32 arrays (see ``parameter_shapes``). Training
4
+ backends hold the same arrays with a leading members axis and run them over
5
+ whole sequences. ``export_step_model`` turns one member into a format-v2
6
+ instruction model: the instruction list processes one token, and the state
7
+ declared under ``"sequence"`` carries the attention caches and feature scanner
8
+ from one token to the next.
9
+ """
10
+
11
+ from dataclasses import asdict, dataclass
12
+
13
+ import numpy as np
14
+
15
+ FORMAT_VERSION = 2
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class SegmentFeatures:
20
+ """Per-token ``(ordinal, index)`` of runs of tokens, each with a learned embedding table.
21
+
22
+ A token belongs to a run when it is in ``members``, or in ``openers`` and
23
+ the previous token is not a member (a sign before a digit). ``ordinal``
24
+ counts the runs started so far; ``index`` is the one-based place inside
25
+ the current run and 0 outside a run. Both clamp to ``rows - 1``.
26
+ """
27
+
28
+ members: tuple
29
+ openers: tuple = ()
30
+ rows: int = 16
31
+
32
+ def __post_init__(self):
33
+ object.__setattr__(self, "members", tuple(int(t) for t in self.members))
34
+ object.__setattr__(self, "openers", tuple(int(t) for t in self.openers))
35
+ if self.rows < 2 or not self.members:
36
+ raise ValueError("segment features need members and at least two rows")
37
+ if set(self.members) & set(self.openers):
38
+ raise ValueError("a token cannot be both a member and an opener")
39
+
40
+
41
+ @dataclass(frozen=True)
42
+ class TransformerConfig:
43
+ """Pre-norm causal transformer.
44
+
45
+ ``heads`` query heads share ``kv_heads`` key/value heads (grouped-query
46
+ attention; query head ``h`` uses key/value head ``h // (heads // kv_heads)``).
47
+ With ``loops`` above one the stack of ``layers`` blocks runs several times
48
+ with shared weights; ``inject`` re-adds the input embedding before each
49
+ later pass, ``pass_embed`` adds a learned vector per pass, ``pass_norms``
50
+ gives every pass its own layer-norm parameters, and ``lora`` is the rank
51
+ of a per-pass correction to each block's query/key/value projection.
52
+ """
53
+
54
+ vocab_size: int
55
+ width: int
56
+ heads: int
57
+ layers: int = 2
58
+ ff_width: int = 0
59
+ kv_heads: int = 0
60
+ loops: int = 1
61
+ inject: bool = False
62
+ pass_embed: bool = False
63
+ pass_norms: bool = False
64
+ lora: int = 0
65
+ tied: bool = True
66
+ features: SegmentFeatures | None = None
67
+ max_positions: int = 192
68
+ epsilon: float = 1e-5
69
+
70
+ def __post_init__(self):
71
+ if isinstance(self.features, dict):
72
+ object.__setattr__(self, "features", SegmentFeatures(**self.features))
73
+ object.__setattr__(self, "ff_width", self.ff_width or self.width)
74
+ object.__setattr__(self, "kv_heads", self.kv_heads or self.heads)
75
+ if self.vocab_size < 2 or self.width < 1 or self.layers < 1 or self.loops < 1:
76
+ raise ValueError("vocab_size, width, layers and loops must be positive")
77
+ if self.width % self.heads or self.heads % self.kv_heads:
78
+ raise ValueError("width must divide into heads, and heads into kv_heads")
79
+ if self.lora < 0 or self.max_positions < 1 or self.ff_width < 1:
80
+ raise ValueError("lora, max_positions and ff_width must be valid sizes")
81
+ if self.features is not None and max(self.features.members + self.features.openers) >= self.vocab_size:
82
+ raise ValueError("segment feature tokens must be vocabulary ids")
83
+
84
+ @property
85
+ def head_dim(self):
86
+ return self.width // self.heads
87
+
88
+ @property
89
+ def qkv_size(self):
90
+ return self.width + 2 * self.kv_heads * self.head_dim
91
+
92
+ @property
93
+ def norm_passes(self):
94
+ return self.loops if self.pass_norms else 1
95
+
96
+ def to_dict(self):
97
+ return asdict(self)
98
+
99
+
100
+ def parameter_shapes(config):
101
+ """Name -> shape of every learned array of one member; linear maps are ``[input, output]``."""
102
+ c, d = config, config.width
103
+ shapes = {"embedding": (c.vocab_size, d)}
104
+ if c.features is not None:
105
+ shapes["ordinal"] = (c.features.rows, d)
106
+ shapes["index"] = (c.features.rows, d)
107
+ if c.pass_embed:
108
+ shapes["passes"] = (c.loops, d)
109
+ for layer in range(c.layers):
110
+ prefix = f"blocks.{layer}."
111
+ shapes[prefix + "norm1"] = (c.norm_passes, 2, d) # Row 0 is the gain, row 1 the bias.
112
+ shapes[prefix + "qkv"] = (d, c.qkv_size)
113
+ shapes[prefix + "qkv_bias"] = (c.qkv_size,)
114
+ if c.lora:
115
+ shapes[prefix + "lora_a"] = (c.loops, d, c.lora)
116
+ shapes[prefix + "lora_b"] = (c.loops, c.lora, c.qkv_size)
117
+ shapes[prefix + "out"] = (d, d)
118
+ shapes[prefix + "out_bias"] = (d,)
119
+ shapes[prefix + "norm2"] = (c.norm_passes, 2, d)
120
+ shapes[prefix + "up"] = (d, c.ff_width)
121
+ shapes[prefix + "up_bias"] = (c.ff_width,)
122
+ shapes[prefix + "down"] = (c.ff_width, d)
123
+ shapes[prefix + "down_bias"] = (d,)
124
+ shapes["norm"] = (2, d)
125
+ if not c.tied:
126
+ shapes["readout"] = (d, c.vocab_size)
127
+ shapes["readout_bias"] = (c.vocab_size,)
128
+ return shapes
129
+
130
+
131
+ def parameter_count(config):
132
+ return sum(int(np.prod(shape)) for shape in parameter_shapes(config).values())
133
+
134
+
135
+ def init_parameters(config, seed=0, members=None):
136
+ """Initial float32 parameters; with ``members`` every array gains a leading members axis.
137
+
138
+ Tables and LoRA inputs are normal with scale ``width ** -0.5``; linear maps
139
+ are uniform in ``±1/sqrt(input)``; norms start as identity; biases, pass
140
+ vectors and LoRA outputs start at zero, so every pass starts shared.
141
+ """
142
+ rng = np.random.default_rng(seed)
143
+ lead = () if members is None else (members,)
144
+ d, out = config.width, {}
145
+ for name, shape in parameter_shapes(config).items():
146
+ leaf = name.rsplit(".", 1)[-1]
147
+ full = lead + shape
148
+ if leaf in ("embedding", "ordinal", "index", "lora_a"):
149
+ value = rng.standard_normal(full) * d ** -0.5
150
+ elif leaf in ("norm1", "norm2", "norm"):
151
+ value = np.zeros(full)
152
+ value[..., 0, :] = 1
153
+ elif leaf in ("qkv", "out", "up", "down", "readout"):
154
+ bound = shape[0] ** -0.5
155
+ value = rng.uniform(-bound, bound, full)
156
+ else:
157
+ value = np.zeros(full)
158
+ out[name] = value.astype(np.float32)
159
+ return out
160
+
161
+
162
+ def check_parameters(config, parameters):
163
+ shapes = parameter_shapes(config)
164
+ if set(parameters) != set(shapes):
165
+ raise ValueError(f"parameter names differ: {sorted(set(parameters) ^ set(shapes))}")
166
+ for name, shape in shapes.items():
167
+ if tuple(np.shape(parameters[name])) != shape:
168
+ raise ValueError(f"{name} has shape {np.shape(parameters[name])}, expected {shape}")
169
+
170
+
171
+ def segment_features(tokens, features):
172
+ """NumPy ``(ordinal, index)`` for token arrays shaped ``[..., time]``."""
173
+ tokens = np.asarray(tokens)
174
+
175
+ def shifted(mask):
176
+ return np.concatenate((np.zeros_like(mask[..., :1]), mask[..., :-1]), -1)
177
+
178
+ member = np.isin(tokens, features.members)
179
+ run = member | (np.isin(tokens, features.openers) & ~shifted(member))
180
+ start = run & ~shifted(run)
181
+ place = np.broadcast_to(np.arange(tokens.shape[-1]), tokens.shape)
182
+ last = np.maximum.accumulate(np.where(start, place, -1), -1)
183
+ index = np.where(run, place - last + 1, 0)
184
+ ordinal = np.cumsum(start, -1)
185
+ limit = features.rows - 1
186
+ return np.minimum(ordinal, limit), np.minimum(index, limit)
187
+
188
+
189
+ class _Program:
190
+ """Accumulates buffers, instructions and deduplicated weight and parameter entries."""
191
+
192
+ def __init__(self):
193
+ self.model = dict(buffer_sizes=[], instructions=[], maps=[], weights=[], bias=[], parameters=[])
194
+ self._entries = {}
195
+
196
+ def buffer(self, size):
197
+ self.model["buffer_sizes"].append(int(size))
198
+ return len(self.model["buffer_sizes"]) - 1
199
+
200
+ def emit(self, **instruction):
201
+ self.model["instructions"].append(instruction)
202
+ return instruction.get("output")
203
+
204
+ def weights(self, key, matrix, bias=None):
205
+ """Index of an ``[output][input]`` matrix, stored once per ``key``."""
206
+ if key not in self._entries:
207
+ matrix = np.asarray(matrix, dtype=np.float32)
208
+ bias = np.zeros(len(matrix), dtype=np.float32) if bias is None else np.asarray(bias, dtype=np.float32)
209
+ self.model["weights"].append(matrix.tolist())
210
+ self.model["bias"].append(bias.tolist())
211
+ self._entries[key] = len(self.model["weights"]) - 1
212
+ return self._entries[key]
213
+
214
+ def parameters(self, key, vector):
215
+ key = ("parameters", key)
216
+ if key not in self._entries:
217
+ self.model["parameters"].append(np.asarray(vector, dtype=np.float32).tolist())
218
+ self._entries[key] = len(self.model["parameters"]) - 1
219
+ return self._entries[key]
220
+
221
+
222
+ def export_step_model(config, parameters, vocabulary=None, metadata=None):
223
+ """Format-v2 instruction model that processes one token of one member.
224
+
225
+ Buffer 0 holds the token id and the last buffer the next-token logits.
226
+ Looped passes are unrolled; their instructions reference the same weight
227
+ entries, and every attention instruction owns one key/value cache slot.
228
+ """
229
+ check_parameters(config, parameters)
230
+ c, p, d = config, parameters, config.width
231
+ program = _Program()
232
+ token = program.buffer(1)
233
+
234
+ def linear(name, source, size, bias=None, activation=None, key=None):
235
+ index = program.weights(key or name, np.asarray(p[name]).T, None if bias is None else p[bias])
236
+ instruction = dict(type="DOT", input=source, output=program.buffer(size), weights=index)
237
+ if activation:
238
+ instruction["activation"] = activation
239
+ return program.emit(**instruction)
240
+
241
+ def norm(name, values, source):
242
+ gain = program.parameters(name + ".gain", values[0])
243
+ bias = program.parameters(name + ".bias", values[1])
244
+ return program.emit(type="LAYER_NORM", input=source, output=program.buffer(d), gain=gain, bias=bias,
245
+ epsilon=c.epsilon)
246
+
247
+ def add(*sources):
248
+ return program.emit(type="ADD_ELEMENTWISE_BUFFERS", input=list(sources), output=program.buffer(d))
249
+
250
+ # The token table doubles as the tied readout matrix, whose bias is the readout bias.
251
+ table = program.weights("embedding", p["embedding"], p["readout_bias"] if c.tied else None)
252
+ embedded = [program.emit(type="EMBEDDING", input=token, output=program.buffer(d), weights=table)]
253
+ if c.features is not None:
254
+ scanned = program.emit(type="SEGMENT_FEATURES", input=token, output=program.buffer(2),
255
+ members=list(c.features.members), openers=list(c.features.openers),
256
+ rows=c.features.rows, scanner=0)
257
+ for column, name in enumerate(("ordinal", "index")):
258
+ embedded.append(program.emit(type="EMBEDDING", input=scanned, internal_input_index=column,
259
+ output=program.buffer(d), weights=program.weights(name, p[name])))
260
+ source = add(*embedded) if len(embedded) > 1 else embedded[0]
261
+
262
+ x, slots = source, []
263
+ for step in range(c.loops):
264
+ if step and c.inject:
265
+ x = add(x, source)
266
+ if c.pass_embed:
267
+ if x == source: # The embedding stays intact for later injection.
268
+ x = program.emit(type="COPY", input=source, output=program.buffer(d), internal_index=0)
269
+ program.emit(type="ADD_ELEMENTWISE", input=x,
270
+ parameters=program.parameters(f"passes.{step}", p["passes"][step]))
271
+ row = step if c.pass_norms else 0
272
+ for layer in range(c.layers):
273
+ prefix = f"blocks.{layer}."
274
+ hidden = norm(f"{prefix}norm1.{row}", p[prefix + "norm1"][row], x)
275
+ qkv = linear(prefix + "qkv", hidden, c.qkv_size, prefix + "qkv_bias")
276
+ if c.lora:
277
+ low = program.emit(type="DOT", input=hidden, output=program.buffer(c.lora),
278
+ weights=program.weights(f"{prefix}lora_a.{step}", p[prefix + "lora_a"][step].T))
279
+ delta = program.emit(type="DOT", input=low, output=program.buffer(c.qkv_size),
280
+ weights=program.weights(f"{prefix}lora_b.{step}", p[prefix + "lora_b"][step].T))
281
+ qkv = program.emit(type="ADD_ELEMENTWISE_BUFFERS", input=[qkv, delta],
282
+ output=program.buffer(c.qkv_size))
283
+ attended = program.emit(type="CAUSAL_ATTENTION", input=qkv, output=program.buffer(d),
284
+ heads=c.heads, kv_heads=c.kv_heads, slot=len(slots))
285
+ slots.append(dict(heads=c.heads, kv_heads=c.kv_heads, head_dim=c.head_dim))
286
+ x = add(x, linear(prefix + "out", attended, d, prefix + "out_bias"))
287
+ hidden = norm(f"{prefix}norm2.{row}", p[prefix + "norm2"][row], x)
288
+ raised = linear(prefix + "up", hidden, c.ff_width, prefix + "up_bias", activation="GELU")
289
+ x = add(x, linear(prefix + "down", raised, d, prefix + "down_bias"))
290
+ final = norm("norm", p["norm"], x)
291
+ if c.tied:
292
+ program.emit(type="DOT", input=final, output=program.buffer(c.vocab_size), weights=table)
293
+ else:
294
+ linear("readout", final, c.vocab_size, "readout_bias")
295
+
296
+ model = program.model
297
+ model["format_version"] = FORMAT_VERSION
298
+ model["features"] = ["token"]
299
+ model["sequence"] = dict(
300
+ vocabulary=list(vocabulary) if vocabulary is not None else [str(i) for i in range(c.vocab_size)],
301
+ max_positions=c.max_positions, kv_slots=slots, scanners=int(c.features is not None),
302
+ config=c.to_dict(), metadata=dict(metadata or {}))
303
+ if len(model["sequence"]["vocabulary"]) != c.vocab_size:
304
+ raise ValueError("vocabulary length differs from vocab_size")
305
+ return model
@@ -0,0 +1,70 @@
1
+ """NumPy reference for sequence models: token-by-token steps with explicit state.
2
+
3
+ This is the ground truth the training backends and the Rust runtime are
4
+ checked against. It runs the exported instruction list once per token on
5
+ float32 arrays, for several sequences at a time.
6
+ """
7
+
8
+ import numpy as np
9
+
10
+ from ..instruction_model import instruction_model_inference, model_arrays
11
+
12
+
13
+ def new_state(model, sequences=1):
14
+ """Empty state for ``sequences`` parallel sequences: attention caches, scanners and position."""
15
+ info = model["sequence"]
16
+ capacity = info["max_positions"]
17
+
18
+ def cache(slot):
19
+ return np.zeros((sequences, capacity, slot["kv_heads"], slot["head_dim"]), dtype=np.float32)
20
+
21
+ def scanner():
22
+ return dict(prev_member=np.zeros(sequences, dtype=bool), prev_run=np.zeros(sequences, dtype=bool),
23
+ index=np.zeros(sequences, dtype=np.int64), ordinal=np.zeros(sequences, dtype=np.int64))
24
+
25
+ return dict(position=0, kv=[dict(keys=cache(slot), values=cache(slot)) for slot in info["kv_slots"]],
26
+ scanners=[scanner() for _ in range(info["scanners"])])
27
+
28
+
29
+ class SequenceSession:
30
+ """Steps a format-v2 sequence model; ``state`` may be copied, kept and restored by the caller."""
31
+
32
+ def __init__(self, model, sequences=1):
33
+ if model.get("format_version") != 2 or "sequence" not in model:
34
+ raise ValueError("not a format-v2 sequence model")
35
+ self.model, self.sequences = model, sequences
36
+ self._arrays = model_arrays(model)
37
+ self.state = new_state(model, sequences)
38
+
39
+ def step(self, tokens):
40
+ """Feeds one token id per sequence and returns next-token logits ``[sequences, vocabulary]``."""
41
+ tokens = np.asarray(tokens, dtype=np.float32).reshape(self.sequences, 1)
42
+ logits = instruction_model_inference(self.model, [tokens], self.state, self._arrays)[-1]
43
+ self.state["position"] += 1
44
+ return logits
45
+
46
+ def run(self, tokens):
47
+ """Logits ``[sequences, time, vocabulary]`` for whole token arrays ``[sequences, time]``."""
48
+ tokens = np.asarray(tokens).reshape(self.sequences, -1)
49
+ return np.stack([self.step(tokens[:, t]) for t in range(tokens.shape[1])], 1)
50
+
51
+
52
+ def add_validation_sequences(model, sequences):
53
+ """Stores token sequences with the reference logits, for runtimes to replay on load."""
54
+ model["validation_sequences"] = [
55
+ dict(tokens=[int(t) for t in tokens], logits=SequenceSession(model).run(tokens)[0].tolist())
56
+ for tokens in sequences
57
+ ]
58
+ return model
59
+
60
+
61
+ def validate_sequence_model(model, atol=1e-4):
62
+ """Replays ``validation_sequences``; returns the largest logit error or raises on a mismatch."""
63
+ worst = 0.0
64
+ for case in model.get("validation_sequences", []):
65
+ expected = np.asarray(case["logits"], dtype=np.float32)
66
+ actual = SequenceSession(model).run(case["tokens"])[0]
67
+ worst = max(worst, float(np.abs(actual - expected).max()))
68
+ if worst > atol or (actual.argmax(-1) != expected.argmax(-1)).any():
69
+ raise AssertionError(f"sequence model differs from its validation logits (max error {worst})")
70
+ return worst