braintrace 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {braintrace-0.2.2 → braintrace-0.2.4}/PKG-INFO +2 -1
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/__init__.py +13 -3
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/__init___test.py +13 -4
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/__init__.py +2 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/_common.py +29 -22
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/base.py +45 -8
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/d_rtrl.py +2 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/d_rtrl_test.py +103 -9
- braintrace-0.2.4/braintrace/_algorithm/e_prop.py +310 -0
- braintrace-0.2.4/braintrace/_algorithm/e_prop_test.py +445 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/graph_executor.py +20 -5
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/graph_executor_test.py +21 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/io_dim_vjp.py +68 -20
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/oracle.py +54 -0
- braintrace-0.2.4/braintrace/_algorithm/oracle_models.py +467 -0
- braintrace-0.2.4/braintrace/_algorithm/oracle_test.py +515 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/ostl.py +28 -7
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/ostl_test.py +115 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/osttp.py +37 -10
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/osttp_test.py +43 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/otpe.py +185 -71
- braintrace-0.2.4/braintrace/_algorithm/otpe_test.py +423 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/ottt.py +125 -27
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/ottt_test.py +118 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/param_dim_vjp.py +423 -184
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/param_dim_vjp_test.py +409 -1
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/pp_prop.py +2 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/tests/conv_vmap_correctness_test.py +11 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/tests/exact_correctness_test.py +63 -7
- braintrace-0.2.4/braintrace/_algorithm/tests/scan_descent_support_test.py +353 -0
- braintrace-0.2.4/braintrace/_algorithm/tests/while_support_test.py +256 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/vjp_base.py +87 -17
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/vjp_graph_executor.py +143 -36
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compatible_imports.py +23 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compile.py +18 -13
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compile_test.py +11 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/__init__.py +7 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/base.py +99 -12
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/base_test.py +178 -3
- braintrace-0.2.4/braintrace/_compiler/canonicalize.py +1068 -0
- braintrace-0.2.4/braintrace/_compiler/canonicalize_test.py +1266 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/diagnostics.py +40 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/diagnostics_test.py +10 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/graph.py +111 -6
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/graph_test.py +81 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/hid_param_op.py +211 -88
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/hidden_group.py +470 -45
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/hidden_group_test.py +486 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/hidden_pertubation.py +186 -39
- braintrace-0.2.4/braintrace/_compiler/hidden_pertubation_test.py +446 -0
- braintrace-0.2.4/braintrace/_compiler/jaxpr_graph.py +312 -0
- braintrace-0.2.4/braintrace/_compiler/jaxpr_graph_test.py +320 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/module_info.py +60 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/module_info_test.py +29 -0
- braintrace-0.2.4/braintrace/_compiler/scan_descent.py +592 -0
- braintrace-0.2.4/braintrace/_compiler/scan_descent_test.py +484 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/scenario_catalog.py +326 -3
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/scenario_catalog_test.py +503 -10
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_grad_exponential.py +4 -2
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_input_data.py +10 -8
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_legacy/_ops.py +2 -2
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_legacy/_ops_test.py +2 -2
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_misc.py +62 -11
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_misc_test.py +53 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_op/__init__.py +28 -2
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_op/_primitive.py +99 -30
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_op/_primitive_test.py +160 -17
- braintrace-0.2.4/braintrace/_op/_registries.py +403 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_op/_registries_test.py +91 -5
- braintrace-0.2.4/braintrace/_op/conv.py +899 -0
- braintrace-0.2.4/braintrace/_op/conv_test.py +1042 -0
- braintrace-0.2.4/braintrace/_op/dense.py +789 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_op/dense_test.py +430 -28
- braintrace-0.2.4/braintrace/_op/einsum.py +340 -0
- braintrace-0.2.4/braintrace/_op/einsum_test.py +496 -0
- braintrace-0.2.4/braintrace/_op/elemwise.py +479 -0
- braintrace-0.2.4/braintrace/_op/elemwise_test.py +485 -0
- braintrace-0.2.4/braintrace/_op/embedding.py +292 -0
- braintrace-0.2.4/braintrace/_op/embedding_test.py +379 -0
- braintrace-0.2.4/braintrace/_op/grouped.py +441 -0
- braintrace-0.2.4/braintrace/_op/grouped_test.py +496 -0
- braintrace-0.2.4/braintrace/_op/lora.py +673 -0
- braintrace-0.2.4/braintrace/_op/lora_test.py +749 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_op/op_rule_oracle_test.py +16 -10
- braintrace-0.2.4/braintrace/_op/sparse.py +536 -0
- braintrace-0.2.4/braintrace/_op/sparse_test.py +886 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_typing.py +7 -1
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_version.py +1 -1
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/nn/__init__.py +13 -5
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/nn/_conv.py +6 -1
- braintrace-0.2.4/braintrace/nn/_embedding.py +81 -0
- braintrace-0.2.4/braintrace/nn/_embedding_test.py +75 -0
- braintrace-0.2.4/braintrace/nn/_linear.py +449 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/nn/_linear_test.py +269 -13
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/nn/_readout.py +10 -6
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/nn/_rnn.py +52 -50
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace.egg-info/PKG-INFO +2 -1
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace.egg-info/SOURCES.txt +16 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace.egg-info/requires.txt +1 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/pyproject.toml +43 -0
- braintrace-0.2.2/braintrace/_algorithm/e_prop.py +0 -245
- braintrace-0.2.2/braintrace/_algorithm/e_prop_test.py +0 -218
- braintrace-0.2.2/braintrace/_algorithm/oracle_models.py +0 -195
- braintrace-0.2.2/braintrace/_algorithm/oracle_test.py +0 -175
- braintrace-0.2.2/braintrace/_algorithm/otpe_test.py +0 -215
- braintrace-0.2.2/braintrace/_compiler/hidden_pertubation_test.py +0 -126
- braintrace-0.2.2/braintrace/_op/_registries.py +0 -136
- braintrace-0.2.2/braintrace/_op/conv.py +0 -528
- braintrace-0.2.2/braintrace/_op/conv_test.py +0 -526
- braintrace-0.2.2/braintrace/_op/dense.py +0 -445
- braintrace-0.2.2/braintrace/_op/elemwise.py +0 -255
- braintrace-0.2.2/braintrace/_op/elemwise_test.py +0 -244
- braintrace-0.2.2/braintrace/_op/lora.py +0 -389
- braintrace-0.2.2/braintrace/_op/lora_test.py +0 -407
- braintrace-0.2.2/braintrace/_op/sparse.py +0 -354
- braintrace-0.2.2/braintrace/_op/sparse_test.py +0 -454
- braintrace-0.2.2/braintrace/nn/_linear.py +0 -267
- {braintrace-0.2.2 → braintrace-0.2.4}/LICENSE +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/README.md +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/_common_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/base_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/io_dim_vjp_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/pp_prop_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/tests/approx_correctness_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/tests/cross_check_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/tests/diagnostic_exploration_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/tests/public_api_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/tests/scan_fusion_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/tests/transform_correctness_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/vjp_base_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_algorithm/vjp_graph_executor_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compatible_imports_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/hid_param_op_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/model4test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/report.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/report_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/tests/cell_relation_guardrail_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/tests/compiler_oracle_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_compiler/tests/compiler_property_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_docs_examples_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_etrace_model_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_grad_exponential_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_input_data_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_legacy/__init__.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_legacy/_params.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_legacy/_params_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_op/op_rule_oracle.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_state_managment.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/_state_managment_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/nn/_conv_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/nn/_readout_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/nn/_rnn_test.py +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace/py.typed +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace.egg-info/dependency_links.txt +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/braintrace.egg-info/top_level.txt +0 -0
- {braintrace-0.2.2 → braintrace-0.2.4}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: braintrace
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: Enabling Scalable Online Learning for Brain Dynamics.
|
|
5
5
|
Author-email: BrainTrace Developers <chao.brain@qq.com>
|
|
6
6
|
License: Apache-2.0 license
|
|
@@ -28,6 +28,7 @@ Description-Content-Type: text/markdown
|
|
|
28
28
|
License-File: LICENSE
|
|
29
29
|
Requires-Dist: brainstate>=0.5.2
|
|
30
30
|
Requires-Dist: brainunit
|
|
31
|
+
Requires-Dist: brainevent
|
|
31
32
|
Requires-Dist: brainpy.state
|
|
32
33
|
Requires-Dist: braintools
|
|
33
34
|
Provides-Extra: cpu
|
|
@@ -66,7 +66,9 @@ Examples
|
|
|
66
66
|
"""
|
|
67
67
|
|
|
68
68
|
|
|
69
|
-
from
|
|
69
|
+
from __future__ import annotations
|
|
70
|
+
|
|
71
|
+
from typing import TYPE_CHECKING, Any
|
|
70
72
|
|
|
71
73
|
from . import nn
|
|
72
74
|
from ._compile import compile
|
|
@@ -92,6 +94,7 @@ from ._algorithm import (
|
|
|
92
94
|
PresynapticTrace,
|
|
93
95
|
)
|
|
94
96
|
from ._compiler import (
|
|
97
|
+
ControlFlowPolicy,
|
|
95
98
|
ETraceGraph,
|
|
96
99
|
compile_etrace_graph,
|
|
97
100
|
HiddenParamOpRelation,
|
|
@@ -113,6 +116,9 @@ from ._compiler import (
|
|
|
113
116
|
from ._op import (
|
|
114
117
|
ETPPrimitive,
|
|
115
118
|
matmul,
|
|
119
|
+
grouped_matmul,
|
|
120
|
+
embedding,
|
|
121
|
+
einsum,
|
|
116
122
|
element_wise,
|
|
117
123
|
conv,
|
|
118
124
|
sparse_matmul,
|
|
@@ -166,6 +172,9 @@ __all__ = [
|
|
|
166
172
|
|
|
167
173
|
# ETP primitives (user API)
|
|
168
174
|
'matmul',
|
|
175
|
+
'grouped_matmul',
|
|
176
|
+
'embedding',
|
|
177
|
+
'einsum',
|
|
169
178
|
'element_wise',
|
|
170
179
|
'conv',
|
|
171
180
|
'sparse_matmul',
|
|
@@ -183,6 +192,7 @@ __all__ = [
|
|
|
183
192
|
'ETraceGraphExecutor',
|
|
184
193
|
|
|
185
194
|
# compiler
|
|
195
|
+
'ControlFlowPolicy',
|
|
186
196
|
'ETraceGraph',
|
|
187
197
|
'compile_etrace_graph',
|
|
188
198
|
'HiddenGroup',
|
|
@@ -244,7 +254,7 @@ _DEPRECATED_LEGACY = {
|
|
|
244
254
|
}
|
|
245
255
|
|
|
246
256
|
|
|
247
|
-
def __getattr__(name):
|
|
257
|
+
def __getattr__(name: str) -> Any:
|
|
248
258
|
if name in _DEPRECATED_LEGACY:
|
|
249
259
|
import warnings
|
|
250
260
|
warnings.warn(
|
|
@@ -258,5 +268,5 @@ def __getattr__(name):
|
|
|
258
268
|
raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
|
|
259
269
|
|
|
260
270
|
|
|
261
|
-
def __dir__():
|
|
271
|
+
def __dir__() -> list[str]:
|
|
262
272
|
return sorted(list(__all__) + list(_DEPRECATED_LEGACY))
|
|
@@ -148,7 +148,14 @@ def test_weight_used_inside_scan_raises_not_implemented():
|
|
|
148
148
|
within a control-flow op. Previously (F-SCAN-WEIGHT) the message construction
|
|
149
149
|
indexed the hidden-path map with the weight var and a KeyError escaped instead;
|
|
150
150
|
the message now renders the weight variable directly, so the intended
|
|
151
|
-
NotImplementedError surfaces.
|
|
151
|
+
NotImplementedError surfaces.
|
|
152
|
+
|
|
153
|
+
Since Phase 2 canonicalization, statically short scans are unrolled at
|
|
154
|
+
extraction time instead of erroring, so this uses a length above
|
|
155
|
+
``ControlFlowPolicy.scan_unroll_limit`` to reach the unsupported-op path
|
|
156
|
+
(after a SCAN_UNROLL_SKIPPED warning). Since Phase 4, structured scan
|
|
157
|
+
descent handles over-limit ETP scans by default, so this pins the
|
|
158
|
+
opt-out path via ``ControlFlowPolicy(scan_descent='off')``."""
|
|
152
159
|
|
|
153
160
|
class ScanModel(brainstate.nn.Module):
|
|
154
161
|
def __init__(self):
|
|
@@ -160,15 +167,17 @@ def test_weight_used_inside_scan_raises_not_implemented():
|
|
|
160
167
|
def update(self, x):
|
|
161
168
|
def body(carry, _):
|
|
162
169
|
return braintrace.matmul(carry, self.w.value), None # weight inside scan
|
|
163
|
-
out, _ = jax.lax.scan(body, self.h.value, xs=None, length=
|
|
170
|
+
out, _ = jax.lax.scan(body, self.h.value, xs=None, length=64)
|
|
164
171
|
self.h.value = jax.nn.tanh(out)
|
|
165
172
|
return self.h.value
|
|
166
173
|
|
|
167
174
|
model = ScanModel()
|
|
168
175
|
brainstate.nn.init_all_states(model, batch_size=1)
|
|
169
|
-
algo = braintrace.D_RTRL(
|
|
176
|
+
algo = braintrace.D_RTRL(
|
|
177
|
+
model, control_flow=braintrace.ControlFlowPolicy(scan_descent='off'))
|
|
170
178
|
with pytest.raises(NotImplementedError):
|
|
171
|
-
|
|
179
|
+
with pytest.warns(UserWarning, match='NOT unrolled'):
|
|
180
|
+
algo.compile_graph(jnp.ones((1, 4), dtype='float32'))
|
|
172
181
|
|
|
173
182
|
|
|
174
183
|
# --- Task 6: legacy + nn deprecations emit DeprecationWarning ----------------
|
|
@@ -21,6 +21,8 @@ and paper-faithful SNN algorithms (EProp, OSTLRecurrent, OSTLFeedforward, OTPE,
|
|
|
21
21
|
OTTT, OSTTP).
|
|
22
22
|
"""
|
|
23
23
|
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
24
26
|
from ._common import FixedRandomFeedback, KappaFilter, PresynapticTrace
|
|
25
27
|
from .base import ETraceAlgorithm, EligibilityTrace
|
|
26
28
|
from .d_rtrl import D_RTRL
|
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
|
|
16
16
|
"""Shared helpers for the SNN online-learning algorithms."""
|
|
17
17
|
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
18
20
|
from functools import partial
|
|
19
21
|
from typing import Any, Dict, Optional
|
|
20
22
|
|
|
@@ -23,7 +25,7 @@ import jax
|
|
|
23
25
|
import jax.numpy as jnp
|
|
24
26
|
import brainunit as u
|
|
25
27
|
|
|
26
|
-
from braintrace._typing import PyTree
|
|
28
|
+
from braintrace._typing import ArrayLike, PyTree
|
|
27
29
|
|
|
28
30
|
__all__ = [
|
|
29
31
|
'PresynapticTrace',
|
|
@@ -42,12 +44,12 @@ class _ZeroResetState(brainstate.ShortTermState):
|
|
|
42
44
|
for ``batch_size`` (or prepended for scalar-shaped states).
|
|
43
45
|
"""
|
|
44
46
|
|
|
45
|
-
def __init__(self, init_value):
|
|
47
|
+
def __init__(self, init_value: Any) -> None:
|
|
46
48
|
super().__init__(init_value)
|
|
47
49
|
self._init_shape = jnp.shape(init_value)
|
|
48
50
|
self._init_dtype = init_value.dtype
|
|
49
51
|
|
|
50
|
-
def reset_state(self, batch_size: Optional[int] = None, **kwargs):
|
|
52
|
+
def reset_state(self, batch_size: Optional[int] = None, **kwargs: Any) -> None:
|
|
51
53
|
"""Re-zero the state at its original shape.
|
|
52
54
|
|
|
53
55
|
Parameters
|
|
@@ -105,13 +107,13 @@ class PresynapticTrace(_ZeroResetState):
|
|
|
105
107
|
|
|
106
108
|
__module__ = 'braintrace'
|
|
107
109
|
|
|
108
|
-
def __init__(self, init_value, leak: float):
|
|
110
|
+
def __init__(self, init_value: Any, leak: float) -> None:
|
|
109
111
|
super().__init__(init_value)
|
|
110
112
|
if not (0.0 < leak < 1.0):
|
|
111
113
|
raise ValueError(f'leak must be in (0, 1); got {leak}')
|
|
112
114
|
self.leak = float(leak)
|
|
113
115
|
|
|
114
|
-
def update(self, x):
|
|
116
|
+
def update(self, x: ArrayLike) -> Any:
|
|
115
117
|
r"""Apply one accumulation step :math:`\hat{a} \leftarrow \lambda \cdot \hat{a} + x`.
|
|
116
118
|
|
|
117
119
|
Parameters
|
|
@@ -129,9 +131,14 @@ class PresynapticTrace(_ZeroResetState):
|
|
|
129
131
|
|
|
130
132
|
|
|
131
133
|
class KappaFilter(_ZeroResetState):
|
|
132
|
-
r"""Low-pass
|
|
134
|
+
r"""Low-pass filter helper state.
|
|
135
|
+
|
|
136
|
+
:class:`~braintrace.EProp` no longer uses this class directly — it filters
|
|
137
|
+
the eligibility trace internally instead. ``KappaFilter`` remains public
|
|
138
|
+
and available for user-side filtering of an output-side (or any other)
|
|
139
|
+
signal outside the algorithm's own hooks.
|
|
133
140
|
|
|
134
|
-
The filter smooths the
|
|
141
|
+
The filter smooths the signal following
|
|
135
142
|
:math:`x_{\mathrm{filt}} \leftarrow (1-\kappa) \cdot x + \kappa \cdot x_{\mathrm{filt}}`.
|
|
136
143
|
|
|
137
144
|
Parameters
|
|
@@ -163,13 +170,13 @@ class KappaFilter(_ZeroResetState):
|
|
|
163
170
|
|
|
164
171
|
__module__ = 'braintrace'
|
|
165
172
|
|
|
166
|
-
def __init__(self, init_value, kappa: float):
|
|
173
|
+
def __init__(self, init_value: Any, kappa: float) -> None:
|
|
167
174
|
super().__init__(init_value)
|
|
168
175
|
if not (0.0 <= kappa < 1.0):
|
|
169
176
|
raise ValueError(f'kappa must be in [0, 1); got {kappa}')
|
|
170
177
|
self.kappa = float(kappa)
|
|
171
178
|
|
|
172
|
-
def update(self, x):
|
|
179
|
+
def update(self, x: ArrayLike) -> Any:
|
|
173
180
|
r"""Apply one low-pass step :math:`x_{\mathrm{filt}} \leftarrow (1-\kappa) x + \kappa\, x_{\mathrm{filt}}`.
|
|
174
181
|
|
|
175
182
|
Parameters
|
|
@@ -230,14 +237,14 @@ class FixedRandomFeedback:
|
|
|
230
237
|
|
|
231
238
|
__module__ = 'braintrace'
|
|
232
239
|
|
|
233
|
-
def __init__(self, n_target: int, n_layer: int, key, init_scale: float = 0.1):
|
|
240
|
+
def __init__(self, n_target: int, n_layer: int, key: Any, init_scale: float = 0.1) -> None:
|
|
234
241
|
self.B = jax.lax.stop_gradient(
|
|
235
|
-
init_scale *
|
|
242
|
+
init_scale * brainstate.random.normal(size=(n_target, n_layer), key=key)
|
|
236
243
|
)
|
|
237
244
|
self.n_target = int(n_target)
|
|
238
245
|
self.n_layer = int(n_layer)
|
|
239
246
|
|
|
240
|
-
def project(self, y_target):
|
|
247
|
+
def project(self, y_target: Any) -> Any:
|
|
241
248
|
"""Project the target onto the frozen feedback matrix.
|
|
242
249
|
|
|
243
250
|
Parameters
|
|
@@ -267,7 +274,7 @@ def extract_y_target(args: tuple, *, index: int = -1) -> Optional[jax.Array]:
|
|
|
267
274
|
def _reset_state_in_a_dict(
|
|
268
275
|
state_dict: Dict[Any, brainstate.State],
|
|
269
276
|
batch_size: Optional[int],
|
|
270
|
-
):
|
|
277
|
+
) -> None:
|
|
271
278
|
"""
|
|
272
279
|
Reset the values in a dictionary of states to zero.
|
|
273
280
|
|
|
@@ -293,7 +300,7 @@ def _reset_state_in_a_dict(
|
|
|
293
300
|
def _zeros_like_batch_or_not(
|
|
294
301
|
batch_size: Optional[int],
|
|
295
302
|
x: jax.Array
|
|
296
|
-
):
|
|
303
|
+
) -> Any:
|
|
297
304
|
"""
|
|
298
305
|
Create a zeros array with the same shape and type as the input array,
|
|
299
306
|
optionally including a batch dimension.
|
|
@@ -325,7 +332,7 @@ def _batched_zeros_like(
|
|
|
325
332
|
batch_size: Optional[int],
|
|
326
333
|
num_state: int, # the number of hidden states
|
|
327
334
|
x: jax.Array # the input array
|
|
328
|
-
):
|
|
335
|
+
) -> Any:
|
|
329
336
|
"""
|
|
330
337
|
Create a batched zeros array with the same shape as the input array,
|
|
331
338
|
extended by the number of hidden states.
|
|
@@ -354,7 +361,7 @@ def _batched_zeros_like(
|
|
|
354
361
|
return u.math.zeros((batch_size, *x.shape, num_state), x.dtype)
|
|
355
362
|
|
|
356
363
|
|
|
357
|
-
def _sum_dim(xs: jax.Array, axis: int = -1):
|
|
364
|
+
def _sum_dim(xs: jax.Array, axis: int = -1) -> Any:
|
|
358
365
|
"""
|
|
359
366
|
Sums the elements along the last dimension of each array in a PyTree.
|
|
360
367
|
|
|
@@ -373,7 +380,7 @@ def _sum_dim(xs: jax.Array, axis: int = -1):
|
|
|
373
380
|
return jax.tree.map(lambda x: u.math.sum(x, axis=axis), xs)
|
|
374
381
|
|
|
375
382
|
|
|
376
|
-
def _unit_safe_add(a, b):
|
|
383
|
+
def _unit_safe_add(a: Any, b: Any) -> Any:
|
|
377
384
|
"""Add two leaves, stripping units only when one side has units and the other does not.
|
|
378
385
|
|
|
379
386
|
Gradient contributions for the same weight may come from paths that
|
|
@@ -390,7 +397,7 @@ def _unit_safe_add(a, b):
|
|
|
390
397
|
return u.math.add(a, b)
|
|
391
398
|
|
|
392
399
|
|
|
393
|
-
def _extract_leaf(pytree_val: PyTree, leaf_idx: int):
|
|
400
|
+
def _extract_leaf(pytree_val: PyTree, leaf_idx: int) -> Any:
|
|
394
401
|
"""Return the leaf at ``leaf_idx`` in ``jax.tree.leaves(pytree_val)``.
|
|
395
402
|
|
|
396
403
|
Bare arrays (treedef with a single leaf) return the array unchanged.
|
|
@@ -409,7 +416,7 @@ def _extract_leaf(pytree_val: PyTree, leaf_idx: int):
|
|
|
409
416
|
def _wrap_leaves_as_pytree(
|
|
410
417
|
reference_pytree: PyTree,
|
|
411
418
|
leaf_grads: Dict[int, jax.Array],
|
|
412
|
-
):
|
|
419
|
+
) -> Any:
|
|
413
420
|
"""Build a pytree matching ``reference_pytree`` with ``leaf_grads``
|
|
414
421
|
inserted at the given leaf indices; any other leaf is zero-filled.
|
|
415
422
|
|
|
@@ -441,7 +448,7 @@ def _wrap_leaves_as_pytree(
|
|
|
441
448
|
|
|
442
449
|
|
|
443
450
|
def _route_grads_by_path(
|
|
444
|
-
relation,
|
|
451
|
+
relation: Any,
|
|
445
452
|
per_key_grads: Dict[str, jax.Array],
|
|
446
453
|
weight_vals: Dict[Any, PyTree],
|
|
447
454
|
target_dict: Dict[Any, PyTree],
|
|
@@ -449,7 +456,7 @@ def _route_grads_by_path(
|
|
|
449
456
|
"""Route per-key gradients from a dict-API rule into per-path pytrees.
|
|
450
457
|
|
|
451
458
|
Both D-RTRL and ES-D-RTRL share this bookkeeping: for each key in
|
|
452
|
-
``per_key_grads`` (returned by ``xy_to_dw`` or ``
|
|
459
|
+
``per_key_grads`` (returned by ``xy_to_dw`` or ``dt_to_t``), look up the
|
|
453
460
|
owning ``ParamState`` path and the leaf index from ``relation``, accumulate
|
|
454
461
|
into ``per_path``, then wrap with ``_wrap_leaves_as_pytree`` and merge into
|
|
455
462
|
``target_dict`` via ``_update_dict``.
|
|
@@ -478,7 +485,7 @@ def _update_dict(
|
|
|
478
485
|
key: Any,
|
|
479
486
|
value: PyTree,
|
|
480
487
|
error_when_no_key: Optional[bool] = False
|
|
481
|
-
):
|
|
488
|
+
) -> None:
|
|
482
489
|
"""Update the dictionary.
|
|
483
490
|
|
|
484
491
|
If the key exists, then add the value to the existing value.
|
|
@@ -19,6 +19,8 @@
|
|
|
19
19
|
|
|
20
20
|
# -*- coding: utf-8 -*-
|
|
21
21
|
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
22
24
|
from typing import Dict, Any, Optional
|
|
23
25
|
|
|
24
26
|
import brainstate
|
|
@@ -79,8 +81,13 @@ class ETraceAlgorithm(brainstate.nn.Module):
|
|
|
79
81
|
The other states.
|
|
80
82
|
is_compiled : bool
|
|
81
83
|
Whether the etrace algorithm has been compiled.
|
|
82
|
-
running_index : brainstate.
|
|
83
|
-
|
|
84
|
+
running_index : brainstate.LongTermState[int]
|
|
85
|
+
Zero-based count of previously-completed ``update()`` calls: it reads
|
|
86
|
+
``0`` during the first call, ``1`` during the second, and so on --
|
|
87
|
+
i.e. the value used *inside* a given ``update()`` is the
|
|
88
|
+
pre-increment count, not the 1-based call number. It is incremented
|
|
89
|
+
by one immediately after each ``update()`` completes, and reset back
|
|
90
|
+
to ``0`` by ``reset_state()``.
|
|
84
91
|
"""
|
|
85
92
|
__module__ = 'braintrace'
|
|
86
93
|
|
|
@@ -89,7 +96,7 @@ class ETraceAlgorithm(brainstate.nn.Module):
|
|
|
89
96
|
model: brainstate.nn.Module,
|
|
90
97
|
graph_executor: ETraceGraphExecutor,
|
|
91
98
|
name: Optional[str] = None,
|
|
92
|
-
):
|
|
99
|
+
) -> None:
|
|
93
100
|
super().__init__(name=name) # type: ignore[call-arg] # brainstate hides Module.__init__ from type checkers
|
|
94
101
|
|
|
95
102
|
# the model
|
|
@@ -210,7 +217,7 @@ class ETraceAlgorithm(brainstate.nn.Module):
|
|
|
210
217
|
assert self._other_states is not None
|
|
211
218
|
return self._other_states
|
|
212
219
|
|
|
213
|
-
def _split_state(self):
|
|
220
|
+
def _split_state(self) -> None:
|
|
214
221
|
# --- the state separation --- #
|
|
215
222
|
#
|
|
216
223
|
# [NOTE]
|
|
@@ -228,7 +235,7 @@ class ETraceAlgorithm(brainstate.nn.Module):
|
|
|
228
235
|
self._other_states
|
|
229
236
|
) = self.graph.module_info.retrieved_model_states.split(brainstate.ParamState, brainstate.HiddenState, ...)
|
|
230
237
|
|
|
231
|
-
def compile_graph(self, *args) -> None:
|
|
238
|
+
def compile_graph(self, *args: Any) -> None:
|
|
232
239
|
r"""
|
|
233
240
|
Compile the eligibility trace graph of the relationship between etrace weights, states and operators.
|
|
234
241
|
|
|
@@ -265,6 +272,36 @@ class ETraceAlgorithm(brainstate.nn.Module):
|
|
|
265
272
|
# --- the model etrace graph -- #
|
|
266
273
|
self.graph_executor.compile_graph(*args)
|
|
267
274
|
|
|
275
|
+
# Structured scan descent (Phase 4): relations discovered inside
|
|
276
|
+
# a descended scan body need a per-substep trace fold, and
|
|
277
|
+
# hidden groups descended from a scan body carry a leading
|
|
278
|
+
# substep axis on their Jacobians — both only handled by
|
|
279
|
+
# algorithms declaring ``_supports_scan_descent = True``. A
|
|
280
|
+
# descended *group* can exist without any descended relation
|
|
281
|
+
# (all body weights routed through plain ops), so check both.
|
|
282
|
+
# Gate here — after the graph is built, before any trace state
|
|
283
|
+
# is initialized against it.
|
|
284
|
+
if (
|
|
285
|
+
(
|
|
286
|
+
any(
|
|
287
|
+
r.control_flow_context is not None
|
|
288
|
+
for r in self.graph.hidden_param_op_relations
|
|
289
|
+
)
|
|
290
|
+
or any(
|
|
291
|
+
g.descent is not None
|
|
292
|
+
for g in self.graph.hidden_groups
|
|
293
|
+
)
|
|
294
|
+
)
|
|
295
|
+
and not getattr(self, '_supports_scan_descent', False)
|
|
296
|
+
):
|
|
297
|
+
raise NotImplementedError(
|
|
298
|
+
f'{type(self).__name__} does not support structured scan '
|
|
299
|
+
f'descent yet (an ETP relation or hidden group was '
|
|
300
|
+
f'discovered inside a scan body). Use an algorithm that '
|
|
301
|
+
f'supports scan descent, or set '
|
|
302
|
+
f"ControlFlowPolicy(scan_descent='off')."
|
|
303
|
+
)
|
|
304
|
+
|
|
268
305
|
# --- the initialization of the states --- #
|
|
269
306
|
self.init_etrace_state(*args)
|
|
270
307
|
|
|
@@ -318,7 +355,7 @@ class ETraceAlgorithm(brainstate.nn.Module):
|
|
|
318
355
|
"""
|
|
319
356
|
return self.graph_executor.show_graph(verbose=verbose, return_msg=return_msg)
|
|
320
357
|
|
|
321
|
-
def __call__(self, *args) -> Any:
|
|
358
|
+
def __call__(self, *args: Any) -> Any:
|
|
322
359
|
"""
|
|
323
360
|
Update the model and the eligibility trace states.
|
|
324
361
|
|
|
@@ -334,7 +371,7 @@ class ETraceAlgorithm(brainstate.nn.Module):
|
|
|
334
371
|
"""
|
|
335
372
|
return self.update(*args)
|
|
336
373
|
|
|
337
|
-
def update(self, *args) -> Any:
|
|
374
|
+
def update(self, *args: Any) -> Any:
|
|
338
375
|
"""
|
|
339
376
|
Update the model and the eligibility trace states.
|
|
340
377
|
|
|
@@ -355,7 +392,7 @@ class ETraceAlgorithm(brainstate.nn.Module):
|
|
|
355
392
|
"""
|
|
356
393
|
raise NotImplementedError
|
|
357
394
|
|
|
358
|
-
def init_etrace_state(self, *args, **kwargs) -> None:
|
|
395
|
+
def init_etrace_state(self, *args: Any, **kwargs: Any) -> None:
|
|
359
396
|
"""
|
|
360
397
|
Initialize the eligibility trace states of the etrace algorithm.
|
|
361
398
|
|
|
@@ -25,7 +25,6 @@ from braintrace._algorithm.d_rtrl import D_RTRL
|
|
|
25
25
|
from braintrace._algorithm.param_dim_vjp import (
|
|
26
26
|
ParamDimVjpAlgorithm,
|
|
27
27
|
_remove_units,
|
|
28
|
-
_fast_solve_contract,
|
|
29
28
|
)
|
|
30
29
|
from braintrace._etrace_model_test import (
|
|
31
30
|
IF_Delta_Dense_Layer,
|
|
@@ -39,7 +38,7 @@ from braintrace._etrace_model_test import (
|
|
|
39
38
|
ALIF_STDExpCu_Dense_Layer,
|
|
40
39
|
ALIF_STPExpCu_Dense_Layer,
|
|
41
40
|
)
|
|
42
|
-
from braintrace._op import etp_mm_p
|
|
41
|
+
from braintrace._op import etp_mm_p, get_fast_path_rules
|
|
43
42
|
|
|
44
43
|
|
|
45
44
|
# ---------------------------------------------------------------------------
|
|
@@ -657,7 +656,7 @@ class TestDRtrlDictTraceStorage:
|
|
|
657
656
|
# Helpers for the trace-update specialization + dtype-knob tests
|
|
658
657
|
# ---------------------------------------------------------------------------
|
|
659
658
|
|
|
660
|
-
def _rnn_mm(seed=0, n=4, bias=False):
|
|
659
|
+
def _rnn_mm(seed=0, n=4, bias=False, weight_fn=None, bias_fn=None):
|
|
661
660
|
"""Batched single-recurrent-weight tanh RNN -> etp_mm_p, num_state==1."""
|
|
662
661
|
|
|
663
662
|
class Net(brainstate.nn.Module):
|
|
@@ -670,7 +669,10 @@ def _rnn_mm(seed=0, n=4, bias=False):
|
|
|
670
669
|
|
|
671
670
|
def update(self, x):
|
|
672
671
|
b = self.b.value if self.b is not None else None
|
|
673
|
-
self.h.value = jax.nn.tanh(braintrace.matmul(
|
|
672
|
+
self.h.value = jax.nn.tanh(braintrace.matmul(
|
|
673
|
+
self.h.value + x, self.w.value, b,
|
|
674
|
+
weight_fn=weight_fn, bias_fn=bias_fn,
|
|
675
|
+
))
|
|
674
676
|
return self.h.value
|
|
675
677
|
|
|
676
678
|
net = Net()
|
|
@@ -678,7 +680,7 @@ def _rnn_mm(seed=0, n=4, bias=False):
|
|
|
678
680
|
return net
|
|
679
681
|
|
|
680
682
|
|
|
681
|
-
def _rnn_mv(seed=0, n=4):
|
|
683
|
+
def _rnn_mv(seed=0, n=4, weight_fn=None):
|
|
682
684
|
"""Unbatched single-recurrent-weight tanh RNN -> etp_mv_p, num_state==1."""
|
|
683
685
|
|
|
684
686
|
class Net(brainstate.nn.Module):
|
|
@@ -690,7 +692,9 @@ def _rnn_mv(seed=0, n=4):
|
|
|
690
692
|
self.h = brainstate.HiddenState(jnp.zeros((n,)))
|
|
691
693
|
|
|
692
694
|
def update(self, x):
|
|
693
|
-
self.h.value = jax.nn.tanh(braintrace.matmul(
|
|
695
|
+
self.h.value = jax.nn.tanh(braintrace.matmul(
|
|
696
|
+
self.h.value + x, self.w.value, weight_fn=weight_fn,
|
|
697
|
+
))
|
|
694
698
|
return self.h.value
|
|
695
699
|
|
|
696
700
|
net = Net()
|
|
@@ -698,6 +702,27 @@ def _rnn_mv(seed=0, n=4):
|
|
|
698
702
|
return net
|
|
699
703
|
|
|
700
704
|
|
|
705
|
+
def _rnn_elemwise(seed=0, n=4, weight_fn=None):
|
|
706
|
+
"""Single element-wise-weight tanh RNN -> etp_elemwise_p, num_state==1."""
|
|
707
|
+
|
|
708
|
+
class Net(brainstate.nn.Module):
|
|
709
|
+
def __init__(self):
|
|
710
|
+
super().__init__()
|
|
711
|
+
self.alpha = brainstate.ParamState(
|
|
712
|
+
0.5 * jax.random.normal(jax.random.PRNGKey(seed), (n,))
|
|
713
|
+
)
|
|
714
|
+
self.h = brainstate.HiddenState(jnp.zeros((1, n)))
|
|
715
|
+
|
|
716
|
+
def update(self, x):
|
|
717
|
+
a = braintrace.element_wise(self.alpha.value, weight_fn=weight_fn)
|
|
718
|
+
self.h.value = jax.nn.tanh(a * (self.h.value + x))
|
|
719
|
+
return self.h.value
|
|
720
|
+
|
|
721
|
+
net = Net()
|
|
722
|
+
brainstate.nn.init_all_states(net, batch_size=1)
|
|
723
|
+
return net
|
|
724
|
+
|
|
725
|
+
|
|
701
726
|
def _run_updates(algo, xs):
|
|
702
727
|
"""Compile, init, run len(xs) eager updates; return etrace_bwg snapshot.
|
|
703
728
|
|
|
@@ -773,6 +798,73 @@ class TestS1RecurrentBroadcastEqualsLegacy:
|
|
|
773
798
|
_assert_etrace_lists_equal(fast, legacy, exact=True)
|
|
774
799
|
|
|
775
800
|
|
|
801
|
+
# ---------------------------------------------------------------------------
|
|
802
|
+
# weight_fn/bias_fn — the fast closed-form trace must equal the rule (legacy)
|
|
803
|
+
# path when a transform hook is present.
|
|
804
|
+
#
|
|
805
|
+
# The fast kernels compute the instant term as the outer product x ⊗ df, which
|
|
806
|
+
# equals ∂h/∂V (the *transformed* weight). With a transform present the true
|
|
807
|
+
# eligibility trace is ∂h/∂W_raw = (x ⊗ df) ⊙ f'(W); the f' factor is supplied
|
|
808
|
+
# only by the rule path's ``xy_to_dw`` (via jax.vjp). So the fast path must fall
|
|
809
|
+
# back to the rules whenever a ``*_fn`` hook is set, otherwise the trace is the
|
|
810
|
+
# gradient w.r.t. the transformed weight and silently drops f'(W).
|
|
811
|
+
# ---------------------------------------------------------------------------
|
|
812
|
+
|
|
813
|
+
class TestTransformFastEqualsLegacy:
|
|
814
|
+
"""With a weight_fn/bias_fn hook, the default (fast) trace must match the
|
|
815
|
+
rule path bit-for-bit — i.e. the fast path falls back to the rules so the
|
|
816
|
+
f'(W) factor from ``xy_to_dw`` is not dropped.
|
|
817
|
+
|
|
818
|
+
Run several steps so the recurrent term (non-zero from the 2nd update) is
|
|
819
|
+
exercised, not just the instant term."""
|
|
820
|
+
|
|
821
|
+
def _xs_mm(self, n=4, steps=4):
|
|
822
|
+
return brainstate.random.randn(steps, 1, n)
|
|
823
|
+
|
|
824
|
+
def _xs_mv(self, n=4, steps=4):
|
|
825
|
+
return brainstate.random.randn(steps, n)
|
|
826
|
+
|
|
827
|
+
def test_mm_weight_fn_fast_equals_legacy(self):
|
|
828
|
+
xs = self._xs_mm()
|
|
829
|
+
wfn = lambda w: w ** 2
|
|
830
|
+
fast = _run_updates(ParamDimVjpAlgorithm(_rnn_mm(weight_fn=wfn), fast_solve=True), xs)
|
|
831
|
+
legacy = _run_updates(ParamDimVjpAlgorithm(_rnn_mm(weight_fn=wfn), fast_solve=False), xs)
|
|
832
|
+
_assert_etrace_lists_equal(fast, legacy, exact=True)
|
|
833
|
+
|
|
834
|
+
def test_mm_weight_fn_and_bias_fn_fast_equals_legacy(self):
|
|
835
|
+
xs = self._xs_mm()
|
|
836
|
+
fast = _run_updates(
|
|
837
|
+
ParamDimVjpAlgorithm(
|
|
838
|
+
_rnn_mm(bias=True, weight_fn=jnp.tanh, bias_fn=lambda b: b * 2.0),
|
|
839
|
+
fast_solve=True), xs)
|
|
840
|
+
legacy = _run_updates(
|
|
841
|
+
ParamDimVjpAlgorithm(
|
|
842
|
+
_rnn_mm(bias=True, weight_fn=jnp.tanh, bias_fn=lambda b: b * 2.0),
|
|
843
|
+
fast_solve=False), xs)
|
|
844
|
+
_assert_etrace_lists_equal(fast, legacy, exact=True)
|
|
845
|
+
|
|
846
|
+
def test_mv_weight_fn_fast_equals_legacy(self):
|
|
847
|
+
xs = self._xs_mv()
|
|
848
|
+
fast = _run_updates(ParamDimVjpAlgorithm(_rnn_mv(weight_fn=jnp.tanh), fast_solve=True), xs)
|
|
849
|
+
legacy = _run_updates(ParamDimVjpAlgorithm(_rnn_mv(weight_fn=jnp.tanh), fast_solve=False), xs)
|
|
850
|
+
_assert_etrace_lists_equal(fast, legacy, exact=True)
|
|
851
|
+
|
|
852
|
+
def test_elemwise_weight_fn_fast_equals_legacy(self):
|
|
853
|
+
xs = self._xs_mm()
|
|
854
|
+
wfn = jnp.tanh
|
|
855
|
+
fast = _run_updates(ParamDimVjpAlgorithm(_rnn_elemwise(weight_fn=wfn), fast_solve=True), xs)
|
|
856
|
+
legacy = _run_updates(ParamDimVjpAlgorithm(_rnn_elemwise(weight_fn=wfn), fast_solve=False), xs)
|
|
857
|
+
_assert_etrace_lists_equal(fast, legacy, exact=True)
|
|
858
|
+
|
|
859
|
+
def test_no_transform_still_uses_fast_path(self):
|
|
860
|
+
"""Guard: without a transform, fast and legacy remain bit-identical (the
|
|
861
|
+
gate must not disable the fast path when no ``*_fn`` hook is present)."""
|
|
862
|
+
xs = self._xs_mm()
|
|
863
|
+
fast = _run_updates(ParamDimVjpAlgorithm(_rnn_mm(), fast_solve=True), xs)
|
|
864
|
+
legacy = _run_updates(ParamDimVjpAlgorithm(_rnn_mm(), fast_solve=False), xs)
|
|
865
|
+
_assert_etrace_lists_equal(fast, legacy, exact=True)
|
|
866
|
+
|
|
867
|
+
|
|
776
868
|
# ---------------------------------------------------------------------------
|
|
777
869
|
# Change A — trace_dtype knob (store the eligibility trace in lower precision)
|
|
778
870
|
# ---------------------------------------------------------------------------
|
|
@@ -838,7 +930,8 @@ class TestTraceDtypeKnob:
|
|
|
838
930
|
|
|
839
931
|
def _alif(n_in=4, n_rec=5):
|
|
840
932
|
"""Adaptive-LIF dense layer -> coupled (V, adaptation) hidden group with
|
|
841
|
-
num_state==2, exercising the S>1 einsum branch of
|
|
933
|
+
num_state==2, exercising the S>1 einsum branch of the dense fast-path
|
|
934
|
+
recurrent rule (``FastPathRules.recurrent``)."""
|
|
842
935
|
model = ALIF_Delta_Dense_Layer(n_in, n_rec)
|
|
843
936
|
brainstate.nn.init_all_states(model)
|
|
844
937
|
return model
|
|
@@ -891,11 +984,12 @@ class TestSolveBatchFold:
|
|
|
891
984
|
'bias': brainstate.random.randn(B, O, S),
|
|
892
985
|
}
|
|
893
986
|
# reference: current per-batch contraction, then explicit batch sum
|
|
894
|
-
|
|
987
|
+
solve = get_fast_path_rules(etp_mm_p).solve
|
|
988
|
+
ref = solve(diag_like, etrace, fold_batch=False)
|
|
895
989
|
ref_w = ref['weight'].sum(axis=0) # (I, O)
|
|
896
990
|
ref_b = ref['bias'].sum(axis=0) # (O,)
|
|
897
991
|
# new: fold the batch axis inside the einsum
|
|
898
|
-
folded =
|
|
992
|
+
folded = solve(diag_like, etrace, fold_batch=True)
|
|
899
993
|
assert folded['weight'].shape == (I, O)
|
|
900
994
|
assert folded['bias'].shape == (O,)
|
|
901
995
|
npt.assert_allclose(folded['weight'], ref_w, atol=1e-6)
|