zeta-scaling-toolkit 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zeta_scaling_toolkit-0.5.0/LICENSE +21 -0
- zeta_scaling_toolkit-0.5.0/PKG-INFO +219 -0
- zeta_scaling_toolkit-0.5.0/README.md +181 -0
- zeta_scaling_toolkit-0.5.0/pyproject.toml +96 -0
- zeta_scaling_toolkit-0.5.0/setup.cfg +4 -0
- zeta_scaling_toolkit-0.5.0/tests/test_audit_fixes.py +280 -0
- zeta_scaling_toolkit-0.5.0/tests/test_edge_cases.py +172 -0
- zeta_scaling_toolkit-0.5.0/tests/test_followup_gaps.py +170 -0
- zeta_scaling_toolkit-0.5.0/tests/test_hardening_regressions.py +141 -0
- zeta_scaling_toolkit-0.5.0/tests/test_hypothesis_properties.py +285 -0
- zeta_scaling_toolkit-0.5.0/tests/test_position_golden.py +92 -0
- zeta_scaling_toolkit-0.5.0/tests/test_properties.py +1259 -0
- zeta_scaling_toolkit-0.5.0/tests/test_toy_copy_task.py +42 -0
- zeta_scaling_toolkit-0.5.0/tests/test_tune_copy_task.py +77 -0
- zeta_scaling_toolkit-0.5.0/tests/test_zeta_toolkit.py +236 -0
- zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit/__init__.py +25 -0
- zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit/core.py +748 -0
- zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit/positions.py +213 -0
- zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit/py.typed +0 -0
- zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit.egg-info/PKG-INFO +219 -0
- zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit.egg-info/SOURCES.txt +22 -0
- zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit.egg-info/dependency_links.txt +1 -0
- zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit.egg-info/requires.txt +13 -0
- zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Zeta Scaling Toolkit Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: zeta-scaling-toolkit
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Experimental PyTorch scaling utilities
|
|
5
|
+
Author: Zeta Scaling Toolkit Contributors
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Slangy0-AFK/zeta-scaling-toolkit
|
|
8
|
+
Project-URL: Documentation, https://github.com/Slangy0-AFK/zeta-scaling-toolkit#readme
|
|
9
|
+
Project-URL: Repository, https://github.com/Slangy0-AFK/zeta-scaling-toolkit.git
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/Slangy0-AFK/zeta-scaling-toolkit/issues
|
|
11
|
+
Keywords: machine-learning,deep-learning,transformers,attention,scaling,pytorch,numerical-stability
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: torch>=2.0.0
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
29
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
30
|
+
Requires-Dist: hypothesis>=6.100.0; extra == "dev"
|
|
31
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
32
|
+
Requires-Dist: mypy>=1.0.0; extra == "dev"
|
|
33
|
+
Provides-Extra: test
|
|
34
|
+
Requires-Dist: pytest>=7.0.0; extra == "test"
|
|
35
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "test"
|
|
36
|
+
Requires-Dist: hypothesis>=6.100.0; extra == "test"
|
|
37
|
+
Dynamic: license-file
|
|
38
|
+
|
|
39
|
+
# Zeta Scaling Toolkit
|
|
40
|
+
|
|
41
|
+
Experimental PyTorch utilities for attention, learning-rate schedules,
|
|
42
|
+
coordinates, and linear initialization. Numerical behavior is covered by the
|
|
43
|
+
linked tests; none of these components is validated as a training improvement
|
|
44
|
+
on large-scale tasks.
|
|
45
|
+
|
|
46
|
+
## Components
|
|
47
|
+
|
|
48
|
+
- **`NonCollapsingAttention`** computes `softmax(tau * cos(q, k)) @ v`, where
|
|
49
|
+
`tau` is learnable, starts at `tau0` (default 10), and is bounded to `[tau_min, tau_max]`
|
|
50
|
+
(defaults 1 and 50) using a logit-sigmoid parameterization. This parameterization
|
|
51
|
+
ensures non-zero gradients throughout the bounded range, allowing the temperature
|
|
52
|
+
parameter to recover if perturbed beyond the bounds. Its normalization and softmax
|
|
53
|
+
calculations use at least float32, and the result is cast to `v.dtype`. Boolean
|
|
54
|
+
masks use `True` to retain a position; floating masks are added to the logits.
|
|
55
|
+
Fully masked query rows return zeros. Shapes, dtypes, devices, and optional
|
|
56
|
+
finite-value checks are validated; an empty key sequence raises `ValueError`,
|
|
57
|
+
while empty query sequences and batches are supported. See [attention property tests](./tests/test_properties.py),
|
|
58
|
+
[edge-case tests](./tests/test_edge_cases.py), and
|
|
59
|
+
[regression tests](./tests/test_hardening_regressions.py).
|
|
60
|
+
- **`AsymptoticLRScheduler`** is the retained public class name for a schedule
|
|
61
|
+
that linearly warms up, follows cosine decay to `floor_ratio * base_lr`, then
|
|
62
|
+
stays at that floor. `floor_ratio` must be in `[0, 1]`, `total_steps >= 1`,
|
|
63
|
+
and `0 <= warmup_steps < total_steps`. Without warmup it is mathematically
|
|
64
|
+
equivalent to `CosineAnnealingLR` with the same floor inside the configured
|
|
65
|
+
horizon, up to floating-point rounding; unlike stock cosine it stays clamped
|
|
66
|
+
to the floor beyond that horizon. See
|
|
67
|
+
[scheduler property and resume tests](./tests/test_properties.py).
|
|
68
|
+
- **`get_isotropic_coords`** returns pixel-centred `(x, y)` coordinates
|
|
69
|
+
normalized by `max(height, width)`. **`get_scale_invariant_vit_coords`**
|
|
70
|
+
returns `(radius, angle)` or, with `smooth_angle=True`,
|
|
71
|
+
`(radius, cos(angle), sin(angle))`. See the
|
|
72
|
+
[coordinate tests](./tests/test_properties.py) and
|
|
73
|
+
[compatibility tests](./tests/test_zeta_toolkit.py).
|
|
74
|
+
- **`make_corrected_linear`** applies orthogonal initialization scaled by a
|
|
75
|
+
gain supported by `torch.nn.init.calculate_gain`; its default is ReLU gain.
|
|
76
|
+
GELU and SiLU are not supported as fixed gains. Use **`calibrate_gain`** with
|
|
77
|
+
representative sample data for sequential GELU/SiLU stacks. See
|
|
78
|
+
[initialization properties](./tests/test_properties.py) and
|
|
79
|
+
[calibration regressions](./tests/test_hardening_regressions.py).
|
|
80
|
+
|
|
81
|
+
Deprecated `A_attn`, `T`, and `A_coords` arguments remain accepted, warn when
|
|
82
|
+
non-default values are supplied, and have no effect. Legacy attention
|
|
83
|
+
checkpoints containing `q_scale` and `k_scale` load with a warning and retain
|
|
84
|
+
the new module's initialized temperature; see
|
|
85
|
+
[compatibility tests](./tests/test_zeta_toolkit.py).
|
|
86
|
+
|
|
87
|
+
## Installation
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
pip install -e .
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
For development and tests:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
pip install -e ".[dev]"
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Quick start
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
import torch
|
|
103
|
+
from zeta_scaling_toolkit import (
|
|
104
|
+
AsymptoticLRScheduler,
|
|
105
|
+
NonCollapsingAttention,
|
|
106
|
+
get_isotropic_coords,
|
|
107
|
+
make_corrected_linear,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
attention = NonCollapsingAttention(d_k=64)
|
|
111
|
+
q, k, v = (torch.randn(2, 128, 64) for _ in range(3))
|
|
112
|
+
output = attention(q, k, v)
|
|
113
|
+
|
|
114
|
+
linear = make_corrected_linear(1024, 4096, nonlinearity="relu")
|
|
115
|
+
optimizer = torch.optim.Adam(linear.parameters(), lr=1e-3)
|
|
116
|
+
scheduler = AsymptoticLRScheduler(optimizer, total_steps=300, warmup_steps=10)
|
|
117
|
+
|
|
118
|
+
coords = get_isotropic_coords(height=224, width=448)
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
## Testing and quality checks
|
|
122
|
+
|
|
123
|
+
`pytest` runs the full deterministic property and regression suite. Coverage can
|
|
124
|
+
be checked with:
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
pytest --cov=zeta_scaling_toolkit --cov-report=term-missing --cov-fail-under=95
|
|
128
|
+
ruff check zeta_scaling_toolkit tests
|
|
129
|
+
mypy --strict zeta_scaling_toolkit
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Run the optional mutation check with `python tools/mutation_check.py`. It
|
|
133
|
+
temporarily injects targeted source changes and runs the full test suite for
|
|
134
|
+
each one, restoring the source afterward. `KILLED` means a test detected the
|
|
135
|
+
injected bug; `SURVIVED` means the suite missed it. This expensive check is not
|
|
136
|
+
part of the default CI matrix.
|
|
137
|
+
|
|
138
|
+
CI runs the CPU test matrix and a nightly Hypothesis profile. CUDA coverage is
|
|
139
|
+
skipped when no CUDA device is available. See [TESTING.md](./TESTING.md).
|
|
140
|
+
Reviewers can see the passing full GitHub Actions matrix at commit
|
|
141
|
+
[719bfb7](https://github.com/Slangy0-AFK/zeta-scaling-toolkit/commit/719bfb73f97e350087cfefa9617f94571a9d02d1).
|
|
142
|
+
|
|
143
|
+
## Real-task evaluation
|
|
144
|
+
|
|
145
|
+
The matched-seed char-level language-model benchmark and its results are
|
|
146
|
+
documented in [benchmarks/RESULTS.md](./benchmarks/RESULTS.md). The 100-step
|
|
147
|
+
table is early-optimisation only. At 500 steps, only the scheduler-only variant
|
|
148
|
+
improved loss on all ten matched seeds; attention-only, initialization-only,
|
|
149
|
+
and the combined variant had higher loss. At 1,000 steps over 20 seeds, the
|
|
150
|
+
scheduler-only variant improved loss on all 20 matched seeds; attention-only
|
|
151
|
+
was worse, while initialization-only and all-components paired intervals
|
|
152
|
+
included zero. These results apply to the measured model and dataset only.
|
|
153
|
+
|
|
154
|
+
## What this can and cannot do
|
|
155
|
+
|
|
156
|
+
- Opt-in log-length temperature scaling compensates for softmax dilution as
|
|
157
|
+
unmasked key count grows. The synthetic measurement and approximation are
|
|
158
|
+
recorded in [benchmarks/RESULTS.md](./benchmarks/RESULTS.md) and tested in
|
|
159
|
+
[test_properties.py](./tests/test_properties.py). This does not establish a
|
|
160
|
+
language-model training benefit.
|
|
161
|
+
- RoPE and ALiBi do not require a learned absolute-position table. Their
|
|
162
|
+
relative-position and numerical properties are covered in
|
|
163
|
+
[test_properties.py](./tests/test_properties.py); extrapolation quality is
|
|
164
|
+
task-dependent and must be measured.
|
|
165
|
+
- Context is not infinite. The optional streaming mask exposes a fixed window
|
|
166
|
+
plus sink tokens and cannot recall information outside that window. Its
|
|
167
|
+
visibility set is tested in [test_properties.py](./tests/test_properties.py).
|
|
168
|
+
The mask alone is dense. `StreamingKVCache` supplies a bounded inference-only
|
|
169
|
+
KV cache when used one token at a time; it detaches cached values and retains
|
|
170
|
+
only sinks plus a recent window. Cache retention and agreement with the dense
|
|
171
|
+
mask are tested in [test_properties.py](./tests/test_properties.py).
|
|
172
|
+
- Attention uses SDPA when one scale applies to every row and chunks query rows
|
|
173
|
+
for row-specific scales or fully masked rows. Dense masks and ALiBi biases
|
|
174
|
+
still use quadratic memory when constructed for a full sequence. None of
|
|
175
|
+
these mechanisms provide unbounded recall.
|
|
176
|
+
- RoPE angles are computed in float64 from int64 positions before casting back
|
|
177
|
+
to the input type. Large positions should not be converted directly to
|
|
178
|
+
float16 or bfloat16; the precision behavior is tested in
|
|
179
|
+
[test_properties.py](./tests/test_properties.py).
|
|
180
|
+
- Language-model measurements use one 32-wide model and one small dataset; the
|
|
181
|
+
synthetic vision task used 500 steps and is deliberately simple. Validated
|
|
182
|
+
on a 1,000-step character-level LM and a separate 500-step synthetic vision
|
|
183
|
+
task, but not on large-scale foundation models. These results do not
|
|
184
|
+
generalize to large-scale training; see
|
|
185
|
+
[benchmarks/RESULTS.md](./benchmarks/RESULTS.md).
|
|
186
|
+
|
|
187
|
+
## Known limitations
|
|
188
|
+
|
|
189
|
+
- The epsilon in normalization makes scale invariance approximate near zero;
|
|
190
|
+
its relative effect becomes small for vector norms around `1e-2` and larger.
|
|
191
|
+
- Attention temperature is bounded to `[tau_min, tau_max]` via a logit-sigmoid
|
|
192
|
+
parameterization; at extreme raw_tau values, sigmoid saturation may produce zero
|
|
193
|
+
gradients, but the parameter can recover for finite perturbations.
|
|
194
|
+
- Fixed-temperature configurations (where tau_min == tau_max) are not supported;
|
|
195
|
+
tau bounds must have tau_min < tau_max.
|
|
196
|
+
- GELU/SiLU require data-dependent `calibrate_gain`; fixed `calculate_gain`
|
|
197
|
+
does not support them.
|
|
198
|
+
- An all-masked attention row returns zeros, not a probability distribution.
|
|
199
|
+
- The `(cos(angle), sin(angle))` representation removes the branch cut away
|
|
200
|
+
from the origin; angle remains undefined at the origin.
|
|
201
|
+
- For rectangular orthogonal matrices with fewer output than input features,
|
|
202
|
+
norm shrinkage of roughly `sqrt(out_features / in_features)` is structurally
|
|
203
|
+
expected. See [initialization properties](./tests/test_properties.py).
|
|
204
|
+
- `calibrate_gain` equalises the whole-batch norm, not each sample's norm. For
|
|
205
|
+
GELU/SiLU in deep stacks without normalisation layers the batch norm can be
|
|
206
|
+
preserved while the median sample's signal collapses (the fixed-seed
|
|
207
|
+
30-layer SiLU characterization test checks for a median per-row ratio below
|
|
208
|
+
0.5). See `test_known_limitation_silu_calibration_*`.
|
|
209
|
+
- `calibrate_gain` emits a `RuntimeWarning` when its target ratio is not reached
|
|
210
|
+
within 2% (for example, bounded activations such as tanh cannot reach 1.0).
|
|
211
|
+
- `calibrate_gain` is now transactional: if validation fails on any layer, all
|
|
212
|
+
parameters revert to their original values.
|
|
213
|
+
- Nothing here has been validated on large-scale training runs.
|
|
214
|
+
|
|
215
|
+
## License and support
|
|
216
|
+
|
|
217
|
+
Python >= 3.9 and PyTorch >= 2.0. MIT License; see [LICENSE](./LICENSE).
|
|
218
|
+
Experimental results are not a substitute for evaluating these components on
|
|
219
|
+
the intended model, data, and training budget.
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
# Zeta Scaling Toolkit
|
|
2
|
+
|
|
3
|
+
Experimental PyTorch utilities for attention, learning-rate schedules,
|
|
4
|
+
coordinates, and linear initialization. Numerical behavior is covered by the
|
|
5
|
+
linked tests; none of these components is validated as a training improvement
|
|
6
|
+
on large-scale tasks.
|
|
7
|
+
|
|
8
|
+
## Components
|
|
9
|
+
|
|
10
|
+
- **`NonCollapsingAttention`** computes `softmax(tau * cos(q, k)) @ v`, where
|
|
11
|
+
`tau` is learnable, starts at `tau0` (default 10), and is bounded to `[tau_min, tau_max]`
|
|
12
|
+
(defaults 1 and 50) using a logit-sigmoid parameterization. This parameterization
|
|
13
|
+
ensures non-zero gradients throughout the bounded range, allowing the temperature
|
|
14
|
+
parameter to recover if perturbed beyond the bounds. Its normalization and softmax
|
|
15
|
+
calculations use at least float32, and the result is cast to `v.dtype`. Boolean
|
|
16
|
+
masks use `True` to retain a position; floating masks are added to the logits.
|
|
17
|
+
Fully masked query rows return zeros. Shapes, dtypes, devices, and optional
|
|
18
|
+
finite-value checks are validated; an empty key sequence raises `ValueError`,
|
|
19
|
+
while empty query sequences and batches are supported. See [attention property tests](./tests/test_properties.py),
|
|
20
|
+
[edge-case tests](./tests/test_edge_cases.py), and
|
|
21
|
+
[regression tests](./tests/test_hardening_regressions.py).
|
|
22
|
+
- **`AsymptoticLRScheduler`** is the retained public class name for a schedule
|
|
23
|
+
that linearly warms up, follows cosine decay to `floor_ratio * base_lr`, then
|
|
24
|
+
stays at that floor. `floor_ratio` must be in `[0, 1]`, `total_steps >= 1`,
|
|
25
|
+
and `0 <= warmup_steps < total_steps`. Without warmup it is mathematically
|
|
26
|
+
equivalent to `CosineAnnealingLR` with the same floor inside the configured
|
|
27
|
+
horizon, up to floating-point rounding; unlike stock cosine it stays clamped
|
|
28
|
+
to the floor beyond that horizon. See
|
|
29
|
+
[scheduler property and resume tests](./tests/test_properties.py).
|
|
30
|
+
- **`get_isotropic_coords`** returns pixel-centred `(x, y)` coordinates
|
|
31
|
+
normalized by `max(height, width)`. **`get_scale_invariant_vit_coords`**
|
|
32
|
+
returns `(radius, angle)` or, with `smooth_angle=True`,
|
|
33
|
+
`(radius, cos(angle), sin(angle))`. See the
|
|
34
|
+
[coordinate tests](./tests/test_properties.py) and
|
|
35
|
+
[compatibility tests](./tests/test_zeta_toolkit.py).
|
|
36
|
+
- **`make_corrected_linear`** applies orthogonal initialization scaled by a
|
|
37
|
+
gain supported by `torch.nn.init.calculate_gain`; its default is ReLU gain.
|
|
38
|
+
GELU and SiLU are not supported as fixed gains. Use **`calibrate_gain`** with
|
|
39
|
+
representative sample data for sequential GELU/SiLU stacks. See
|
|
40
|
+
[initialization properties](./tests/test_properties.py) and
|
|
41
|
+
[calibration regressions](./tests/test_hardening_regressions.py).
|
|
42
|
+
|
|
43
|
+
Deprecated `A_attn`, `T`, and `A_coords` arguments remain accepted, warn when
|
|
44
|
+
non-default values are supplied, and have no effect. Legacy attention
|
|
45
|
+
checkpoints containing `q_scale` and `k_scale` load with a warning and retain
|
|
46
|
+
the new module's initialized temperature; see
|
|
47
|
+
[compatibility tests](./tests/test_zeta_toolkit.py).
|
|
48
|
+
|
|
49
|
+
## Installation
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install -e .
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
For development and tests:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install -e ".[dev]"
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Quick start
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
import torch
|
|
65
|
+
from zeta_scaling_toolkit import (
|
|
66
|
+
AsymptoticLRScheduler,
|
|
67
|
+
NonCollapsingAttention,
|
|
68
|
+
get_isotropic_coords,
|
|
69
|
+
make_corrected_linear,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
attention = NonCollapsingAttention(d_k=64)
|
|
73
|
+
q, k, v = (torch.randn(2, 128, 64) for _ in range(3))
|
|
74
|
+
output = attention(q, k, v)
|
|
75
|
+
|
|
76
|
+
linear = make_corrected_linear(1024, 4096, nonlinearity="relu")
|
|
77
|
+
optimizer = torch.optim.Adam(linear.parameters(), lr=1e-3)
|
|
78
|
+
scheduler = AsymptoticLRScheduler(optimizer, total_steps=300, warmup_steps=10)
|
|
79
|
+
|
|
80
|
+
coords = get_isotropic_coords(height=224, width=448)
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Testing and quality checks
|
|
84
|
+
|
|
85
|
+
`pytest` runs the full deterministic property and regression suite. Coverage can
|
|
86
|
+
be checked with:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
pytest --cov=zeta_scaling_toolkit --cov-report=term-missing --cov-fail-under=95
|
|
90
|
+
ruff check zeta_scaling_toolkit tests
|
|
91
|
+
mypy --strict zeta_scaling_toolkit
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Run the optional mutation check with `python tools/mutation_check.py`. It
|
|
95
|
+
temporarily injects targeted source changes and runs the full test suite for
|
|
96
|
+
each one, restoring the source afterward. `KILLED` means a test detected the
|
|
97
|
+
injected bug; `SURVIVED` means the suite missed it. This expensive check is not
|
|
98
|
+
part of the default CI matrix.
|
|
99
|
+
|
|
100
|
+
CI runs the CPU test matrix and a nightly Hypothesis profile. CUDA coverage is
|
|
101
|
+
skipped when no CUDA device is available. See [TESTING.md](./TESTING.md).
|
|
102
|
+
Reviewers can see the passing full GitHub Actions matrix at commit
|
|
103
|
+
[719bfb7](https://github.com/Slangy0-AFK/zeta-scaling-toolkit/commit/719bfb73f97e350087cfefa9617f94571a9d02d1).
|
|
104
|
+
|
|
105
|
+
## Real-task evaluation
|
|
106
|
+
|
|
107
|
+
The matched-seed char-level language-model benchmark and its results are
|
|
108
|
+
documented in [benchmarks/RESULTS.md](./benchmarks/RESULTS.md). The 100-step
|
|
109
|
+
table is early-optimisation only. At 500 steps, only the scheduler-only variant
|
|
110
|
+
improved loss on all ten matched seeds; attention-only, initialization-only,
|
|
111
|
+
and the combined variant had higher loss. At 1,000 steps over 20 seeds, the
|
|
112
|
+
scheduler-only variant improved loss on all 20 matched seeds; attention-only
|
|
113
|
+
was worse, while initialization-only and all-components paired intervals
|
|
114
|
+
included zero. These results apply to the measured model and dataset only.
|
|
115
|
+
|
|
116
|
+
## What this can and cannot do
|
|
117
|
+
|
|
118
|
+
- Opt-in log-length temperature scaling compensates for softmax dilution as
|
|
119
|
+
unmasked key count grows. The synthetic measurement and approximation are
|
|
120
|
+
recorded in [benchmarks/RESULTS.md](./benchmarks/RESULTS.md) and tested in
|
|
121
|
+
[test_properties.py](./tests/test_properties.py). This does not establish a
|
|
122
|
+
language-model training benefit.
|
|
123
|
+
- RoPE and ALiBi do not require a learned absolute-position table. Their
|
|
124
|
+
relative-position and numerical properties are covered in
|
|
125
|
+
[test_properties.py](./tests/test_properties.py); extrapolation quality is
|
|
126
|
+
task-dependent and must be measured.
|
|
127
|
+
- Context is not infinite. The optional streaming mask exposes a fixed window
|
|
128
|
+
plus sink tokens and cannot recall information outside that window. Its
|
|
129
|
+
visibility set is tested in [test_properties.py](./tests/test_properties.py).
|
|
130
|
+
The mask alone is dense. `StreamingKVCache` supplies a bounded inference-only
|
|
131
|
+
KV cache when used one token at a time; it detaches cached values and retains
|
|
132
|
+
only sinks plus a recent window. Cache retention and agreement with the dense
|
|
133
|
+
mask are tested in [test_properties.py](./tests/test_properties.py).
|
|
134
|
+
- Attention uses SDPA when one scale applies to every row and chunks query rows
|
|
135
|
+
for row-specific scales or fully masked rows. Dense masks and ALiBi biases
|
|
136
|
+
still use quadratic memory when constructed for a full sequence. None of
|
|
137
|
+
these mechanisms provide unbounded recall.
|
|
138
|
+
- RoPE angles are computed in float64 from int64 positions before casting back
|
|
139
|
+
to the input type. Large positions should not be converted directly to
|
|
140
|
+
float16 or bfloat16; the precision behavior is tested in
|
|
141
|
+
[test_properties.py](./tests/test_properties.py).
|
|
142
|
+
- Language-model measurements use one 32-wide model and one small dataset; the
|
|
143
|
+
synthetic vision task used 500 steps and is deliberately simple. Validated
|
|
144
|
+
on a 1,000-step character-level LM and a separate 500-step synthetic vision
|
|
145
|
+
task, but not on large-scale foundation models. These results do not
|
|
146
|
+
generalize to large-scale training; see
|
|
147
|
+
[benchmarks/RESULTS.md](./benchmarks/RESULTS.md).
|
|
148
|
+
|
|
149
|
+
## Known limitations
|
|
150
|
+
|
|
151
|
+
- The epsilon in normalization makes scale invariance approximate near zero;
|
|
152
|
+
its relative effect becomes small for vector norms around `1e-2` and larger.
|
|
153
|
+
- Attention temperature is bounded to `[tau_min, tau_max]` via a logit-sigmoid
|
|
154
|
+
parameterization; at extreme raw_tau values, sigmoid saturation may produce zero
|
|
155
|
+
gradients, but the parameter can recover for finite perturbations.
|
|
156
|
+
- Fixed-temperature configurations (where tau_min == tau_max) are not supported;
|
|
157
|
+
tau bounds must have tau_min < tau_max.
|
|
158
|
+
- GELU/SiLU require data-dependent `calibrate_gain`; fixed `calculate_gain`
|
|
159
|
+
does not support them.
|
|
160
|
+
- An all-masked attention row returns zeros, not a probability distribution.
|
|
161
|
+
- The `(cos(angle), sin(angle))` representation removes the branch cut away
|
|
162
|
+
from the origin; angle remains undefined at the origin.
|
|
163
|
+
- For rectangular orthogonal matrices with fewer output than input features,
|
|
164
|
+
norm shrinkage of roughly `sqrt(out_features / in_features)` is structurally
|
|
165
|
+
expected. See [initialization properties](./tests/test_properties.py).
|
|
166
|
+
- `calibrate_gain` equalises the whole-batch norm, not each sample's norm. For
|
|
167
|
+
GELU/SiLU in deep stacks without normalisation layers the batch norm can be
|
|
168
|
+
preserved while the median sample's signal collapses (the fixed-seed
|
|
169
|
+
30-layer SiLU characterization test checks for a median per-row ratio below
|
|
170
|
+
0.5). See `test_known_limitation_silu_calibration_*`.
|
|
171
|
+
- `calibrate_gain` emits a `RuntimeWarning` when its target ratio is not reached
|
|
172
|
+
within 2% (for example, bounded activations such as tanh cannot reach 1.0).
|
|
173
|
+
- `calibrate_gain` is now transactional: if validation fails on any layer, all
|
|
174
|
+
parameters revert to their original values.
|
|
175
|
+
- Nothing here has been validated on large-scale training runs.
|
|
176
|
+
|
|
177
|
+
## License and support
|
|
178
|
+
|
|
179
|
+
Python >= 3.9 and PyTorch >= 2.0. MIT License; see [LICENSE](./LICENSE).
|
|
180
|
+
Experimental results are not a substitute for evaluating these components on
|
|
181
|
+
the intended model, data, and training budget.
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=65.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "zeta-scaling-toolkit"
|
|
7
|
+
version = "0.5.0"
|
|
8
|
+
description = "Experimental PyTorch scaling utilities"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Zeta Scaling Toolkit Contributors"}
|
|
14
|
+
]
|
|
15
|
+
keywords = [
|
|
16
|
+
"machine-learning",
|
|
17
|
+
"deep-learning",
|
|
18
|
+
"transformers",
|
|
19
|
+
"attention",
|
|
20
|
+
"scaling",
|
|
21
|
+
"pytorch",
|
|
22
|
+
"numerical-stability"
|
|
23
|
+
]
|
|
24
|
+
classifiers = [
|
|
25
|
+
"Development Status :: 3 - Alpha",
|
|
26
|
+
"Intended Audience :: Developers",
|
|
27
|
+
"Intended Audience :: Science/Research",
|
|
28
|
+
"License :: OSI Approved :: MIT License",
|
|
29
|
+
"Programming Language :: Python :: 3",
|
|
30
|
+
"Programming Language :: Python :: 3.9",
|
|
31
|
+
"Programming Language :: Python :: 3.10",
|
|
32
|
+
"Programming Language :: Python :: 3.11",
|
|
33
|
+
"Programming Language :: Python :: 3.12",
|
|
34
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
35
|
+
"Topic :: Software Development :: Libraries :: Python Modules"
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
dependencies = [
|
|
39
|
+
"torch>=2.0.0"
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
[project.optional-dependencies]
|
|
43
|
+
dev = [
|
|
44
|
+
"pytest>=7.0.0",
|
|
45
|
+
"pytest-cov>=4.0.0",
|
|
46
|
+
"hypothesis>=6.100.0",
|
|
47
|
+
"ruff>=0.1.0",
|
|
48
|
+
"mypy>=1.0.0"
|
|
49
|
+
]
|
|
50
|
+
test = [
|
|
51
|
+
"pytest>=7.0.0",
|
|
52
|
+
"pytest-cov>=4.0.0",
|
|
53
|
+
"hypothesis>=6.100.0"
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
[project.urls]
|
|
57
|
+
Homepage = "https://github.com/Slangy0-AFK/zeta-scaling-toolkit"
|
|
58
|
+
Documentation = "https://github.com/Slangy0-AFK/zeta-scaling-toolkit#readme"
|
|
59
|
+
Repository = "https://github.com/Slangy0-AFK/zeta-scaling-toolkit.git"
|
|
60
|
+
"Bug Tracker" = "https://github.com/Slangy0-AFK/zeta-scaling-toolkit/issues"
|
|
61
|
+
|
|
62
|
+
[tool.setuptools]
|
|
63
|
+
packages = ["zeta_scaling_toolkit"]
|
|
64
|
+
|
|
65
|
+
[tool.setuptools.package-data]
|
|
66
|
+
zeta_scaling_toolkit = ["py.typed"]
|
|
67
|
+
|
|
68
|
+
[tool.black]
|
|
69
|
+
line-length = 100
|
|
70
|
+
target-version = ["py39", "py310", "py311", "py312"]
|
|
71
|
+
|
|
72
|
+
[tool.ruff]
|
|
73
|
+
line-length = 100
|
|
74
|
+
target-version = "py39"
|
|
75
|
+
|
|
76
|
+
[tool.ruff.lint]
|
|
77
|
+
select = ["E", "F", "W", "I", "N", "UP"]
|
|
78
|
+
ignore = ["E501", "N802", "N803", "N806", "N812"]
|
|
79
|
+
|
|
80
|
+
[tool.mypy]
|
|
81
|
+
python_version = "3.10"
|
|
82
|
+
strict = true
|
|
83
|
+
files = ["zeta_scaling_toolkit"]
|
|
84
|
+
warn_unused_configs = true
|
|
85
|
+
|
|
86
|
+
[tool.pytest.ini_options]
|
|
87
|
+
testpaths = ["tests"]
|
|
88
|
+
python_files = ["test_*.py", "*_test.py"]
|
|
89
|
+
addopts = "-v --strict-config --strict-markers"
|
|
90
|
+
filterwarnings = [
|
|
91
|
+
"error",
|
|
92
|
+
"ignore:Failed to initialize NumPy:UserWarning",
|
|
93
|
+
]
|
|
94
|
+
markers = [
|
|
95
|
+
"cuda: requires a CUDA device",
|
|
96
|
+
]
|