zeta-scaling-toolkit 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. zeta_scaling_toolkit-0.5.0/LICENSE +21 -0
  2. zeta_scaling_toolkit-0.5.0/PKG-INFO +219 -0
  3. zeta_scaling_toolkit-0.5.0/README.md +181 -0
  4. zeta_scaling_toolkit-0.5.0/pyproject.toml +96 -0
  5. zeta_scaling_toolkit-0.5.0/setup.cfg +4 -0
  6. zeta_scaling_toolkit-0.5.0/tests/test_audit_fixes.py +280 -0
  7. zeta_scaling_toolkit-0.5.0/tests/test_edge_cases.py +172 -0
  8. zeta_scaling_toolkit-0.5.0/tests/test_followup_gaps.py +170 -0
  9. zeta_scaling_toolkit-0.5.0/tests/test_hardening_regressions.py +141 -0
  10. zeta_scaling_toolkit-0.5.0/tests/test_hypothesis_properties.py +285 -0
  11. zeta_scaling_toolkit-0.5.0/tests/test_position_golden.py +92 -0
  12. zeta_scaling_toolkit-0.5.0/tests/test_properties.py +1259 -0
  13. zeta_scaling_toolkit-0.5.0/tests/test_toy_copy_task.py +42 -0
  14. zeta_scaling_toolkit-0.5.0/tests/test_tune_copy_task.py +77 -0
  15. zeta_scaling_toolkit-0.5.0/tests/test_zeta_toolkit.py +236 -0
  16. zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit/__init__.py +25 -0
  17. zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit/core.py +748 -0
  18. zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit/positions.py +213 -0
  19. zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit/py.typed +0 -0
  20. zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit.egg-info/PKG-INFO +219 -0
  21. zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit.egg-info/SOURCES.txt +22 -0
  22. zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit.egg-info/dependency_links.txt +1 -0
  23. zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit.egg-info/requires.txt +13 -0
  24. zeta_scaling_toolkit-0.5.0/zeta_scaling_toolkit.egg-info/top_level.txt +1 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Zeta Scaling Toolkit Contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,219 @@
1
+ Metadata-Version: 2.4
2
+ Name: zeta-scaling-toolkit
3
+ Version: 0.5.0
4
+ Summary: Experimental PyTorch scaling utilities
5
+ Author: Zeta Scaling Toolkit Contributors
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/Slangy0-AFK/zeta-scaling-toolkit
8
+ Project-URL: Documentation, https://github.com/Slangy0-AFK/zeta-scaling-toolkit#readme
9
+ Project-URL: Repository, https://github.com/Slangy0-AFK/zeta-scaling-toolkit.git
10
+ Project-URL: Bug Tracker, https://github.com/Slangy0-AFK/zeta-scaling-toolkit/issues
11
+ Keywords: machine-learning,deep-learning,transformers,attention,scaling,pytorch,numerical-stability
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.9
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
22
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
23
+ Requires-Python: >=3.9
24
+ Description-Content-Type: text/markdown
25
+ License-File: LICENSE
26
+ Requires-Dist: torch>=2.0.0
27
+ Provides-Extra: dev
28
+ Requires-Dist: pytest>=7.0.0; extra == "dev"
29
+ Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
30
+ Requires-Dist: hypothesis>=6.100.0; extra == "dev"
31
+ Requires-Dist: ruff>=0.1.0; extra == "dev"
32
+ Requires-Dist: mypy>=1.0.0; extra == "dev"
33
+ Provides-Extra: test
34
+ Requires-Dist: pytest>=7.0.0; extra == "test"
35
+ Requires-Dist: pytest-cov>=4.0.0; extra == "test"
36
+ Requires-Dist: hypothesis>=6.100.0; extra == "test"
37
+ Dynamic: license-file
38
+
39
+ # Zeta Scaling Toolkit
40
+
41
+ Experimental PyTorch utilities for attention, learning-rate schedules,
42
+ coordinates, and linear initialization. Numerical behavior is covered by the
43
+ linked tests; none of these components is validated as a training improvement
44
+ on large-scale tasks.
45
+
46
+ ## Components
47
+
48
+ - **`NonCollapsingAttention`** computes `softmax(tau * cos(q, k)) @ v`, where
49
+ `tau` is learnable, starts at `tau0` (default 10), and is bounded to `[tau_min, tau_max]`
50
+ (defaults 1 and 50) using a logit-sigmoid parameterization. This parameterization
51
+ ensures non-zero gradients throughout the bounded range, allowing the temperature
52
+ parameter to recover if perturbed beyond the bounds. Its normalization and softmax
53
+ calculations use at least float32, and the result is cast to `v.dtype`. Boolean
54
+ masks use `True` to retain a position; floating masks are added to the logits.
55
+ Fully masked query rows return zeros. Shapes, dtypes, devices, and optional
56
+ finite-value checks are validated; an empty key sequence raises `ValueError`,
57
+ while empty query sequences and batches are supported. See [attention property tests](./tests/test_properties.py),
58
+ [edge-case tests](./tests/test_edge_cases.py), and
59
+ [regression tests](./tests/test_hardening_regressions.py).
60
+ - **`AsymptoticLRScheduler`** is the retained public class name for a schedule
61
+ that linearly warms up, follows cosine decay to `floor_ratio * base_lr`, then
62
+ stays at that floor. `floor_ratio` must be in `[0, 1]`, `total_steps >= 1`,
63
+ and `0 <= warmup_steps < total_steps`. Without warmup it is mathematically
64
+ equivalent to `CosineAnnealingLR` with the same floor inside the configured
65
+ horizon, up to floating-point rounding; unlike stock cosine it stays clamped
66
+ to the floor beyond that horizon. See
67
+ [scheduler property and resume tests](./tests/test_properties.py).
68
+ - **`get_isotropic_coords`** returns pixel-centred `(x, y)` coordinates
69
+ normalized by `max(height, width)`. **`get_scale_invariant_vit_coords`**
70
+ returns `(radius, angle)` or, with `smooth_angle=True`,
71
+ `(radius, cos(angle), sin(angle))`. See the
72
+ [coordinate tests](./tests/test_properties.py) and
73
+ [compatibility tests](./tests/test_zeta_toolkit.py).
74
+ - **`make_corrected_linear`** applies orthogonal initialization scaled by a
75
+ gain supported by `torch.nn.init.calculate_gain`; its default is ReLU gain.
76
+ GELU and SiLU are not supported as fixed gains. Use **`calibrate_gain`** with
77
+ representative sample data for sequential GELU/SiLU stacks. See
78
+ [initialization properties](./tests/test_properties.py) and
79
+ [calibration regressions](./tests/test_hardening_regressions.py).
80
+
81
+ Deprecated `A_attn`, `T`, and `A_coords` arguments remain accepted, warn when
82
+ non-default values are supplied, and have no effect. Legacy attention
83
+ checkpoints containing `q_scale` and `k_scale` load with a warning and retain
84
+ the new module's initialized temperature; see
85
+ [compatibility tests](./tests/test_zeta_toolkit.py).
86
+
87
+ ## Installation
88
+
89
+ ```bash
90
+ pip install -e .
91
+ ```
92
+
93
+ For development and tests:
94
+
95
+ ```bash
96
+ pip install -e ".[dev]"
97
+ ```
98
+
99
+ ## Quick start
100
+
101
+ ```python
102
+ import torch
103
+ from zeta_scaling_toolkit import (
104
+ AsymptoticLRScheduler,
105
+ NonCollapsingAttention,
106
+ get_isotropic_coords,
107
+ make_corrected_linear,
108
+ )
109
+
110
+ attention = NonCollapsingAttention(d_k=64)
111
+ q, k, v = (torch.randn(2, 128, 64) for _ in range(3))
112
+ output = attention(q, k, v)
113
+
114
+ linear = make_corrected_linear(1024, 4096, nonlinearity="relu")
115
+ optimizer = torch.optim.Adam(linear.parameters(), lr=1e-3)
116
+ scheduler = AsymptoticLRScheduler(optimizer, total_steps=300, warmup_steps=10)
117
+
118
+ coords = get_isotropic_coords(height=224, width=448)
119
+ ```
120
+
121
+ ## Testing and quality checks
122
+
123
+ `pytest` runs the full deterministic property and regression suite. Coverage can
124
+ be checked with:
125
+
126
+ ```bash
127
+ pytest --cov=zeta_scaling_toolkit --cov-report=term-missing --cov-fail-under=95
128
+ ruff check zeta_scaling_toolkit tests
129
+ mypy --strict zeta_scaling_toolkit
130
+ ```
131
+
132
+ Run the optional mutation check with `python tools/mutation_check.py`. It
133
+ temporarily injects targeted source changes and runs the full test suite for
134
+ each one, restoring the source afterward. `KILLED` means a test detected the
135
+ injected bug; `SURVIVED` means the suite missed it. This expensive check is not
136
+ part of the default CI matrix.
137
+
138
+ CI runs the CPU test matrix and a nightly Hypothesis profile. CUDA coverage is
139
+ skipped when no CUDA device is available. See [TESTING.md](./TESTING.md).
140
+ Reviewers can see the passing full GitHub Actions matrix at commit
141
+ [719bfb7](https://github.com/Slangy0-AFK/zeta-scaling-toolkit/commit/719bfb73f97e350087cfefa9617f94571a9d02d1).
142
+
143
+ ## Real-task evaluation
144
+
145
+ The matched-seed char-level language-model benchmark and its results are
146
+ documented in [benchmarks/RESULTS.md](./benchmarks/RESULTS.md). The 100-step
147
+ table is early-optimisation only. At 500 steps, only the scheduler-only variant
148
+ improved loss on all ten matched seeds; attention-only, initialization-only,
149
+ and the combined variant had higher loss. At 1,000 steps over 20 seeds, the
150
+ scheduler-only variant improved loss on all 20 matched seeds; attention-only
151
+ was worse, while initialization-only and all-components paired intervals
152
+ included zero. These results apply to the measured model and dataset only.
153
+
154
+ ## What this can and cannot do
155
+
156
+ - Opt-in log-length temperature scaling compensates for softmax dilution as
157
+ unmasked key count grows. The synthetic measurement and approximation are
158
+ recorded in [benchmarks/RESULTS.md](./benchmarks/RESULTS.md) and tested in
159
+ [test_properties.py](./tests/test_properties.py). This does not establish a
160
+ language-model training benefit.
161
+ - RoPE and ALiBi do not require a learned absolute-position table. Their
162
+ relative-position and numerical properties are covered in
163
+ [test_properties.py](./tests/test_properties.py); extrapolation quality is
164
+ task-dependent and must be measured.
165
+ - Context is not infinite. The optional streaming mask exposes a fixed window
166
+ plus sink tokens and cannot recall information outside that window. Its
167
+ visibility set is tested in [test_properties.py](./tests/test_properties.py).
168
+ The mask alone is dense. `StreamingKVCache` supplies a bounded inference-only
169
+ KV cache when used one token at a time; it detaches cached values and retains
170
+ only sinks plus a recent window. Cache retention and agreement with the dense
171
+ mask are tested in [test_properties.py](./tests/test_properties.py).
172
+ - Attention uses SDPA when one scale applies to every row and chunks query rows
173
+ for row-specific scales or fully masked rows. Dense masks and ALiBi biases
174
+ still use quadratic memory when constructed for a full sequence. None of
175
+ these mechanisms provide unbounded recall.
176
+ - RoPE angles are computed in float64 from int64 positions before casting back
177
+ to the input type. Large positions should not be converted directly to
178
+ float16 or bfloat16; the precision behavior is tested in
179
+ [test_properties.py](./tests/test_properties.py).
180
+ - Language-model measurements use one 32-wide model and one small dataset; the
181
+ synthetic vision task used 500 steps and is deliberately simple. Validated
182
+ on a 1,000-step character-level LM and a separate 500-step synthetic vision
183
+ task, but not on large-scale foundation models. These results do not
184
+ generalize to large-scale training; see
185
+ [benchmarks/RESULTS.md](./benchmarks/RESULTS.md).
186
+
187
+ ## Known limitations
188
+
189
+ - The epsilon in normalization makes scale invariance approximate near zero;
190
+ its relative effect becomes small for vector norms around `1e-2` and larger.
191
+ - Attention temperature is bounded to `[tau_min, tau_max]` via a logit-sigmoid
192
+ parameterization; at extreme raw_tau values, sigmoid saturation may produce zero
193
+ gradients, but the parameter can recover for finite perturbations.
194
+ - Fixed-temperature configurations (where tau_min == tau_max) are not supported;
195
+ tau bounds must have tau_min < tau_max.
196
+ - GELU/SiLU require data-dependent `calibrate_gain`; fixed `calculate_gain`
197
+ does not support them.
198
+ - An all-masked attention row returns zeros, not a probability distribution.
199
+ - The `(cos(angle), sin(angle))` representation removes the branch cut away
200
+ from the origin; angle remains undefined at the origin.
201
+ - For rectangular orthogonal matrices with fewer output than input features,
202
+ norm shrinkage of roughly `sqrt(out_features / in_features)` is structurally
203
+ expected. See [initialization properties](./tests/test_properties.py).
204
+ - `calibrate_gain` equalises the whole-batch norm, not each sample's norm. For
205
+ GELU/SiLU in deep stacks without normalisation layers the batch norm can be
206
+ preserved while the median sample's signal collapses (the fixed-seed
207
+ 30-layer SiLU characterization test checks for a median per-row ratio below
208
+ 0.5). See `test_known_limitation_silu_calibration_*`.
209
+ - `calibrate_gain` emits a `RuntimeWarning` when its target ratio is not reached
210
+ within 2% (for example, bounded activations such as tanh cannot reach 1.0).
211
+ - `calibrate_gain` is now transactional: if validation fails on any layer, all
212
+ parameters revert to their original values.
213
+ - Nothing here has been validated on large-scale training runs.
214
+
215
+ ## License and support
216
+
217
+ Python >= 3.9 and PyTorch >= 2.0. MIT License; see [LICENSE](./LICENSE).
218
+ Experimental results are not a substitute for evaluating these components on
219
+ the intended model, data, and training budget.
@@ -0,0 +1,181 @@
1
+ # Zeta Scaling Toolkit
2
+
3
+ Experimental PyTorch utilities for attention, learning-rate schedules,
4
+ coordinates, and linear initialization. Numerical behavior is covered by the
5
+ linked tests; none of these components is validated as a training improvement
6
+ on large-scale tasks.
7
+
8
+ ## Components
9
+
10
+ - **`NonCollapsingAttention`** computes `softmax(tau * cos(q, k)) @ v`, where
11
+ `tau` is learnable, starts at `tau0` (default 10), and is bounded to `[tau_min, tau_max]`
12
+ (defaults 1 and 50) using a logit-sigmoid parameterization. This parameterization
13
+ ensures non-zero gradients throughout the bounded range, allowing the temperature
14
+ parameter to recover if perturbed beyond the bounds. Its normalization and softmax
15
+ calculations use at least float32, and the result is cast to `v.dtype`. Boolean
16
+ masks use `True` to retain a position; floating masks are added to the logits.
17
+ Fully masked query rows return zeros. Shapes, dtypes, devices, and optional
18
+ finite-value checks are validated; an empty key sequence raises `ValueError`,
19
+ while empty query sequences and batches are supported. See [attention property tests](./tests/test_properties.py),
20
+ [edge-case tests](./tests/test_edge_cases.py), and
21
+ [regression tests](./tests/test_hardening_regressions.py).
22
+ - **`AsymptoticLRScheduler`** is the retained public class name for a schedule
23
+ that linearly warms up, follows cosine decay to `floor_ratio * base_lr`, then
24
+ stays at that floor. `floor_ratio` must be in `[0, 1]`, `total_steps >= 1`,
25
+ and `0 <= warmup_steps < total_steps`. Without warmup it is mathematically
26
+ equivalent to `CosineAnnealingLR` with the same floor inside the configured
27
+ horizon, up to floating-point rounding; unlike stock cosine it stays clamped
28
+ to the floor beyond that horizon. See
29
+ [scheduler property and resume tests](./tests/test_properties.py).
30
+ - **`get_isotropic_coords`** returns pixel-centred `(x, y)` coordinates
31
+ normalized by `max(height, width)`. **`get_scale_invariant_vit_coords`**
32
+ returns `(radius, angle)` or, with `smooth_angle=True`,
33
+ `(radius, cos(angle), sin(angle))`. See the
34
+ [coordinate tests](./tests/test_properties.py) and
35
+ [compatibility tests](./tests/test_zeta_toolkit.py).
36
+ - **`make_corrected_linear`** applies orthogonal initialization scaled by a
37
+ gain supported by `torch.nn.init.calculate_gain`; its default is ReLU gain.
38
+ GELU and SiLU are not supported as fixed gains. Use **`calibrate_gain`** with
39
+ representative sample data for sequential GELU/SiLU stacks. See
40
+ [initialization properties](./tests/test_properties.py) and
41
+ [calibration regressions](./tests/test_hardening_regressions.py).
42
+
43
+ Deprecated `A_attn`, `T`, and `A_coords` arguments remain accepted, warn when
44
+ non-default values are supplied, and have no effect. Legacy attention
45
+ checkpoints containing `q_scale` and `k_scale` load with a warning and retain
46
+ the new module's initialized temperature; see
47
+ [compatibility tests](./tests/test_zeta_toolkit.py).
48
+
49
+ ## Installation
50
+
51
+ ```bash
52
+ pip install -e .
53
+ ```
54
+
55
+ For development and tests:
56
+
57
+ ```bash
58
+ pip install -e ".[dev]"
59
+ ```
60
+
61
+ ## Quick start
62
+
63
+ ```python
64
+ import torch
65
+ from zeta_scaling_toolkit import (
66
+ AsymptoticLRScheduler,
67
+ NonCollapsingAttention,
68
+ get_isotropic_coords,
69
+ make_corrected_linear,
70
+ )
71
+
72
+ attention = NonCollapsingAttention(d_k=64)
73
+ q, k, v = (torch.randn(2, 128, 64) for _ in range(3))
74
+ output = attention(q, k, v)
75
+
76
+ linear = make_corrected_linear(1024, 4096, nonlinearity="relu")
77
+ optimizer = torch.optim.Adam(linear.parameters(), lr=1e-3)
78
+ scheduler = AsymptoticLRScheduler(optimizer, total_steps=300, warmup_steps=10)
79
+
80
+ coords = get_isotropic_coords(height=224, width=448)
81
+ ```
82
+
83
+ ## Testing and quality checks
84
+
85
+ `pytest` runs the full deterministic property and regression suite. Coverage can
86
+ be checked with:
87
+
88
+ ```bash
89
+ pytest --cov=zeta_scaling_toolkit --cov-report=term-missing --cov-fail-under=95
90
+ ruff check zeta_scaling_toolkit tests
91
+ mypy --strict zeta_scaling_toolkit
92
+ ```
93
+
94
+ Run the optional mutation check with `python tools/mutation_check.py`. It
95
+ temporarily injects targeted source changes and runs the full test suite for
96
+ each one, restoring the source afterward. `KILLED` means a test detected the
97
+ injected bug; `SURVIVED` means the suite missed it. This expensive check is not
98
+ part of the default CI matrix.
99
+
100
+ CI runs the CPU test matrix and a nightly Hypothesis profile. CUDA coverage is
101
+ skipped when no CUDA device is available. See [TESTING.md](./TESTING.md).
102
+ Reviewers can see the passing full GitHub Actions matrix at commit
103
+ [719bfb7](https://github.com/Slangy0-AFK/zeta-scaling-toolkit/commit/719bfb73f97e350087cfefa9617f94571a9d02d1).
104
+
105
+ ## Real-task evaluation
106
+
107
+ The matched-seed char-level language-model benchmark and its results are
108
+ documented in [benchmarks/RESULTS.md](./benchmarks/RESULTS.md). The 100-step
109
+ table is early-optimisation only. At 500 steps, only the scheduler-only variant
110
+ improved loss on all ten matched seeds; attention-only, initialization-only,
111
+ and the combined variant had higher loss. At 1,000 steps over 20 seeds, the
112
+ scheduler-only variant improved loss on all 20 matched seeds; attention-only
113
+ was worse, while initialization-only and all-components paired intervals
114
+ included zero. These results apply to the measured model and dataset only.
115
+
116
+ ## What this can and cannot do
117
+
118
+ - Opt-in log-length temperature scaling compensates for softmax dilution as
119
+ unmasked key count grows. The synthetic measurement and approximation are
120
+ recorded in [benchmarks/RESULTS.md](./benchmarks/RESULTS.md) and tested in
121
+ [test_properties.py](./tests/test_properties.py). This does not establish a
122
+ language-model training benefit.
123
+ - RoPE and ALiBi do not require a learned absolute-position table. Their
124
+ relative-position and numerical properties are covered in
125
+ [test_properties.py](./tests/test_properties.py); extrapolation quality is
126
+ task-dependent and must be measured.
127
+ - Context is not infinite. The optional streaming mask exposes a fixed window
128
+ plus sink tokens and cannot recall information outside that window. Its
129
+ visibility set is tested in [test_properties.py](./tests/test_properties.py).
130
+ The mask alone is dense. `StreamingKVCache` supplies a bounded inference-only
131
+ KV cache when used one token at a time; it detaches cached values and retains
132
+ only sinks plus a recent window. Cache retention and agreement with the dense
133
+ mask are tested in [test_properties.py](./tests/test_properties.py).
134
+ - Attention uses SDPA when one scale applies to every row and chunks query rows
135
+ for row-specific scales or fully masked rows. Dense masks and ALiBi biases
136
+ still use quadratic memory when constructed for a full sequence. None of
137
+ these mechanisms provide unbounded recall.
138
+ - RoPE angles are computed in float64 from int64 positions before casting back
139
+ to the input type. Large positions should not be converted directly to
140
+ float16 or bfloat16; the precision behavior is tested in
141
+ [test_properties.py](./tests/test_properties.py).
142
+ - Language-model measurements use one 32-wide model and one small dataset; the
143
+ synthetic vision task used 500 steps and is deliberately simple. Validated
144
+ on a 1,000-step character-level LM and a separate 500-step synthetic vision
145
+ task, but not on large-scale foundation models. These results do not
146
+ generalize to large-scale training; see
147
+ [benchmarks/RESULTS.md](./benchmarks/RESULTS.md).
148
+
149
+ ## Known limitations
150
+
151
+ - The epsilon in normalization makes scale invariance approximate near zero;
152
+ its relative effect becomes small for vector norms around `1e-2` and larger.
153
+ - Attention temperature is bounded to `[tau_min, tau_max]` via a logit-sigmoid
154
+ parameterization; at extreme raw_tau values, sigmoid saturation may produce zero
155
+ gradients, but the parameter can recover for finite perturbations.
156
+ - Fixed-temperature configurations (where tau_min == tau_max) are not supported;
157
+ tau bounds must have tau_min < tau_max.
158
+ - GELU/SiLU require data-dependent `calibrate_gain`; fixed `calculate_gain`
159
+ does not support them.
160
+ - An all-masked attention row returns zeros, not a probability distribution.
161
+ - The `(cos(angle), sin(angle))` representation removes the branch cut away
162
+ from the origin; angle remains undefined at the origin.
163
+ - For rectangular orthogonal matrices with fewer output than input features,
164
+ norm shrinkage of roughly `sqrt(out_features / in_features)` is structurally
165
+ expected. See [initialization properties](./tests/test_properties.py).
166
+ - `calibrate_gain` equalises the whole-batch norm, not each sample's norm. For
167
+ GELU/SiLU in deep stacks without normalisation layers the batch norm can be
168
+ preserved while the median sample's signal collapses (the fixed-seed
169
+ 30-layer SiLU characterization test checks for a median per-row ratio below
170
+ 0.5). See `test_known_limitation_silu_calibration_*`.
171
+ - `calibrate_gain` emits a `RuntimeWarning` when its target ratio is not reached
172
+ within 2% (for example, bounded activations such as tanh cannot reach 1.0).
173
+ - `calibrate_gain` is now transactional: if validation fails on any layer, all
174
+ parameters revert to their original values.
175
+ - Nothing here has been validated on large-scale training runs.
176
+
177
+ ## License and support
178
+
179
+ Python >= 3.9 and PyTorch >= 2.0. MIT License; see [LICENSE](./LICENSE).
180
+ Experimental results are not a substitute for evaluating these components on
181
+ the intended model, data, and training budget.
@@ -0,0 +1,96 @@
1
+ [build-system]
2
+ requires = ["setuptools>=65.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "zeta-scaling-toolkit"
7
+ version = "0.5.0"
8
+ description = "Experimental PyTorch scaling utilities"
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = {text = "MIT"}
12
+ authors = [
13
+ {name = "Zeta Scaling Toolkit Contributors"}
14
+ ]
15
+ keywords = [
16
+ "machine-learning",
17
+ "deep-learning",
18
+ "transformers",
19
+ "attention",
20
+ "scaling",
21
+ "pytorch",
22
+ "numerical-stability"
23
+ ]
24
+ classifiers = [
25
+ "Development Status :: 3 - Alpha",
26
+ "Intended Audience :: Developers",
27
+ "Intended Audience :: Science/Research",
28
+ "License :: OSI Approved :: MIT License",
29
+ "Programming Language :: Python :: 3",
30
+ "Programming Language :: Python :: 3.9",
31
+ "Programming Language :: Python :: 3.10",
32
+ "Programming Language :: Python :: 3.11",
33
+ "Programming Language :: Python :: 3.12",
34
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
35
+ "Topic :: Software Development :: Libraries :: Python Modules"
36
+ ]
37
+
38
+ dependencies = [
39
+ "torch>=2.0.0"
40
+ ]
41
+
42
+ [project.optional-dependencies]
43
+ dev = [
44
+ "pytest>=7.0.0",
45
+ "pytest-cov>=4.0.0",
46
+ "hypothesis>=6.100.0",
47
+ "ruff>=0.1.0",
48
+ "mypy>=1.0.0"
49
+ ]
50
+ test = [
51
+ "pytest>=7.0.0",
52
+ "pytest-cov>=4.0.0",
53
+ "hypothesis>=6.100.0"
54
+ ]
55
+
56
+ [project.urls]
57
+ Homepage = "https://github.com/Slangy0-AFK/zeta-scaling-toolkit"
58
+ Documentation = "https://github.com/Slangy0-AFK/zeta-scaling-toolkit#readme"
59
+ Repository = "https://github.com/Slangy0-AFK/zeta-scaling-toolkit.git"
60
+ "Bug Tracker" = "https://github.com/Slangy0-AFK/zeta-scaling-toolkit/issues"
61
+
62
+ [tool.setuptools]
63
+ packages = ["zeta_scaling_toolkit"]
64
+
65
+ [tool.setuptools.package-data]
66
+ zeta_scaling_toolkit = ["py.typed"]
67
+
68
+ [tool.black]
69
+ line-length = 100
70
+ target-version = ["py39", "py310", "py311", "py312"]
71
+
72
+ [tool.ruff]
73
+ line-length = 100
74
+ target-version = "py39"
75
+
76
+ [tool.ruff.lint]
77
+ select = ["E", "F", "W", "I", "N", "UP"]
78
+ ignore = ["E501", "N802", "N803", "N806", "N812"]
79
+
80
+ [tool.mypy]
81
+ python_version = "3.10"
82
+ strict = true
83
+ files = ["zeta_scaling_toolkit"]
84
+ warn_unused_configs = true
85
+
86
+ [tool.pytest.ini_options]
87
+ testpaths = ["tests"]
88
+ python_files = ["test_*.py", "*_test.py"]
89
+ addopts = "-v --strict-config --strict-markers"
90
+ filterwarnings = [
91
+ "error",
92
+ "ignore:Failed to initialize NumPy:UserWarning",
93
+ ]
94
+ markers = [
95
+ "cuda: requires a CUDA device",
96
+ ]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+