astro-emulators-toolkit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. astro_emulators_toolkit/__init__.py +23 -0
  2. astro_emulators_toolkit/_typing.py +13 -0
  3. astro_emulators_toolkit/_vendor/__init__.py +1 -0
  4. astro_emulators_toolkit/_vendor/get_soap_jax.py +51 -0
  5. astro_emulators_toolkit/_vendor/soap_jax/LICENSE +21 -0
  6. astro_emulators_toolkit/_vendor/soap_jax/README.txt +37 -0
  7. astro_emulators_toolkit/_vendor/soap_jax/__init__.py +4 -0
  8. astro_emulators_toolkit/_vendor/soap_jax/soap.py +480 -0
  9. astro_emulators_toolkit/_version.py +1 -0
  10. astro_emulators_toolkit/bundle/__init__.py +12 -0
  11. astro_emulators_toolkit/bundle/artifact.py +175 -0
  12. astro_emulators_toolkit/bundle/bundle.py +83 -0
  13. astro_emulators_toolkit/bundle/extras.py +328 -0
  14. astro_emulators_toolkit/bundle/hub.py +45 -0
  15. astro_emulators_toolkit/bundle/integrity.py +364 -0
  16. astro_emulators_toolkit/bundle/metadata.py +361 -0
  17. astro_emulators_toolkit/bundle/minmax_sidecars.py +291 -0
  18. astro_emulators_toolkit/bundle/provenance.py +52 -0
  19. astro_emulators_toolkit/bundle/readme.py +68 -0
  20. astro_emulators_toolkit/bundle/release.py +524 -0
  21. astro_emulators_toolkit/bundle/safetensors_io.py +68 -0
  22. astro_emulators_toolkit/bundle/versions.py +15 -0
  23. astro_emulators_toolkit/config/__init__.py +34 -0
  24. astro_emulators_toolkit/config/io.py +298 -0
  25. astro_emulators_toolkit/config/parsing.py +18 -0
  26. astro_emulators_toolkit/config/schema.py +681 -0
  27. astro_emulators_toolkit/data/__init__.py +32 -0
  28. astro_emulators_toolkit/data/array_dataset.py +159 -0
  29. astro_emulators_toolkit/data/loader.py +123 -0
  30. astro_emulators_toolkit/data/mapped_dataset.py +46 -0
  31. astro_emulators_toolkit/data/npy_table.py +102 -0
  32. astro_emulators_toolkit/data/preprocess.py +11 -0
  33. astro_emulators_toolkit/data/protocols.py +93 -0
  34. astro_emulators_toolkit/data/subset.py +47 -0
  35. astro_emulators_toolkit/data/toy.py +170 -0
  36. astro_emulators_toolkit/data/toy_utils.py +163 -0
  37. astro_emulators_toolkit/emulator.py +979 -0
  38. astro_emulators_toolkit/emulator_runtime.py +68 -0
  39. astro_emulators_toolkit/experimental/__init__.py +3 -0
  40. astro_emulators_toolkit/experimental/models/__init__.py +76 -0
  41. astro_emulators_toolkit/experimental/models/explicit_wavelength_mlp.py +180 -0
  42. astro_emulators_toolkit/experimental/models/mlp_2d_regression.py +73 -0
  43. astro_emulators_toolkit/experimental/models/siren.py +128 -0
  44. astro_emulators_toolkit/experimental/runtime_adapters.py +442 -0
  45. astro_emulators_toolkit/experimental/tasks/__init__.py +25 -0
  46. astro_emulators_toolkit/experimental/tasks/classification.py +131 -0
  47. astro_emulators_toolkit/inference/__init__.py +3 -0
  48. astro_emulators_toolkit/inference/compose.py +319 -0
  49. astro_emulators_toolkit/io_trees.py +431 -0
  50. astro_emulators_toolkit/models/__init__.py +86 -0
  51. astro_emulators_toolkit/models/activations.py +25 -0
  52. astro_emulators_toolkit/models/cannon.py +62 -0
  53. astro_emulators_toolkit/models/canonical_wrappers.py +82 -0
  54. astro_emulators_toolkit/models/mlp.py +111 -0
  55. astro_emulators_toolkit/models/runtime_adapters.py +840 -0
  56. astro_emulators_toolkit/models/transformer_payne.py +794 -0
  57. astro_emulators_toolkit/models/transformer_payne_batch.py +566 -0
  58. astro_emulators_toolkit/optimizers/__init__.py +344 -0
  59. astro_emulators_toolkit/optimizers/_soap.py +5 -0
  60. astro_emulators_toolkit/presets/__init__.py +11 -0
  61. astro_emulators_toolkit/presets/cannon.py +50 -0
  62. astro_emulators_toolkit/presets/mlp.py +148 -0
  63. astro_emulators_toolkit/presets/transformer_payne.py +133 -0
  64. astro_emulators_toolkit/py.typed +1 -0
  65. astro_emulators_toolkit/resolver.py +181 -0
  66. astro_emulators_toolkit/spec.py +371 -0
  67. astro_emulators_toolkit/tasks/__init__.py +23 -0
  68. astro_emulators_toolkit/tasks/common.py +105 -0
  69. astro_emulators_toolkit/tasks/regression.py +351 -0
  70. astro_emulators_toolkit/training/__init__.py +28 -0
  71. astro_emulators_toolkit/training/callbacks.py +363 -0
  72. astro_emulators_toolkit/training/checkpoint.py +77 -0
  73. astro_emulators_toolkit/training/fit_backends.py +401 -0
  74. astro_emulators_toolkit/training/paths.py +5 -0
  75. astro_emulators_toolkit/training/solvers.py +226 -0
  76. astro_emulators_toolkit/training/state.py +29 -0
  77. astro_emulators_toolkit/training/trainer.py +347 -0
  78. astro_emulators_toolkit/utils/__init__.py +3 -0
  79. astro_emulators_toolkit/utils/tree.py +35 -0
  80. astro_emulators_toolkit-0.1.0.dist-info/METADATA +1130 -0
  81. astro_emulators_toolkit-0.1.0.dist-info/RECORD +84 -0
  82. astro_emulators_toolkit-0.1.0.dist-info/WHEEL +5 -0
  83. astro_emulators_toolkit-0.1.0.dist-info/licenses/LICENSE +21 -0
  84. astro_emulators_toolkit-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,23 @@
1
+ from ._version import __version__
2
+ from .config.io import load_config, save_config
3
+ from .config.schema import RootConfig
4
+ from .io_trees import denormalize_tree, normalize_tree
5
+
6
+
7
+ __all__ = [
8
+ "Emulator",
9
+ "RootConfig",
10
+ "load_config",
11
+ "save_config",
12
+ "normalize_tree",
13
+ "denormalize_tree",
14
+ "__version__",
15
+ ]
16
+
17
+
18
+ def __getattr__(name: str):
19
+ if name == "Emulator":
20
+ from .emulator import Emulator
21
+
22
+ return Emulator
23
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,13 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Any, Protocol, TypeAlias
4
+
5
+
6
+ PytreeDict: TypeAlias = dict[str, Any]
7
+ AffineLeafSpec: TypeAlias = dict[str, Any]
8
+ AffineLeafSpecs: TypeAlias = dict[str, AffineLeafSpec]
9
+
10
+
11
+ class SupportsFromDict(Protocol):
12
+ @classmethod
13
+ def from_dict(cls, d: dict[str, Any]) -> Any: ...
@@ -0,0 +1 @@
1
+ """Vendored third-party sources used as internal implementation details."""
@@ -0,0 +1,51 @@
1
+ """Refresh the vendored SOAP_JAX snapshot from the pinned upstream commit.
2
+
3
+ This script documents how the vendored copy in
4
+ `astro_emulators_toolkit._vendor.soap_jax` was obtained. It clones the public
5
+ upstream repository, checks out the pinned commit, and copies the Python
6
+ sources plus license into this package.
7
+
8
+ The local `README.txt` is maintained by this repository and is intentionally
9
+ not overwritten by the refresh step.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import shutil
15
+ import subprocess
16
+ import tempfile
17
+ from pathlib import Path
18
+
19
+
20
+ UPSTREAM_URL = "https://github.com/haydn-jones/SOAP_JAX"
21
+ UPSTREAM_COMMIT = "ddddc25724dfd629b0cd01584eca1e32ea8ac4de"
22
+ VENDOR_ROOT = Path(__file__).resolve().parent / "soap_jax"
23
+ UPSTREAM_PACKAGE_DIR = Path("src") / "soap_jax"
24
+ UPSTREAM_FILES = ("__init__.py", "soap.py")
25
+
26
+
27
+ def refresh_vendor_tree() -> None:
28
+ VENDOR_ROOT.mkdir(parents=True, exist_ok=True)
29
+
30
+ with tempfile.TemporaryDirectory(prefix="soap_jax_vendor_") as tmp_dir:
31
+ repo_dir = Path(tmp_dir) / "SOAP_JAX"
32
+
33
+ subprocess.run(["git", "clone", UPSTREAM_URL, str(repo_dir)], check=True)
34
+ subprocess.run(
35
+ ["git", "-C", str(repo_dir), "checkout", UPSTREAM_COMMIT], check=True
36
+ )
37
+
38
+ source_root = repo_dir / UPSTREAM_PACKAGE_DIR
39
+ for filename in UPSTREAM_FILES:
40
+ shutil.copy2(source_root / filename, VENDOR_ROOT / filename)
41
+ shutil.copy2(repo_dir / "LICENSE", VENDOR_ROOT / "LICENSE")
42
+
43
+
44
+ def main() -> None:
45
+ refresh_vendor_tree()
46
+ print(f"Vendored SOAP_JAX from {UPSTREAM_URL} at {UPSTREAM_COMMIT}.")
47
+ print(f"Updated files under: {VENDOR_ROOT}")
48
+
49
+
50
+ if __name__ == "__main__":
51
+ main()
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2024 Haydn Jones
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,37 @@
1
+ Vendored SOAP_JAX
2
+ =================
3
+
4
+ This directory contains a vendored copy of the public `SOAP_JAX` project by
5
+ Haydn Jones. Many thanks to the upstream author for publishing and maintaining
6
+ the implementation.
7
+
8
+ Upstream repository:
9
+ https://github.com/haydn-jones/SOAP_JAX
10
+
11
+ Pinned upstream commit:
12
+ ddddc25724dfd629b0cd01584eca1e32ea8ac4de
13
+
14
+ What is included here:
15
+ - `__init__.py`
16
+ - `soap.py`
17
+ - `LICENSE`
18
+
19
+ Why this is vendored:
20
+ - SOAP is an internal optimizer implementation detail for this toolkit.
21
+ - `uv sync` already installs this project editable, so keeping SOAP in `src/`
22
+ avoids an extra separately-installed package.
23
+ - This keeps the integration surface small while preserving upstream
24
+ provenance.
25
+ - It should make a future migration simpler if comparable functionality lands
26
+ directly in Optax and we want to swap implementations behind
27
+ `astro_emulators_toolkit.optimizers`.
28
+
29
+ Local adjustments:
30
+ - The vendored file keeps the upstream algorithm and structure.
31
+ - Type-only dependencies on `chex` and `jaxtyping` were removed to keep the
32
+ toolkit dependency surface smaller.
33
+
34
+ Refresh workflow:
35
+ - Run `python -m astro_emulators_toolkit._vendor.get_soap_jax` to re-fetch the
36
+ pinned upstream snapshot.
37
+ - Review the diff afterward before committing any update.
@@ -0,0 +1,4 @@
1
+ from .soap import scale_by_soap as scale_by_soap
2
+ from .soap import soap as soap
3
+
4
+ __all__ = ["scale_by_soap", "soap"]
@@ -0,0 +1,480 @@
1
+ """Vendored SOAP_JAX implementation.
2
+
3
+ Source:
4
+ https://github.com/haydn-jones/SOAP_JAX
5
+
6
+ Pinned upstream commit:
7
+ ddddc25724dfd629b0cd01584eca1e32ea8ac4de
8
+
9
+ Minimal local edits:
10
+ - vendored under the toolkit package
11
+ - removed type-only dependencies on `chex` and `jaxtyping`
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from itertools import chain
17
+ from typing import Any, Callable, Iterable, NamedTuple
18
+
19
+ import jax
20
+ import jax.numpy as jnp
21
+ import jax.tree_util as jtu
22
+ import optax
23
+ import optax.tree_utils as otu
24
+ from optax import GradientTransformation, Updates
25
+
26
+ Array = Any
27
+ ArrayDType = Any
28
+ Numeric = Any
29
+ PreconditionerMatrix = Array | None
30
+
31
+
32
+ @jtu.register_pytree_node_class
33
+ class Preconditioner:
34
+ """Per-parameter preconditioner matrices (None for dimensions exceeding max_precond_dim)."""
35
+
36
+ __slots__ = ("matrices",)
37
+
38
+ def __init__(self, matrices: Iterable[PreconditionerMatrix]):
39
+ self.matrices = tuple(matrices)
40
+
41
+ def tree_flatten(self) -> tuple[tuple[PreconditionerMatrix, ...], None]:
42
+ return (self.matrices, None)
43
+
44
+ @classmethod
45
+ def tree_unflatten(
46
+ cls, aux_data: None, children: tuple[PreconditionerMatrix, ...]
47
+ ) -> "Preconditioner":
48
+ return cls(children)
49
+
50
+ def map(
51
+ self, fn: Callable[[PreconditionerMatrix], PreconditionerMatrix]
52
+ ) -> "Preconditioner":
53
+ return Preconditioner(fn(m) for m in self.matrices)
54
+
55
+
56
+ class SOAPState(NamedTuple):
57
+ count: Array
58
+ exp_avg: Updates
59
+ exp_avg_sq: Updates
60
+ GG: Updates
61
+ Q: Updates
62
+
63
+
64
+ class PostDecayState(NamedTuple):
65
+ count: Array
66
+
67
+
68
+ def soap(
69
+ learning_rate: optax.ScalarOrSchedule = 3e-3,
70
+ b1: float = 0.95,
71
+ b2: float = 0.95,
72
+ shampoo_beta: float = -1,
73
+ eps: float = 1e-8,
74
+ weight_decay: float = 0.0,
75
+ correct_bias: bool = True,
76
+ precondition_frequency: int = 10,
77
+ max_precond_dim: int = 10000,
78
+ precondition_1d: bool = False,
79
+ precision: jax.lax.PrecisionLike = jax.lax.Precision.HIGHEST,
80
+ mu_dtype: ArrayDType | None = None,
81
+ qr_dtype: ArrayDType = jnp.float32,
82
+ ) -> optax.GradientTransformationExtraArgs:
83
+ """SOAP optimizer based on the public SOAP_JAX implementation."""
84
+
85
+ return optax.chain(
86
+ scale_by_soap(
87
+ b1=b1,
88
+ b2=b2,
89
+ shampoo_beta=shampoo_beta,
90
+ eps=eps,
91
+ correct_bias=correct_bias,
92
+ precondition_frequency=precondition_frequency,
93
+ max_precond_dim=max_precond_dim,
94
+ precondition_1d=precondition_1d,
95
+ precision=precision,
96
+ mu_dtype=mu_dtype,
97
+ qr_dtype=qr_dtype,
98
+ ),
99
+ optax.scale_by_learning_rate(learning_rate),
100
+ add_decayed_weights_post(weight_decay, learning_rate),
101
+ )
102
+
103
+
104
+ def scale_by_soap(
105
+ b1: float = 0.95,
106
+ b2: float = 0.95,
107
+ shampoo_beta: float = -1,
108
+ eps: float = 1e-8,
109
+ correct_bias: bool = True,
110
+ precondition_frequency: int = 10,
111
+ max_precond_dim: int = 10000,
112
+ precondition_1d: bool = False,
113
+ precision: jax.lax.PrecisionLike = jax.lax.Precision.HIGHEST,
114
+ mu_dtype: ArrayDType | None = None,
115
+ qr_dtype: ArrayDType = jnp.float32,
116
+ ) -> GradientTransformation:
117
+ """SOAP gradient transformation based on the public SOAP_JAX implementation."""
118
+
119
+ if not (0 <= b1 < 1):
120
+ raise ValueError("b1 must be in [0, 1)")
121
+ if not (0 <= b2 < 1):
122
+ raise ValueError("b2 must be in [0, 1)")
123
+ if shampoo_beta >= 1 or (shampoo_beta < 0 and shampoo_beta != -1):
124
+ raise ValueError("shampoo_beta must be in [0, 1) or -1")
125
+ if eps <= 0:
126
+ raise ValueError("eps must be positive")
127
+ if precondition_frequency <= 0:
128
+ raise ValueError("precondition_frequency must be a positive integer")
129
+ if max_precond_dim <= 0:
130
+ raise ValueError("max_precond_dim must be a positive integer")
131
+
132
+ shampoo_beta = shampoo_beta if shampoo_beta >= 0 else b2
133
+
134
+ def init_fn(params: Updates) -> SOAPState:
135
+ exp_avg = otu.tree_zeros_like(params, dtype=mu_dtype)
136
+ exp_avg_sq = otu.tree_zeros_like(params, dtype=mu_dtype)
137
+ GG = jtu.tree_map(
138
+ lambda p: init_conditioner(p, max_precond_dim, precondition_1d, qr_dtype),
139
+ params,
140
+ )
141
+ Q = jtu.tree_map(
142
+ lambda p: init_conditioner(p, max_precond_dim, precondition_1d, qr_dtype),
143
+ params,
144
+ )
145
+ return SOAPState(
146
+ count=jnp.zeros([], jnp.int32),
147
+ exp_avg=exp_avg,
148
+ exp_avg_sq=exp_avg_sq,
149
+ GG=GG,
150
+ Q=Q,
151
+ )
152
+
153
+ def init_step(
154
+ updates: Updates,
155
+ state: SOAPState,
156
+ ) -> tuple[Updates, SOAPState]:
157
+ new_GG = jtu.tree_map(
158
+ lambda grad, gg: update_preconditioner(grad, gg, shampoo_beta, precision),
159
+ updates,
160
+ state.GG,
161
+ is_leaf=_is_preconditioner,
162
+ )
163
+
164
+ new_Q = jtu.tree_map(
165
+ lambda gg: gg.map(lambda m: get_orthogonal_matrix(m, qr_dtype)),
166
+ new_GG,
167
+ is_leaf=_is_preconditioner,
168
+ )
169
+
170
+ new_updates = otu.tree_zeros_like(updates)
171
+
172
+ return new_updates, state._replace(GG=new_GG, Q=new_Q)
173
+
174
+ def update_step(
175
+ updates: Updates,
176
+ state: SOAPState,
177
+ ) -> tuple[Updates, SOAPState]:
178
+ grad_projected = jtu.tree_map(
179
+ lambda grad, q: project(grad, q, precision),
180
+ updates,
181
+ state.Q,
182
+ is_leaf=_is_preconditioner,
183
+ )
184
+
185
+ exp_avg = otu.tree_update_moment(grad_projected, state.exp_avg, b1, 1)
186
+ exp_avg_sq = otu.tree_update_moment_per_elem_norm(
187
+ grad_projected, state.exp_avg_sq, b2, 2
188
+ )
189
+ if mu_dtype is not None:
190
+ exp_avg = otu.tree_cast(exp_avg, mu_dtype)
191
+ exp_avg_sq = otu.tree_cast(exp_avg_sq, mu_dtype)
192
+
193
+ norm_updates = jtu.tree_map(
194
+ lambda e_avg, e_avg_sq, q: project_back(
195
+ e_avg / (jnp.sqrt(e_avg_sq) + jnp.asarray(eps, dtype=e_avg_sq.dtype)),
196
+ q,
197
+ precision,
198
+ ),
199
+ exp_avg,
200
+ exp_avg_sq,
201
+ state.Q,
202
+ is_leaf=_is_preconditioner,
203
+ )
204
+
205
+ if correct_bias:
206
+ effective_step = state.count - 1
207
+ bc1 = 1 - b1**effective_step
208
+ bc2 = 1 - b2**effective_step
209
+ corr = jnp.sqrt(bc2) / bc1
210
+ norm_updates = jtu.tree_map(
211
+ lambda p: p * jnp.asarray(corr, dtype=p.dtype), norm_updates
212
+ )
213
+
214
+ new_GG = jtu.tree_map(
215
+ lambda grad, gg: update_preconditioner(grad, gg, shampoo_beta, precision),
216
+ updates,
217
+ state.GG,
218
+ is_leaf=_is_preconditioner,
219
+ )
220
+
221
+ def refresh_preconditioner() -> tuple[Updates, Updates, Updates]:
222
+ new_Q_and_exp_avg_sq = jtu.tree_map(
223
+ lambda e, gg, q: get_orthogonal_matrix_QR(
224
+ gg, q, e, precision, qr_dtype
225
+ ),
226
+ exp_avg_sq,
227
+ new_GG,
228
+ state.Q,
229
+ is_leaf=_is_preconditioner,
230
+ )
231
+ new_Q = jtu.tree_map(
232
+ lambda _, x: x[0],
233
+ updates,
234
+ new_Q_and_exp_avg_sq,
235
+ )
236
+ new_exp_avg_sq = jtu.tree_map(
237
+ lambda _, x: x[1],
238
+ updates,
239
+ new_Q_and_exp_avg_sq,
240
+ )
241
+ new_exp_avg = jtu.tree_map(
242
+ lambda e, old_q, new_q: project(
243
+ project_back(e, old_q, precision), new_q, precision
244
+ ),
245
+ exp_avg,
246
+ state.Q,
247
+ new_Q,
248
+ is_leaf=_is_preconditioner,
249
+ )
250
+ return new_Q, new_exp_avg_sq, new_exp_avg
251
+
252
+ def keep_preconditioner() -> tuple[Updates, Updates, Updates]:
253
+ return state.Q, exp_avg_sq, exp_avg
254
+
255
+ new_Q, exp_avg_sq, exp_avg = jax.lax.cond(
256
+ (state.count - 1) % precondition_frequency == 0,
257
+ refresh_preconditioner,
258
+ keep_preconditioner,
259
+ )
260
+
261
+ new_state = SOAPState(
262
+ count=state.count,
263
+ exp_avg=exp_avg,
264
+ exp_avg_sq=exp_avg_sq,
265
+ GG=new_GG,
266
+ Q=new_Q,
267
+ )
268
+
269
+ return norm_updates, new_state
270
+
271
+ def update_fn(
272
+ updates: Updates, state: SOAPState, params: Updates | None = None
273
+ ) -> tuple[Updates, SOAPState]:
274
+ del params
275
+ count_inc = jnp.asarray(optax.safe_int32_increment(state.count))
276
+ state = state._replace(count=count_inc)
277
+
278
+ updates, new_state = jax.lax.cond(
279
+ count_inc == 1,
280
+ lambda: init_step(updates, state),
281
+ lambda: update_step(updates, state),
282
+ )
283
+
284
+ return updates, new_state
285
+
286
+ return optax.GradientTransformation(init_fn, update_fn) # type: ignore[return-value]
287
+
288
+
289
+ def add_decayed_weights_post(
290
+ weight_decay: float,
291
+ learning_rate: optax.ScalarOrSchedule,
292
+ ) -> GradientTransformation:
293
+ if weight_decay == 0.0:
294
+ return optax.identity()
295
+
296
+ def init_fn(params: Updates) -> PostDecayState:
297
+ del params
298
+ return PostDecayState(count=jnp.zeros([], jnp.int32))
299
+
300
+ def update_fn(
301
+ updates: Updates,
302
+ state: PostDecayState,
303
+ params: Updates | None = None,
304
+ ) -> tuple[Updates, PostDecayState]:
305
+ if params is None:
306
+ raise ValueError("add_decayed_weights_post requires parameters.")
307
+
308
+ count_inc = jnp.asarray(optax.safe_int32_increment(state.count))
309
+ lr = _resolve_learning_rate(learning_rate, count_inc)
310
+ decay = lr * weight_decay
311
+ decay = jnp.where(count_inc == 1, jnp.zeros_like(decay), decay)
312
+ updates = jtu.tree_map(lambda u, p: u - decay * p, updates, params)
313
+ return updates, state._replace(count=count_inc)
314
+
315
+ return optax.GradientTransformation(init_fn, update_fn) # type: ignore[return-value]
316
+
317
+
318
+ def update_preconditioner(
319
+ grad: Array,
320
+ GG: Preconditioner,
321
+ beta: float,
322
+ precision: jax.lax.PrecisionLike = jax.lax.Precision.HIGHEST,
323
+ ) -> Preconditioner:
324
+ if grad.ndim == 1:
325
+ if GG.matrices[0] is None:
326
+ return GG
327
+ return Preconditioner(
328
+ [
329
+ lerp(
330
+ GG.matrices[0],
331
+ jnp.matmul(grad[:, None], grad[None, :], precision=precision),
332
+ 1 - beta,
333
+ )
334
+ ]
335
+ )
336
+
337
+ new_GG = []
338
+ for idx, gg in enumerate(GG.matrices):
339
+ if gg is None:
340
+ new_GG.append(None)
341
+ continue
342
+
343
+ outer_product = jnp.tensordot(
344
+ grad,
345
+ grad,
346
+ axes=[[*chain(range(idx), range(idx + 1, len(grad.shape)))]] * 2,
347
+ precision=precision,
348
+ )
349
+ new_GG.append(lerp(gg, outer_product, 1 - beta))
350
+
351
+ return Preconditioner(new_GG)
352
+
353
+
354
+ def project(
355
+ grad: Array,
356
+ Q: Preconditioner,
357
+ precision: jax.lax.PrecisionLike = jax.lax.Precision.HIGHEST,
358
+ ) -> Array:
359
+ for mat in Q.matrices:
360
+ if mat is not None:
361
+ grad = jnp.tensordot(
362
+ grad,
363
+ mat,
364
+ axes=((0,), (0,)),
365
+ precision=precision,
366
+ )
367
+ else:
368
+ permute_order = list(range(1, len(grad.shape))) + [0]
369
+ grad = jnp.transpose(grad, permute_order)
370
+
371
+ return grad
372
+
373
+
374
+ def project_back(
375
+ grad: Array,
376
+ Q: Preconditioner,
377
+ precision: jax.lax.PrecisionLike = jax.lax.Precision.HIGHEST,
378
+ ) -> Array:
379
+ for mat in Q.matrices:
380
+ if mat is not None:
381
+ grad = jnp.tensordot(
382
+ grad,
383
+ mat,
384
+ axes=((0,), (1,)),
385
+ precision=precision,
386
+ )
387
+ else:
388
+ grad = jnp.moveaxis(grad, 0, -1)
389
+
390
+ return grad
391
+
392
+
393
+ def get_orthogonal_matrix(gg: Array | None, qr_dtype: ArrayDType) -> Array | None:
394
+ if gg is None:
395
+ return None
396
+
397
+ gg_mat = gg.astype(qr_dtype) if gg.dtype != qr_dtype else gg
398
+ jitter = jnp.asarray(1e-30, dtype=qr_dtype)
399
+ _, eigh = jnp.linalg.eigh(
400
+ gg_mat + jitter * jnp.eye(gg_mat.shape[0], dtype=qr_dtype)
401
+ )
402
+ q = jnp.flip(eigh, axis=1)
403
+ if q.dtype != gg.dtype:
404
+ q = q.astype(gg.dtype)
405
+ return q
406
+
407
+
408
+ def get_orthogonal_matrix_QR(
409
+ GG: Preconditioner,
410
+ Q: Preconditioner,
411
+ exp_avg_sq: Array,
412
+ precision: jax.lax.PrecisionLike = jax.lax.Precision.HIGHEST,
413
+ qr_dtype: ArrayDType = jnp.float32,
414
+ ) -> tuple[Preconditioner, Array]:
415
+ final_Q = []
416
+ for ind, (m, o) in enumerate(zip(GG.matrices, Q.matrices)):
417
+ if m is None or o is None:
418
+ final_Q.append(None)
419
+ continue
420
+
421
+ m_mat = m.astype(qr_dtype) if m.dtype != qr_dtype else m
422
+ o_mat = o.astype(qr_dtype) if o.dtype != qr_dtype else o
423
+ est_eig = jnp.diag(
424
+ jnp.matmul(
425
+ jnp.matmul(o_mat.T, m_mat, precision=precision),
426
+ o_mat,
427
+ precision=precision,
428
+ )
429
+ )
430
+ sort_idx = jnp.argsort(est_eig, descending=True)
431
+ exp_avg_sq = jnp.take(exp_avg_sq, sort_idx, axis=ind)
432
+ o_mat = o_mat[:, sort_idx]
433
+ power_iter = jnp.matmul(m_mat, o_mat, precision=precision)
434
+ q_new, _ = jnp.linalg.qr(power_iter)
435
+
436
+ if q_new.dtype != m.dtype:
437
+ q_new = q_new.astype(m.dtype)
438
+ final_Q.append(q_new)
439
+
440
+ return Preconditioner(final_Q), exp_avg_sq
441
+
442
+
443
+ def lerp(
444
+ start: Array,
445
+ end: Array,
446
+ weight: Numeric,
447
+ ) -> Array:
448
+ return start + weight * (end - start)
449
+
450
+
451
+ def init_conditioner(
452
+ p: Array,
453
+ max_precond_dim: int,
454
+ precondition_1d: bool,
455
+ dtype: ArrayDType,
456
+ ) -> Preconditioner:
457
+ if p.ndim == 1:
458
+ if not precondition_1d or p.shape[0] > max_precond_dim:
459
+ return Preconditioner([None])
460
+ return Preconditioner([jnp.zeros((p.shape[0], p.shape[0]), dtype=dtype)])
461
+
462
+ return Preconditioner(
463
+ [
464
+ jnp.zeros((s, s), dtype=dtype) if s <= max_precond_dim else None
465
+ for s in p.shape
466
+ ]
467
+ )
468
+
469
+
470
+ def _is_preconditioner(value: object) -> bool:
471
+ return isinstance(value, Preconditioner)
472
+
473
+
474
+ def _resolve_learning_rate(
475
+ learning_rate: optax.ScalarOrSchedule, count: Array
476
+ ) -> Array:
477
+ if callable(learning_rate):
478
+ return learning_rate(count)
479
+
480
+ return jnp.asarray(learning_rate)
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
@@ -0,0 +1,12 @@
1
+ # src/astro_emulators_toolkit/bundle/__init__.py
2
+ from .release import (
3
+ load_bundle_fingerprint_evaluation,
4
+ prepare_bundle_release,
5
+ verify_bundle_fingerprint_evaluation,
6
+ )
7
+
8
+ __all__ = [
9
+ "load_bundle_fingerprint_evaluation",
10
+ "prepare_bundle_release",
11
+ "verify_bundle_fingerprint_evaluation",
12
+ ]