astro-emulators-toolkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- astro_emulators_toolkit/__init__.py +23 -0
- astro_emulators_toolkit/_typing.py +13 -0
- astro_emulators_toolkit/_vendor/__init__.py +1 -0
- astro_emulators_toolkit/_vendor/get_soap_jax.py +51 -0
- astro_emulators_toolkit/_vendor/soap_jax/LICENSE +21 -0
- astro_emulators_toolkit/_vendor/soap_jax/README.txt +37 -0
- astro_emulators_toolkit/_vendor/soap_jax/__init__.py +4 -0
- astro_emulators_toolkit/_vendor/soap_jax/soap.py +480 -0
- astro_emulators_toolkit/_version.py +1 -0
- astro_emulators_toolkit/bundle/__init__.py +12 -0
- astro_emulators_toolkit/bundle/artifact.py +175 -0
- astro_emulators_toolkit/bundle/bundle.py +83 -0
- astro_emulators_toolkit/bundle/extras.py +328 -0
- astro_emulators_toolkit/bundle/hub.py +45 -0
- astro_emulators_toolkit/bundle/integrity.py +364 -0
- astro_emulators_toolkit/bundle/metadata.py +361 -0
- astro_emulators_toolkit/bundle/minmax_sidecars.py +291 -0
- astro_emulators_toolkit/bundle/provenance.py +52 -0
- astro_emulators_toolkit/bundle/readme.py +68 -0
- astro_emulators_toolkit/bundle/release.py +524 -0
- astro_emulators_toolkit/bundle/safetensors_io.py +68 -0
- astro_emulators_toolkit/bundle/versions.py +15 -0
- astro_emulators_toolkit/config/__init__.py +34 -0
- astro_emulators_toolkit/config/io.py +298 -0
- astro_emulators_toolkit/config/parsing.py +18 -0
- astro_emulators_toolkit/config/schema.py +681 -0
- astro_emulators_toolkit/data/__init__.py +32 -0
- astro_emulators_toolkit/data/array_dataset.py +159 -0
- astro_emulators_toolkit/data/loader.py +123 -0
- astro_emulators_toolkit/data/mapped_dataset.py +46 -0
- astro_emulators_toolkit/data/npy_table.py +102 -0
- astro_emulators_toolkit/data/preprocess.py +11 -0
- astro_emulators_toolkit/data/protocols.py +93 -0
- astro_emulators_toolkit/data/subset.py +47 -0
- astro_emulators_toolkit/data/toy.py +170 -0
- astro_emulators_toolkit/data/toy_utils.py +163 -0
- astro_emulators_toolkit/emulator.py +979 -0
- astro_emulators_toolkit/emulator_runtime.py +68 -0
- astro_emulators_toolkit/experimental/__init__.py +3 -0
- astro_emulators_toolkit/experimental/models/__init__.py +76 -0
- astro_emulators_toolkit/experimental/models/explicit_wavelength_mlp.py +180 -0
- astro_emulators_toolkit/experimental/models/mlp_2d_regression.py +73 -0
- astro_emulators_toolkit/experimental/models/siren.py +128 -0
- astro_emulators_toolkit/experimental/runtime_adapters.py +442 -0
- astro_emulators_toolkit/experimental/tasks/__init__.py +25 -0
- astro_emulators_toolkit/experimental/tasks/classification.py +131 -0
- astro_emulators_toolkit/inference/__init__.py +3 -0
- astro_emulators_toolkit/inference/compose.py +319 -0
- astro_emulators_toolkit/io_trees.py +431 -0
- astro_emulators_toolkit/models/__init__.py +86 -0
- astro_emulators_toolkit/models/activations.py +25 -0
- astro_emulators_toolkit/models/cannon.py +62 -0
- astro_emulators_toolkit/models/canonical_wrappers.py +82 -0
- astro_emulators_toolkit/models/mlp.py +111 -0
- astro_emulators_toolkit/models/runtime_adapters.py +840 -0
- astro_emulators_toolkit/models/transformer_payne.py +794 -0
- astro_emulators_toolkit/models/transformer_payne_batch.py +566 -0
- astro_emulators_toolkit/optimizers/__init__.py +344 -0
- astro_emulators_toolkit/optimizers/_soap.py +5 -0
- astro_emulators_toolkit/presets/__init__.py +11 -0
- astro_emulators_toolkit/presets/cannon.py +50 -0
- astro_emulators_toolkit/presets/mlp.py +148 -0
- astro_emulators_toolkit/presets/transformer_payne.py +133 -0
- astro_emulators_toolkit/py.typed +1 -0
- astro_emulators_toolkit/resolver.py +181 -0
- astro_emulators_toolkit/spec.py +371 -0
- astro_emulators_toolkit/tasks/__init__.py +23 -0
- astro_emulators_toolkit/tasks/common.py +105 -0
- astro_emulators_toolkit/tasks/regression.py +351 -0
- astro_emulators_toolkit/training/__init__.py +28 -0
- astro_emulators_toolkit/training/callbacks.py +363 -0
- astro_emulators_toolkit/training/checkpoint.py +77 -0
- astro_emulators_toolkit/training/fit_backends.py +401 -0
- astro_emulators_toolkit/training/paths.py +5 -0
- astro_emulators_toolkit/training/solvers.py +226 -0
- astro_emulators_toolkit/training/state.py +29 -0
- astro_emulators_toolkit/training/trainer.py +347 -0
- astro_emulators_toolkit/utils/__init__.py +3 -0
- astro_emulators_toolkit/utils/tree.py +35 -0
- astro_emulators_toolkit-0.1.0.dist-info/METADATA +1130 -0
- astro_emulators_toolkit-0.1.0.dist-info/RECORD +84 -0
- astro_emulators_toolkit-0.1.0.dist-info/WHEEL +5 -0
- astro_emulators_toolkit-0.1.0.dist-info/licenses/LICENSE +21 -0
- astro_emulators_toolkit-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from ._version import __version__
|
|
2
|
+
from .config.io import load_config, save_config
|
|
3
|
+
from .config.schema import RootConfig
|
|
4
|
+
from .io_trees import denormalize_tree, normalize_tree
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"Emulator",
|
|
9
|
+
"RootConfig",
|
|
10
|
+
"load_config",
|
|
11
|
+
"save_config",
|
|
12
|
+
"normalize_tree",
|
|
13
|
+
"denormalize_tree",
|
|
14
|
+
"__version__",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def __getattr__(name: str):
|
|
19
|
+
if name == "Emulator":
|
|
20
|
+
from .emulator import Emulator
|
|
21
|
+
|
|
22
|
+
return Emulator
|
|
23
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any, Protocol, TypeAlias
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
PytreeDict: TypeAlias = dict[str, Any]
|
|
7
|
+
AffineLeafSpec: TypeAlias = dict[str, Any]
|
|
8
|
+
AffineLeafSpecs: TypeAlias = dict[str, AffineLeafSpec]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class SupportsFromDict(Protocol):
|
|
12
|
+
@classmethod
|
|
13
|
+
def from_dict(cls, d: dict[str, Any]) -> Any: ...
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Vendored third-party sources used as internal implementation details."""
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Refresh the vendored SOAP_JAX snapshot from the pinned upstream commit.
|
|
2
|
+
|
|
3
|
+
This script documents how the vendored copy in
|
|
4
|
+
`astro_emulators_toolkit._vendor.soap_jax` was obtained. It clones the public
|
|
5
|
+
upstream repository, checks out the pinned commit, and copies the Python
|
|
6
|
+
sources plus license into this package.
|
|
7
|
+
|
|
8
|
+
The local `README.txt` is maintained by this repository and is intentionally
|
|
9
|
+
not overwritten by the refresh step.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import shutil
|
|
15
|
+
import subprocess
|
|
16
|
+
import tempfile
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
UPSTREAM_URL = "https://github.com/haydn-jones/SOAP_JAX"
|
|
21
|
+
UPSTREAM_COMMIT = "ddddc25724dfd629b0cd01584eca1e32ea8ac4de"
|
|
22
|
+
VENDOR_ROOT = Path(__file__).resolve().parent / "soap_jax"
|
|
23
|
+
UPSTREAM_PACKAGE_DIR = Path("src") / "soap_jax"
|
|
24
|
+
UPSTREAM_FILES = ("__init__.py", "soap.py")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def refresh_vendor_tree() -> None:
|
|
28
|
+
VENDOR_ROOT.mkdir(parents=True, exist_ok=True)
|
|
29
|
+
|
|
30
|
+
with tempfile.TemporaryDirectory(prefix="soap_jax_vendor_") as tmp_dir:
|
|
31
|
+
repo_dir = Path(tmp_dir) / "SOAP_JAX"
|
|
32
|
+
|
|
33
|
+
subprocess.run(["git", "clone", UPSTREAM_URL, str(repo_dir)], check=True)
|
|
34
|
+
subprocess.run(
|
|
35
|
+
["git", "-C", str(repo_dir), "checkout", UPSTREAM_COMMIT], check=True
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
source_root = repo_dir / UPSTREAM_PACKAGE_DIR
|
|
39
|
+
for filename in UPSTREAM_FILES:
|
|
40
|
+
shutil.copy2(source_root / filename, VENDOR_ROOT / filename)
|
|
41
|
+
shutil.copy2(repo_dir / "LICENSE", VENDOR_ROOT / "LICENSE")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def main() -> None:
|
|
45
|
+
refresh_vendor_tree()
|
|
46
|
+
print(f"Vendored SOAP_JAX from {UPSTREAM_URL} at {UPSTREAM_COMMIT}.")
|
|
47
|
+
print(f"Updated files under: {VENDOR_ROOT}")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
if __name__ == "__main__":
|
|
51
|
+
main()
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Haydn Jones
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
Vendored SOAP_JAX
|
|
2
|
+
=================
|
|
3
|
+
|
|
4
|
+
This directory contains a vendored copy of the public `SOAP_JAX` project by
|
|
5
|
+
Haydn Jones. Many thanks to the upstream author for publishing and maintaining
|
|
6
|
+
the implementation.
|
|
7
|
+
|
|
8
|
+
Upstream repository:
|
|
9
|
+
https://github.com/haydn-jones/SOAP_JAX
|
|
10
|
+
|
|
11
|
+
Pinned upstream commit:
|
|
12
|
+
ddddc25724dfd629b0cd01584eca1e32ea8ac4de
|
|
13
|
+
|
|
14
|
+
What is included here:
|
|
15
|
+
- `__init__.py`
|
|
16
|
+
- `soap.py`
|
|
17
|
+
- `LICENSE`
|
|
18
|
+
|
|
19
|
+
Why this is vendored:
|
|
20
|
+
- SOAP is an internal optimizer implementation detail for this toolkit.
|
|
21
|
+
- `uv sync` already installs this project editable, so keeping SOAP in `src/`
|
|
22
|
+
avoids an extra separately-installed package.
|
|
23
|
+
- This keeps the integration surface small while preserving upstream
|
|
24
|
+
provenance.
|
|
25
|
+
- It should make a future migration simpler if comparable functionality lands
|
|
26
|
+
directly in Optax and we want to swap implementations behind
|
|
27
|
+
`astro_emulators_toolkit.optimizers`.
|
|
28
|
+
|
|
29
|
+
Local adjustments:
|
|
30
|
+
- The vendored file keeps the upstream algorithm and structure.
|
|
31
|
+
- Type-only dependencies on `chex` and `jaxtyping` were removed to keep the
|
|
32
|
+
toolkit dependency surface smaller.
|
|
33
|
+
|
|
34
|
+
Refresh workflow:
|
|
35
|
+
- Run `python -m astro_emulators_toolkit._vendor.get_soap_jax` to re-fetch the
|
|
36
|
+
pinned upstream snapshot.
|
|
37
|
+
- Review the diff afterward before committing any update.
|
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
"""Vendored SOAP_JAX implementation.
|
|
2
|
+
|
|
3
|
+
Source:
|
|
4
|
+
https://github.com/haydn-jones/SOAP_JAX
|
|
5
|
+
|
|
6
|
+
Pinned upstream commit:
|
|
7
|
+
ddddc25724dfd629b0cd01584eca1e32ea8ac4de
|
|
8
|
+
|
|
9
|
+
Minimal local edits:
|
|
10
|
+
- vendored under the toolkit package
|
|
11
|
+
- removed type-only dependencies on `chex` and `jaxtyping`
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from itertools import chain
|
|
17
|
+
from typing import Any, Callable, Iterable, NamedTuple
|
|
18
|
+
|
|
19
|
+
import jax
|
|
20
|
+
import jax.numpy as jnp
|
|
21
|
+
import jax.tree_util as jtu
|
|
22
|
+
import optax
|
|
23
|
+
import optax.tree_utils as otu
|
|
24
|
+
from optax import GradientTransformation, Updates
|
|
25
|
+
|
|
26
|
+
Array = Any
|
|
27
|
+
ArrayDType = Any
|
|
28
|
+
Numeric = Any
|
|
29
|
+
PreconditionerMatrix = Array | None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@jtu.register_pytree_node_class
|
|
33
|
+
class Preconditioner:
|
|
34
|
+
"""Per-parameter preconditioner matrices (None for dimensions exceeding max_precond_dim)."""
|
|
35
|
+
|
|
36
|
+
__slots__ = ("matrices",)
|
|
37
|
+
|
|
38
|
+
def __init__(self, matrices: Iterable[PreconditionerMatrix]):
|
|
39
|
+
self.matrices = tuple(matrices)
|
|
40
|
+
|
|
41
|
+
def tree_flatten(self) -> tuple[tuple[PreconditionerMatrix, ...], None]:
|
|
42
|
+
return (self.matrices, None)
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def tree_unflatten(
|
|
46
|
+
cls, aux_data: None, children: tuple[PreconditionerMatrix, ...]
|
|
47
|
+
) -> "Preconditioner":
|
|
48
|
+
return cls(children)
|
|
49
|
+
|
|
50
|
+
def map(
|
|
51
|
+
self, fn: Callable[[PreconditionerMatrix], PreconditionerMatrix]
|
|
52
|
+
) -> "Preconditioner":
|
|
53
|
+
return Preconditioner(fn(m) for m in self.matrices)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class SOAPState(NamedTuple):
|
|
57
|
+
count: Array
|
|
58
|
+
exp_avg: Updates
|
|
59
|
+
exp_avg_sq: Updates
|
|
60
|
+
GG: Updates
|
|
61
|
+
Q: Updates
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class PostDecayState(NamedTuple):
|
|
65
|
+
count: Array
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def soap(
|
|
69
|
+
learning_rate: optax.ScalarOrSchedule = 3e-3,
|
|
70
|
+
b1: float = 0.95,
|
|
71
|
+
b2: float = 0.95,
|
|
72
|
+
shampoo_beta: float = -1,
|
|
73
|
+
eps: float = 1e-8,
|
|
74
|
+
weight_decay: float = 0.0,
|
|
75
|
+
correct_bias: bool = True,
|
|
76
|
+
precondition_frequency: int = 10,
|
|
77
|
+
max_precond_dim: int = 10000,
|
|
78
|
+
precondition_1d: bool = False,
|
|
79
|
+
precision: jax.lax.PrecisionLike = jax.lax.Precision.HIGHEST,
|
|
80
|
+
mu_dtype: ArrayDType | None = None,
|
|
81
|
+
qr_dtype: ArrayDType = jnp.float32,
|
|
82
|
+
) -> optax.GradientTransformationExtraArgs:
|
|
83
|
+
"""SOAP optimizer based on the public SOAP_JAX implementation."""
|
|
84
|
+
|
|
85
|
+
return optax.chain(
|
|
86
|
+
scale_by_soap(
|
|
87
|
+
b1=b1,
|
|
88
|
+
b2=b2,
|
|
89
|
+
shampoo_beta=shampoo_beta,
|
|
90
|
+
eps=eps,
|
|
91
|
+
correct_bias=correct_bias,
|
|
92
|
+
precondition_frequency=precondition_frequency,
|
|
93
|
+
max_precond_dim=max_precond_dim,
|
|
94
|
+
precondition_1d=precondition_1d,
|
|
95
|
+
precision=precision,
|
|
96
|
+
mu_dtype=mu_dtype,
|
|
97
|
+
qr_dtype=qr_dtype,
|
|
98
|
+
),
|
|
99
|
+
optax.scale_by_learning_rate(learning_rate),
|
|
100
|
+
add_decayed_weights_post(weight_decay, learning_rate),
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def scale_by_soap(
|
|
105
|
+
b1: float = 0.95,
|
|
106
|
+
b2: float = 0.95,
|
|
107
|
+
shampoo_beta: float = -1,
|
|
108
|
+
eps: float = 1e-8,
|
|
109
|
+
correct_bias: bool = True,
|
|
110
|
+
precondition_frequency: int = 10,
|
|
111
|
+
max_precond_dim: int = 10000,
|
|
112
|
+
precondition_1d: bool = False,
|
|
113
|
+
precision: jax.lax.PrecisionLike = jax.lax.Precision.HIGHEST,
|
|
114
|
+
mu_dtype: ArrayDType | None = None,
|
|
115
|
+
qr_dtype: ArrayDType = jnp.float32,
|
|
116
|
+
) -> GradientTransformation:
|
|
117
|
+
"""SOAP gradient transformation based on the public SOAP_JAX implementation."""
|
|
118
|
+
|
|
119
|
+
if not (0 <= b1 < 1):
|
|
120
|
+
raise ValueError("b1 must be in [0, 1)")
|
|
121
|
+
if not (0 <= b2 < 1):
|
|
122
|
+
raise ValueError("b2 must be in [0, 1)")
|
|
123
|
+
if shampoo_beta >= 1 or (shampoo_beta < 0 and shampoo_beta != -1):
|
|
124
|
+
raise ValueError("shampoo_beta must be in [0, 1) or -1")
|
|
125
|
+
if eps <= 0:
|
|
126
|
+
raise ValueError("eps must be positive")
|
|
127
|
+
if precondition_frequency <= 0:
|
|
128
|
+
raise ValueError("precondition_frequency must be a positive integer")
|
|
129
|
+
if max_precond_dim <= 0:
|
|
130
|
+
raise ValueError("max_precond_dim must be a positive integer")
|
|
131
|
+
|
|
132
|
+
shampoo_beta = shampoo_beta if shampoo_beta >= 0 else b2
|
|
133
|
+
|
|
134
|
+
def init_fn(params: Updates) -> SOAPState:
|
|
135
|
+
exp_avg = otu.tree_zeros_like(params, dtype=mu_dtype)
|
|
136
|
+
exp_avg_sq = otu.tree_zeros_like(params, dtype=mu_dtype)
|
|
137
|
+
GG = jtu.tree_map(
|
|
138
|
+
lambda p: init_conditioner(p, max_precond_dim, precondition_1d, qr_dtype),
|
|
139
|
+
params,
|
|
140
|
+
)
|
|
141
|
+
Q = jtu.tree_map(
|
|
142
|
+
lambda p: init_conditioner(p, max_precond_dim, precondition_1d, qr_dtype),
|
|
143
|
+
params,
|
|
144
|
+
)
|
|
145
|
+
return SOAPState(
|
|
146
|
+
count=jnp.zeros([], jnp.int32),
|
|
147
|
+
exp_avg=exp_avg,
|
|
148
|
+
exp_avg_sq=exp_avg_sq,
|
|
149
|
+
GG=GG,
|
|
150
|
+
Q=Q,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
def init_step(
|
|
154
|
+
updates: Updates,
|
|
155
|
+
state: SOAPState,
|
|
156
|
+
) -> tuple[Updates, SOAPState]:
|
|
157
|
+
new_GG = jtu.tree_map(
|
|
158
|
+
lambda grad, gg: update_preconditioner(grad, gg, shampoo_beta, precision),
|
|
159
|
+
updates,
|
|
160
|
+
state.GG,
|
|
161
|
+
is_leaf=_is_preconditioner,
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
new_Q = jtu.tree_map(
|
|
165
|
+
lambda gg: gg.map(lambda m: get_orthogonal_matrix(m, qr_dtype)),
|
|
166
|
+
new_GG,
|
|
167
|
+
is_leaf=_is_preconditioner,
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
new_updates = otu.tree_zeros_like(updates)
|
|
171
|
+
|
|
172
|
+
return new_updates, state._replace(GG=new_GG, Q=new_Q)
|
|
173
|
+
|
|
174
|
+
def update_step(
|
|
175
|
+
updates: Updates,
|
|
176
|
+
state: SOAPState,
|
|
177
|
+
) -> tuple[Updates, SOAPState]:
|
|
178
|
+
grad_projected = jtu.tree_map(
|
|
179
|
+
lambda grad, q: project(grad, q, precision),
|
|
180
|
+
updates,
|
|
181
|
+
state.Q,
|
|
182
|
+
is_leaf=_is_preconditioner,
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
exp_avg = otu.tree_update_moment(grad_projected, state.exp_avg, b1, 1)
|
|
186
|
+
exp_avg_sq = otu.tree_update_moment_per_elem_norm(
|
|
187
|
+
grad_projected, state.exp_avg_sq, b2, 2
|
|
188
|
+
)
|
|
189
|
+
if mu_dtype is not None:
|
|
190
|
+
exp_avg = otu.tree_cast(exp_avg, mu_dtype)
|
|
191
|
+
exp_avg_sq = otu.tree_cast(exp_avg_sq, mu_dtype)
|
|
192
|
+
|
|
193
|
+
norm_updates = jtu.tree_map(
|
|
194
|
+
lambda e_avg, e_avg_sq, q: project_back(
|
|
195
|
+
e_avg / (jnp.sqrt(e_avg_sq) + jnp.asarray(eps, dtype=e_avg_sq.dtype)),
|
|
196
|
+
q,
|
|
197
|
+
precision,
|
|
198
|
+
),
|
|
199
|
+
exp_avg,
|
|
200
|
+
exp_avg_sq,
|
|
201
|
+
state.Q,
|
|
202
|
+
is_leaf=_is_preconditioner,
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
if correct_bias:
|
|
206
|
+
effective_step = state.count - 1
|
|
207
|
+
bc1 = 1 - b1**effective_step
|
|
208
|
+
bc2 = 1 - b2**effective_step
|
|
209
|
+
corr = jnp.sqrt(bc2) / bc1
|
|
210
|
+
norm_updates = jtu.tree_map(
|
|
211
|
+
lambda p: p * jnp.asarray(corr, dtype=p.dtype), norm_updates
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
new_GG = jtu.tree_map(
|
|
215
|
+
lambda grad, gg: update_preconditioner(grad, gg, shampoo_beta, precision),
|
|
216
|
+
updates,
|
|
217
|
+
state.GG,
|
|
218
|
+
is_leaf=_is_preconditioner,
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
def refresh_preconditioner() -> tuple[Updates, Updates, Updates]:
|
|
222
|
+
new_Q_and_exp_avg_sq = jtu.tree_map(
|
|
223
|
+
lambda e, gg, q: get_orthogonal_matrix_QR(
|
|
224
|
+
gg, q, e, precision, qr_dtype
|
|
225
|
+
),
|
|
226
|
+
exp_avg_sq,
|
|
227
|
+
new_GG,
|
|
228
|
+
state.Q,
|
|
229
|
+
is_leaf=_is_preconditioner,
|
|
230
|
+
)
|
|
231
|
+
new_Q = jtu.tree_map(
|
|
232
|
+
lambda _, x: x[0],
|
|
233
|
+
updates,
|
|
234
|
+
new_Q_and_exp_avg_sq,
|
|
235
|
+
)
|
|
236
|
+
new_exp_avg_sq = jtu.tree_map(
|
|
237
|
+
lambda _, x: x[1],
|
|
238
|
+
updates,
|
|
239
|
+
new_Q_and_exp_avg_sq,
|
|
240
|
+
)
|
|
241
|
+
new_exp_avg = jtu.tree_map(
|
|
242
|
+
lambda e, old_q, new_q: project(
|
|
243
|
+
project_back(e, old_q, precision), new_q, precision
|
|
244
|
+
),
|
|
245
|
+
exp_avg,
|
|
246
|
+
state.Q,
|
|
247
|
+
new_Q,
|
|
248
|
+
is_leaf=_is_preconditioner,
|
|
249
|
+
)
|
|
250
|
+
return new_Q, new_exp_avg_sq, new_exp_avg
|
|
251
|
+
|
|
252
|
+
def keep_preconditioner() -> tuple[Updates, Updates, Updates]:
|
|
253
|
+
return state.Q, exp_avg_sq, exp_avg
|
|
254
|
+
|
|
255
|
+
new_Q, exp_avg_sq, exp_avg = jax.lax.cond(
|
|
256
|
+
(state.count - 1) % precondition_frequency == 0,
|
|
257
|
+
refresh_preconditioner,
|
|
258
|
+
keep_preconditioner,
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
new_state = SOAPState(
|
|
262
|
+
count=state.count,
|
|
263
|
+
exp_avg=exp_avg,
|
|
264
|
+
exp_avg_sq=exp_avg_sq,
|
|
265
|
+
GG=new_GG,
|
|
266
|
+
Q=new_Q,
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
return norm_updates, new_state
|
|
270
|
+
|
|
271
|
+
def update_fn(
|
|
272
|
+
updates: Updates, state: SOAPState, params: Updates | None = None
|
|
273
|
+
) -> tuple[Updates, SOAPState]:
|
|
274
|
+
del params
|
|
275
|
+
count_inc = jnp.asarray(optax.safe_int32_increment(state.count))
|
|
276
|
+
state = state._replace(count=count_inc)
|
|
277
|
+
|
|
278
|
+
updates, new_state = jax.lax.cond(
|
|
279
|
+
count_inc == 1,
|
|
280
|
+
lambda: init_step(updates, state),
|
|
281
|
+
lambda: update_step(updates, state),
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
return updates, new_state
|
|
285
|
+
|
|
286
|
+
return optax.GradientTransformation(init_fn, update_fn) # type: ignore[return-value]
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def add_decayed_weights_post(
|
|
290
|
+
weight_decay: float,
|
|
291
|
+
learning_rate: optax.ScalarOrSchedule,
|
|
292
|
+
) -> GradientTransformation:
|
|
293
|
+
if weight_decay == 0.0:
|
|
294
|
+
return optax.identity()
|
|
295
|
+
|
|
296
|
+
def init_fn(params: Updates) -> PostDecayState:
|
|
297
|
+
del params
|
|
298
|
+
return PostDecayState(count=jnp.zeros([], jnp.int32))
|
|
299
|
+
|
|
300
|
+
def update_fn(
|
|
301
|
+
updates: Updates,
|
|
302
|
+
state: PostDecayState,
|
|
303
|
+
params: Updates | None = None,
|
|
304
|
+
) -> tuple[Updates, PostDecayState]:
|
|
305
|
+
if params is None:
|
|
306
|
+
raise ValueError("add_decayed_weights_post requires parameters.")
|
|
307
|
+
|
|
308
|
+
count_inc = jnp.asarray(optax.safe_int32_increment(state.count))
|
|
309
|
+
lr = _resolve_learning_rate(learning_rate, count_inc)
|
|
310
|
+
decay = lr * weight_decay
|
|
311
|
+
decay = jnp.where(count_inc == 1, jnp.zeros_like(decay), decay)
|
|
312
|
+
updates = jtu.tree_map(lambda u, p: u - decay * p, updates, params)
|
|
313
|
+
return updates, state._replace(count=count_inc)
|
|
314
|
+
|
|
315
|
+
return optax.GradientTransformation(init_fn, update_fn) # type: ignore[return-value]
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def update_preconditioner(
|
|
319
|
+
grad: Array,
|
|
320
|
+
GG: Preconditioner,
|
|
321
|
+
beta: float,
|
|
322
|
+
precision: jax.lax.PrecisionLike = jax.lax.Precision.HIGHEST,
|
|
323
|
+
) -> Preconditioner:
|
|
324
|
+
if grad.ndim == 1:
|
|
325
|
+
if GG.matrices[0] is None:
|
|
326
|
+
return GG
|
|
327
|
+
return Preconditioner(
|
|
328
|
+
[
|
|
329
|
+
lerp(
|
|
330
|
+
GG.matrices[0],
|
|
331
|
+
jnp.matmul(grad[:, None], grad[None, :], precision=precision),
|
|
332
|
+
1 - beta,
|
|
333
|
+
)
|
|
334
|
+
]
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
new_GG = []
|
|
338
|
+
for idx, gg in enumerate(GG.matrices):
|
|
339
|
+
if gg is None:
|
|
340
|
+
new_GG.append(None)
|
|
341
|
+
continue
|
|
342
|
+
|
|
343
|
+
outer_product = jnp.tensordot(
|
|
344
|
+
grad,
|
|
345
|
+
grad,
|
|
346
|
+
axes=[[*chain(range(idx), range(idx + 1, len(grad.shape)))]] * 2,
|
|
347
|
+
precision=precision,
|
|
348
|
+
)
|
|
349
|
+
new_GG.append(lerp(gg, outer_product, 1 - beta))
|
|
350
|
+
|
|
351
|
+
return Preconditioner(new_GG)
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def project(
|
|
355
|
+
grad: Array,
|
|
356
|
+
Q: Preconditioner,
|
|
357
|
+
precision: jax.lax.PrecisionLike = jax.lax.Precision.HIGHEST,
|
|
358
|
+
) -> Array:
|
|
359
|
+
for mat in Q.matrices:
|
|
360
|
+
if mat is not None:
|
|
361
|
+
grad = jnp.tensordot(
|
|
362
|
+
grad,
|
|
363
|
+
mat,
|
|
364
|
+
axes=((0,), (0,)),
|
|
365
|
+
precision=precision,
|
|
366
|
+
)
|
|
367
|
+
else:
|
|
368
|
+
permute_order = list(range(1, len(grad.shape))) + [0]
|
|
369
|
+
grad = jnp.transpose(grad, permute_order)
|
|
370
|
+
|
|
371
|
+
return grad
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def project_back(
|
|
375
|
+
grad: Array,
|
|
376
|
+
Q: Preconditioner,
|
|
377
|
+
precision: jax.lax.PrecisionLike = jax.lax.Precision.HIGHEST,
|
|
378
|
+
) -> Array:
|
|
379
|
+
for mat in Q.matrices:
|
|
380
|
+
if mat is not None:
|
|
381
|
+
grad = jnp.tensordot(
|
|
382
|
+
grad,
|
|
383
|
+
mat,
|
|
384
|
+
axes=((0,), (1,)),
|
|
385
|
+
precision=precision,
|
|
386
|
+
)
|
|
387
|
+
else:
|
|
388
|
+
grad = jnp.moveaxis(grad, 0, -1)
|
|
389
|
+
|
|
390
|
+
return grad
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def get_orthogonal_matrix(gg: Array | None, qr_dtype: ArrayDType) -> Array | None:
|
|
394
|
+
if gg is None:
|
|
395
|
+
return None
|
|
396
|
+
|
|
397
|
+
gg_mat = gg.astype(qr_dtype) if gg.dtype != qr_dtype else gg
|
|
398
|
+
jitter = jnp.asarray(1e-30, dtype=qr_dtype)
|
|
399
|
+
_, eigh = jnp.linalg.eigh(
|
|
400
|
+
gg_mat + jitter * jnp.eye(gg_mat.shape[0], dtype=qr_dtype)
|
|
401
|
+
)
|
|
402
|
+
q = jnp.flip(eigh, axis=1)
|
|
403
|
+
if q.dtype != gg.dtype:
|
|
404
|
+
q = q.astype(gg.dtype)
|
|
405
|
+
return q
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def get_orthogonal_matrix_QR(
|
|
409
|
+
GG: Preconditioner,
|
|
410
|
+
Q: Preconditioner,
|
|
411
|
+
exp_avg_sq: Array,
|
|
412
|
+
precision: jax.lax.PrecisionLike = jax.lax.Precision.HIGHEST,
|
|
413
|
+
qr_dtype: ArrayDType = jnp.float32,
|
|
414
|
+
) -> tuple[Preconditioner, Array]:
|
|
415
|
+
final_Q = []
|
|
416
|
+
for ind, (m, o) in enumerate(zip(GG.matrices, Q.matrices)):
|
|
417
|
+
if m is None or o is None:
|
|
418
|
+
final_Q.append(None)
|
|
419
|
+
continue
|
|
420
|
+
|
|
421
|
+
m_mat = m.astype(qr_dtype) if m.dtype != qr_dtype else m
|
|
422
|
+
o_mat = o.astype(qr_dtype) if o.dtype != qr_dtype else o
|
|
423
|
+
est_eig = jnp.diag(
|
|
424
|
+
jnp.matmul(
|
|
425
|
+
jnp.matmul(o_mat.T, m_mat, precision=precision),
|
|
426
|
+
o_mat,
|
|
427
|
+
precision=precision,
|
|
428
|
+
)
|
|
429
|
+
)
|
|
430
|
+
sort_idx = jnp.argsort(est_eig, descending=True)
|
|
431
|
+
exp_avg_sq = jnp.take(exp_avg_sq, sort_idx, axis=ind)
|
|
432
|
+
o_mat = o_mat[:, sort_idx]
|
|
433
|
+
power_iter = jnp.matmul(m_mat, o_mat, precision=precision)
|
|
434
|
+
q_new, _ = jnp.linalg.qr(power_iter)
|
|
435
|
+
|
|
436
|
+
if q_new.dtype != m.dtype:
|
|
437
|
+
q_new = q_new.astype(m.dtype)
|
|
438
|
+
final_Q.append(q_new)
|
|
439
|
+
|
|
440
|
+
return Preconditioner(final_Q), exp_avg_sq
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def lerp(
|
|
444
|
+
start: Array,
|
|
445
|
+
end: Array,
|
|
446
|
+
weight: Numeric,
|
|
447
|
+
) -> Array:
|
|
448
|
+
return start + weight * (end - start)
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def init_conditioner(
|
|
452
|
+
p: Array,
|
|
453
|
+
max_precond_dim: int,
|
|
454
|
+
precondition_1d: bool,
|
|
455
|
+
dtype: ArrayDType,
|
|
456
|
+
) -> Preconditioner:
|
|
457
|
+
if p.ndim == 1:
|
|
458
|
+
if not precondition_1d or p.shape[0] > max_precond_dim:
|
|
459
|
+
return Preconditioner([None])
|
|
460
|
+
return Preconditioner([jnp.zeros((p.shape[0], p.shape[0]), dtype=dtype)])
|
|
461
|
+
|
|
462
|
+
return Preconditioner(
|
|
463
|
+
[
|
|
464
|
+
jnp.zeros((s, s), dtype=dtype) if s <= max_precond_dim else None
|
|
465
|
+
for s in p.shape
|
|
466
|
+
]
|
|
467
|
+
)
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _is_preconditioner(value: object) -> bool:
|
|
471
|
+
return isinstance(value, Preconditioner)
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _resolve_learning_rate(
|
|
475
|
+
learning_rate: optax.ScalarOrSchedule, count: Array
|
|
476
|
+
) -> Array:
|
|
477
|
+
if callable(learning_rate):
|
|
478
|
+
return learning_rate(count)
|
|
479
|
+
|
|
480
|
+
return jnp.asarray(learning_rate)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# src/astro_emulators_toolkit/bundle/__init__.py
|
|
2
|
+
from .release import (
|
|
3
|
+
load_bundle_fingerprint_evaluation,
|
|
4
|
+
prepare_bundle_release,
|
|
5
|
+
verify_bundle_fingerprint_evaluation,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"load_bundle_fingerprint_evaluation",
|
|
10
|
+
"prepare_bundle_release",
|
|
11
|
+
"verify_bundle_fingerprint_evaluation",
|
|
12
|
+
]
|