laplacekit 0.1a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- laplacekit/__init__.py +169 -0
- laplacekit/_health.py +461 -0
- laplacekit/core/__init__.py +263 -0
- laplacekit/core/_functorch.py +55 -0
- laplacekit/core/_pytree.py +213 -0
- laplacekit/core/_seed.py +169 -0
- laplacekit/core/curvature.py +1032 -0
- laplacekit/core/fit.py +896 -0
- laplacekit/core/layout.py +150 -0
- laplacekit/core/likelihoods.py +869 -0
- laplacekit/core/model.py +100 -0
- laplacekit/core/ops.py +1583 -0
- laplacekit/core/posteriors.py +2013 -0
- laplacekit/core/predictive.py +140 -0
- laplacekit/core/probe.py +457 -0
- laplacekit/core/random.py +96 -0
- laplacekit/core/serialize.py +655 -0
- laplacekit/core/subsets.py +260 -0
- laplacekit/core/tree.py +5 -0
- laplacekit/core/tuning.py +401 -0
- laplacekit/core/types.py +390 -0
- laplacekit/curvature.py +1687 -0
- laplacekit/curvlinops.py +616 -0
- laplacekit/data.py +213 -0
- laplacekit/distributed.py +5 -0
- laplacekit/front.py +510 -0
- laplacekit/function_space.py +1031 -0
- laplacekit/likelihoods.py +759 -0
- laplacekit/objectives.py +388 -0
- laplacekit/predictive.py +1095 -0
- laplacekit/priors.py +934 -0
- laplacekit/py.typed +0 -0
- laplacekit/subsets.py +181 -0
- laplacekit/testing.py +1325 -0
- laplacekit/tuning.py +386 -0
- laplacekit-0.1a0.dist-info/METADATA +183 -0
- laplacekit-0.1a0.dist-info/RECORD +38 -0
- laplacekit-0.1a0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
"""Core protocols and shared machinery."""
|
|
2
|
+
|
|
3
|
+
from laplacekit.core._functorch import is_transformed
|
|
4
|
+
from laplacekit.core._pytree import as_pytree, pytree, static, tree_leaves, tree_map
|
|
5
|
+
from laplacekit.core.curvature import (
|
|
6
|
+
BlockRule,
|
|
7
|
+
BlockStat,
|
|
8
|
+
Curvature,
|
|
9
|
+
Done,
|
|
10
|
+
FitContext,
|
|
11
|
+
HasTemplate,
|
|
12
|
+
HessianApprox,
|
|
13
|
+
Pass,
|
|
14
|
+
PerBlockCurvature,
|
|
15
|
+
Staged,
|
|
16
|
+
StagedPerBlock,
|
|
17
|
+
Stat,
|
|
18
|
+
SumStat,
|
|
19
|
+
chain,
|
|
20
|
+
per_block,
|
|
21
|
+
stat_dtype,
|
|
22
|
+
)
|
|
23
|
+
from laplacekit.core.fit import (
|
|
24
|
+
CurvatureFit,
|
|
25
|
+
Finished,
|
|
26
|
+
FitStat,
|
|
27
|
+
Running,
|
|
28
|
+
StagedCurvatureFit,
|
|
29
|
+
all_reduce,
|
|
30
|
+
fit_curvature,
|
|
31
|
+
run,
|
|
32
|
+
)
|
|
33
|
+
from laplacekit.core.layout import ROOT_BLOCK, Layout
|
|
34
|
+
from laplacekit.core.likelihoods import (
|
|
35
|
+
MAX_DENSE_EVENT,
|
|
36
|
+
HasDistribution,
|
|
37
|
+
HasMask,
|
|
38
|
+
HasOutputCheck,
|
|
39
|
+
HasOutputHessian,
|
|
40
|
+
HasSample,
|
|
41
|
+
HasTargetCheck,
|
|
42
|
+
LateLikelihood,
|
|
43
|
+
Likelihood,
|
|
44
|
+
LikelihoodTree,
|
|
45
|
+
MaskedOutputHessian,
|
|
46
|
+
OutputHessian,
|
|
47
|
+
can_sample,
|
|
48
|
+
check_likelihood,
|
|
49
|
+
curvature_scale,
|
|
50
|
+
default_log_prob,
|
|
51
|
+
default_sample,
|
|
52
|
+
distribution,
|
|
53
|
+
log_lik,
|
|
54
|
+
log_prob,
|
|
55
|
+
observed_output_hessian,
|
|
56
|
+
output_hessian,
|
|
57
|
+
sample,
|
|
58
|
+
)
|
|
59
|
+
from laplacekit.core.model import ModelFn, tied_names
|
|
60
|
+
from laplacekit.core.ops import (
|
|
61
|
+
BlockDiag,
|
|
62
|
+
Dense,
|
|
63
|
+
Diag,
|
|
64
|
+
HasBlocks,
|
|
65
|
+
Joint,
|
|
66
|
+
KronEigenbasis,
|
|
67
|
+
LowRankPlusDiag,
|
|
68
|
+
Operator,
|
|
69
|
+
with_eigen,
|
|
70
|
+
)
|
|
71
|
+
from laplacekit.core.posteriors import (
|
|
72
|
+
DEFAULT_OUTPUT_SAMPLES,
|
|
73
|
+
FactorCov,
|
|
74
|
+
FunctionPosterior,
|
|
75
|
+
HasCovMany,
|
|
76
|
+
HasExpand,
|
|
77
|
+
HasVersions,
|
|
78
|
+
LaplacePosterior,
|
|
79
|
+
OutputCov,
|
|
80
|
+
OutputGaussian,
|
|
81
|
+
PerInputCov,
|
|
82
|
+
Posterior,
|
|
83
|
+
Prior,
|
|
84
|
+
PriorWithCovariance,
|
|
85
|
+
SampleCov,
|
|
86
|
+
Strategy,
|
|
87
|
+
TunablePrior,
|
|
88
|
+
WeightPosterior,
|
|
89
|
+
check_function_args,
|
|
90
|
+
check_location,
|
|
91
|
+
check_outputs_args,
|
|
92
|
+
exact_fits,
|
|
93
|
+
mean_outputs,
|
|
94
|
+
param_versions,
|
|
95
|
+
predict_outputs,
|
|
96
|
+
resolve_strategy,
|
|
97
|
+
sample_outputs,
|
|
98
|
+
)
|
|
99
|
+
from laplacekit.core.predictive import HasDefaultLink, HasSampleAverage, LinkApprox, Predictive
|
|
100
|
+
from laplacekit.core.probe import probe
|
|
101
|
+
from laplacekit.core.random import generator_for_batch, spawn_generator
|
|
102
|
+
from laplacekit.core.serialize import load, save, serializable
|
|
103
|
+
from laplacekit.core.subsets import Selection, Subset, select
|
|
104
|
+
from laplacekit.core.tuning import (
|
|
105
|
+
HasLikelihood,
|
|
106
|
+
HasPrior,
|
|
107
|
+
HyperTuner,
|
|
108
|
+
Objective,
|
|
109
|
+
TunableLikelihood,
|
|
110
|
+
check_fit_time,
|
|
111
|
+
check_tunable,
|
|
112
|
+
detach_leaves,
|
|
113
|
+
fill_leaves,
|
|
114
|
+
hyperparameters,
|
|
115
|
+
likelihood_hyperparameters,
|
|
116
|
+
minimize_on_logs,
|
|
117
|
+
prior_hyperparameters,
|
|
118
|
+
replace_leaves,
|
|
119
|
+
)
|
|
120
|
+
from laplacekit.core.types import (
|
|
121
|
+
Batch,
|
|
122
|
+
Capabilities,
|
|
123
|
+
CapabilityError,
|
|
124
|
+
Inputs,
|
|
125
|
+
Outputs,
|
|
126
|
+
Params,
|
|
127
|
+
PyTree,
|
|
128
|
+
Requirements,
|
|
129
|
+
StructureError,
|
|
130
|
+
Targets,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
__all__ = [
|
|
134
|
+
"MAX_DENSE_EVENT",
|
|
135
|
+
"HasDefaultLink",
|
|
136
|
+
"HasDistribution",
|
|
137
|
+
"HasMask",
|
|
138
|
+
"HasOutputCheck",
|
|
139
|
+
"HasOutputHessian",
|
|
140
|
+
"HasSample",
|
|
141
|
+
"HasSampleAverage",
|
|
142
|
+
"HasTargetCheck",
|
|
143
|
+
"LateLikelihood",
|
|
144
|
+
"Likelihood",
|
|
145
|
+
"LikelihoodTree",
|
|
146
|
+
"MaskedOutputHessian",
|
|
147
|
+
"OutputHessian",
|
|
148
|
+
"can_sample",
|
|
149
|
+
"check_likelihood",
|
|
150
|
+
"curvature_scale",
|
|
151
|
+
"default_log_prob",
|
|
152
|
+
"default_sample",
|
|
153
|
+
"distribution",
|
|
154
|
+
"log_lik",
|
|
155
|
+
"log_prob",
|
|
156
|
+
"observed_output_hessian",
|
|
157
|
+
"output_hessian",
|
|
158
|
+
"sample",
|
|
159
|
+
"DEFAULT_OUTPUT_SAMPLES",
|
|
160
|
+
"Batch",
|
|
161
|
+
"Posterior",
|
|
162
|
+
"BlockDiag",
|
|
163
|
+
"HasBlocks",
|
|
164
|
+
"BlockStat",
|
|
165
|
+
"Capabilities",
|
|
166
|
+
"CapabilityError",
|
|
167
|
+
"Curvature",
|
|
168
|
+
"Done",
|
|
169
|
+
"CurvatureFit",
|
|
170
|
+
"Finished",
|
|
171
|
+
"FitStat",
|
|
172
|
+
"Running",
|
|
173
|
+
"StagedCurvatureFit",
|
|
174
|
+
"all_reduce",
|
|
175
|
+
"fit_curvature",
|
|
176
|
+
"run",
|
|
177
|
+
"is_transformed",
|
|
178
|
+
"BlockRule",
|
|
179
|
+
"FitContext",
|
|
180
|
+
"HasTemplate",
|
|
181
|
+
"Pass",
|
|
182
|
+
"PerBlockCurvature",
|
|
183
|
+
"Staged",
|
|
184
|
+
"StagedPerBlock",
|
|
185
|
+
"Stat",
|
|
186
|
+
"SumStat",
|
|
187
|
+
"HessianApprox",
|
|
188
|
+
"chain",
|
|
189
|
+
"per_block",
|
|
190
|
+
"stat_dtype",
|
|
191
|
+
"detach_leaves",
|
|
192
|
+
"HasLikelihood",
|
|
193
|
+
"HasPrior",
|
|
194
|
+
"HyperTuner",
|
|
195
|
+
"TunableLikelihood",
|
|
196
|
+
"check_fit_time",
|
|
197
|
+
"check_tunable",
|
|
198
|
+
"fill_leaves",
|
|
199
|
+
"hyperparameters",
|
|
200
|
+
"likelihood_hyperparameters",
|
|
201
|
+
"minimize_on_logs",
|
|
202
|
+
"prior_hyperparameters",
|
|
203
|
+
"replace_leaves",
|
|
204
|
+
"HasCovMany",
|
|
205
|
+
"HasExpand",
|
|
206
|
+
"Dense",
|
|
207
|
+
"Diag",
|
|
208
|
+
"FactorCov",
|
|
209
|
+
"FunctionPosterior",
|
|
210
|
+
"HasVersions",
|
|
211
|
+
"Inputs",
|
|
212
|
+
"Joint",
|
|
213
|
+
"KronEigenbasis",
|
|
214
|
+
"LaplacePosterior",
|
|
215
|
+
"Layout",
|
|
216
|
+
"ROOT_BLOCK",
|
|
217
|
+
"LinkApprox",
|
|
218
|
+
"LowRankPlusDiag",
|
|
219
|
+
"ModelFn",
|
|
220
|
+
"Objective",
|
|
221
|
+
"Operator",
|
|
222
|
+
"OutputCov",
|
|
223
|
+
"OutputGaussian",
|
|
224
|
+
"Outputs",
|
|
225
|
+
"Params",
|
|
226
|
+
"PerInputCov",
|
|
227
|
+
"SampleCov",
|
|
228
|
+
"Predictive",
|
|
229
|
+
"PriorWithCovariance",
|
|
230
|
+
"PyTree",
|
|
231
|
+
"Requirements",
|
|
232
|
+
"StructureError",
|
|
233
|
+
"Selection",
|
|
234
|
+
"Subset",
|
|
235
|
+
"select",
|
|
236
|
+
"Targets",
|
|
237
|
+
"TunablePrior",
|
|
238
|
+
"Strategy",
|
|
239
|
+
"Prior",
|
|
240
|
+
"WeightPosterior",
|
|
241
|
+
"as_pytree",
|
|
242
|
+
"check_function_args",
|
|
243
|
+
"check_outputs_args",
|
|
244
|
+
"check_location",
|
|
245
|
+
"exact_fits",
|
|
246
|
+
"mean_outputs",
|
|
247
|
+
"predict_outputs",
|
|
248
|
+
"param_versions",
|
|
249
|
+
"resolve_strategy",
|
|
250
|
+
"sample_outputs",
|
|
251
|
+
"probe",
|
|
252
|
+
"pytree",
|
|
253
|
+
"load",
|
|
254
|
+
"save",
|
|
255
|
+
"serializable",
|
|
256
|
+
"static",
|
|
257
|
+
"tree_leaves",
|
|
258
|
+
"tree_map",
|
|
259
|
+
"tied_names",
|
|
260
|
+
"with_eigen",
|
|
261
|
+
"generator_for_batch",
|
|
262
|
+
"spawn_generator",
|
|
263
|
+
]
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Reads of torch's private functorch state.
|
|
2
|
+
|
|
3
|
+
torch has no public API for these, so they live here, each guarded by a test in
|
|
4
|
+
`tests/core/test_fit.py` or `tests/core/test_seed.py`.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from contextlib import AbstractContextManager
|
|
8
|
+
|
|
9
|
+
import torch
|
|
10
|
+
from torch import Tensor
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def transformed() -> bool:
|
|
14
|
+
"""Whether a `torch.func` transform (`vmap`, `grad`, ...) is running.
|
|
15
|
+
|
|
16
|
+
Tensors under one are wrapped and have no storage. This reads the private functorch
|
|
17
|
+
interpreter stack (guarded by `test_transformed_reads_the_functorch_interpreter_stack`).
|
|
18
|
+
|
|
19
|
+
Returns:
|
|
20
|
+
True inside a transform.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
return torch._C._functorch.peek_interpreter_stack() is not None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def is_transformed(t: Tensor, /) -> bool:
|
|
27
|
+
"""Whether `t` is wrapped by a `torch.func` transform (a `vmap`, `grad` or `jvp` level).
|
|
28
|
+
|
|
29
|
+
A wrapped tensor's values can't be read, so a check on its data (`bool(t.any())`) must be
|
|
30
|
+
skipped. This reads the private `torch._C._functorch.is_functorch_wrapped_tensor`
|
|
31
|
+
(guarded by `test_is_transformed_reads_functorch_wrappers`).
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
t: The tensor. One created outside the transform and only closed over is not
|
|
35
|
+
wrapped.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
True when `t` is wrapped.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
return torch._C._functorch.is_functorch_wrapped_tensor(t)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def outside_transforms() -> AbstractContextManager[None]:
|
|
45
|
+
"""A block that `torch.func` transforms don't see, for drawing a seed under `vmap`.
|
|
46
|
+
|
|
47
|
+
`vmap` refuses random operations by default. A seed is drawn once per call, not per
|
|
48
|
+
vmapped example, so it is drawn with functorch disabled. This uses the private
|
|
49
|
+
`torch._C._DisableFuncTorch` (guarded by `test_draw_works_under_transforms`).
|
|
50
|
+
|
|
51
|
+
Returns:
|
|
52
|
+
A context manager.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
return torch._C._DisableFuncTorch()
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
"""The one place that touches torch's (private) pytree registry.
|
|
2
|
+
|
|
3
|
+
`@pytree` turns a class into a frozen dataclass whose fields are pytree children,
|
|
4
|
+
except fields marked `static()`, which go into the treedef context. That makes
|
|
5
|
+
`torch.func.grad` and `vmap` see the tensors inside `@pytree` objects.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import dataclasses
|
|
9
|
+
from collections.abc import Callable, Mapping
|
|
10
|
+
from typing import Any, cast, dataclass_transform, overload
|
|
11
|
+
|
|
12
|
+
import torch.utils._pytree as _pt
|
|
13
|
+
|
|
14
|
+
_STATIC = "laplacekit_static"
|
|
15
|
+
|
|
16
|
+
PYTREE_CLASSES: dict[str, type] = {}
|
|
17
|
+
"""Every `@pytree` class, by `"module:qualname"`: the classes `lk.load` may rebuild."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@overload
|
|
21
|
+
def static[T](*, default: T, kw_only: bool = False) -> T: ...
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@overload
|
|
25
|
+
def static[T](*, default_factory: Callable[[], T], kw_only: bool = False) -> T: ...
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@overload
|
|
29
|
+
def static(*, kw_only: bool = False) -> Any: ...
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def static(**kwargs: Any) -> Any:
|
|
33
|
+
"""Mark a field as static: part of the structure, not a pytree leaf.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
**kwargs: Passed to `dataclasses.field`: `default` or `default_factory`, and
|
|
37
|
+
`kw_only=True` for a field that only a keyword sets.
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
A dataclass field.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
return dataclasses.field(metadata={_STATIC: True}, **kwargs)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass_transform(frozen_default=True, field_specifiers=(dataclasses.field, static))
|
|
47
|
+
def pytree[C: type](cls: C) -> C:
|
|
48
|
+
"""Make `cls` a frozen dataclass registered as a torch pytree node.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
cls: The class to decorate. Its fields marked `static()` go into the treedef
|
|
52
|
+
context; the others are children.
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
The frozen dataclass, registered.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
own_repr = "__repr__" in cls.__dict__
|
|
59
|
+
dc: type[Any] = dataclasses.dataclass(frozen=True)(cls)
|
|
60
|
+
fs = dataclasses.fields(dc)
|
|
61
|
+
|
|
62
|
+
if not own_repr:
|
|
63
|
+
dc.__repr__ = _compact_repr
|
|
64
|
+
children = tuple(f.name for f in fs if not f.metadata.get(_STATIC, False))
|
|
65
|
+
statics = tuple(f.name for f in fs if f.metadata.get(_STATIC, False))
|
|
66
|
+
|
|
67
|
+
def flatten(obj: Any) -> tuple[list[Any], tuple[Any, ...]]:
|
|
68
|
+
return [getattr(obj, n) for n in children], tuple(getattr(obj, n) for n in statics)
|
|
69
|
+
|
|
70
|
+
def unflatten(values: Any, context: tuple[Any, ...]) -> Any:
|
|
71
|
+
obj = object.__new__(dc)
|
|
72
|
+
|
|
73
|
+
for n, v in zip(children, values, strict=True):
|
|
74
|
+
object.__setattr__(obj, n, v)
|
|
75
|
+
|
|
76
|
+
for n, v in zip(statics, context, strict=True):
|
|
77
|
+
object.__setattr__(obj, n, v)
|
|
78
|
+
|
|
79
|
+
return obj
|
|
80
|
+
|
|
81
|
+
def flatten_with_keys(obj: Any) -> tuple[list[tuple[Any, Any]], tuple[Any, ...]]:
|
|
82
|
+
vals, ctx = flatten(obj)
|
|
83
|
+
return [(_pt.GetAttrKey(n), v) for n, v in zip(children, vals, strict=True)], ctx
|
|
84
|
+
|
|
85
|
+
PYTREE_CLASSES[f"{dc.__module__}:{dc.__qualname__}"] = dc
|
|
86
|
+
|
|
87
|
+
_pt.register_pytree_node(
|
|
88
|
+
dc,
|
|
89
|
+
flatten,
|
|
90
|
+
unflatten,
|
|
91
|
+
serialized_type_name=f"{dc.__module__}.{dc.__qualname__}",
|
|
92
|
+
flatten_with_keys_fn=flatten_with_keys,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
return cast(C, dc)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _is_default(f: dataclasses.Field[Any], value: Any) -> bool:
|
|
99
|
+
if f.default is not dataclasses.MISSING:
|
|
100
|
+
default = f.default
|
|
101
|
+
elif f.default_factory is not dataclasses.MISSING:
|
|
102
|
+
default = f.default_factory()
|
|
103
|
+
else:
|
|
104
|
+
return False
|
|
105
|
+
|
|
106
|
+
try:
|
|
107
|
+
same = value == default
|
|
108
|
+
except Exception: # noqa: BLE001 # a tensor against a default: not comparable
|
|
109
|
+
return False
|
|
110
|
+
|
|
111
|
+
return same is True
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _compact_repr(self: Any) -> str:
|
|
115
|
+
"""`Name(field=value, ...)` with the fields at their defaults left out, as `nn.Module`."""
|
|
116
|
+
|
|
117
|
+
shown = ", ".join(
|
|
118
|
+
f"{f.name}={getattr(self, f.name)!r}"
|
|
119
|
+
for f in dataclasses.fields(self)
|
|
120
|
+
if f.repr and not _is_default(f, getattr(self, f.name))
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
return f"{type(self).__qualname__.rsplit('.', 1)[-1]}({shown})"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def tree_map(fn: Callable[..., Any], tree: Any, *rest: Any) -> Any:
|
|
127
|
+
"""Apply `fn` to the leaves of `tree`.
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
fn: Called on each leaf of `tree`, followed by the matching leaves of `rest`.
|
|
131
|
+
tree: The tree.
|
|
132
|
+
*rest: Trees with the same structure as `tree`.
|
|
133
|
+
|
|
134
|
+
Returns:
|
|
135
|
+
A tree like `tree` holding the results.
|
|
136
|
+
"""
|
|
137
|
+
|
|
138
|
+
return _pt.tree_map(fn, tree, *rest)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def tree_leaves(tree: Any) -> list[Any]:
|
|
142
|
+
"""Flat list of the leaves of `tree`.
|
|
143
|
+
|
|
144
|
+
Args:
|
|
145
|
+
tree: The tree.
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
The leaves. Static fields are excluded.
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
return _pt.tree_leaves(tree)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def tree_leaves_with_path(tree: Any) -> list[tuple[str, Any]]:
|
|
155
|
+
"""The leaves of `tree`, each with its path.
|
|
156
|
+
|
|
157
|
+
Args:
|
|
158
|
+
tree: The tree.
|
|
159
|
+
|
|
160
|
+
Returns:
|
|
161
|
+
`(path, leaf)` pairs, the path as a string (e.g. `.curvature.sum['0.weight']`).
|
|
162
|
+
"""
|
|
163
|
+
|
|
164
|
+
return [(_pt.keystr(path), leaf) for path, leaf in _pt.tree_flatten_with_path(tree)[0]]
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def tree_map_with_path(fn: Callable[[str, Any], Any], tree: Any) -> Any:
|
|
168
|
+
"""Apply `fn(path, leaf)` to the leaves of `tree`.
|
|
169
|
+
|
|
170
|
+
Args:
|
|
171
|
+
fn: Called with each leaf's path (a string, as in `tree_leaves_with_path`) and the
|
|
172
|
+
leaf.
|
|
173
|
+
tree: The tree.
|
|
174
|
+
|
|
175
|
+
Returns:
|
|
176
|
+
A tree like `tree` holding the results.
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
return _pt.tree_map_with_path(lambda path, leaf: fn(_pt.keystr(path), leaf), tree)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _opaque(x: Any) -> bool:
|
|
183
|
+
"""Whether `x` is a mapping torch's pytree treats as a leaf.
|
|
184
|
+
|
|
185
|
+
That is any `Mapping` whose type is not a registered node (a `UserDict`, a transformers
|
|
186
|
+
`BatchEncoding`, a dict subclass).
|
|
187
|
+
"""
|
|
188
|
+
|
|
189
|
+
return isinstance(x, Mapping) and type(x) not in _pt.SUPPORTED_NODES
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def as_pytree(tree: Any) -> Any:
|
|
193
|
+
"""Turn each mapping that torch's pytree would treat as a leaf into a plain `dict`.
|
|
194
|
+
|
|
195
|
+
This works at any depth, so `vmap` and `tree_map` see the mapping's tensors.
|
|
196
|
+
|
|
197
|
+
Args:
|
|
198
|
+
tree: The tree.
|
|
199
|
+
|
|
200
|
+
Returns:
|
|
201
|
+
The converted tree. A tree without such a mapping comes back as it is.
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
if not any(_opaque(x) for x in _pt.tree_leaves(tree, is_leaf=_opaque)):
|
|
205
|
+
return tree
|
|
206
|
+
|
|
207
|
+
def plain(x: Any) -> Any:
|
|
208
|
+
if not _opaque(x):
|
|
209
|
+
return x
|
|
210
|
+
|
|
211
|
+
return {k: as_pytree(v) for k, v in cast(Mapping[Any, Any], x).items()}
|
|
212
|
+
|
|
213
|
+
return _pt.tree_map(plain, tree, is_leaf=_opaque)
|
laplacekit/core/_seed.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Seeds behind the `generator` arguments. Private.
|
|
2
|
+
|
|
3
|
+
A top-level call draws one seed, from its `generator` or from torch's global RNG, and
|
|
4
|
+
derives the rest from it: the fit folds in the pass index and the examples before each
|
|
5
|
+
batch, then splits for `prepare` and `update`; `PerBlockCurvature` folds in each member's position;
|
|
6
|
+
under `torch.distributed`, the batch seed folds in the rank. A batch's draws then depend
|
|
7
|
+
only on the seed, its pass and the examples before it, and a member's draws only on its
|
|
8
|
+
position, not on the other members.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import dataclasses
|
|
12
|
+
from collections.abc import Generator
|
|
13
|
+
from contextlib import AbstractContextManager, contextmanager
|
|
14
|
+
|
|
15
|
+
import torch
|
|
16
|
+
import torch.distributed as dist
|
|
17
|
+
|
|
18
|
+
from laplacekit.core._functorch import outside_transforms
|
|
19
|
+
|
|
20
|
+
_MASK = (1 << 64) - 1
|
|
21
|
+
_GOLDEN = 0x9E3779B97F4A7C15
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _mix(z: int) -> int:
|
|
25
|
+
"""SplitMix64 finalizer."""
|
|
26
|
+
|
|
27
|
+
z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & _MASK
|
|
28
|
+
z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & _MASK
|
|
29
|
+
|
|
30
|
+
return z ^ (z >> 31)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclasses.dataclass(frozen=True)
|
|
34
|
+
class _Seed:
|
|
35
|
+
"""A 64-bit seed that derives others and seeds torch generators.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
value: The seed, taken modulo $2^{64}$.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
value: int
|
|
42
|
+
|
|
43
|
+
def __post_init__(self) -> None:
|
|
44
|
+
object.__setattr__(self, "value", self.value & _MASK)
|
|
45
|
+
|
|
46
|
+
@staticmethod
|
|
47
|
+
def draw(generator: torch.Generator | None = None, /) -> "_Seed":
|
|
48
|
+
"""One seed drawn from `generator`, or from torch's global RNG when `None`.
|
|
49
|
+
|
|
50
|
+
It works under `torch.func` transforms: the seed is one draw for the whole call.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
generator: The generator to draw from. It advances by one draw.
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
A new seed.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
device = generator.device if generator is not None else torch.device("cpu")
|
|
60
|
+
low, high = -(1 << 63), (1 << 63) - 1
|
|
61
|
+
|
|
62
|
+
with outside_transforms():
|
|
63
|
+
t = torch.randint(low, high, (), dtype=torch.int64, generator=generator, device=device)
|
|
64
|
+
|
|
65
|
+
return _Seed(int(t.item()))
|
|
66
|
+
|
|
67
|
+
def fold_in(self, data: int, /) -> "_Seed":
|
|
68
|
+
"""A new seed from this one and `data`. The same inputs give the same seed.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
data: Any integer.
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
The derived seed.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
return _Seed(_mix(self.value ^ ((data * _GOLDEN) & _MASK)))
|
|
78
|
+
|
|
79
|
+
def split(self, n: int, /) -> tuple["_Seed", ...]:
|
|
80
|
+
"""`n` independent seeds, the i-th equal to `fold_in(i)`.
|
|
81
|
+
|
|
82
|
+
Args:
|
|
83
|
+
n: Number of seeds.
|
|
84
|
+
|
|
85
|
+
Returns:
|
|
86
|
+
The seeds.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
return tuple(self.fold_in(i) for i in range(n))
|
|
90
|
+
|
|
91
|
+
def generator(self, device: torch.device | str | None = None, /) -> torch.Generator:
|
|
92
|
+
"""A fresh generator seeded from this seed.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
device: The generator's device. `None` means the CPU.
|
|
96
|
+
|
|
97
|
+
Returns:
|
|
98
|
+
A new `torch.Generator`.
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
g = torch.Generator(device=device if device is not None else "cpu")
|
|
102
|
+
|
|
103
|
+
return g.manual_seed(self.value % (1 << 63))
|
|
104
|
+
|
|
105
|
+
def torch_rng(self, device: torch.device | None = None, /) -> AbstractContextManager[None]:
|
|
106
|
+
"""Seed torch's global RNG from this seed inside the block, restoring it after.
|
|
107
|
+
|
|
108
|
+
For code that can't take a generator, such as `Distribution.sample()`. The CPU
|
|
109
|
+
generator is always seeded. No other device is touched unless `device` names one.
|
|
110
|
+
|
|
111
|
+
Args:
|
|
112
|
+
device: The accelerator the draws are made on. Its generator is forked and
|
|
113
|
+
seeded too. `None` or a CPU device seeds only the CPU generator.
|
|
114
|
+
|
|
115
|
+
Returns:
|
|
116
|
+
A context manager. On exit, the forked generators get their old state back.
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
return self._torch_rng(device)
|
|
120
|
+
|
|
121
|
+
@contextmanager
|
|
122
|
+
def _torch_rng(self, device: torch.device | None) -> Generator[None]:
|
|
123
|
+
seed = self.value % (1 << 63)
|
|
124
|
+
|
|
125
|
+
# Not `torch.manual_seed`: it seeds every accelerator without restoring them, and
|
|
126
|
+
# its lazy device calls dominate the cost of a CPU draw.
|
|
127
|
+
if device is None or device.type == "cpu":
|
|
128
|
+
with torch.random.fork_rng(devices=[]):
|
|
129
|
+
torch.default_generator.manual_seed(seed)
|
|
130
|
+
yield
|
|
131
|
+
|
|
132
|
+
return
|
|
133
|
+
|
|
134
|
+
# The device-generic accelerator API picks the device; the device module only
|
|
135
|
+
# seeds its current one, which every backend (cuda, xpu, mps) supports.
|
|
136
|
+
acc = torch.accelerator
|
|
137
|
+
index = device.index if device.index is not None else acc.current_device_index()
|
|
138
|
+
|
|
139
|
+
with torch.random.fork_rng(devices=[index], device_type=device.type):
|
|
140
|
+
torch.default_generator.manual_seed(seed)
|
|
141
|
+
|
|
142
|
+
with acc.device_index(index):
|
|
143
|
+
torch.get_device_module(device.type).manual_seed(seed)
|
|
144
|
+
|
|
145
|
+
yield
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _batch_seeds(seed: _Seed, count: int, /) -> tuple[_Seed, _Seed]:
|
|
149
|
+
"""The `(prepare, update)` seeds of the batch that starts at example `count`.
|
|
150
|
+
|
|
151
|
+
`seed` is the pass's seed. Under `torch.distributed` the rank is folded in, so ranks
|
|
152
|
+
with equal shards draw differently.
|
|
153
|
+
|
|
154
|
+
Args:
|
|
155
|
+
seed: The pass's seed.
|
|
156
|
+
count: Examples seen in this pass before the batch.
|
|
157
|
+
|
|
158
|
+
Returns:
|
|
159
|
+
Two independent seeds.
|
|
160
|
+
"""
|
|
161
|
+
|
|
162
|
+
s = seed.fold_in(count)
|
|
163
|
+
|
|
164
|
+
if dist.is_available() and dist.is_initialized():
|
|
165
|
+
s = s.fold_in(dist.get_rank())
|
|
166
|
+
|
|
167
|
+
prepare, update = s.split(2)
|
|
168
|
+
|
|
169
|
+
return prepare, update
|