torchzero 0.3.10__py3-none-any.whl → 0.3.11__py3-none-any.whl

torchzero/utils/tensorlist.py

@@ -19,7 +19,7 @@ from typing_extensions import Self, TypeAlias, Unpack
 
 import torch
 from .ops import where_
-from .python_tools import generic_eq, zipmap
+from .python_tools import zipmap, generic_ne
 from .numberlist import NumberList, as_numberlist, maybe_numberlist
 
 
@@ -217,6 +217,12 @@ class TensorList(list[torch.Tensor | Any]):
         """Returns a TensorList with all elements for which `fn` returned True."""
         return self.__class__(i for i in self if fn(i, *args, **kwargs))
 
+    def filter_by_list(self, s: Sequence[bool]):
+        """returns a new TensorList with all elements where corresponding elements in :code:`s` are True."""
+        if len(self) != len(s):
+            raise ValueError(f"{len(self) = }, {len(s) = }")
+        return self.__class__(i for i, boolean in zip(self, s) if boolean)
+
     def zipmap(self, fn: Callable, other: Any | list | tuple, *args, **kwargs):
         """If `other` is list/tuple, applies `fn` to this TensorList zipped with `other`.
         Otherwise applies `fn` to this TensorList and `other`.
@@ -319,7 +325,8 @@ class TensorList(list[torch.Tensor | Any]):
     def global_sum(self) -> torch.Tensor: return builtins.sum(self.sum()) # pyright:ignore[reportArgumentType,reportReturnType]
     def global_std(self) -> torch.Tensor: return torch.std(self.to_vec())
     def global_var(self) -> torch.Tensor: return torch.var(self.to_vec())
-    def global_vector_norm(self, ord:float = 2) -> torch.Tensor:
+    def global_vector_norm(self, ord:float | Literal['mean_abs'] = 2) -> torch.Tensor:
+        if ord == 'mean_abs': return self.abs().global_mean()
         return torch.linalg.vector_norm(self.to_vec(), ord = ord) # pylint:disable = not-callable
     def global_any(self): return builtins.any(self.any())
     def global_all(self): return builtins.all(self.all())
@@ -425,11 +432,11 @@ class TensorList(list[torch.Tensor | Any]):
         return self
 
     def lazy_add(self, other: int | float | list[int | float] | tuple[int | float]):
-        if generic_eq(other, 0): return self
-        return self.add(other)
+        if generic_ne(other, 0): return self.add(other)
+        return self
     def lazy_add_(self, other: int | float | list[int | float] | tuple[int | float]):
-        if generic_eq(other, 0): return self
-        return self.add_(other)
+        if generic_ne(other, 0): return self.add_(other)
+        return self
 
     @overload
     def sub(self, other: _TensorSeq, alpha: _Scalar = 1): ...
@@ -449,11 +456,11 @@ class TensorList(list[torch.Tensor | Any]):
         return self
 
     def lazy_sub(self, other: int | float | list[int | float] | tuple[int | float]):
-        if generic_eq(other, 0): return self
-        return self.sub(other)
+        if generic_ne(other, 0): return self.sub(other)
+        return self
     def lazy_sub_(self, other: int | float | list[int | float] | tuple[int | float]):
-        if generic_eq(other, 0): return self
-        return self.sub_(other)
+        if generic_ne(other, 0): return self.sub_(other)
+        return self
 
     def neg(self): return self.__class__(torch._foreach_neg(self))
     def neg_(self):
@@ -467,13 +474,13 @@ class TensorList(list[torch.Tensor | Any]):
 
     # TODO: benchmark
     def lazy_mul(self, other: int | float | list[int | float] | tuple[int | float], clone=False):
-        if generic_eq(other, 1):
-            if clone: return self.clone()
-            return self
-        return self * other
+        if generic_ne(other, 1):
+            return self * other
+        if clone: return self.clone()
+        return self
     def lazy_mul_(self, other: int | float | list[int | float] | tuple[int | float]):
-        if generic_eq(other, 1): return self
-        return self.mul_(other)
+        if generic_ne(other, 1): return self.mul_(other)
+        return self
 
     def div(self, other: _STOrSTSeq) -> Self: return self.__class__(torch._foreach_div(self, other))
     def div_(self, other: _STOrSTSeq):
@@ -481,11 +488,11 @@ class TensorList(list[torch.Tensor | Any]):
         return self
 
     def lazy_div(self, other: int | float | list[int | float] | tuple[int | float]):
-        if generic_eq(other, 1): return self
-        return self / other
+        if generic_ne(other, 1): return self / other
+        return self
     def lazy_div_(self, other: int | float | list[int | float] | tuple[int | float]):
-        if generic_eq(other, 1): return self
-        return self.div_(other)
+        if generic_ne(other, 1): return self.div_(other)
+        return self
 
     def pow(self, exponent: "_Scalar | _STSeq"): return self.__class__(torch._foreach_pow(self, exponent))
     def pow_(self, exponent: "_Scalar | _STSeq"):
@@ -627,7 +634,8 @@ class TensorList(list[torch.Tensor | Any]):
         if dim is None: dim = ()
         return self.__class__(i.amin(dim=dim, keepdim=keepdim) for i in self)
 
-    def norm(self, ord: _Scalar, dtype=None):
+    def norm(self, ord: _Scalar|Literal["mean_abs"], dtype=None):
+        if isinstance(ord, str): return self.abs().mean()
         return self.__class__(torch._foreach_norm(self, ord, dtype))
 
     def mean(self, dim: _Dim = None, keepdim = False) -> Self | Any:
@@ -782,7 +790,7 @@ class TensorList(list[torch.Tensor | Any]):
         for t, o in zip(self, other): t.copysign_(o)
         return self
 
-    def graft(self, magnitude: "_TensorSeq", tensorwise=False, ord: float = 2, eps = 1e-6, strength: float | _ScalarSeq = 1):
+    def graft(self, magnitude: "_TensorSeq", tensorwise=False, ord: float | Literal['mean_abs'] = 2, eps = 1e-6, strength: float | _ScalarSeq = 1):
         if not isinstance(magnitude, TensorList): magnitude = TensorList(magnitude)
         if tensorwise:
             norm_self = self.norm(ord)
@@ -791,11 +799,11 @@ class TensorList(list[torch.Tensor | Any]):
             norm_self = self.global_vector_norm(ord)
             norm_other = magnitude.global_vector_norm(ord)
 
-        if not generic_eq(strength, 1): norm_other.lerp_(norm_self, 1-maybe_numberlist(strength)) # pyright:ignore[reportCallIssue,reportArgumentType]
+        if generic_ne(strength, 1): norm_other.lerp_(norm_self, 1-maybe_numberlist(strength)) # pyright:ignore[reportCallIssue,reportArgumentType]
 
         return self * (norm_other / norm_self.clip_(min=eps))
 
-    def graft_(self, magnitude: "_TensorSeq", tensorwise=False, ord: float = 2, eps = 1e-6, strength: float | _ScalarSeq = 1):
+    def graft_(self, magnitude: "_TensorSeq", tensorwise=False, ord: float | Literal['mean_abs'] = 2, eps = 1e-6, strength: float | _ScalarSeq = 1):
         if not isinstance(magnitude, TensorList): magnitude = TensorList(magnitude)
         if tensorwise:
             norm_self = self.norm(ord)
@@ -804,7 +812,7 @@ class TensorList(list[torch.Tensor | Any]):
             norm_self = self.global_vector_norm(ord)
             norm_other = magnitude.global_vector_norm(ord)
 
-        if not generic_eq(strength, 1): norm_other.lerp_(norm_self, 1-maybe_numberlist(strength)) # pyright:ignore[reportCallIssue,reportArgumentType]
+        if generic_ne(strength, 1): norm_other.lerp_(norm_self, 1-maybe_numberlist(strength)) # pyright:ignore[reportCallIssue,reportArgumentType]
 
         return self.mul_(norm_other / norm_self.clip_(min=eps))
 
@@ -897,7 +905,7 @@ class TensorList(list[torch.Tensor | Any]):
         if eps!=0: std.add_(eps)
         return self.sub_(self.mean(dim = dim, keepdim=True)).div_(std)
 
-    def _clip_multiplier(self, min: "_Scalar | _ScalarSeq | None"= None, max: "_Scalar | _ScalarSeq | None" = None, tensorwise: bool = True, ord:float = 2):
+    def _clip_multiplier(self, min: "_Scalar | _ScalarSeq | None"= None, max: "_Scalar | _ScalarSeq | None" = None, tensorwise: bool = True, ord:float|Literal["mean_abs"] = 2):
         """calculate multipler to clip self norm to min and max"""
         if tensorwise:
             self_norm = self.norm(ord)
@@ -918,12 +926,12 @@ class TensorList(list[torch.Tensor | Any]):
 
         return mul
 
-    def clip_norm(self, min: "_Scalar | _ScalarSeq | None"= None, max: "_Scalar | _ScalarSeq | None" = None, tensorwise: bool = True, ord:float = 2):
+    def clip_norm(self, min: "_Scalar | _ScalarSeq | None"= None, max: "_Scalar | _ScalarSeq | None" = None, tensorwise: bool = True, ord:float|Literal["mean_abs"] = 2):
         """clips norm of each tensor to (min, max) range"""
         if min is None and max is None: return self
         return self * self._clip_multiplier(min, max, tensorwise, ord)
 
-    def clip_norm_(self, min: "_Scalar | _ScalarSeq | None"= None, max: "_Scalar | _ScalarSeq | None" = None, tensorwise: bool = True, ord:float = 2):
+    def clip_norm_(self, min: "_Scalar | _ScalarSeq | None"= None, max: "_Scalar | _ScalarSeq | None" = None, tensorwise: bool = True, ord:float|Literal["mean_abs"] = 2):
         """clips norm of each tensor to (min, max) range"""
         if min is None and max is None: return self
         return self.mul_(self._clip_multiplier(min, max, tensorwise, ord))
@@ -1057,6 +1065,10 @@ def generic_numel(x: torch.Tensor | TensorList) -> int:
     if isinstance(x, torch.Tensor): return x.numel()
     return x.global_numel()
 
+def generic_finfo_eps(x: torch.Tensor | TensorList) -> float:
+    if isinstance(x, torch.Tensor): return torch.finfo(x.dtype).eps
+    return torch.finfo(x[0].dtype).eps
+
 @overload
 def generic_zeros_like(x: torch.Tensor) -> torch.Tensor: ...
 @overload

{torchzero-0.3.10.dist-info → torchzero-0.3.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: torchzero
-Version: 0.3.10
+Version: 0.3.11
 Summary: Modular optimization library for PyTorch.
 Author-email: Ivan Nikishev <nkshv2@gmail.com>
 License: MIT License
@@ -45,8 +45,6 @@ Dynamic: license-file
 
 `torchzero` is a PyTorch library providing a highly modular framework for creating and experimenting with a huge number of various optimization algorithms - various momentum techniques, gradient clipping, gradient approximations, line searches, quasi newton methods and more. All algorithms are implemented as modules that can be chained together freely.
 
-NOTE: torchzero is in active development, currently docs are in a state of flux.
-
 ## Installation
 
 ```bash
@@ -113,31 +111,21 @@ for epoch in range(100):
 `torchzero` provides a huge number of various modules:
 
 * **Optimizers**: Optimization algorithms.
-  * `Adam`.
-  * `Shampoo`.
-  * `SOAP` (my current recommendation).
-  * `Muon`.
-  * `SophiaH`.
-  * `Adagrad` and `FullMatrixAdagrad`.
-  * `Lion`.
-  * `RMSprop`.
-  * `OrthoGrad`.
-  * `Rprop`.
+  * `Adam`, `Adan`, `Adagrad`, `ESGD`, `FullMatrixAdagrad`, `LMAdagrad`, `AdaHessian`, `AdaptiveHeavyBall`, `OrthoGrad`, `Lion`, `MARS`, `MatrixMomentum`, `AdaptiveMatrixMomentum`, `Muon`, `RMSprop`, `Rprop`, `SAM`, `ASAM`, `MSAM`, `Shampoo`, `SOAP`, `SophiaH`.
 
   Additionally many other optimizers can be easily defined via modules:
   * Grams: `[tz.m.Adam(), tz.m.GradSign()]`
   * LaProp: `[tz.m.RMSprop(), tz.m.EMA(0.9)]`
   * Signum: `[tz.m.HeavyBall(), tz.m.Sign()]`
-  * Full matrix version of any diagonal optimizer, like Adam: `tz.m.FullMatrixAdagrad(beta=0.999, inner=tz.m.EMA(0.9))`
+  * Efficient full-matrix version of any diagonal optimizer, like Adam: `[tz.m.LMAdagrad(beta=0.999, inner=tz.m.EMA(0.9)), tz.m.Debias(0.9, 0.999)]`
   * Cautious version of any optimizer, like SOAP: `[tz.m.SOAP(), tz.m.Cautious()]`
 
 * **Momentum**:
-  * `NAG`: Nesterov Accelerated Gradient.
   * `HeavyBall`: Classic momentum (Polyak's momentum).
+  * `NAG`: Nesterov Accelerated Gradient.
   * `EMA`: Exponential moving average.
-  * `Averaging` (`Medianveraging`, `WeightedAveraging`): Simple, median, or weighted averaging of updates.
+  * `Averaging` (`MedianAveraging`, `WeightedAveraging`): Simple, median, or weighted averaging of updates.
   * `Cautious`, `ScaleByGradCosineSimilarity`: Momentum cautioning.
-  * `MatrixMomentum`, `AdaptiveMatrixMomentum`: Second order momentum.
 
 * **Stabilization**: Gradient stabilization techniques.
   * `ClipNorm`: Clips gradient L2 norm.
@@ -154,32 +142,42 @@ for epoch in range(100):
 
 * **Second order**: Second order methods.
   * `Newton`: Classic Newton's method.
-  * `NewtonCG`: Matrix-free newton's method with conjugate gradient solver.
+  * `InverseFreeNewton`: Inverse-free version of Newton's method.
+  * `NewtonCG`: Matrix-free newton's method with conjugate gradient or minimal residual solvers.
+  * `TruncatedNewtonCG`: Steihaug-Toint Trust-region NewtonCG via a truncated CG solver.
   * `NystromSketchAndSolve`: Nyström sketch-and-solve method.
-  * `NystromPCG`: NewtonCG with Nyström preconditioning (usually beats NewtonCG).
+  * `NystromPCG`: NewtonCG with Nyström preconditioning.
   * `HigherOrderNewton`: Higher order Newton's method with trust region.
 
 * **Quasi-Newton**: Approximate second-order optimization methods.
   * `LBFGS`: Limited-memory BFGS.
   * `LSR1`: Limited-memory SR1.
   * `OnlineLBFGS`: Online LBFGS.
-  * `BFGS`, `DFP`, `PSB`, `SR1`, `SSVM`, `BroydenBad`, `BroydenGood`, `ColumnUpdatingMethod`, `FletcherVMM`, `GradientCorrection`, `Greenstadt1`, `Greenstadt2`, `Horisho`, `McCormick`, `Pearson`, `ProjectedNewtonRaphson`, `ThomasOptimalMethod`: Classic full-matrix quasi-newton methods.
+  * `BFGS`, `DFP`, `ICUM`, `PSB`, `SR1`, `SSVM`, `BroydenBad`, `BroydenGood`, `FletcherVMM`, `GradientCorrection`, `Greenstadt1`, `Greenstadt2`, `Horisho`, `McCormick`, `NewSSM`, `Pearson`, `ProjectedNewtonRaphson`, `ThomasOptimalMethod`, `ShorR`: Full-matrix quasi-newton methods.
+  * `DiagonalBFGS`, `DiagonalSR1`, `DiagonalQuasiCauchi`, `DiagonalWeightedQuasiCauchi`, `DNRTR`, `NewDQN`: Diagonal quasi-newton methods.
   * `PolakRibiere`, `FletcherReeves`, `HestenesStiefel`, `DaiYuan`, `LiuStorey`, `ConjugateDescent`, `HagerZhang`, `HybridHS_DY`, `ProjectedGradientMethod`: Conjugate gradient methods.
 
+* **Trust Region** Trust region can work with exact hessian or any of the quasi-newton methods (L-BFGS support is WIP)
+  * `TrustCG`: Trust-region, uses a Steihaug-Toint truncated CG solver.
+  * `CubicRegularization`: Cubic regularization, works better with exact hessian.
+
 * **Line Search**:
   * `Backtracking`, `AdaptiveBacktracking`: Backtracking line searches (adaptive is my own).
   * `StrongWolfe`: Cubic interpolation line search satisfying strong Wolfe conditions.
   * `ScipyMinimizeScalar`: Wrapper for SciPy's scalar minimization for line search.
-  * `TrustRegion`: First order trust region method.
 
 * **Learning Rate**:
   * `LR`: Controls learning rate and adds support for LR schedulers.
-  * `PolyakStepSize`: Polyak's method.
-  * `Warmup`: Learning rate warmup.
+  * `PolyakStepSize`: Polyak's subgradient method.
+  * `BarzilaiBorwein`: Barzilai-Borwein step-size.
+  * `Warmup`, `WarmupNormCLip`: Learning rate warmup.
 
 * **Projections**: This can implement things like GaLore but I haven't done that yet.
-  * `FFTProjection`, `DCTProjection`: Use any update rule in Fourier or DCT domain (doesn't seem to help though).
-  * `VectorProjection`, `TensorizeProjection`, `BlockPartition`, `TensorNormsProjection`: Structural projection methods (for block BFGS etc.).
+  <!-- * `FFTProjection`, `DCTProjection`: Use any update rule in Fourier or DCT domain (doesn't seem to help though).
+  * `VectorProjection`, `TensorizeProjection`, `BlockPartition`, `TensorNormsProjection`: Structural projection methods (for block BFGS etc.). -->
+  This is WIP
+  * `To`: this casts everything to any other dtype and device for other modules, e.g. if you want better precision
+  * `ViewAsReal`: put if you have complex paramters.
 
 * **Smoothing**: Smoothing-based optimization methods.
   * `LaplacianSmoothing`: Laplacian smoothing for gradients (implements Laplacian Smooth GD).
@@ -195,6 +193,8 @@ for epoch in range(100):
 
 * **Experimental**: various horrible atrocities
 
+A complete list of modules is available in the [documentation](https://torchzero.readthedocs.io/en/latest/autoapi/torchzero/modules/index.html).
+
 ## Advanced Usage
 
 ### Closure
@@ -321,6 +321,7 @@ class HeavyBall(Module):
         super().__init__(defaults)
 
     def step(self, var: Var):
+        # Var object holds all attributes used for optimization - parameters, gradient, update, etc.
         # a module takes a Var object, modifies it or creates a new one, and returns it
         # Var has a bunch of attributes, including parameters, gradients, update, closure, loss
         # for now we are only interested in update, and we will apply the heavyball rule to it.
@@ -352,28 +353,52 @@ class HeavyBall(Module):
         return var
 ```
 
-There are a some specialized base modules that make it much easier to implement some specific things.
+More in-depth guide will be available in the documentation in the future.
+
+## Other stuff
 
-* `GradApproximator` for gradient approximations
-* `LineSearch` for line searches
-* `Projection` for projections like GaLore or into fourier domain.
-* `QuasiNewtonH` for full-matrix quasi-newton methods that update hessian inverse approximation (because they are all very similar)
-* `ConguateGradientBase` for conjugate gradient methods, basically the only difference is how beta is calculated.
+There are also wrappers providing `torch.optim.Optimizer` interface for various other libraries. When using those, make sure closure has `backward` argument as described in **Advanced Usage**.
 
-The documentation on how to actually use them is to write itself in the near future.
+---
 
-## License
+### Scipy
 
-This project is licensed under the MIT License
+#### torchzero.optim.wrappers.scipy.ScipyMinimize
 
-## Project Links
+A wrapper for `scipy.optimize.minimize` with gradients and hessians supplied by pytorch autograd. Scipy provides implementations of the following methods: `'nelder-mead', 'powell', 'cg', 'bfgs', 'newton-cg', 'l-bfgs-b', 'tnc', 'cobyla', 'cobyqa', 'slsqp', 'trust-constr', 'dogleg', 'trust-ncg', 'trust-exact', 'trust-krylov'`.
 
-TODO (there are docs but from very old version)
+#### torchzero.optim.wrappers.scipy.ScipyDE, ScipyDualAnnealing, ScipySHGO, ScipyDIRECT, ScipyBrute
 
-## Other stuff
+Equivalent wrappers for other derivative free solvers available in `scipy.optimize`
+
+---
+
+### NLOpt
+
+#### torchzero.optim.wrappers.nlopt.NLOptWrapper
 
-There are also wrappers providing `torch.optim.Optimizer` interface for for `scipy.optimize`, NLOpt and Nevergrad.
+A wrapper for [NLOpt](https://github.com/stevengj/nlopt) with gradients supplied by pytorch autograd. NLOpt is another popular library with many gradient based and gradient free [algorithms](https://nlopt.readthedocs.io/en/latest/NLopt_Algorithms/)
+
+---
+
+### Nevergrad
+
+#### torchzero.optim.wrappers.nevergrad.NevergradWrapper
+
+A wrapper for [nevergrad](https://facebookresearch.github.io/nevergrad/) which has a huge library of gradient free [algorithms](https://facebookresearch.github.io/nevergrad/optimizers_ref.html#optimizers)
+
+---
+
+### fast-cma-es
+
+#### torchzero.optim.wrappers.fcmaes.FcmaesWrapper
+
+A wrapper for [fast-cma-es](https://github.com/dietmarwo/fast-cma-es), which implements various gradient free algorithms. Notably it includes [BITEOPT](https://github.com/avaneev/biteopt) which seems to have very good performance in benchmarks.
+
+# License
+
+This project is licensed under the MIT License
 
-They are in `torchzero.optim.wrappers.scipy.ScipyMinimize`, `torchzero.optim.wrappers.nlopt.NLOptOptimizer`, and `torchzero.optim.wrappers.nevergrad.NevergradOptimizer`. Make sure closure has `backward` argument as described in **Advanced Usage**.
+# Project Links
 
-Apparently <https://github.com/avaneev/biteopt> is diabolical so I will add a wrapper for it too very soon.
+The documentation is available at <https://torchzero.readthedocs.io/en/latest/>

torchzero-0.3.11.dist-info/RECORD

@@ -0,0 +1,159 @@
+docs/source/conf.py,sha256=Kd0Uyu6WnhSHEyTbOEjxoaUg4sAu0AxN19raSARtltE,1883
+docs/source/docstring template.py,sha256=lIf4Jdkxd-Vr0vOuL9IOTCMOxw5ENsmZDLXKv1eO9ns,1585
+tests/test_identical.py,sha256=PJnQtSO3aHZYMQolHmoB26BEUPD_Gpmdh2_M0tfUfm0,11502
+tests/test_module.py,sha256=qX3rjdSJsbA8JO17bPTUIDspe7bg2dogqxMw__KV7SU,2039
+tests/test_opts.py,sha256=pAeyDIT0Q4SXBZqR9W_IUjwAEBcMnYr3zE0N4R0xn8w,42509
+tests/test_tensorlist.py,sha256=SwzLKLrs2ppMtm_7UrfTDTlD-ObZd7JQ_FNHbp059tc,72460
+tests/test_utils_optimizer.py,sha256=bvC0Ehvs2L8fohpyIF5Vfr9OKTycpnODWLPflXilU1c,8414
+tests/test_vars.py,sha256=2BoawNdDAnnNh_vv49_peJMnHvaQjp_sfnca1nosTWY,6766
+torchzero/__init__.py,sha256=L7IJ1qZ3o8E9oRwlJZBK2_2yII_eeGEk57Of6EfVbrk,112
+torchzero/core/__init__.py,sha256=Zib_4is13LFAabp_7VU8QXZpQEEZGzsH94vgRI0HxAg,150
+torchzero/core/module.py,sha256=BfU4YMjwLrwcz24XAfL-cZx05cESIimViKUStJKBEHM,32872
+torchzero/core/transform.py,sha256=sBgEyQVm141v99lnosusNIMWaReuWKuMyzkJha_WwKg,16440
+torchzero/modules/__init__.py,sha256=0Gk6XK32FKxtiW9rh-0Plql2dghHn3Ms1F-Ymn4oVzw,386
+torchzero/modules/functional.py,sha256=hmJaxB7U9X9nsT1Z5aPSqsw5HsQfL2ns1YS8AWdul6c,6948
+torchzero/modules/clipping/__init__.py,sha256=ZaffMF7mIRK6hZSfuZadgjNTX6hF5ANiLBny2w3S7I8,250
+torchzero/modules/clipping/clipping.py,sha256=6d-LPCI4zqlcV9fXK8rtRLiReyt8lMeQhmt1gsqNljs,14897
+torchzero/modules/clipping/ema_clipping.py,sha256=PNUTvixvc0wdjtWzja6pEzXbNpyXtGxj_H15umWx4zc,6608
+torchzero/modules/clipping/growth_clipping.py,sha256=mHn6BQqfHFWnfVjYG_Dokb3VjfSK2QVGsy2rf0Z-RMY,6657
+torchzero/modules/experimental/__init__.py,sha256=qV-VaBnRsLFtv6T6R9Imkd1G81QR4O-9_kDbCAwJXeY,1464
+torchzero/modules/experimental/absoap.py,sha256=U3nLAV_vxl6HjJhqi8FlK8K6AMLoiZ-deykEshhnCC0,9916
+torchzero/modules/experimental/adadam.py,sha256=PARjM2kRmJ7ifYsI83tADKCuvSZYAoT2vR4Gj2aZ-SA,4103
+torchzero/modules/experimental/adamY.py,sha256=Rr9vXjFPWTfIHnnhGQAfVAQnfANNgcrFm_R8vJsU1to,4043
+torchzero/modules/experimental/adam_lambertw.py,sha256=FXZiTJKVRbXSu9-_boZGYoCqBlh2035mwsagq75qyeA,5323
+torchzero/modules/experimental/adaptive_step_size.py,sha256=OJseQX9sd9F58pMC5JbVNm7PtovMXL4sMwQg3jooVtg,3494
+torchzero/modules/experimental/adasoap.py,sha256=vcgWEgDdqmgimt5bGgvznCnxkkathGO0engd1xo7M4s,7491
+torchzero/modules/experimental/cosine.py,sha256=0Cc42Wd1sMrjm-YxmpcwCCsGpLv3H83rL-XAtrgZhb4,9155
+torchzero/modules/experimental/cubic_adam.py,sha256=wHJKm9bO24Xvtwunz_1Kz7mGi_C-syupixiDaBnYx2Q,2787
+torchzero/modules/experimental/curveball.py,sha256=JdgojuSYLNe9u3bmqcYrFm8brUD4kvKm9XYx78GzpKI,3257
+torchzero/modules/experimental/dct.py,sha256=Iv8ZxGhTOIm3NHS4zxoFG9K9BEwtrJqsKApctiIjnxg,2463
+torchzero/modules/experimental/eigendescent.py,sha256=Pdz7QUbM3pD3DTsTC0nZ0AfOe2pj-WVPPkbnw8lDZ3c,4725
+torchzero/modules/experimental/etf.py,sha256=ul167I1qAbYeTmTPG_WFLLlE1MEsNXxVsTWd9s2YC9g,6125
+torchzero/modules/experimental/exp_adam.py,sha256=yhR5-NGflbEJrSAe0ps4xgAM-eFI-gAdS6cgZIJDgaI,4100
+torchzero/modules/experimental/expanded_lbfgs.py,sha256=M58cCaeLZXGqZwyaeGhi-UAyCsnnJvLAYIZ64r0tQNE,5649
+torchzero/modules/experimental/fft.py,sha256=YEUKdAXNX8BCZYXKV5uWWU8aTlGjpFTUSpIEwIG-_fM,3050
+torchzero/modules/experimental/gradmin.py,sha256=UixSLdca4ekYHOipEivdXfBAV-uEL9TZm5nCFXVaNco,3684
+torchzero/modules/experimental/hnewton.py,sha256=_Gv4O2x0qYBxGtkCuYuzL21VuI5wTn1sTEegk17d6X4,3036
+torchzero/modules/experimental/modular_lbfgs.py,sha256=d40yRi6NN2Au7-UQ1akMkET0PWhEFAhGKAYoQBDmqFQ,10671
+torchzero/modules/experimental/newton_solver.py,sha256=3dZ7FG-2vGxJKkFF9P2LCs-LI_epcvZbyNtJOtw47pg,3055
+torchzero/modules/experimental/newtonnewton.py,sha256=cRL4dKsDAN8tHPyHQkLbTGxkHfemCU6re-n4odV3Ik4,3324
+torchzero/modules/experimental/parabolic_search.py,sha256=2GgE4cq5QkJYZprADIplQfbPWRJRGFmToYTScJkR0tg,6328
+torchzero/modules/experimental/reduce_outward_lr.py,sha256=ui_39wNdf5J2FOQtQFk0WUA8DuicwEp0kepccbq8rI0,1309
+torchzero/modules/experimental/structural_projections.py,sha256=lrySQZOq7VhL_VqU7dIJRsypxA16cUliQYkj5-N2B2I,4187
+torchzero/modules/experimental/subspace_preconditioners.py,sha256=RdG-RoPF6AiFVphrVlb6egNyYI0e_eHoENUWqKJ4icQ,5170
+torchzero/modules/experimental/tensor_adagrad.py,sha256=y29i6BGXwv9lwrTRDzq2YRSngQmfZnreRIeH1NGzpBo,1572
+torchzero/modules/grad_approximation/__init__.py,sha256=DVFjf0cXuF70NA0nJ2WklpP01PQgrRZxUjUQjjQeSos,195
+torchzero/modules/grad_approximation/fdm.py,sha256=K_D0fKwspg21Opo2xTG4I34gLDmcaYBp5NUzlaQnjxQ,4490
+torchzero/modules/grad_approximation/forward_gradient.py,sha256=AoezoYxXii2gKpIGO7BOZkLb2weYwxrWAKpHL7hrW9Y,4313
+torchzero/modules/grad_approximation/grad_approximator.py,sha256=HO-XaNRF3ZwMduBP02V0oabmSRgqmDGPlKkWfDVDPW8,4740
+torchzero/modules/grad_approximation/rfdm.py,sha256=omarcZyMgJomJwxQ_b7ulE6eK6aW3JP_Sh-jcX5DhR4,23434
+torchzero/modules/higher_order/__init__.py,sha256=W94CY8K1NFxs9TPi415UssKVKz5MV_bH9adax1uZsYM,50
+torchzero/modules/higher_order/higher_order_newton.py,sha256=_v5v0WY07CvZn9QPIS89FxEZ2tNfd8Bkamt1o12_mLQ,12255
+torchzero/modules/line_search/__init__.py,sha256=9ja1Dspfuzu9UxGbU5-t0bFeBcdwoX9Fl_aSMR-AXnQ,219
+torchzero/modules/line_search/adaptive.py,sha256=Uj7lAIzpgy89ddlwA4VcEEIfcNJSbGA5HH3ncuzHrTU,2926
+torchzero/modules/line_search/backtracking.py,sha256=dyXgfrIJ_IO7W4p8GqJNPc4r_igU4X4ljLCLNKyY2Tw,8246
+torchzero/modules/line_search/line_search.py,sha256=_u59XYFkRsIKuT1H4Bz7qAHr3Ldzxbup71OeqDGxMfs,9724
+torchzero/modules/line_search/polynomial.py,sha256=KlK0d9qaphxS0s8B5rlt-yIUYNuV-5O24STcx4vN2Ic,9056
+torchzero/modules/line_search/scipy.py,sha256=eGplW1L8kQKdRbt9PPpvZ6MMekDq5KsjurhSpN9QCnY,2301
+torchzero/modules/line_search/strong_wolfe.py,sha256=F5962HTHdPWgvWHwnUofCqFxfKsCu5p8Ic-aRbn7wVg,8458
+torchzero/modules/misc/__init__.py,sha256=cZpMkZQubuzquhFZV-yELrDMznqhhCibmr0CBOR0ZpU,693
+torchzero/modules/misc/debug.py,sha256=iuWg5egoMnG6y3Cyd423xS7BRVYiwZq9575d7A7U3Dg,1652
+torchzero/modules/misc/escape.py,sha256=1XgNmT4pOptaXHSWEONkUPpcYnIujm5gdK6n_-zmw20,1821
+torchzero/modules/misc/gradient_accumulation.py,sha256=6yXRUxD_f3Zfx83UyCvPJ-56XN4GJjEQcNIDlvFtuuY,2590
+torchzero/modules/misc/misc.py,sha256=VTQZAcfQBo2yudy1u1lyHhmaAmQlxzVcZTHcXXnUeTM,13470
+torchzero/modules/misc/multistep.py,sha256=rAPCALSHXjVNxR8d1CA3RFP_xnN6j5KksjB6yl8vtng,5585
+torchzero/modules/misc/regularization.py,sha256=R8ya7HEF2MLtcAr7GS9IjXwJ4xh0lJWMdWMIRfwL42s,6279
+torchzero/modules/misc/split.py,sha256=ebc95OZjC-Vs73JeTkL--eZrtKijg7lPN0hmD0Whfxc,3195
+torchzero/modules/misc/switch.py,sha256=72mfY_uIVyTllwuR21_K7QC8IQFP7JMKzH4K2nAx0Wc,3726
+torchzero/modules/momentum/__init__.py,sha256=tI2I5zSQB7aTwEn371wvUTy2O2n_-KVCafjBv-OMsYE,545
+torchzero/modules/momentum/averaging.py,sha256=gZRjHb443HuFF03p3Oh2rfgh2Qu8sJBxc_8NR-ircaA,3241
+torchzero/modules/momentum/cautious.py,sha256=QP3Sqc8nMb3xTDDDfGwFn5AWvN4EI5U-CCcZb-F5oX0,8266
+torchzero/modules/momentum/ema.py,sha256=9OdMF20RYnEkwe9Xu2dCAAiI0qY2MQvhS87bKP7ptTI,10755
+torchzero/modules/momentum/experimental.py,sha256=WnM9FUKPxyFNiKU6Ip7wqqYxHbXuaMKOcLjjomfENb4,6916
+torchzero/modules/momentum/matrix_momentum.py,sha256=gZeTJZbhgixCOkE9Jyowtva58hl5vsH9iTqGC54FWFs,8047
+torchzero/modules/momentum/momentum.py,sha256=Yx35jtbLb1syVFcTiNSoZPoUPmdsUy3QpoNWcN4sC9w,2664
+torchzero/modules/ops/__init__.py,sha256=1q9CBo6OXWXDgyjvKKTlG0EdP4ASIvkWFXtd6LOuU88,1083
+torchzero/modules/ops/accumulate.py,sha256=kyjiC9M9fugpG5Pc07XUi6GEWBvRi8iJ-7_Mb1SXQzE,3665
+torchzero/modules/ops/binary.py,sha256=mIeaa3v5Bk7mwzSTC0jGMLhKf-Ujg6aFbSia2yo-3JQ,12199
+torchzero/modules/ops/multi.py,sha256=DpabTYj0sic5dmosnmj7lgIX3dbmcgl0h9XfzKpbaus,8918
+torchzero/modules/ops/reduce.py,sha256=uLCq493hFy_Ib22GjIKtMHTTObK3RDmubGHTVqgFgg8,6339
+torchzero/modules/ops/unary.py,sha256=EFA_A834KmA6Ec3pZWH5XxZ9OzAhZZudwAwsP4GWZA0,5476
+torchzero/modules/ops/utility.py,sha256=9Skxkt4RO79OBdw95wOKhqKN2RMdZg9emO7R9q2d5oU,3767
+torchzero/modules/optimizers/__init__.py,sha256=IJaLoZ39rbB4GSW9rLKrfSCh5FsAkFy2ww5MhJ6MYnE,817
+torchzero/modules/optimizers/adagrad.py,sha256=p-DWbhGuuogldiFPNxxQfJ8AA5Tsd4UwGOIyX7GT0WE,5892
+torchzero/modules/optimizers/adahessian.py,sha256=vOJfwGi7ypfi7vifCMJfGew-McdGJKQM3TmkT-OUgI0,8682
+torchzero/modules/optimizers/adam.py,sha256=SkJ7UJ1BOAgfregmzYDFo_3cgPNke_RK9B58hOal_Zg,3954
+torchzero/modules/optimizers/adan.py,sha256=aOG6KGLU4oHYeQn3JB-A4NQ-279QpHA7firY3kkhFR4,3311
+torchzero/modules/optimizers/adaptive_heavyball.py,sha256=DnkWHA0GBLIKCq8nWh76fZA6PnJ3eKsJDBXWKnZ_uIs,2127
+torchzero/modules/optimizers/esgd.py,sha256=WXwYPA-qTA_QW9h4NDwNaly9gbi1uvMQ-5fSuLqnPkQ,6413
+torchzero/modules/optimizers/ladagrad.py,sha256=HQb7LuZnG8SvS8JWqu7JJz_owlkyT-fnqeICrJBQxbc,7314
+torchzero/modules/optimizers/lion.py,sha256=XFyglRNdnP1l8CmEZ7L_ZB8HWiR03BsZ_PEFCvHijb8,1127
+torchzero/modules/optimizers/mars.py,sha256=7tr32x2eQNu8ZVQAPnLIkM2kkYp7S57uiDywTdqy1uY,2710
+torchzero/modules/optimizers/msam.py,sha256=nvoo6smewR3hiCCymZQiB3DlCvLBGxfxlovJF2bwwsc,6588
+torchzero/modules/optimizers/muon.py,sha256=AZKpmkVUjukXtI7Pb9PKDEeycreLF6qYlIMSbV_9IuA,10463
+torchzero/modules/optimizers/orthograd.py,sha256=KbQuudjKgYVJcq1jRW_YmR2pPnwmAwyx9X_vrJAJgN4,2029
+torchzero/modules/optimizers/rmsprop.py,sha256=ugZLfH4dXvHTxawtGWQL6xSfsjBDl_t1s29aFN9FMuY,4345
+torchzero/modules/optimizers/rprop.py,sha256=nFpnqcXevGkUcPWERDX9gsiBCGgOi4pyPFloL68zwPY,11984
+torchzero/modules/optimizers/sam.py,sha256=yEhXAS3v62nhAvs63RZ80VfZ93MaQ0cyMQziFdy6e2U,5711
+torchzero/modules/optimizers/shampoo.py,sha256=m_XOvo2Eb1HP8QqYFPsT0rgczJ8HqKjh67QmtaY9dVg,9544
+torchzero/modules/optimizers/soap.py,sha256=MXQ8fdBzLyFtgW34fnmY3hQqv3q4QwEthho9kK-72VE,11305
+torchzero/modules/optimizers/sophia_h.py,sha256=dgQwjij5R4zdESYoKhc4BMhb6dKkDuEvjlL4bDdeQtw,7213
+torchzero/modules/projections/__init__.py,sha256=4LfmBEu_eM4YWmcWQVH4CdI1H0ucCIHDH9tTGigjVPY,136
+torchzero/modules/projections/cast.py,sha256=FJx2Tt1lbQRnOC5wxx3LbOnacLfUluFP6QOXLUCIEPY,2174
+torchzero/modules/projections/galore.py,sha256=GDJ7hf6cdk_Iu2qW0rWaQwYLQAxQEe27FEfOiZvFXHo,252
+torchzero/modules/projections/projection.py,sha256=PU2e9LNfVMnNrXnBDt-hdr5pVtl0TpgiB4b92WUguSs,14005
+torchzero/modules/quasi_newton/__init__.py,sha256=guTCpbAffZyupnThdPxAsLULAmPP3vdPaNfPCe9KB9Y,854
+torchzero/modules/quasi_newton/cg.py,sha256=HCfza5UInco7_hYT8s3duNRTmBdjbw5jscWLKNUiS8w,14453
+torchzero/modules/quasi_newton/diagonal_quasi_newton.py,sha256=bMvIcWifYlJX83UtXFESMw7OdA4AO7tJwlHZwkc5wx0,6555
+torchzero/modules/quasi_newton/lbfgs.py,sha256=BmE5sOFLFoJDlpoSphM5VowMgt7wtEFihbLkdylDXhM,10638
+torchzero/modules/quasi_newton/lsr1.py,sha256=a19a9ABqMiTVJmXe6Woc0sJ1kkhQa3Y6QDouaUNnPt0,7873
+torchzero/modules/quasi_newton/quasi_newton.py,sha256=hKJ9Irmh2pKNfB7Wen4MrDfMrbvzp00FTcPlpFvJLDU,48582
+torchzero/modules/quasi_newton/trust_region.py,sha256=cxOEDeZ8ZhG_w7QXGYnTsF-t5g5zZ39q9Uxb2IXWgAY,15213
+torchzero/modules/second_order/__init__.py,sha256=Trje1qM65yp8WWzuRm-tMTRqfKi4wpI7f8yyZWjhPCw,152
+torchzero/modules/second_order/newton.py,sha256=94LGrQo5Q8aC5DI9S6RSXF0stVcgWzq3JnE9l_BsVUw,12875
+torchzero/modules/second_order/newton_cg.py,sha256=l8FX9vQSVCSkpk5a-M2wEBBjQoODF-T07GFW_tjJxkM,14890
+torchzero/modules/second_order/nystrom.py,sha256=yAJijWCl-K8k63YSJUqE_kXEIFmL_FjDghVjQoutAXo,11352
+torchzero/modules/smoothing/__init__.py,sha256=tUTGN0A-EQC7xuLV2AuHFWk-t7D6jIJlpV_3qyfRqLk,80
+torchzero/modules/smoothing/gaussian.py,sha256=iTsWlMNHuDLoxPRIsm2pAb5cS8OqdRJwCsw-vUTVmpE,7887
+torchzero/modules/smoothing/laplacian.py,sha256=05Y6ft0GHRGkfSDV-g8vlaTlZTiXMr79xDagJb126ug,5302
+torchzero/modules/step_size/__init__.py,sha256=Z8NpB9RYIXhcNx11NWixa7mORPiT4nI1mKQGA7JfC6g,122
+torchzero/modules/step_size/adaptive.py,sha256=3qQr1aaPYEJlkiDSQbuVQ_OVkOq-W4LL7PkHFFgwP2c,4845
+torchzero/modules/step_size/lr.py,sha256=I9-aIxei4Y2XnlOoCKvec2r__cTY_JTwBDlMf2O5D2A,5908
+torchzero/modules/weight_decay/__init__.py,sha256=7UHAiiimsbQ_dHlxxcW87G5cCQFom9Uh_733W_23PWU,93
+torchzero/modules/weight_decay/weight_decay.py,sha256=2MhWRyryplDtB61QyKN7KqBa3mEkhtqXhij8LGR-mYA,5464
+torchzero/modules/wrappers/__init__.py,sha256=6b5Ac-8u18IVp_Jnw1T1xQExwpQhpQ0JwNV9GyC_Yj8,31
+torchzero/modules/wrappers/optim_wrapper.py,sha256=B8ZwZf-qzChBfbx-cwL8Rez4AgH7FzvsT7N1S2SUiR8,4417
+torchzero/optim/__init__.py,sha256=aXf7EkywqYiR50I4QeeVXro9aBhKiqfbY_BCia59sgU,46
+torchzero/optim/utility/__init__.py,sha256=pUacok4XmebfxofE-QWZLgViajsU-3JkXcWi9OS-Jrw,24
+torchzero/optim/utility/split.py,sha256=ZbazNuMTYunm75V_5ard0A_LletGaYAg-Pm2rANJKrE,1610
+torchzero/optim/wrappers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+torchzero/optim/wrappers/directsearch.py,sha256=GQ2nzy9ADqbV_QUMN3IaYecZ0Pzx_3mAasSB4fryTBE,11362
+torchzero/optim/wrappers/fcmaes.py,sha256=o_FchMtDsrEj9XRonHHeyVHPAXTHaU244SzlldgEzLg,4250
+torchzero/optim/wrappers/mads.py,sha256=Zi9u3vNlgNsCaIdYLl2_jgRA_dQrmVAuG0V31BFuct4,3087
+torchzero/optim/wrappers/nevergrad.py,sha256=U_ZAHD_nEsJZ71cJ8TQ_DOZcTmS06EEvUPvaaDRSxWI,4901
+torchzero/optim/wrappers/nlopt.py,sha256=AaVEKfjbrt5DFION44_-g-jQAoVi4lCvBBPU5UDGO9Q,8151
+torchzero/optim/wrappers/optuna.py,sha256=ZZ66aXEypSJMVomphbzHNJnmIOyXS9tqE89YZBPpIuo,2331
+torchzero/optim/wrappers/scipy.py,sha256=Td1AvpLDEPqPVW6IpHbkVW4CpNiUU9r_eyc3qJVHZAY,19352
+torchzero/utils/__init__.py,sha256=4JMKzF3qICE9PSfgXAwb3cPswM5f1JUutWwviev2-0k,875
+torchzero/utils/compile.py,sha256=N8AWLv_7oBUHYornmvvx_L4uynjiD-x5Hj1tBwei3-w,5127
+torchzero/utils/derivatives.py,sha256=IIn4stpMMJxYmGKh1JCH4Gha_a4w8Z5G04uVz2BwMP4,16995
+torchzero/utils/numberlist.py,sha256=iMoqz4IzXy-aE9bqVYJ21GV6pl0z-NeTsXR-LaI8C24,6229
+torchzero/utils/ops.py,sha256=n4Su1sbgTzlHczuPEHkuWenTtNBCa_MvlQ_hCZkIPnQ,314
+torchzero/utils/optimizer.py,sha256=r52qu6pEcRH4lCXVlLxW5IweA6L-VrQj6RCMfdhzRpw,12466
+torchzero/utils/optuna_tools.py,sha256=F-1Xg0n_29MVEb6lqgUFFNIl9BNJ6MOdIJPduoNH4JU,1325
+torchzero/utils/params.py,sha256=nQo270aOURU7rJ_D102y2pSXbzhJPK0Z_ehx4mZBMes,5784
+torchzero/utils/python_tools.py,sha256=NEyDVJfLBbdwh5m49qiOdIr0NffZRqKhaJ-cktviD1o,3243
+torchzero/utils/tensorlist.py,sha256=WvjhPzGbgRySAsUBFQ7b-39V9rm7jbR1VOeYZQXiiKw,53925
+torchzero/utils/torch_tools.py,sha256=ohqnnZRlqdfp5PAfMSbQDIEKygW0_ARjxSEBp3Zo9nU,4756
+torchzero/utils/linalg/__init__.py,sha256=tsUt20_rbA_3pV6NK7yCkGoX1l0D9ayMKwZeySsYxHw,291
+torchzero/utils/linalg/benchmark.py,sha256=wiIMn-GY2xxWbHVf8CPbJddUPeUPq9OUDkvbp1iILYI,479
+torchzero/utils/linalg/matrix_funcs.py,sha256=-LecWrPWbJvfeCgIzUhfWARa2aSZvJ12lHX7Jno38O4,3099
+torchzero/utils/linalg/orthogonalize.py,sha256=mDCkET7qgDZqf_y6oPYAK3d2L5HrB8gzOFPl0YoONaY,399
+torchzero/utils/linalg/qr.py,sha256=L-RXuYV-SIHI-Llq4y1rQ_Tz-yamds0_QNZeHapbjNE,2507
+torchzero/utils/linalg/solve.py,sha256=JF0i_eJTBRKCs7CONUOV7coPjE46NC5nMaz2JotrvSE,11232
+torchzero/utils/linalg/svd.py,sha256=wBxl-JSciINV-N6zvM4SGdveqMr6idq51h68LyQQRYg,660
+torchzero-0.3.11.dist-info/licenses/LICENSE,sha256=r9ZciAoZoqKC_FNADE0ORukj1p1XhLXEbegdsAyqhJs,1087
+torchzero-0.3.11.dist-info/METADATA,sha256=Czo-sKnlVxQ75MhY3D61oD8lusASV0ez_l697dyJBNc,15797
+torchzero-0.3.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+torchzero-0.3.11.dist-info/top_level.txt,sha256=YDdpIOb7HyKV9THOtOYsFFMTbxvCO0kiol4-83tDj-A,21
+torchzero-0.3.11.dist-info/RECORD,,