torchzero 0.3.11__py3-none-any.whl → 0.3.14__py3-none-any.whl

torchzero/utils/tensorlist.py

@@ -11,17 +11,19 @@ in an optimizer when you have to create one from parameters on each step. The so
 it once beforehand, but then you won't be able to easily support parameter groups and per-parameter states.
 """
 import builtins
-from collections.abc import Callable, Sequence, Iterable, Generator, Iterator
 import math
 import operator
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Generator, Iterable, Iterator, Sequence
 from typing import Any, Literal, TypedDict, overload
-from typing_extensions import Self, TypeAlias, Unpack
 
 import torch
-from .ops import where_
-from .python_tools import zipmap, generic_ne
-from .numberlist import NumberList, as_numberlist, maybe_numberlist
+from typing_extensions import Self, TypeAlias, Unpack
 
+from .metrics import Metrics, evaluate_metric, calculate_metric_list
+from .numberlist import NumberList, as_numberlist, maybe_numberlist
+from .ops import where_
+from .python_tools import generic_ne, zipmap
 
 _Scalar = int | float | bool | complex
 _TensorSeq = list[torch.Tensor] | tuple[torch.Tensor, ...]
@@ -33,6 +35,7 @@ _STOrSTSeq = _Scalar | torch.Tensor | _ScalarSeq | _TensorSeq
 _Dim = int | list[int] | tuple[int,...] | Literal['global'] | None
 
 Distributions = Literal['normal', 'gaussian', 'uniform', 'sphere', 'rademacher']
+
 class _NewTensorKwargs(TypedDict, total = False):
     memory_format: Any
     dtype: Any
@@ -325,9 +328,20 @@ class TensorList(list[torch.Tensor | Any]):
     def global_sum(self) -> torch.Tensor: return builtins.sum(self.sum()) # pyright:ignore[reportArgumentType,reportReturnType]
     def global_std(self) -> torch.Tensor: return torch.std(self.to_vec())
     def global_var(self) -> torch.Tensor: return torch.var(self.to_vec())
-    def global_vector_norm(self, ord:float | Literal['mean_abs'] = 2) -> torch.Tensor:
-        if ord == 'mean_abs': return self.abs().global_mean()
-        return torch.linalg.vector_norm(self.to_vec(), ord = ord) # pylint:disable = not-callable
+
+    def global_vector_norm(self, ord:float = 2) -> torch.Tensor:
+        # return torch.linalg.vector_norm(self.to_vec(), ord = ord) # pylint:disable = not-callable
+        if ord == 1: return self.global_sum()
+        if ord % 2 == 0: return self.pow(ord).global_sum().pow(1/ord)
+        if ord == torch.inf: return self.abs().global_max()
+        if ord == -torch.inf: return self.abs().global_min()
+        if ord == 0: return (self != 0).global_sum().to(self[0].dtype)
+
+        return self.abs().pow_(ord).global_sum().pow(1/ord)
+
+    def global_metric(self, metric: Metrics) -> torch.Tensor:
+        return evaluate_metric(self, metric)
+
     def global_any(self): return builtins.any(self.any())
     def global_all(self): return builtins.all(self.all())
     def global_numel(self) -> int: return builtins.sum(self.numel())
@@ -358,31 +372,54 @@ class TensorList(list[torch.Tensor | Any]):
 
     def randint_like(self, low: "_Scalar | _ScalarSeq", high: "_Scalar | _ScalarSeq", **kwargs: Unpack[_NewTensorKwargs]):
         return self.zipmap_args(torch.randint_like, low, high, **kwargs)
+
     def uniform_like(self, low: "_Scalar | _ScalarSeq" = 0, high: "_Scalar | _ScalarSeq" = 1, generator=None, **kwargs: Unpack[_NewTensorKwargs]):
         res = self.empty_like(**kwargs)
         res.uniform_(low, high, generator=generator)
         return res
+
     def sphere_like(self, radius: "_Scalar | _ScalarSeq", generator=None, **kwargs: Unpack[_NewTensorKwargs]) -> Self:
         r = self.randn_like(generator=generator, **kwargs)
-        return (r * radius) / r.global_vector_norm()
+        return r.mul_(maybe_numberlist(radius) / r.global_vector_norm())
+
     def bernoulli(self, generator = None):
         return self.__class__(torch.bernoulli(i, generator=generator) for i in self)
+
     def bernoulli_like(self, p: "_Scalar | _ScalarSeq" = 0.5, generator = None, **kwargs: Unpack[_NewTensorKwargs]):
         """p is probability of a 1, other values will be 0."""
         return self.__class__(torch.bernoulli(i, generator = generator) for i in self.full_like(p, **kwargs))
+
     def rademacher_like(self, p: "_Scalar | _ScalarSeq" = 0.5, generator = None, **kwargs: Unpack[_NewTensorKwargs]):
         """p is probability of a 1, other values will be -1."""
         return self.bernoulli_like(p, generator=generator, **kwargs).mul_(2).sub_(1)
 
-    def sample_like(self, eps: "_Scalar | _ScalarSeq" = 1, distribution: Distributions = 'normal', generator=None, **kwargs: Unpack[_NewTensorKwargs]):
+    def sample_like(self, distribution: Distributions = 'normal', variance: "_Scalar | _ScalarSeq | Sequence | None" = None, generator=None, **kwargs: Unpack[_NewTensorKwargs]):
         """Sample around 0."""
-        if distribution in ('normal', 'gaussian'): return self.randn_like(generator=generator, **kwargs) * eps
+        if isinstance(variance, Sequence):
+            if all(v is None for v in variance): variance = None
+            else: variance = [v if v is not None else 1 for v in variance]
+
+        if distribution in ('normal', 'gaussian'):
+            ret = self.randn_like(generator=generator, **kwargs)
+            if variance is not None: ret *= variance
+            return ret
+
         if distribution == 'uniform':
-            if isinstance(eps, (list,tuple)):
-                return self.uniform_like([-i/2 for i in eps], [i/2 for i in eps], generator=generator, **kwargs)
-            return self.uniform_like(-eps/2, eps/2, generator=generator, **kwargs)
-        if distribution == 'sphere': return self.sphere_like(eps, generator=generator, **kwargs)
-        if distribution == 'rademacher': return self.rademacher_like(generator=generator, **kwargs) * eps
+            b = 1
+            if variance is not None:
+                b = ((12 * maybe_numberlist(variance)) ** 0.5) / 2
+            return self.uniform_like(-b, b, generator=generator, **kwargs)
+
+        if distribution == 'sphere':
+            if variance is None: radius = 1
+            else: radius = maybe_numberlist(variance) * math.sqrt(self.global_numel())
+            return self.sphere_like(radius, generator=generator, **kwargs)
+
+        if distribution == 'rademacher':
+            ret = self.rademacher_like(generator=generator, **kwargs)
+            if variance is not None: ret *= variance
+            return ret
+
         raise ValueError(f'Unknow distribution {distribution}')
 
     def eq(self, other: _STOrSTSeq): return self.zipmap(torch.eq, other)
@@ -504,6 +541,11 @@ class TensorList(list[torch.Tensor | Any]):
         torch._foreach_pow_(input, self)
         return self
 
+    def square(self): return self.__class__(torch._foreach_pow(self, 2))
+    def square_(self):
+        torch._foreach_pow_(self, 2)
+        return self
+
     def sqrt(self): return self.__class__(torch._foreach_sqrt(self))
     def sqrt_(self):
         torch._foreach_sqrt_(self)
@@ -634,10 +676,12 @@ class TensorList(list[torch.Tensor | Any]):
         if dim is None: dim = ()
         return self.__class__(i.amin(dim=dim, keepdim=keepdim) for i in self)
 
-    def norm(self, ord: _Scalar|Literal["mean_abs"], dtype=None):
-        if isinstance(ord, str): return self.abs().mean()
+    def norm(self, ord: float, dtype=None):
         return self.__class__(torch._foreach_norm(self, ord, dtype))
 
+    def metric(self, metric: Metrics) -> "TensorList":
+        return calculate_metric_list(self, metric)
+
     def mean(self, dim: _Dim = None, keepdim = False) -> Self | Any:
         if dim == 'global': return self._global_fn(keepdim, self.global_mean)
         return self.__class__(i.mean(dim=dim, keepdim=keepdim) for i in self)
@@ -790,27 +834,27 @@ class TensorList(list[torch.Tensor | Any]):
         for t, o in zip(self, other): t.copysign_(o)
         return self
 
-    def graft(self, magnitude: "_TensorSeq", tensorwise=False, ord: float | Literal['mean_abs'] = 2, eps = 1e-6, strength: float | _ScalarSeq = 1):
+    def graft(self, magnitude: "_TensorSeq", tensorwise=False, ord: Metrics = 2, eps = 1e-6, strength: float | _ScalarSeq = 1):
         if not isinstance(magnitude, TensorList): magnitude = TensorList(magnitude)
         if tensorwise:
-            norm_self = self.norm(ord)
-            norm_other = magnitude.norm(ord)
+            norm_self = self.metric(ord)
+            norm_other = magnitude.metric(ord)
         else:
-            norm_self = self.global_vector_norm(ord)
-            norm_other = magnitude.global_vector_norm(ord)
+            norm_self = self.global_metric(ord)
+            norm_other = magnitude.global_metric(ord)
 
         if generic_ne(strength, 1): norm_other.lerp_(norm_self, 1-maybe_numberlist(strength)) # pyright:ignore[reportCallIssue,reportArgumentType]
 
         return self * (norm_other / norm_self.clip_(min=eps))
 
-    def graft_(self, magnitude: "_TensorSeq", tensorwise=False, ord: float | Literal['mean_abs'] = 2, eps = 1e-6, strength: float | _ScalarSeq = 1):
+    def graft_(self, magnitude: "_TensorSeq", tensorwise=False, ord: Metrics = 2, eps = 1e-6, strength: float | _ScalarSeq = 1):
         if not isinstance(magnitude, TensorList): magnitude = TensorList(magnitude)
         if tensorwise:
-            norm_self = self.norm(ord)
-            norm_other = magnitude.norm(ord)
+            norm_self = self.metric(ord)
+            norm_other = magnitude.metric(ord)
         else:
-            norm_self = self.global_vector_norm(ord)
-            norm_other = magnitude.global_vector_norm(ord)
+            norm_self = self.global_metric(ord)
+            norm_other = magnitude.global_metric(ord)
 
         if generic_ne(strength, 1): norm_other.lerp_(norm_self, 1-maybe_numberlist(strength)) # pyright:ignore[reportCallIssue,reportArgumentType]
 
@@ -905,14 +949,14 @@ class TensorList(list[torch.Tensor | Any]):
         if eps!=0: std.add_(eps)
         return self.sub_(self.mean(dim = dim, keepdim=True)).div_(std)
 
-    def _clip_multiplier(self, min: "_Scalar | _ScalarSeq | None"= None, max: "_Scalar | _ScalarSeq | None" = None, tensorwise: bool = True, ord:float|Literal["mean_abs"] = 2):
+    def _clip_multiplier(self, min: "_Scalar | _ScalarSeq | None"= None, max: "_Scalar | _ScalarSeq | None" = None, tensorwise: bool = True, ord:Metrics = 2):
         """calculate multipler to clip self norm to min and max"""
         if tensorwise:
-            self_norm = self.norm(ord)
+            self_norm = self.metric(ord)
             self_norm.masked_fill_(self_norm == 0, 1)
 
         else:
-            self_norm = self.global_vector_norm(ord)
+            self_norm = self.global_metric(ord)
             if self_norm == 0: return 1
 
         mul = 1
@@ -926,12 +970,12 @@ class TensorList(list[torch.Tensor | Any]):
 
         return mul
 
-    def clip_norm(self, min: "_Scalar | _ScalarSeq | None"= None, max: "_Scalar | _ScalarSeq | None" = None, tensorwise: bool = True, ord:float|Literal["mean_abs"] = 2):
+    def clip_norm(self, min: "_Scalar | _ScalarSeq | None"= None, max: "_Scalar | _ScalarSeq | None" = None, tensorwise: bool = True, ord:Metrics = 2):
         """clips norm of each tensor to (min, max) range"""
         if min is None and max is None: return self
         return self * self._clip_multiplier(min, max, tensorwise, ord)
 
-    def clip_norm_(self, min: "_Scalar | _ScalarSeq | None"= None, max: "_Scalar | _ScalarSeq | None" = None, tensorwise: bool = True, ord:float|Literal["mean_abs"] = 2):
+    def clip_norm_(self, min: "_Scalar | _ScalarSeq | None"= None, max: "_Scalar | _ScalarSeq | None" = None, tensorwise: bool = True, ord:Metrics = 2):
         """clips norm of each tensor to (min, max) range"""
         if min is None and max is None: return self
         return self.mul_(self._clip_multiplier(min, max, tensorwise, ord))
@@ -990,6 +1034,15 @@ class TensorList(list[torch.Tensor | Any]):
     #     """sets index in flattened view"""
     #     return self.clone().flatset_(idx, value)
 
+    def flat_get(self, idx: int):
+        cur = 0
+        for tensor in self:
+            numel = tensor.numel()
+            if idx < cur + numel:
+                return tensor.view(-1)[cur-idx]
+            cur += numel
+        raise IndexError(idx)
+
     def flat_set_(self, idx: int, value: Any):
         """sets index in flattened view"""
         cur = 0
@@ -1065,10 +1118,19 @@ def generic_numel(x: torch.Tensor | TensorList) -> int:
     if isinstance(x, torch.Tensor): return x.numel()
     return x.global_numel()
 
+
+def generic_finfo(x: torch.Tensor | TensorList) -> torch.finfo:
+    if isinstance(x, torch.Tensor): return torch.finfo(x.dtype)
+    return torch.finfo(x[0].dtype)
+
 def generic_finfo_eps(x: torch.Tensor | TensorList) -> float:
     if isinstance(x, torch.Tensor): return torch.finfo(x.dtype).eps
     return torch.finfo(x[0].dtype).eps
 
+def generic_finfo_tiny(x: torch.Tensor | TensorList) -> float:
+    if isinstance(x, torch.Tensor): return torch.finfo(x.dtype).tiny
+    return torch.finfo(x[0].dtype).tiny
+
 @overload
 def generic_zeros_like(x: torch.Tensor) -> torch.Tensor: ...
 @overload
@@ -1081,7 +1143,8 @@ def generic_vector_norm(x: torch.Tensor | TensorList, ord=2) -> torch.Tensor:
     if isinstance(x, torch.Tensor): return torch.linalg.vector_norm(x, ord=ord) # pylint:disable=not-callable
     return x.global_vector_norm(ord)
 
-
+def generic_metric(x: torch.Tensor | TensorList, metric: Metrics) -> torch.Tensor:
+    return evaluate_metric(x, metric)
 
 @overload
 def generic_randn_like(x: torch.Tensor) -> torch.Tensor: ...
@@ -1091,3 +1154,11 @@ def generic_randn_like(x: torch.Tensor | TensorList):
     if isinstance(x, torch.Tensor): return torch.randn_like(x)
     return x.randn_like()
 
+
+def generic_sum(x: torch.Tensor | TensorList) -> torch.Tensor:
+    if isinstance(x, torch.Tensor): return x.sum()
+    return x.global_sum()
+
+def generic_max(x: torch.Tensor | TensorList) -> torch.Tensor:
+    if isinstance(x, torch.Tensor): return x.max()
+    return x.global_max()

torchzero/utils/torch_tools.py

@@ -7,10 +7,15 @@ import numpy as np
 import torch
 
 
-def totensor(x):
-    if isinstance(x, torch.Tensor): return x
-    if isinstance(x, np.ndarray): return torch.from_numpy(x)
-    return torch.from_numpy(np.asarray(x))
+def totensor(x, device=None, dtype=None):
+    if device is None and dtype is None:
+        if isinstance(x, torch.Tensor): return x
+        if isinstance(x, np.ndarray): return torch.from_numpy(x)
+        return torch.from_numpy(np.asarray(x))
+
+    if isinstance(x, torch.Tensor): return x.to(device=device, dtype=dtype)
+    if isinstance(x, np.ndarray): return torch.as_tensor(x, device=device, dtype=dtype)
+    return torch.as_tensor(np.asarray(x), device=device, dtype=dtype)
 
 def tonumpy(x):
     if isinstance(x, np.ndarray): return x

torchzero-0.3.14.dist-info/METADATA

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: torchzero
+Version: 0.3.14
+Summary: Modular optimization library for PyTorch.
+Author-email: Ivan Nikishev <nkshv2@gmail.com>
+Project-URL: Homepage, https://github.com/inikishev/torchzero
+Project-URL: Repository, https://github.com/inikishev/torchzero
+Project-URL: Issues, https://github.com/inikishev/torchzero/isses
+Keywords: optimization,optimizers,torch,neural networks,zeroth order,second order
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: torch
+Requires-Dist: numpy
+Requires-Dist: typing_extensions

torchzero-0.3.14.dist-info/RECORD

@@ -0,0 +1,167 @@
+tests/test_identical.py,sha256=PJnQtSO3aHZYMQolHmoB26BEUPD_Gpmdh2_M0tfUfm0,11502
+tests/test_module.py,sha256=qX3rjdSJsbA8JO17bPTUIDspe7bg2dogqxMw__KV7SU,2039
+tests/test_opts.py,sha256=XsOTxiKJQJOUr47d1nzyQ1WFclBrhXYKQMdEVN66bWs,43687
+tests/test_tensorlist.py,sha256=pWXQE-vEq08EGJSKWgsTgo-7QjjkavOJ5BlWUm241qI,72434
+tests/test_utils_optimizer.py,sha256=bvC0Ehvs2L8fohpyIF5Vfr9OKTycpnODWLPflXilU1c,8414
+tests/test_vars.py,sha256=2BoawNdDAnnNh_vv49_peJMnHvaQjp_sfnca1nosTWY,6766
+torchzero/__init__.py,sha256=aIH-cCTXnDr90cKUPhM8bv-uE69Hzjlf0jlYspYf0ZM,120
+torchzero/core/__init__.py,sha256=aYyQt-CHzWT6hGUt5KVjRZZr2lsX5I1XvbWpzaAv3VE,151
+torchzero/core/module.py,sha256=chn9NZWdgYekzVPDppvp2REMp4WugiMhOjQU3tys6ZU,40651
+torchzero/core/reformulation.py,sha256=jppgzXBtqdsc7ot6_Gr38vJbbrhG1Gs4vC32y7iB4BA,2387
+torchzero/core/transform.py,sha256=xRDpsZj0H1QcFdO-t2mNMNOYoqqnRHiI3K1YluWwCVk,17097
+torchzero/modules/__init__.py,sha256=3lGta9P0N3cWdVcruCBJ7uqu4DfLTPCKI_mlOZT6Z_o,615
+torchzero/modules/functional.py,sha256=E_d6hLT2_xdE-3AhQ4AthDYK5uZULbF10iHI09Z3_yk,7921
+torchzero/modules/adaptive/__init__.py,sha256=5L2dlEJV6HKBnYhgd7wo2yGi0WPd9qmpw9XS5wOQOq8,944
+torchzero/modules/adaptive/adagrad.py,sha256=0qXC5F4PuOsgLjRXQUWBoiq0AUixsvOP1uDbEeRIcNs,12531
+torchzero/modules/adaptive/adahessian.py,sha256=rWxgDiBMd6MK64mRjZwiudDF07is5AFj_MCEvBD7h8U,8670
+torchzero/modules/adaptive/adam.py,sha256=4lWSe__tdyRv0rfkUda1qa_NH36DIW3sd8td98bK6XI,3829
+torchzero/modules/adaptive/adan.py,sha256=Dt_gibyrGtWDIUCaSF6RFIxu2xwiF9fCfrpkoD-CaUM,2825
+torchzero/modules/adaptive/adaptive_heavyball.py,sha256=xQQw1Vx-NgQd_ouK14J1p5ijd5lEn3sAN0hVJAL0j8U,2024
+torchzero/modules/adaptive/aegd.py,sha256=_4ASgDX8__DPnnBE_RncnMqM4rItM7Eji4EZzSGGq5I,1876
+torchzero/modules/adaptive/esgd.py,sha256=DtrN3hZhGK4LgMxmcjQCBtEO-5hLZrAdnvps6f8WQ2A,6416
+torchzero/modules/adaptive/lion.py,sha256=XFyglRNdnP1l8CmEZ7L_ZB8HWiR03BsZ_PEFCvHijb8,1127
+torchzero/modules/adaptive/lmadagrad.py,sha256=rMs7vrgiwOJgWo-OZXkGu32X561edEhhaxwgbY2NTnk,7176
+torchzero/modules/adaptive/mars.py,sha256=iOkyY3r52btp7Cry7WN0AB4arpn4N9b_Hg6S55XC6Q8,2255
+torchzero/modules/adaptive/matrix_momentum.py,sha256=ZMSdGNSHgyUgtJwjKzK7PZDwMrPw5wZJVu0K4Xa-SpI,6693
+torchzero/modules/adaptive/msam.py,sha256=locqM2jiC3AbGCDCo6T40GF3iaVV2svrkzoi0hD2cJI,6663
+torchzero/modules/adaptive/muon.py,sha256=5Asgj03s6JXXrO-p5Qgn3D8bVwbDEqCq7hxNy4joQDE,10335
+torchzero/modules/adaptive/natural_gradient.py,sha256=5qRehh-iAeZ4hjfOR-gfsObbMsJZnipzCkG-yptkrH0,6349
+torchzero/modules/adaptive/orthograd.py,sha256=KbQuudjKgYVJcq1jRW_YmR2pPnwmAwyx9X_vrJAJgN4,2029
+torchzero/modules/adaptive/rmsprop.py,sha256=ugZLfH4dXvHTxawtGWQL6xSfsjBDl_t1s29aFN9FMuY,4345
+torchzero/modules/adaptive/rprop.py,sha256=VDnLPKxw8ECihyUeNVE8cyDll_Ut3k3_NqoLgpgxxLA,11818
+torchzero/modules/adaptive/sam.py,sha256=LnOPNZnIUsis0402RHnA-fTPkNM8baUR9HR50pF_BtM,5696
+torchzero/modules/adaptive/shampoo.py,sha256=r7V4I5_Ve1YVOS3HhO2k5cZvJT1lPHTVApV3iVJVceA,9711
+torchzero/modules/adaptive/soap.py,sha256=roQLBthNNNmTYgeJPi_LxZY-r4m6REeUo0_DZknYU50,10662
+torchzero/modules/adaptive/sophia_h.py,sha256=lSK8uVdOxBAhU2jE6fyIx1YgqEQyZG-Fv9o2TniAZzk,7179
+torchzero/modules/clipping/__init__.py,sha256=ZaffMF7mIRK6hZSfuZadgjNTX6hF5ANiLBny2w3S7I8,250
+torchzero/modules/clipping/clipping.py,sha256=t98M3QKZKqXJ3_tzXXIiG4EOYMaHqLYrMZ-6zmRuy-k,14331
+torchzero/modules/clipping/ema_clipping.py,sha256=Ki0LPNUwPoE825A5rSE7SxGQMiI3nO3iwnjKQ486iaI,6611
+torchzero/modules/clipping/growth_clipping.py,sha256=mHn6BQqfHFWnfVjYG_Dokb3VjfSK2QVGsy2rf0Z-RMY,6657
+torchzero/modules/conjugate_gradient/__init__.py,sha256=G5WcVoiQYupRBeqjI4lCraGeXNSvWT-_-ynpcE6NQS8,184
+torchzero/modules/conjugate_gradient/cg.py,sha256=L49p_wBeyQ3pZmaPM1vLx7uWmZBKtIhMf7Uv2ggUbI4,14534
+torchzero/modules/experimental/__init__.py,sha256=blI-OhpQAC6-Ho1uxUq-t7Mm9CAMnNMXkBDmXul8tbc,729
+torchzero/modules/experimental/curveball.py,sha256=JdgojuSYLNe9u3bmqcYrFm8brUD4kvKm9XYx78GzpKI,3257
+torchzero/modules/experimental/dct.py,sha256=CW-Y2gcjlHlxtIx7SekUOfw2EzujA6v0LcjDYGAfh6M,2433
+torchzero/modules/experimental/fft.py,sha256=s95EzvK4-ZJdwZbVhtqwirY9eVy7v6mFDRMgoLY9wjo,3020
+torchzero/modules/experimental/gradmin.py,sha256=hKTOG7tk6FnG8t-7OmTAhGTGSDdONzP1JvCRPRqaKt0,3740
+torchzero/modules/experimental/l_infinity.py,sha256=nhYusM3YYbc0ptaSf9zlrsqY8EgxlHm9OejJ6VV0qtM,4750
+torchzero/modules/experimental/momentum.py,sha256=VqZc14EGVO_KUABPLRIBlvHdgg-64o-4heMQH0vW5vY,5233
+torchzero/modules/experimental/newton_solver.py,sha256=0HnDBlrBLvUgS4hWmkJqyw0M7UPFp3kU3SFq2xZVYhQ,5454
+torchzero/modules/experimental/newtonnewton.py,sha256=a0XXvlVe37z2MMcQ4TeGbbWX9OuYp_5b-21jq3o1z3E,3823
+torchzero/modules/experimental/reduce_outward_lr.py,sha256=ui_39wNdf5J2FOQtQFk0WUA8DuicwEp0kepccbq8rI0,1309
+torchzero/modules/experimental/scipy_newton_cg.py,sha256=8nKBaHHmqdU9F1kVPn2QVFUTx2_I8Jsfqxix1v-qoL0,4073
+torchzero/modules/experimental/spsa1.py,sha256=tytlOaKSyhvoElErI-MzhbL0fIm3K4d_M1Kpbpb9jbw,3622
+torchzero/modules/experimental/structural_projections.py,sha256=rxJFG5F23dOiK_8KqKyvoSMLWqAOXtVGHSwfRqH22Wg,4185
+torchzero/modules/grad_approximation/__init__.py,sha256=_mQ2sWvnMfqc3RQcVmZuBlphtLZCO7z819abGY6kYuM,196
+torchzero/modules/grad_approximation/fdm.py,sha256=zx70GZDQmhe43bZP5Mbbl31xsMOsGO43kznoQDbqxJo,4372
+torchzero/modules/grad_approximation/forward_gradient.py,sha256=pCOsvt4ZZtzsIlAeXLmpS8vNXYewA_Gh7uyz1_1yROs,4011
+torchzero/modules/grad_approximation/grad_approximator.py,sha256=x8vlweBrfJ6SmhMHvI_C8UZGzlS3AnmlulvqnSzm6iY,4437
+torchzero/modules/grad_approximation/rfdm.py,sha256=zgzZYJDAq6HsOMeHePUWlnnJWYRpqey77nHMaHh8140,19217
+torchzero/modules/higher_order/__init__.py,sha256=iaoIrmR9DJE9QHt9PeZNCWqIYDe-86h1IjkaumR4qF0,51
+torchzero/modules/higher_order/higher_order_newton.py,sha256=2r1wuhdi57pbo8akQE88O8R-Y79BtiwD1WQIShh1rjQ,12967
+torchzero/modules/least_squares/__init__.py,sha256=mJwE2IXVB3mn_7BzsmDNKhfyViCV8GOrqHJJjz04HR4,41
+torchzero/modules/least_squares/gn.py,sha256=23AB6AWAl5IuBj4Vd3boQ6ndk0pO3ovaF9EiY1a1XWs,5094
+torchzero/modules/line_search/__init__.py,sha256=mFWgkcgfMkL2NKj3CLbuwee3e8WHBOaXs-wtx3oTW58,216
+torchzero/modules/line_search/_polyinterp.py,sha256=qIhcLjOlpB6NHU0oiUGMncwQxWNfy8757orsbzjkp6s,10882
+torchzero/modules/line_search/adaptive.py,sha256=8Ip5F5PpsDLgg6TwB_E7zIZheycd78coRg4u7cpO3Cg,3795
+torchzero/modules/line_search/backtracking.py,sha256=Mhx8_UT_Mr1gASYHUorBJ38E4YlcM9LpW9YrJHYfLXU,9049
+torchzero/modules/line_search/line_search.py,sha256=lmtjr9Zpz9RYJXoYaJnpXkBSIdcN6DdwGKKXTCmcJNU,13294
+torchzero/modules/line_search/scipy.py,sha256=xQ80h9cSyF4Iorq_1NoJglu_Bx4_KeojulBIxvwU6gQ,2836
+torchzero/modules/line_search/strong_wolfe.py,sha256=21IMtenhBlrw_edtwizzLZ01PF4-rU4M2oGOCUA2udc,14936
+torchzero/modules/misc/__init__.py,sha256=UYY9CeNepnC8H1LnFa829ux5MEjtGZ9zql624IbCFX8,825
+torchzero/modules/misc/debug.py,sha256=6pFAGYANjCPGIZH_4ghpUYYTEsT5jr7PMB9KLuPP4p8,1532
+torchzero/modules/misc/escape.py,sha256=qfEdKLD5rejqrmvyHrI5BRQq8js9UF2-Axs_C0KFyWA,1866
+torchzero/modules/misc/gradient_accumulation.py,sha256=mBWa5CBCZwp4TrtOyjWI3VnHag4gum4WBM2WFhvHqW4,4891
+torchzero/modules/misc/homotopy.py,sha256=hihLETE4dNZ27zatqPR_qT3kGX-AXbC7oBWRDbFQo58,1939
+torchzero/modules/misc/misc.py,sha256=feI-IQlxhIoAbsSRTjE4SbGez1c2Uu9-WA_nkK7iiqQ,15411
+torchzero/modules/misc/multistep.py,sha256=RtDFIeTHu4RcERvlKEP4_10-lpRZOgbnBeSah92dQ7A,6323
+torchzero/modules/misc/regularization.py,sha256=SkQ0_Ybtv9IEGI9QGdvNZaja5bAyc1x-j_1gvYIVepI,6105
+torchzero/modules/misc/split.py,sha256=JcXVB4xk3h55YT2OAdepVsRoE1PD7bqX6NmJ2IxBgAI,4013
+torchzero/modules/misc/switch.py,sha256=p758heAnv-PkoslpafL35Yp7mlvPmDVSe1mWiuuD8Mk,3711
+torchzero/modules/momentum/__init__.py,sha256=AKWC4HIkN9ZJwN38dJvVJkFEhiP9r93G-kMDokBfsj8,281
+torchzero/modules/momentum/averaging.py,sha256=vDW8tgGsEuBXF_BTUYHB_j--TIVam9j0nZdp_x8TkxY,3229
+torchzero/modules/momentum/cautious.py,sha256=x506a3lUETRpxPWqXLFJVFBH1gmLqIfqL5J-hFdEvOo,8051
+torchzero/modules/momentum/momentum.py,sha256=q3n0BvQURuSBzA9vn1ZrH-n7Nsr0AS-38VJuwraQPY0,4495
+torchzero/modules/ops/__init__.py,sha256=9UHaXs9aaKc0ewAhicTlDmj42bSC_vddMOD0eYuUj_8,1226
+torchzero/modules/ops/accumulate.py,sha256=kyjiC9M9fugpG5Pc07XUi6GEWBvRi8iJ-7_Mb1SXQzE,3665
+torchzero/modules/ops/binary.py,sha256=2hV2oruaq5Asu4Ts8X8yiZQ-07fU0RGpRy3-vifXqXY,12151
+torchzero/modules/ops/higher_level.py,sha256=E76zgSHlhVpHLrXhnVwelIQFm1IKn0IFcVq7DOJw0es,9037
+torchzero/modules/ops/multi.py,sha256=YC3rBTmPRwF5aEPDNsyTK4J_JEAbmE7oBmF7W-VOV3A,8588
+torchzero/modules/ops/reduce.py,sha256=kALG7X8q02sWpo1skpXjS0r875gwq6mrhLZbFfYaZoA,6324
+torchzero/modules/ops/unary.py,sha256=EFA_A834KmA6Ec3pZWH5XxZ9OzAhZZudwAwsP4GWZA0,5476
+torchzero/modules/ops/utility.py,sha256=_k9S59i6IYOzzfIQlToQ9mlDseTTAS_49wujUxMGXZo,4105
+torchzero/modules/projections/__init__.py,sha256=4LfmBEu_eM4YWmcWQVH4CdI1H0ucCIHDH9tTGigjVPY,136
+torchzero/modules/projections/cast.py,sha256=FJx2Tt1lbQRnOC5wxx3LbOnacLfUluFP6QOXLUCIEPY,2174
+torchzero/modules/projections/galore.py,sha256=GDJ7hf6cdk_Iu2qW0rWaQwYLQAxQEe27FEfOiZvFXHo,252
+torchzero/modules/projections/projection.py,sha256=itkkb2UmMqbdtWKjUUg6gbFJfCEIZAskC0HCvom-6sc,14084
+torchzero/modules/quasi_newton/__init__.py,sha256=HxXENs3O6nFRfCvUJhWPK9f8_A6iMwB6UF1Zold12UQ,515
+torchzero/modules/quasi_newton/damping.py,sha256=K1DVqqKiAs6-F3JQh5jlKNb79oJdObqnKWwHHRl6boQ,2813
+torchzero/modules/quasi_newton/diagonal_quasi_newton.py,sha256=Zx-tlFRa89GhoSP7RFJdLQJPiqPCL7rWaV7WJoQ1YCs,6930
+torchzero/modules/quasi_newton/lbfgs.py,sha256=fzCjV5YsLo_uJTVG3vosPHsvDc97mLKueK6fxOHLb8I,11195
+torchzero/modules/quasi_newton/lsr1.py,sha256=D3_yV5xtgklMlU4fAL1-sH82-1tNl3K2F12ZBZyLQGM,8512
+torchzero/modules/quasi_newton/quasi_newton.py,sha256=-xUGPld8Y0MHwN6qsmDihLbUbulU0T1z8jf2mZhNpcE,44529
+torchzero/modules/restarts/__init__.py,sha256=7282ePwN_I0vSeLPYS4TTclE9ZU7pL6UpyRp5ydgdSg,134
+torchzero/modules/restarts/restars.py,sha256=ZN8kzufkOKredI54HJ9cSANAYY2yN4gAggqAVtEBCyA,9084
+torchzero/modules/second_order/__init__.py,sha256=lTGccDNVwPuMevMeKi5O0a9cl24Rn9tk7VkC6jvlGYc,233
+torchzero/modules/second_order/multipoint.py,sha256=Ilzo0Ddd3iApegceu7cHSMGim9ZH5QS4-2uBtrKXC6k,8581
+torchzero/modules/second_order/newton.py,sha256=PAPbJzssx0Ji328BFOEzeJZPd3IubJTPHs6ZhqS_nW8,15663
+torchzero/modules/second_order/newton_cg.py,sha256=6KmYr-U5gmuTGoVZA8Gt2-rfLwnjNakjKkBKEuBffTk,16738
+torchzero/modules/second_order/nystrom.py,sha256=yAJijWCl-K8k63YSJUqE_kXEIFmL_FjDghVjQoutAXo,11352
+torchzero/modules/smoothing/__init__.py,sha256=RYxCLLfG2onBbMUToaoedsr20rXaayyBt7Ov8OxULrU,80
+torchzero/modules/smoothing/laplacian.py,sha256=05Y6ft0GHRGkfSDV-g8vlaTlZTiXMr79xDagJb126ug,5302
+torchzero/modules/smoothing/sampling.py,sha256=zI5bATytQmCqm_UgAQbfA9tNRgrZaKLfUb0B-kzKRHU,12867
+torchzero/modules/step_size/__init__.py,sha256=jG0qXpIn17oYXL8b34UjiEbkl002hj3FqJk1uQ5bkCg,136
+torchzero/modules/step_size/adaptive.py,sha256=HvffW3m1NnpMTpps0QjJTvbblSODxxWMBBFTbNwp0vM,14482
+torchzero/modules/step_size/lr.py,sha256=I9-aIxei4Y2XnlOoCKvec2r__cTY_JTwBDlMf2O5D2A,5908
+torchzero/modules/termination/__init__.py,sha256=LkXBiOOYD4ce1Lemj0Vx9BCm_KhRTQTMvm-PD4lQwTs,344
+torchzero/modules/termination/termination.py,sha256=BXU3R04caBc8rFJ4v_yJjgGi1X4iA11eYwlbiJfxexI,6637
+torchzero/modules/trust_region/__init__.py,sha256=kWke9FB41-EpjdXCPk8VBwZhpgYalOWSKDI1XWe0yYg,204
+torchzero/modules/trust_region/cubic_regularization.py,sha256=gbKOR5zBo3t9i-sW23DCtTQwZrBubuFy_VuafrLaeUw,6718
+torchzero/modules/trust_region/dogleg.py,sha256=zwFR49gghxztVGEETF2D4AkeGgHkQRbHGGelav3GuFg,3619
+torchzero/modules/trust_region/levenberg_marquardt.py,sha256=s4XBXK8LwHuwyanOJFtgmOLkxEiBMhBVzch0J2_dIFk,5055
+torchzero/modules/trust_region/trust_cg.py,sha256=F6G0hTXv6Ry0swO_4fx3ecxYWxMr72nrmwJPbFDpqH4,4459
+torchzero/modules/trust_region/trust_region.py,sha256=eimCFViJSzoubrRmDluCon6mfcyT7PQA0yRPu4FlO2Q,12872
+torchzero/modules/variance_reduction/__init__.py,sha256=3pwPWZpjgz1btfLJ3rEaK7Wl8B1pDh0HIf0kvD_NJH8,22
+torchzero/modules/variance_reduction/svrg.py,sha256=9pBjPY4EMkGyfj68gXqPi1GJIolUVl5zyNtlZInCKKo,8635
+torchzero/modules/weight_decay/__init__.py,sha256=7UHAiiimsbQ_dHlxxcW87G5cCQFom9Uh_733W_23PWU,93
+torchzero/modules/weight_decay/weight_decay.py,sha256=Y7kE_j0GRF8ceJ9SS6qykQ8a23X2OTDCjJ9VklOQSEw,5415
+torchzero/modules/wrappers/__init__.py,sha256=6b5Ac-8u18IVp_Jnw1T1xQExwpQhpQ0JwNV9GyC_Yj8,31
+torchzero/modules/wrappers/optim_wrapper.py,sha256=B8ZwZf-qzChBfbx-cwL8Rez4AgH7FzvsT7N1S2SUiR8,4417
+torchzero/modules/zeroth_order/__init__.py,sha256=1ADUiOHVHzvIP4TpH7_ILmeW2heidfikbf6d5g_1RzY,18
+torchzero/modules/zeroth_order/cd.py,sha256=SwjwoAqX86-JnVHIwKAE7g_tqm0EvEUUNuLM4T5mKXE,4876
+torchzero/optim/__init__.py,sha256=aXf7EkywqYiR50I4QeeVXro9aBhKiqfbY_BCia59sgU,46
+torchzero/optim/root.py,sha256=gGtAJ9qBoSNV58EKzUGZ8J3lyKGUF8BEw34Zfprppdo,2273
+torchzero/optim/utility/__init__.py,sha256=pUacok4XmebfxofE-QWZLgViajsU-3JkXcWi9OS-Jrw,24
+torchzero/optim/utility/split.py,sha256=kraPCLAewX2uLbD_9R2dIrcF-kpUuT9IcxPeVrAARvA,1672
+torchzero/optim/wrappers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+torchzero/optim/wrappers/directsearch.py,sha256=rimJIB2RrVzLpRPQKhzkrMQ4bTAEU3NEOT4pJQNIAHE,11309
+torchzero/optim/wrappers/fcmaes.py,sha256=jKmmBKEwguYiJdvTRmAp5JSilxcUhtpRoKlzmp-lyWE,4251
+torchzero/optim/wrappers/mads.py,sha256=Zi9u3vNlgNsCaIdYLl2_jgRA_dQrmVAuG0V31BFuct4,3087
+torchzero/optim/wrappers/nevergrad.py,sha256=U_ZAHD_nEsJZ71cJ8TQ_DOZcTmS06EEvUPvaaDRSxWI,4901
+torchzero/optim/wrappers/nlopt.py,sha256=RuLKretljjAjTZ9tXY3FyEDuB7mAboeGOQBupWfzPc4,8105
+torchzero/optim/wrappers/optuna.py,sha256=pIXkC5NVmEnUQ4jsGaz6Gv9uYOZM9rnxME4UGkeolsE,2393
+torchzero/optim/wrappers/scipy.py,sha256=A4yeQRdB9f65UrJ2g80NfqqMc6zUyr9js40TUESCHPg,21535
+torchzero/utils/__init__.py,sha256=7S4VRTkfS-0uI8HOR0EFIjiEcKrmYK7LEhTocIgki6c,1112
+torchzero/utils/compile.py,sha256=Dozox91tcShUJ3L320TTbJrcuA-l4WVegLAQujRqy94,5132
+torchzero/utils/derivatives.py,sha256=zJ0xyedvlIwgAYMa1F5BBfyrkvgjXy7v7evvl6QAlT0,17195
+torchzero/utils/metrics.py,sha256=XPpOvY257tb4mN3Sje1AVNlQkOXiW24_lXXdtd0JYok,3130
+torchzero/utils/numberlist.py,sha256=iMoqz4IzXy-aE9bqVYJ21GV6pl0z-NeTsXR-LaI8C24,6229
+torchzero/utils/ops.py,sha256=n4Su1sbgTzlHczuPEHkuWenTtNBCa_MvlQ_hCZkIPnQ,314
+torchzero/utils/optimizer.py,sha256=pzGIddJtsR_oH3mb3GOsWHi02Sb62ms4NCaWUOk0Clo,12470
+torchzero/utils/optuna_tools.py,sha256=F-1Xg0n_29MVEb6lqgUFFNIl9BNJ6MOdIJPduoNH4JU,1325
+torchzero/utils/params.py,sha256=nQo270aOURU7rJ_D102y2pSXbzhJPK0Z_ehx4mZBMes,5784
+torchzero/utils/python_tools.py,sha256=QxLZ2PKgp4R2zI_C1qNOX_u4eIcuT0wNBpBM5YEIYuU,3428
+torchzero/utils/tensorlist.py,sha256=nIWBME3fUQPsr4buvtV3LaJgSXPEG_Xb58KAzfjwK-k,56064
+torchzero/utils/torch_tools.py,sha256=DsHaSRGZ3-IuySZJTrkojTbaMMlttJFe0hFvB2xnl2U,5069
+torchzero/utils/linalg/__init__.py,sha256=cNoTJOPeqbNn9l7_HAAen2rlehGS3DyY5SveInG3Stc,328
+torchzero/utils/linalg/benchmark.py,sha256=wiIMn-GY2xxWbHVf8CPbJddUPeUPq9OUDkvbp1iILYI,479
+torchzero/utils/linalg/linear_operator.py,sha256=uJUxvOVHpG3U3GNx61JGa_uM8GqzsNZmA_z7P0RwZ5E,12747
+torchzero/utils/linalg/matrix_funcs.py,sha256=BKQK_oIG35R6yGxU80eBG0VkyY2EgxywqbhvU7JhWm4,3109
+torchzero/utils/linalg/orthogonalize.py,sha256=BpuDiAPrsJMUpTNBMCntBNA8-O2nozLxY5ZbCfRlEFY,444
+torchzero/utils/linalg/qr.py,sha256=5tbPEV9I6X69r5ACWF9XeqjZTUtUql2145uoGjlJNDs,2517
+torchzero/utils/linalg/solve.py,sha256=R5lPTzHn2sgvRy4MRp-Ngl0sypSGLRLHJjf1oKKAJD0,14395
+torchzero/utils/linalg/svd.py,sha256=wBxl-JSciINV-N6zvM4SGdveqMr6idq51h68LyQQRYg,660
+torchzero-0.3.14.dist-info/METADATA,sha256=Pd9XeJLSPuNQtb-dVJvnIiDOX5Mdr9b9ihpfw6rxBpQ,565
+torchzero-0.3.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+torchzero-0.3.14.dist-info/top_level.txt,sha256=ETW_iE2ubg0oMyef_h-ayB5i1OOZZd4SNdR3ltIbHe0,16
+torchzero-0.3.14.dist-info/RECORD,,

{torchzero-0.3.11.dist-info → torchzero-0.3.14.dist-info}/top_level.txt

@@ -1,3 +1,2 @@
-docs
 tests
 torchzero

docs/source/conf.py

@@ -1,59 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# For the full list of built-in configuration values, see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Project information -----------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
-import sys, os
-#sys.path.insert(0, os.path.abspath('.../src'))
-
-project = 'torchzero'
-copyright = '2025, Ivan Nikishev'
-author = 'Ivan Nikishev'
-
-# -- General configuration ---------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
-
-# https://sphinx-intro-tutorial.readthedocs.io/en/latest/sphinx_extensions.html
-extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.autosummary',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.autosectionlabel',
-    'sphinx.ext.githubpages',
-    'sphinx.ext.napoleon',
-    'autoapi.extension',
-    "myst_nb",
-
-    # 'sphinx_rtd_theme',
-]
-autosummary_generate = True
-autoapi_dirs = ['../../torchzero']
-autoapi_type = "python"
-# autoapi_ignore = ["*/tensorlist.py"]
-
-# https://sphinx-autoapi.readthedocs.io/en/latest/reference/config.html#confval-autoapi_options
-autoapi_options = [
-    "members",
-    "undoc-members",
-    "show-inheritance",
-    "show-module-summary",
-    "imported-members",
-]
-
-
-templates_path = ['_templates']
-exclude_patterns = []
-
-# -- Options for HTML output -------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
-
-#html_theme = 'alabaster'
-html_theme = 'sphinx_rtd_theme'
-html_static_path = ['_static']
-
-
-# OTHER STUFF I FOUND ON THE INTERNET AND PUT THERE HOPING IT DOES SOMETHING USEFUL
-source_suffix = ['.rst', '.md']
-master_doc = 'index'

docs/source/docstring template.py

@@ -1,46 +0,0 @@
-class MyModule:
-    """[One-line summary of the class].
-
-    [A more detailed description of the class, explaining its purpose, how it
-    works, and its typical use cases. You can use multiple paragraphs.]
-
-    .. note::
-        [Optional: Add important notes, warnings, or usage guidelines here.
-        For example, you could mention if a closure is required, discuss
-        stability, or highlight performance characteristics. Use the `.. note::`
-        directive to make it stand out in the documentation.]
-
-    Args:
-        param1 (type, optional):
-            [Description of the first parameter. Use :code:`backticks` for
-            inline code like variable names or specific values like ``"autograd"``.
-            Explain what the parameter does.] Defaults to [value].
-        param2 (type):
-            [Description of a mandatory parameter (no "optional" or "Defaults to").]
-        **kwargs:
-            [If you accept keyword arguments, describe what they are used for.]
-
-    Examples:
-        [A title or short sentence describing the first example]:
-
-        .. code-block:: python
-
-            opt = tz.Modular(
-                model.parameters(),
-                ...
-            )
-
-        [A title or short sentence for a second, different example]:
-
-        .. code-block:: python
-
-            opt = tz.Modular(
-                model.parameters(),
-                ...
-            )
-
-    References:
-        - [Optional: A citation for a relevant paper, book, or algorithm.]
-        - [Optional: A link to a blog post or website with more information.]
-
-    """