lucid-dl 2.7.6__tar.gz → 2.7.8__tar.gz

{lucid_dl-2.7.6/lucid_dl.egg-info → lucid_dl-2.7.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lucid-dl
-Version: 2.7.6
+Version: 2.7.8
 Summary: Lumerico's Comprehensive Interface for Deep Learning
 Home-page: https://github.com/ChanLumerico/lucid
 Author: ChanLumerico
@@ -46,6 +46,12 @@ Whether you're a student, educator, or an advanced researcher seeking to demysti
 
 - Now supports [**`Safetensors`**](https://github.com/huggingface/safetensors) for Lucid neural module porting along with the legacy `.lcd` format
 
+- Added multiple `nn` utilities:
+
+  - `nn.util.grad_norm` - Returns the global norm of the gradients
+  - `nn.util.clip_grad_norm` - Rescales the gradients based on the global norm
+  - `nn.util.clip_grad_value` - Rescales the gradients based on their values.
+
 - Implemented **EfficientDet**: `lucid.models.EfficientDet` with variants from `D0` to `D7`
 
 ## 🔧 How to Install

{lucid_dl-2.7.6 → lucid_dl-2.7.8}/README.md

@@ -18,6 +18,12 @@ Whether you're a student, educator, or an advanced researcher seeking to demysti
 
 - Now supports [**`Safetensors`**](https://github.com/huggingface/safetensors) for Lucid neural module porting along with the legacy `.lcd` format
 
+- Added multiple `nn` utilities:
+
+  - `nn.util.grad_norm` - Returns the global norm of the gradients
+  - `nn.util.clip_grad_norm` - Rescales the gradients based on the global norm
+  - `nn.util.clip_grad_value` - Rescales the gradients based on their values.
+
 - Implemented **EfficientDet**: `lucid.models.EfficientDet` with variants from `D0` to `D7`
 
 ## 🔧 How to Install

{lucid_dl-2.7.6 → lucid_dl-2.7.8}/lucid/_tensor/tensor.py

@@ -397,28 +397,33 @@ class Tensor(_TensorBase):
         return hash(id(self))
 
     def __deepcopy__(self, *args: Any) -> Self:
+        cls = self.__class__
         copied_data = Tensor.copy_data(self.data)
 
-        new_tensor = Tensor(
-            copied_data,
-            requires_grad=self.requires_grad,
-            keep_grad=self.keep_grad,
-            dtype=self.dtype,
-            device=self.device,
-        )
+        if cls is Tensor:
+            new = Tensor(
+                copied_data, self.requires_grad, self.keep_grad, self.dtype, self.device
+            )
+        else:
+            base = Tensor(copied_data, dtype=self.dtype, device=self.device)
+            new = cls(base)
 
-        if self.grad is not None:
-            new_tensor.grad = Tensor.copy_grad(self.grad)
+        if self.grad is not None and (
+            self.keep_grad or getattr(new, "keep_grad", False)
+        ):
+            new.grad = Tensor.copy_grad(self.grad)
+        else:
+            new.grad = None
 
-        new_tensor._op = self._op
-        new_tensor._backward_op = self._backward_op
-        new_tensor._prev = self._prev.copy()
-        new_tensor._backward_hooks = self._backward_hooks.copy()
+        new._op = None
+        new._backward_op = _noop
+        new._prev = []
+        new._backward_hooks = []
 
-        new_tensor._is_free = self._is_free
-        new_tensor._is_bool_tensor = self._is_bool_tensor
+        new._is_free = self._is_free
+        new._is_bool_tensor = self._is_bool_tensor
 
-        return new_tensor
+        return new
 
     def __bool__(self) -> bool:
         if self.data.size != 1:

{lucid_dl-2.7.6 → lucid_dl-2.7.8}/lucid/nn/__init__.py

@@ -4,3 +4,4 @@ from lucid.nn.modules import *
 from lucid.nn.fused import *
 
 import lucid.nn.init as init
+import lucid.nn.util as util

lucid_dl-2.7.8/lucid/nn/util.py

@@ -0,0 +1,60 @@
+from typing import Iterable
+
+import lucid
+
+from lucid._tensor import Tensor
+from lucid.types import _Scalar
+
+
+__all__ = ["grad_norm", "clip_grad_norm", "clip_grad_value"]
+
+
+def _as_iter(parameters: Iterable[Tensor] | Tensor) -> list[Tensor]:
+    if isinstance(parameters, Tensor):
+        return [parameters]
+    return list(parameters)
+
+
+def grad_norm(parameters: Iterable[Tensor] | Tensor, norm_type: int = 2) -> Tensor:
+    parameters = _as_iter(parameters)
+    device = parameters[0].device
+
+    params: list[Tensor] = [p for p in parameters if p.grad is not None]
+    if not params:
+        return Tensor(0.0, device=device)
+
+    norm_pow_sum = 0.0
+    for p in params:
+        param_norm = lucid.linalg.norm(lucid.ravel(p.grad), ord=norm_type).item()
+        norm_pow_sum += param_norm**norm_type
+
+    total_norm = norm_pow_sum ** (1.0 / norm_type)
+    return Tensor(total_norm, device=device)
+
+
+def clip_grad_norm(
+    parameters: Iterable[Tensor] | Tensor,
+    max_norm: _Scalar,
+    norm_type: int = 2,
+    eps: float = 1e-7,
+) -> float:
+    params: list[Tensor] = [p for p in _as_iter(parameters) if p.grad is not None]
+    total_norm = grad_norm(params, norm_type=norm_type)
+
+    clip_coef = float(max_norm) / (total_norm.item() + eps)
+    if clip_coef < 1.0:
+        for p in params:
+            p.grad = p.grad * clip_coef
+
+    return total_norm
+
+
+def clip_grad_value(parameters: Iterable[Tensor] | Tensor, clip_value: _Scalar) -> None:
+    params = [p for p in _as_iter(parameters) if p.grad is not None]
+    if not params:
+        return
+
+    lo, hi = -float(clip_value), float(clip_value)
+    for p in params:
+        g_clip = lucid.clip(p.grad, lo, hi).data
+        p.grad = g_clip

{lucid_dl-2.7.6 → lucid_dl-2.7.8/lucid_dl.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lucid-dl
-Version: 2.7.6
+Version: 2.7.8
 Summary: Lumerico's Comprehensive Interface for Deep Learning
 Home-page: https://github.com/ChanLumerico/lucid
 Author: ChanLumerico
@@ -46,6 +46,12 @@ Whether you're a student, educator, or an advanced researcher seeking to demysti
 
 - Now supports [**`Safetensors`**](https://github.com/huggingface/safetensors) for Lucid neural module porting along with the legacy `.lcd` format
 
+- Added multiple `nn` utilities:
+
+  - `nn.util.grad_norm` - Returns the global norm of the gradients
+  - `nn.util.clip_grad_norm` - Rescales the gradients based on the global norm
+  - `nn.util.clip_grad_value` - Rescales the gradients based on their values.
+
 - Implemented **EfficientDet**: `lucid.models.EfficientDet` with variants from `D0` to `D7`
 
 ## 🔧 How to Install

{lucid_dl-2.7.6 → lucid_dl-2.7.8}/lucid_dl.egg-info/SOURCES.txt

@@ -78,6 +78,7 @@ lucid/nn/__init__.py
 lucid/nn/fused.py
 lucid/nn/module.py
 lucid/nn/parameter.py
+lucid/nn/util.py
 lucid/nn/functional/__init__.py
 lucid/nn/functional/_activation.py
 lucid/nn/functional/_attention.py

{lucid_dl-2.7.6 → lucid_dl-2.7.8}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="lucid-dl",
-    version="2.7.6",
+    version="2.7.8",
     author="ChanLumerico",
     author_email="greensox284@gmail.com",
     description="Lumerico's Comprehensive Interface for Deep Learning",