lucid-dl 2.7.6__py3-none-any.whl → 2.7.8__py3-none-any.whl

lucid/_tensor/tensor.py

@@ -397,28 +397,33 @@ class Tensor(_TensorBase):
         return hash(id(self))
 
     def __deepcopy__(self, *args: Any) -> Self:
+        cls = self.__class__
         copied_data = Tensor.copy_data(self.data)
 
-        new_tensor = Tensor(
-            copied_data,
-            requires_grad=self.requires_grad,
-            keep_grad=self.keep_grad,
-            dtype=self.dtype,
-            device=self.device,
-        )
+        if cls is Tensor:
+            new = Tensor(
+                copied_data, self.requires_grad, self.keep_grad, self.dtype, self.device
+            )
+        else:
+            base = Tensor(copied_data, dtype=self.dtype, device=self.device)
+            new = cls(base)
 
-        if self.grad is not None:
-            new_tensor.grad = Tensor.copy_grad(self.grad)
+        if self.grad is not None and (
+            self.keep_grad or getattr(new, "keep_grad", False)
+        ):
+            new.grad = Tensor.copy_grad(self.grad)
+        else:
+            new.grad = None
 
-        new_tensor._op = self._op
-        new_tensor._backward_op = self._backward_op
-        new_tensor._prev = self._prev.copy()
-        new_tensor._backward_hooks = self._backward_hooks.copy()
+        new._op = None
+        new._backward_op = _noop
+        new._prev = []
+        new._backward_hooks = []
 
-        new_tensor._is_free = self._is_free
-        new_tensor._is_bool_tensor = self._is_bool_tensor
+        new._is_free = self._is_free
+        new._is_bool_tensor = self._is_bool_tensor
 
-        return new_tensor
+        return new
 
     def __bool__(self) -> bool:
         if self.data.size != 1:

lucid/nn/__init__.py

@@ -4,3 +4,4 @@ from lucid.nn.modules import *
 from lucid.nn.fused import *
 
 import lucid.nn.init as init
+import lucid.nn.util as util

lucid/nn/util.py

@@ -0,0 +1,60 @@
+from typing import Iterable
+
+import lucid
+
+from lucid._tensor import Tensor
+from lucid.types import _Scalar
+
+
+__all__ = ["grad_norm", "clip_grad_norm", "clip_grad_value"]
+
+
+def _as_iter(parameters: Iterable[Tensor] | Tensor) -> list[Tensor]:
+    if isinstance(parameters, Tensor):
+        return [parameters]
+    return list(parameters)
+
+
+def grad_norm(parameters: Iterable[Tensor] | Tensor, norm_type: int = 2) -> Tensor:
+    parameters = _as_iter(parameters)
+    device = parameters[0].device
+
+    params: list[Tensor] = [p for p in parameters if p.grad is not None]
+    if not params:
+        return Tensor(0.0, device=device)
+
+    norm_pow_sum = 0.0
+    for p in params:
+        param_norm = lucid.linalg.norm(lucid.ravel(p.grad), ord=norm_type).item()
+        norm_pow_sum += param_norm**norm_type
+
+    total_norm = norm_pow_sum ** (1.0 / norm_type)
+    return Tensor(total_norm, device=device)
+
+
+def clip_grad_norm(
+    parameters: Iterable[Tensor] | Tensor,
+    max_norm: _Scalar,
+    norm_type: int = 2,
+    eps: float = 1e-7,
+) -> float:
+    params: list[Tensor] = [p for p in _as_iter(parameters) if p.grad is not None]
+    total_norm = grad_norm(params, norm_type=norm_type)
+
+    clip_coef = float(max_norm) / (total_norm.item() + eps)
+    if clip_coef < 1.0:
+        for p in params:
+            p.grad = p.grad * clip_coef
+
+    return total_norm
+
+
+def clip_grad_value(parameters: Iterable[Tensor] | Tensor, clip_value: _Scalar) -> None:
+    params = [p for p in _as_iter(parameters) if p.grad is not None]
+    if not params:
+        return
+
+    lo, hi = -float(clip_value), float(clip_value)
+    for p in params:
+        g_clip = lucid.clip(p.grad, lo, hi).data
+        p.grad = g_clip

{lucid_dl-2.7.6.dist-info → lucid_dl-2.7.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lucid-dl
-Version: 2.7.6
+Version: 2.7.8
 Summary: Lumerico's Comprehensive Interface for Deep Learning
 Home-page: https://github.com/ChanLumerico/lucid
 Author: ChanLumerico
@@ -46,6 +46,12 @@ Whether you're a student, educator, or an advanced researcher seeking to demysti
 
 - Now supports [**`Safetensors`**](https://github.com/huggingface/safetensors) for Lucid neural module porting along with the legacy `.lcd` format
 
+- Added multiple `nn` utilities:
+
+  - `nn.util.grad_norm` - Returns the global norm of the gradients
+  - `nn.util.clip_grad_norm` - Rescales the gradients based on the global norm
+  - `nn.util.clip_grad_value` - Rescales the gradients based on their values.
+
 - Implemented **EfficientDet**: `lucid.models.EfficientDet` with variants from `D0` to `D7`
 
 ## 🔧 How to Install

{lucid_dl-2.7.6.dist-info → lucid_dl-2.7.8.dist-info}/RECORD

@@ -10,7 +10,7 @@ lucid/_func/bfunc.py,sha256=OEgtBWFL2S46UTtNU93ekYCe37Pj4oS8MEnJt-rcids,20571
 lucid/_func/gfunc.py,sha256=_gEJnYnAYhcN6aV1Z6ArIOGTgd0NEB2_eM86tE6uq-Q,4866
 lucid/_func/ufunc.py,sha256=OucEx6QdXzO2_y8Xie8Deh5IC7wnt2EIQcgCAJraWqs,30425
 lucid/_tensor/__init__.py,sha256=wFWAMhTnQwThNiBEIT4fcw4ryIm8A4AoR-m9KDhklOQ,40
-lucid/_tensor/tensor.py,sha256=GnoJzsaDg_nL9pBHTiaID-JwlUaLvNgRcC9O0QGTqLQ,13717
+lucid/_tensor/tensor.py,sha256=HSpV9XUpMI79gw1vlmuoGh7Bkl3qFR_4alOjoq4Lq70,13819
 lucid/_tensor/tensor_ops.py,sha256=5XtqcPbrkOb87pVTSOMNaje9K78tgLpsNdV8XXnbpHc,3991
 lucid/_util/__init__.py,sha256=NgOleItHJGVLdJlKHKfpzuSl3vofzJpNsZByHAYJmKs,6838
 lucid/_util/func.py,sha256=jPpjDw-pdK9bw2U3ACEtgCbV47j6Dp9ycbkvkHKefJo,44383
@@ -71,10 +71,11 @@ lucid/models/objdet/yolo/yolo_v3.py,sha256=B5U42Npwfg8nSgU9E261zf0cbQS9RVYrX1ADD
 lucid/models/objdet/yolo/yolo_v4.py,sha256=RFbBumreXmy6s8IYZvUuhW0893ss8sx_8Vgi6KbBKWo,21467
 lucid/models/seq2seq/__init__.py,sha256=wjsrhj4H_AcqwwbebAN8b68QBA8L6p1_12dkG2995-w,27
 lucid/models/seq2seq/transformer.py,sha256=y5rerCs1s6jXTsVvbgscWScKpQKuSu1fezsBe7PNTRA,3513
-lucid/nn/__init__.py,sha256=Kc6_wlpWo0_AtywX8aEWtzjKb0ju2c2cKGNsEY9ho4E,153
+lucid/nn/__init__.py,sha256=_hk6KltQIJuWXowXstMSu3TjiaTP8zMLNvGpjnA9Mpw,182
 lucid/nn/fused.py,sha256=ZGOQmDThaGNQLC59y3M7s993K_K09ce6IZP8cFX8FUE,5498
 lucid/nn/module.py,sha256=XvFWJ8NqXeZpr3RmKBQBz5eqT535Oi_7DaPN1Zi9gJc,21971
 lucid/nn/parameter.py,sha256=jDaWukWecCcH9ri65SefNls66MmyTyucFolWbzSjapc,856
+lucid/nn/util.py,sha256=Yw1iBSPrGV_r_F51qpqLYdafNE_hyaA0DPWYP-rjaig,1699
 lucid/nn/functional/__init__.py,sha256=90Zi7jClPOiiSYx-Qkg0QTideKD6GigbWON9eFCoxzg,13869
 lucid/nn/functional/_activation.py,sha256=nQVwArvPuwkUpLMLCNABTw96Zgw9VsPB8SyXCL6t2LM,1331
 lucid/nn/functional/_attention.py,sha256=nrZF3-2AR03kNo1PGNszujhWlAVcab_FNQwOCWZT47I,946
@@ -119,8 +120,8 @@ lucid/visual/__init__.py,sha256=6TuFDfmXTwpLyHl7_KqBfdzW6zqHjGzIFvymjFPlvjI,21
 lucid/visual/graph.py,sha256=YjpIDM_lloZARw3sCBiXPl_hT5A2gTk2fEHvwvJWXTk,4599
 lucid/weights/__init__.py,sha256=z1AikA3rOEeckWGkYWlcZkxNlJo9Xwa39PL6ly3hWnc,8801
 lucid/weights/__init__.pyi,sha256=lFonYC3cUx2Idolf3AEPnjFcyqcn3UDU84oJlZafqLY,3013
-lucid_dl-2.7.6.dist-info/licenses/LICENSE,sha256=vxRFYnVD1IeYtsvw-KmoElfqrjxKHv1h9YTvsG54loQ,1065
-lucid_dl-2.7.6.dist-info/METADATA,sha256=GgdsxwpPv_EGIOIARff6VweVKfgDi_A8vs9aHvFQcCI,11260
-lucid_dl-2.7.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lucid_dl-2.7.6.dist-info/top_level.txt,sha256=uzP_qBx9iNWIHKJRlElYcBLYVqMpdm9Q1Ma63QPYbFc,6
-lucid_dl-2.7.6.dist-info/RECORD,,
+lucid_dl-2.7.8.dist-info/licenses/LICENSE,sha256=vxRFYnVD1IeYtsvw-KmoElfqrjxKHv1h9YTvsG54loQ,1065
+lucid_dl-2.7.8.dist-info/METADATA,sha256=t8DZ8085SXcg8SH2ibdQH5K9zq2toJ3B4qwm-SpLkAk,11519
+lucid_dl-2.7.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lucid_dl-2.7.8.dist-info/top_level.txt,sha256=uzP_qBx9iNWIHKJRlElYcBLYVqMpdm9Q1Ma63QPYbFc,6
+lucid_dl-2.7.8.dist-info/RECORD,,