lucid-dl 2.7.6__py3-none-any.whl → 2.7.7__py3-none-any.whl

lucid/nn/__init__.py

@@ -4,3 +4,4 @@ from lucid.nn.modules import *
 from lucid.nn.fused import *
 
 import lucid.nn.init as init
+import lucid.nn.util as util

lucid/nn/util.py

@@ -0,0 +1,60 @@
+from typing import Iterable
+
+import lucid
+
+from lucid._tensor import Tensor
+from lucid.types import _Scalar
+
+
+__all__ = ["grad_norm", "clip_grad_norm", "clip_grad_value"]
+
+
+def _as_iter(parameters: Iterable[Tensor] | Tensor) -> list[Tensor]:
+    if isinstance(parameters, Tensor):
+        return [parameters]
+    return list(parameters)
+
+
+def grad_norm(parameters: Iterable[Tensor] | Tensor, norm_type: int = 2) -> Tensor:
+    parameters = _as_iter(parameters)
+    device = parameters[0].device
+
+    params: list[Tensor] = [p for p in parameters if p.grad is not None]
+    if not params:
+        return Tensor(0.0, device=device)
+
+    norm_pow_sum = 0.0
+    for p in params:
+        param_norm = lucid.linalg.norm(lucid.ravel(p.grad), ord=norm_type).item()
+        norm_pow_sum += param_norm**norm_type
+
+    total_norm = norm_pow_sum ** (1.0 / norm_type)
+    return Tensor(total_norm, device=device)
+
+
+def clip_grad_norm(
+    parameters: Iterable[Tensor] | Tensor,
+    max_norm: _Scalar,
+    norm_type: int = 2,
+    eps: float = 1e-7,
+) -> float:
+    params: list[Tensor] = [p for p in _as_iter(parameters) if p.grad is not None]
+    total_norm = grad_norm(params, norm_type=norm_type)
+
+    clip_coef = float(max_norm) / (total_norm.item() + eps)
+    if clip_coef < 1.0:
+        for p in params:
+            p.grad = p.grad * clip_coef
+
+    return total_norm
+
+
+def clip_grad_value(parameters: Iterable[Tensor] | Tensor, clip_value: _Scalar) -> None:
+    params = [p for p in _as_iter(parameters) if p.grad is not None]
+    if not params:
+        return
+
+    lo, hi = -float(clip_value), float(clip_value)
+    for p in params:
+        g_clip = lucid.clip(p.grad, lo, hi).data
+        p.grad = g_clip

{lucid_dl-2.7.6.dist-info → lucid_dl-2.7.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lucid-dl
-Version: 2.7.6
+Version: 2.7.7
 Summary: Lumerico's Comprehensive Interface for Deep Learning
 Home-page: https://github.com/ChanLumerico/lucid
 Author: ChanLumerico

{lucid_dl-2.7.6.dist-info → lucid_dl-2.7.7.dist-info}/RECORD

@@ -71,10 +71,11 @@ lucid/models/objdet/yolo/yolo_v3.py,sha256=B5U42Npwfg8nSgU9E261zf0cbQS9RVYrX1ADD
 lucid/models/objdet/yolo/yolo_v4.py,sha256=RFbBumreXmy6s8IYZvUuhW0893ss8sx_8Vgi6KbBKWo,21467
 lucid/models/seq2seq/__init__.py,sha256=wjsrhj4H_AcqwwbebAN8b68QBA8L6p1_12dkG2995-w,27
 lucid/models/seq2seq/transformer.py,sha256=y5rerCs1s6jXTsVvbgscWScKpQKuSu1fezsBe7PNTRA,3513
-lucid/nn/__init__.py,sha256=Kc6_wlpWo0_AtywX8aEWtzjKb0ju2c2cKGNsEY9ho4E,153
+lucid/nn/__init__.py,sha256=_hk6KltQIJuWXowXstMSu3TjiaTP8zMLNvGpjnA9Mpw,182
 lucid/nn/fused.py,sha256=ZGOQmDThaGNQLC59y3M7s993K_K09ce6IZP8cFX8FUE,5498
 lucid/nn/module.py,sha256=XvFWJ8NqXeZpr3RmKBQBz5eqT535Oi_7DaPN1Zi9gJc,21971
 lucid/nn/parameter.py,sha256=jDaWukWecCcH9ri65SefNls66MmyTyucFolWbzSjapc,856
+lucid/nn/util.py,sha256=Yw1iBSPrGV_r_F51qpqLYdafNE_hyaA0DPWYP-rjaig,1699
 lucid/nn/functional/__init__.py,sha256=90Zi7jClPOiiSYx-Qkg0QTideKD6GigbWON9eFCoxzg,13869
 lucid/nn/functional/_activation.py,sha256=nQVwArvPuwkUpLMLCNABTw96Zgw9VsPB8SyXCL6t2LM,1331
 lucid/nn/functional/_attention.py,sha256=nrZF3-2AR03kNo1PGNszujhWlAVcab_FNQwOCWZT47I,946
@@ -119,8 +120,8 @@ lucid/visual/__init__.py,sha256=6TuFDfmXTwpLyHl7_KqBfdzW6zqHjGzIFvymjFPlvjI,21
 lucid/visual/graph.py,sha256=YjpIDM_lloZARw3sCBiXPl_hT5A2gTk2fEHvwvJWXTk,4599
 lucid/weights/__init__.py,sha256=z1AikA3rOEeckWGkYWlcZkxNlJo9Xwa39PL6ly3hWnc,8801
 lucid/weights/__init__.pyi,sha256=lFonYC3cUx2Idolf3AEPnjFcyqcn3UDU84oJlZafqLY,3013
-lucid_dl-2.7.6.dist-info/licenses/LICENSE,sha256=vxRFYnVD1IeYtsvw-KmoElfqrjxKHv1h9YTvsG54loQ,1065
-lucid_dl-2.7.6.dist-info/METADATA,sha256=GgdsxwpPv_EGIOIARff6VweVKfgDi_A8vs9aHvFQcCI,11260
-lucid_dl-2.7.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lucid_dl-2.7.6.dist-info/top_level.txt,sha256=uzP_qBx9iNWIHKJRlElYcBLYVqMpdm9Q1Ma63QPYbFc,6
-lucid_dl-2.7.6.dist-info/RECORD,,
+lucid_dl-2.7.7.dist-info/licenses/LICENSE,sha256=vxRFYnVD1IeYtsvw-KmoElfqrjxKHv1h9YTvsG54loQ,1065
+lucid_dl-2.7.7.dist-info/METADATA,sha256=UVi3p93MdqSoYHvUkdil5C12wx5sfZy5Obw4Hkd7pxs,11260
+lucid_dl-2.7.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lucid_dl-2.7.7.dist-info/top_level.txt,sha256=uzP_qBx9iNWIHKJRlElYcBLYVqMpdm9Q1Ma63QPYbFc,6
+lucid_dl-2.7.7.dist-info/RECORD,,