lucid-dl 2.7.5__tar.gz → 2.7.7__tar.gz

{lucid_dl-2.7.5 → lucid_dl-2.7.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lucid-dl
-Version: 2.7.5
+Version: 2.7.7
 Summary: Lumerico's Comprehensive Interface for Deep Learning
 Home-page: https://github.com/ChanLumerico/lucid
 Author: ChanLumerico

{lucid_dl-2.7.5 → lucid_dl-2.7.7}/lucid/nn/__init__.py

@@ -4,3 +4,4 @@ from lucid.nn.modules import *
 from lucid.nn.fused import *
 
 import lucid.nn.init as init
+import lucid.nn.util as util

{lucid_dl-2.7.5 → lucid_dl-2.7.7}/lucid/nn/module.py

@@ -118,7 +118,7 @@ class Module:
         for param in self.parameters():
             param.zero()
 
-    def forward(self, *args: Any, **kwargs: Any) -> Tensor | tuple[Tensor, ...]:
+    def forward(self) -> Tensor | tuple[Tensor, ...]:
         raise NotImplementedError(
             "The forward method must be implemented by the subclass."
         )
@@ -204,7 +204,7 @@ class Module:
                 destination=destination, prefix=prefix + name + ".", keep_vars=keep_vars
             )
 
-        for key in destination.keys():
+        for key in list(destination.keys()):
             if key in self._state_dict_pass_attr:
                 del destination[key]
 
@@ -229,10 +229,8 @@ class Module:
             if key in own_state:
                 attr = own_state[key]
                 if isinstance(attr, (nn.Parameter, nn.Buffer)):
-                    if isinstance(value, Tensor):
-                        attr.data = value.data
-                    else:
-                        attr.data = value
+                    value_t = Tensor(value, device=self.device)
+                    attr.data = value_t.data
                 else:
                     setattr(self, key, value)
             elif strict:

lucid_dl-2.7.7/lucid/nn/util.py

@@ -0,0 +1,60 @@
+from typing import Iterable
+
+import lucid
+
+from lucid._tensor import Tensor
+from lucid.types import _Scalar
+
+
+__all__ = ["grad_norm", "clip_grad_norm", "clip_grad_value"]
+
+
+def _as_iter(parameters: Iterable[Tensor] | Tensor) -> list[Tensor]:
+    if isinstance(parameters, Tensor):
+        return [parameters]
+    return list(parameters)
+
+
+def grad_norm(parameters: Iterable[Tensor] | Tensor, norm_type: int = 2) -> Tensor:
+    parameters = _as_iter(parameters)
+    device = parameters[0].device
+
+    params: list[Tensor] = [p for p in parameters if p.grad is not None]
+    if not params:
+        return Tensor(0.0, device=device)
+
+    norm_pow_sum = 0.0
+    for p in params:
+        param_norm = lucid.linalg.norm(lucid.ravel(p.grad), ord=norm_type).item()
+        norm_pow_sum += param_norm**norm_type
+
+    total_norm = norm_pow_sum ** (1.0 / norm_type)
+    return Tensor(total_norm, device=device)
+
+
+def clip_grad_norm(
+    parameters: Iterable[Tensor] | Tensor,
+    max_norm: _Scalar,
+    norm_type: int = 2,
+    eps: float = 1e-7,
+) -> float:
+    params: list[Tensor] = [p for p in _as_iter(parameters) if p.grad is not None]
+    total_norm = grad_norm(params, norm_type=norm_type)
+
+    clip_coef = float(max_norm) / (total_norm.item() + eps)
+    if clip_coef < 1.0:
+        for p in params:
+            p.grad = p.grad * clip_coef
+
+    return total_norm
+
+
+def clip_grad_value(parameters: Iterable[Tensor] | Tensor, clip_value: _Scalar) -> None:
+    params = [p for p in _as_iter(parameters) if p.grad is not None]
+    if not params:
+        return
+
+    lo, hi = -float(clip_value), float(clip_value)
+    for p in params:
+        g_clip = lucid.clip(p.grad, lo, hi).data
+        p.grad = g_clip

lucid_dl-2.7.7/lucid/optim/_base.py

@@ -0,0 +1,142 @@
+from collections import defaultdict
+from typing import Any, Iterable
+from abc import ABC, abstractmethod
+import copy
+
+import lucid.nn as nn
+
+from lucid.types import _OptimClosure
+
+
+class Optimizer(ABC):
+    def __init__(
+        self, params: Iterable[nn.Parameter], defaults: dict[str, Any]
+    ) -> None:
+        if not isinstance(params, Iterable):
+            raise TypeError("params should be an iterable of Parameters.")
+
+        param_list = list(params)
+        for p in param_list:
+            if not isinstance(p, nn.Parameter):
+                raise TypeError(f"Expected nn.Parameter, got {type(p).__name__}.")
+
+        self.defaults: dict[str, Any] = dict(defaults)
+        self.param_groups: list[dict[str, Any]] = self.param_groups_setup(
+            param_list, self.defaults
+        )
+        self.state: dict[nn.Parameter, dict[str, Any]] = defaultdict(dict)
+
+    @abstractmethod
+    def step(self, closure: _OptimClosure | None = None) -> Any | None:
+        raise NotImplementedError("The step method must be implemented by subclasses.")
+
+    def zero_grad(self) -> None:
+        for group in self.param_groups:
+            for param in group["params"]:
+                if isinstance(param, nn.Parameter):
+                    param.zero_grad()
+
+    def param_groups_setup(
+        self, params: list[nn.Parameter], defaults: dict[str, Any]
+    ) -> list[dict[str, Any]]:
+        return [{"params": list(params), **defaults}]
+
+    def add_param_group(self, param_group: dict[str, Any]) -> None:
+        if "params" not in param_group:
+            raise ValueError("param_group must have a 'params' key.")
+
+        params = list(param_group["params"])
+        if len(params) == 0:
+            raise ValueError("param_group['params'] must be non-empty.")
+
+        for p in params:
+            if not isinstance(p, nn.Parameter):
+                raise TypeError(
+                    f"Expected nn.Parameter in param_group, got {type(p).__name__}."
+                )
+
+        existing = set()
+        for g in self.param_groups:
+            existing.update(g["params"])
+
+        if any(p in existing for p in params):
+            raise ValueError("Some parameters appear in more than one parameter group.")
+
+        filled = {
+            **self.defaults,
+            **{k: v for k, v in param_group.items() if k != "params"},
+        }
+        filled["params"] = params
+        self.param_groups.append(filled)
+
+    def _flat_params(self) -> list[nn.Parameter]:
+        flat: list[nn.Parameter] = []
+        for g in self.param_groups:
+            flat.extend(g["params"])
+
+        return flat
+
+    def state_dict(self) -> dict:
+        param_to_idx: dict[nn.Parameter, int] = {}
+        for idx, p in enumerate(self._flat_params()):
+            if p not in param_to_idx:
+                param_to_idx[p] = idx
+
+        packed_state: dict[int, dict[str, Any]] = {}
+        for p, st in self.state.items():
+            if p in param_to_idx:
+                packed_state[param_to_idx[p]] = copy.deepcopy(st)
+
+        packed_groups: list[dict[str, Any]] = []
+        for g in self.param_groups:
+            new_g: dict[str, Any] = {}
+
+            for k, v in g.items():
+                if k == "params":
+                    new_g[k] = [param_to_idx[p] for p in v]
+                else:
+                    new_g[k] = copy.deepcopy(v)
+
+            packed_groups.append(new_g)
+
+        return {"state": packed_state, "param_groups": packed_groups}
+
+    def load_state_dict(self, state_dict: dict) -> None:
+        if (
+            not isinstance(state_dict, dict)
+            or "state" not in state_dict
+            or "param_groups" not in state_dict
+        ):
+            raise TypeError("Invalid state_dict format for Optimizer.")
+
+        saved_groups = state_dict["param_groups"]
+        saved_state = state_dict["state"]
+
+        current_params = self._flat_params()
+        n_current = len(current_params)
+
+        new_groups: list[dict[str, Any]] = []
+        for sg in saved_groups:
+            if "params" not in sg:
+                raise KeyError("Saved param_group missing 'params'.")
+            indices: list[int] = list(sg["params"])
+
+            if any(i < 0 or i >= n_current for i in indices):
+                raise IndexError("Saved state refers to parameter index out of range.")
+
+            params = [current_params[i] for i in indices]
+            ng = {
+                k: (params if k == "params" else copy.deepcopy(v))
+                for k, v in sg.items()
+            }
+            new_groups.append(ng)
+
+        self.param_groups = new_groups
+
+        self.state = defaultdict(dict)
+        for i, p in enumerate(self._flat_params()):
+            if i in saved_state:
+                self.state[p] = copy.deepcopy(saved_state[i])
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}({self.defaults})"

lucid_dl-2.7.7/lucid/optim/lr_scheduler/_base.py

@@ -0,0 +1,85 @@
+from typing import Any
+from abc import ABC, abstractmethod
+
+from lucid.optim import Optimizer
+
+
+class LRScheduler(ABC):
+    def __init__(
+        self, optimizer: Optimizer, last_epoch: int = -1, verbose: bool = False
+    ) -> None:
+        if not hasattr(optimizer, "param_groups"):
+            raise TypeError(f"{type(optimizer).__name__} is not a valid optimizer.")
+
+        self.optimizer = optimizer
+        self.last_epoch = last_epoch
+        self.verbose = verbose
+        self.base_lrs: list[float] = [float(g["lr"]) for g in optimizer.param_groups]
+
+        self._step_count = 0
+        self._last_lr: list[float] = [float(g["lr"]) for g in optimizer.param_groups]
+
+    @abstractmethod
+    def get_lr(self) -> list[float]:
+        raise NotImplementedError
+
+    def step(self, epoch: int | None = None) -> None:
+        if epoch is None:
+            self.last_epoch += 1
+        else:
+            self.last_epoch = int(epoch)
+        self._step_count += 1
+
+        new_lrs = self.get_lr()
+        if len(new_lrs) != len(self.optimizer.param_groups):
+            raise ValueError(
+                f"get_lr returned {len(new_lrs)} values, "
+                f"but optimizer has {len(self.optimizer.param_groups)} param groups."
+            )
+
+        for group, lr in zip(self.optimizer.param_groups, new_lrs):
+            group["lr"] = float(lr)
+
+        self._last_lr = [float(g["lr"]) for g in self.optimizer.param_groups]
+
+        if self.verbose:
+            print(
+                f"Epoch {self.last_epoch}: setting learning rates to {self._last_lr}."
+            )
+
+    def state_dict(self) -> dict[str, Any]:
+        return {
+            "last_epoch": int(self.last_epoch),
+            "base_lrs": [float(x) for x in self.base_lrs],
+            "_step_count": int(self._step_count),
+            "_last_lr": [float(x) for x in self._last_lr],
+            "_group_count": len(self.optimizer.param_groups),
+        }
+
+    def load_state_dict(self, state_dict: dict[str, Any]) -> None:
+        required = {"last_epoch", "base_lrs", "_step_count", "_last_lr"}
+        missing = required - set(state_dict)
+        if missing:
+            raise KeyError(f"Missing keys in scheduler state_dict: {missing}")
+
+        saved_group_count = int(
+            state_dict.get("_group_count", len(state_dict["_last_lr"]))
+        )
+        current_group_count = len(self.optimizer.param_groups)
+        if saved_group_count != current_group_count:
+            raise ValueError(
+                "Cannot load scheduler state: param group count mismatch "
+                f"(saved={saved_group_count}, current={current_group_count})."
+            )
+
+        self.last_epoch = int(state_dict["last_epoch"])
+        self.base_lrs = [float(x) for x in state_dict["base_lrs"]]
+        self._step_count = int(state_dict["_step_count"])
+        self._last_lr = [float(x) for x in state_dict["_last_lr"]]
+
+        for group, lr in zip(self.optimizer.param_groups, self._last_lr):
+            group["lr"] = float(lr)
+
+    @property
+    def last_lr(self) -> list[float]:
+        return list(self._last_lr)

{lucid_dl-2.7.5 → lucid_dl-2.7.7}/lucid/port.py

@@ -5,14 +5,13 @@ from typing import Literal
 
 from lucid._tensor import Tensor
 from lucid.nn import Module
-from lucid.types import _NumPyArray
 
 
 __all__ = ["save", "load"]
 
-_LucidPortable = Tensor | Module | OrderedDict
+_LucidPortable = Tensor | Module | OrderedDict | dict
 
-FORMAT_VERSION: float = 1.0
+FORMAT_VERSION: float = 1.1
 
 EXTENSIONS = Literal[".lct", ".lcd", ".safetensors"]
 
@@ -30,7 +29,7 @@ def save(obj: _LucidPortable, path: Path | str, safetensors: bool = False) -> Pa
     if path.suffix == "":
         if isinstance(obj, Tensor):
             path = path.with_suffix(".lct")
-        elif isinstance(obj, (Module, OrderedDict)):
+        elif isinstance(obj, (Module, OrderedDict, dict)):
             path = (
                 path.with_suffix(".safetensors")
                 if safetensors
@@ -56,10 +55,14 @@ def save(obj: _LucidPortable, path: Path | str, safetensors: bool = False) -> Pa
     elif suffix == ".lcd":
         if isinstance(obj, Module):
             obj = obj.state_dict()
-        if not isinstance(obj, OrderedDict):
-            raise TypeError("Expected a state_dict (OrderedDict) for .lcd file.")
+        if not isinstance(obj, (OrderedDict, dict)):
+            raise TypeError("Expected a state_dict for .lcd file.")
 
-        data = {"type": "OrderedDict", "format_version": FORMAT_VERSION, "content": obj}
+        data = {
+            "type": type(obj).__name__,
+            "format_version": FORMAT_VERSION,
+            "content": obj,
+        }
 
     elif suffix == ".safetensors":
         try:
@@ -72,10 +75,8 @@ def save(obj: _LucidPortable, path: Path | str, safetensors: bool = False) -> Pa
 
         if isinstance(obj, Module):
             obj = obj.state_dict()
-        if not isinstance(obj, OrderedDict):
-            raise TypeError(
-                "Expected a state_dict (OrderedDict) for .safetensors file."
-            )
+        if not isinstance(obj, (OrderedDict, dict)):
+            raise TypeError("Expected a state_dict for .safetensors file.")
 
         save_file(obj, str(path))
         return path.resolve()
@@ -122,7 +123,7 @@ def load(path: Path | str) -> _LucidPortable:
         array = data["content"]
         return Tensor(array)
 
-    elif file_type == "OrderedDict":
+    elif file_type in {"OrderedDict", "dict"}:
         return data["content"]
 
     else:

{lucid_dl-2.7.5 → lucid_dl-2.7.7}/lucid_dl.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lucid-dl
-Version: 2.7.5
+Version: 2.7.7
 Summary: Lumerico's Comprehensive Interface for Deep Learning
 Home-page: https://github.com/ChanLumerico/lucid
 Author: ChanLumerico

{lucid_dl-2.7.5 → lucid_dl-2.7.7}/lucid_dl.egg-info/SOURCES.txt

@@ -78,6 +78,7 @@ lucid/nn/__init__.py
 lucid/nn/fused.py
 lucid/nn/module.py
 lucid/nn/parameter.py
+lucid/nn/util.py
 lucid/nn/functional/__init__.py
 lucid/nn/functional/_activation.py
 lucid/nn/functional/_attention.py

{lucid_dl-2.7.5 → lucid_dl-2.7.7}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="lucid-dl",
-    version="2.7.5",
+    version="2.7.7",
     author="ChanLumerico",
     author_email="greensox284@gmail.com",
     description="Lumerico's Comprehensive Interface for Deep Learning",

lucid_dl-2.7.5/lucid/optim/_base.py

@@ -1,57 +0,0 @@
-from collections import defaultdict
-from typing import Any, Iterable, OrderedDict
-from abc import ABC, abstractmethod
-import copy
-
-import lucid.nn as nn
-
-from lucid.types import _OptimClosure
-
-
-class Optimizer(ABC):
-    def __init__(
-        self, params: Iterable[nn.Parameter], defaults: dict[str, Any]
-    ) -> None:
-        super().__init__()
-        if not isinstance(params, Iterable):
-            raise TypeError("params should be an iterable of Parameters.")
-
-        params = list(params)
-        self.param_groups = self.param_groups_setup(params, defaults)
-        self.defaults = defaults
-        self.state: dict[nn.Parameter, dict[str, Any]] = defaultdict(dict)
-
-    def param_groups_setup(
-        self, params: list[nn.Parameter], defaults: dict[str, Any]
-    ) -> list[dict[str, Any]]:
-        return [{"params": list(params), **defaults}]
-
-    @abstractmethod
-    def step(self, closure: _OptimClosure | None = None) -> Any | None:
-        raise NotImplementedError("The step method must be implemented by subclasses.")
-
-    def zero_grad(self) -> None:
-        for group in self.param_groups:
-            for param in group["params"]:
-                param.zero_grad()
-
-    def add_param_group(self, param_group: dict[str, Any]) -> None:
-        for group in self.param_groups:
-            if set(group["params"]) & set(param_group["params"]):
-                raise ValueError(
-                    "Some parameters appear in more than one parameter group."
-                )
-        self.param_groups.append(param_group)
-
-    def state_dict(self) -> OrderedDict:
-        return {
-            "state": copy.deepcopy(self.state),
-            "param_groups": copy.deepcopy(self.param_groups),
-        }
-
-    def load_state_dict(self, state_dict: OrderedDict) -> None:
-        self.state = defaultdict(dict, copy.deepcopy(state_dict["state"]))
-        self.param_groups = copy.deepcopy(state_dict["param_groups"])
-
-    def __repr__(self) -> str:
-        return f"{type(self).__name__}({self.param_groups})"

lucid_dl-2.7.5/lucid/optim/lr_scheduler/_base.py

@@ -1,63 +0,0 @@
-from typing import Any
-from abc import ABC, abstractmethod
-
-from lucid.optim import Optimizer
-
-
-class LRScheduler(ABC):
-    def __init__(
-        self, optimizer: Optimizer, last_epoch: int = -1, verbose: bool = False
-    ) -> None:
-        super().__init__()
-        if not hasattr(optimizer, "param_groups"):
-            raise TypeError(f"{type(optimizer).__name__} is not a valid optimizer.")
-
-        self.optimizer = optimizer
-        self.last_epoch = last_epoch
-        self.verbose = verbose
-        self.base_lrs = [group["lr"] for group in optimizer.param_groups]
-
-        self._step_count = 0
-        self._last_lr = [group["lr"] for group in optimizer.param_groups]
-
-    @abstractmethod
-    def get_lr(self) -> list[float]:
-        raise NotImplementedError("get_lr must be implemented in subclasses.")
-
-    def step(self, epoch: int | None = None) -> None:
-        if epoch is not None:
-            self.last_epoch = epoch
-        else:
-            self._step_count += 1
-            self.last_epoch = self._step_count
-
-        new_lrs = self.get_lr()
-        for param_group, lr in zip(self.optimizer.param_groups, new_lrs):
-            param_group["lr"] = lr
-
-        self._last_lr = new_lrs
-
-        if self.verbose:
-            print(f"Epoch {self.last_epoch}: setting learning rates to {new_lrs}.")
-
-    def state_dict(self) -> dict[str, Any]:
-        return {
-            "last_epoch": self.last_epoch,
-            "base_lrs": self.base_lrs,
-            "_step_count": self._step_count,
-            "_last_lr": self._last_lr,
-        }
-
-    def load_state_dict(self, state_dict: dict[str, Any]) -> None:
-        self.last_epoch = state_dict["last_epoch"]
-        self.base_lrs = state_dict["base_lrs"]
-
-        self._step_count = state_dict["_step_count"]
-        self._last_lr = state_dict["_last_lr"]
-
-        for param_group, lr in zip(self.optimizer.param_groups, self._last_lr):
-            param_group["lr"] = lr
-
-    @property
-    def last_lr(self) -> list[float]:
-        return self._last_lr