lucid-dl 2.11.5__tar.gz → 2.12.0__tar.gz

{lucid_dl-2.11.5 → lucid_dl-2.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lucid-dl
-Version: 2.11.5
+Version: 2.12.0
 Summary: Lumerico's Comprehensive Interface for Deep Learning
 Home-page: https://github.com/ChanLumerico/lucid
 Author: ChanLumerico

lucid_dl-2.12.0/lucid/datasets/__init__.py

@@ -0,0 +1,2 @@
+from .mnist import *
+from .cifar import *

lucid_dl-2.12.0/lucid/datasets/cifar.py

@@ -0,0 +1,365 @@
+import pandas as pd
+import numpy as np
+import openml
+import math
+
+from typing import SupportsIndex, Tuple, ClassVar
+from pathlib import Path
+import re
+
+import lucid
+from lucid._tensor import Tensor
+
+from ._base import DatasetBase
+
+
+__all__ = ["CIFAR10", "CIFAR100"]
+
+
+class CIFAR10(DatasetBase):
+    OPENML_ID: ClassVar[int] = 40927
+
+    def __init__(
+        self,
+        root: str | Path,
+        train: bool | None = True,
+        download: bool | None = False,
+        transform: lucid.nn.Module | None = None,
+        target_transform: lucid.nn.Module | None = None,
+        test_size: float = 0.2,
+        to_tensor: bool = True,
+        *,
+        cache: bool = True,
+        scale: float | None = None,
+        resize: tuple[int, int] | None = None,
+        normalize: tuple[tuple[float, ...], tuple[float, ...]] | None = None,
+        cache_preprocessed: bool = True,
+        preprocess_dtype: lucid.Numeric = lucid.Float16,
+        preprocess_chunk_size: int = 4096,
+    ) -> None:
+        self.cache = cache
+        self.scale = scale
+        self.resize = resize
+        self.normalize = normalize
+        self.cache_preprocessed = cache_preprocessed
+        self.preprocess_dtype = preprocess_dtype
+        self.preprocess_chunk_size = preprocess_chunk_size
+
+        super().__init__(
+            root=root,
+            train=train,
+            download=download,
+            transform=transform,
+            target_transform=target_transform,
+            test_size=test_size,
+            to_tensor=to_tensor,
+        )
+
+    def _download(self) -> None:
+        try:
+            dataset = openml.datasets.get_dataset(self.OPENML_ID)
+            df, _, _, _ = dataset.get_data(dataset_format="dataframe")
+            df.to_csv(self.root / "CIFAR10.csv", index=False)
+
+        except Exception as e:
+            raise RuntimeError(f"Failed to download the CIFAR-10 dataset. Error: {e}")
+
+    def _cache_key(self) -> str:
+        parts: list[str] = []
+        if self.scale is not None:
+            parts.append(f"s{self.scale:g}")
+        if self.resize is not None:
+            parts.append(f"r{self.resize[0]}x{self.resize[1]}")
+        if self.normalize is not None:
+            mean, std = self.normalize
+            parts.append("m" + ",".join(f"{v:g}" for v in mean))
+            parts.append("v" + ",".join(f"{v:g}" for v in std))
+        if not parts:
+            return "raw"
+        key = "_".join(parts)
+        return re.sub(r"[^a-zA-Z0-9_,.x-]+", "_", key)
+
+    def _raw_cache_path(self) -> Path:
+        return self.root / "CIFAR10_uint8.npz"
+
+    def _proc_cache_path(self) -> Path:
+        dtype_name = str(self.preprocess_dtype)
+        return self.root / f"CIFAR10_{self._cache_key()}_{dtype_name}.npz"
+
+    def _ensure_raw_cache(self) -> tuple[np.ndarray, np.ndarray]:
+        raw_path = self._raw_cache_path()
+        if self.cache and raw_path.exists():
+            with np.load(raw_path) as npz:
+                images = npz["images"]
+                labels = npz["labels"]
+            return images, labels
+
+        csv_path = self.root / "CIFAR10.csv"
+        if not csv_path.exists():
+            raise RuntimeError(
+                f"CIFAR-10 dataset CSV file not found at {csv_path}. "
+                + "Use `download=True`."
+            )
+
+        df = pd.read_csv(csv_path)
+        labels = df["class"].values.astype(np.int32)
+        images = df.drop(columns=["class"]).values.astype(np.uint8, copy=False)
+        images = images.reshape(-1, 3, 32, 32)
+
+        if self.cache:
+            np.savez_compressed(raw_path, images=images, labels=labels)
+
+        return images, labels
+
+    def _maybe_preprocess_and_cache(
+        self, images_uint8: np.ndarray, labels_int32: np.ndarray
+    ) -> tuple[np.ndarray, np.ndarray]:
+        if self.resize is None and self.scale is None and self.normalize is None:
+            return images_uint8.astype(np.float32), labels_int32
+
+        proc_path = self._proc_cache_path()
+        if self.cache and self.cache_preprocessed and proc_path.exists():
+            with np.load(proc_path) as npz:
+                images = npz["images"]
+                labels = npz["labels"]
+            return images, labels
+
+        from lucid.transforms import Compose, Resize, Normalize
+
+        class _Scale(lucid.nn.Module):
+            def __init__(self, factor: float) -> None:
+                super().__init__()
+                self.factor = factor
+
+            def forward(self, x: Tensor) -> Tensor:
+                return x * self.factor
+
+        transforms: list[lucid.nn.Module] = []
+        if self.resize is not None:
+            transforms.append(Resize(self.resize))
+        if self.scale is not None:
+            transforms.append(_Scale(self.scale))
+        if self.normalize is not None:
+            mean, std = self.normalize
+            transforms.append(Normalize(mean=mean, std=std))
+
+        transform = Compose(transforms)
+        n = images_uint8.shape[0]
+        out_h, out_w = self.resize if self.resize is not None else (32, 32)
+
+        out_dtype = np.float16 if self.preprocess_dtype == lucid.Float16 else np.float32
+        out_images = np.empty((n, 3, out_h, out_w), dtype=out_dtype)
+
+        for start in range(0, n, self.preprocess_chunk_size):
+            end = min(start + self.preprocess_chunk_size, n)
+            chunk = images_uint8[start:end].astype(np.float32)
+            x = lucid.to_tensor(chunk, dtype=lucid.Float32)
+            x = transform(x)
+            out_images[start:end] = x.numpy().astype(out_dtype, copy=False)
+
+        if self.cache and self.cache_preprocessed:
+            np.savez_compressed(proc_path, images=out_images, labels=labels_int32)
+
+        return out_images, labels_int32
+
+    def _load_data(self, split: str) -> Tuple[Tensor, Tensor]:
+        images, labels = self._ensure_raw_cache()
+        images, labels = self._maybe_preprocess_and_cache(images, labels)
+
+        train_size = int(math.floor(len(images) * (1 - self.test_size)))
+        if split == "train":
+            images, labels = images[:train_size], labels[:train_size]
+        else:
+            images, labels = images[train_size:], labels[train_size:]
+
+        if self.to_tensor:
+            images = lucid.to_tensor(images, dtype=lucid.Float32)
+            labels = lucid.to_tensor(labels, dtype=lucid.Int32)
+
+        return images, labels
+
+    def __getitem__(self, index: SupportsIndex) -> Tuple[Tensor, Tensor]:
+        image = self.data[index]
+        label = self.targets[index]
+
+        if self.transform:
+            image = self.transform(image)
+        if self.target_transform:
+            label = self.target_transform(label)
+
+        return image, label
+
+
+class CIFAR100(DatasetBase):
+    OPENML_ID: ClassVar[int] = 41983
+
+    def __init__(
+        self,
+        root: str | Path,
+        train: bool | None = True,
+        download: bool | None = False,
+        transform: lucid.nn.Module | None = None,
+        target_transform: lucid.nn.Module | None = None,
+        test_size: float = 0.2,
+        to_tensor: bool = True,
+        *,
+        cache: bool = True,
+        scale: float | None = None,
+        resize: tuple[int, int] | None = None,
+        normalize: tuple[tuple[float, ...], tuple[float, ...]] | None = None,
+        cache_preprocessed: bool = True,
+        preprocess_dtype: lucid.Numeric = lucid.Float16,
+        preprocess_chunk_size: int = 4096,
+    ) -> None:
+        self.cache = cache
+        self.scale = scale
+        self.resize = resize
+        self.normalize = normalize
+        self.cache_preprocessed = cache_preprocessed
+        self.preprocess_dtype = preprocess_dtype
+        self.preprocess_chunk_size = preprocess_chunk_size
+
+        super().__init__(
+            root=root,
+            train=train,
+            download=download,
+            transform=transform,
+            target_transform=target_transform,
+            test_size=test_size,
+            to_tensor=to_tensor,
+        )
+
+    def _download(self) -> None:
+        try:
+            dataset = openml.datasets.get_dataset(self.OPENML_ID)
+            df, _, _, _ = dataset.get_data(dataset_format="dataframe")
+            df.to_csv(self.root / "CIFAR100.csv", index=False)
+
+        except Exception as e:
+            raise RuntimeError(f"Failed to download the CIFAR-100 dataset. Error: {e}")
+
+    def _cache_key(self) -> str:
+        parts: list[str] = []
+        if self.scale is not None:
+            parts.append(f"s{self.scale:g}")
+        if self.resize is not None:
+            parts.append(f"r{self.resize[0]}x{self.resize[1]}")
+        if self.normalize is not None:
+            mean, std = self.normalize
+            parts.append("m" + ",".join(f"{v:g}" for v in mean))
+            parts.append("v" + ",".join(f"{v:g}" for v in std))
+        if not parts:
+            return "raw"
+
+        key = "_".join(parts)
+        return re.sub(r"[^a-zA-Z0-9_,.x-]+", "_", key)
+
+    def _raw_cache_path(self) -> Path:
+        return self.root / "CIFAR100_uint8.npz"
+
+    def _proc_cache_path(self) -> Path:
+        dtype_name = str(self.preprocess_dtype)
+        return self.root / f"CIFAR100_{self._cache_key()}_{dtype_name}.npz"
+
+    def _ensure_raw_cache(self) -> tuple[np.ndarray, np.ndarray]:
+        raw_path = self._raw_cache_path()
+        if self.cache and raw_path.exists():
+            with np.load(raw_path) as npz:
+                images = npz["images"]
+                labels = npz["labels"]
+            return images, labels
+
+        csv_path = self.root / "CIFAR100.csv"
+        if not csv_path.exists():
+            raise RuntimeError(
+                f"CIFAR-100 dataset CSV file not found at {csv_path}. "
+                + "Use `download=True`."
+            )
+
+        df = pd.read_csv(csv_path)
+        labels = df["class"].values.astype(np.int32)
+        images = df.drop(columns=["class"]).values.astype(np.uint8, copy=False)
+        images = images.reshape(-1, 3, 32, 32)
+
+        if self.cache:
+            np.savez_compressed(raw_path, images=images, labels=labels)
+
+        return images, labels
+
+    def _maybe_preprocess_and_cache(
+        self, images_uint8: np.ndarray, labels_int32: np.ndarray
+    ) -> tuple[np.ndarray, np.ndarray]:
+        if self.resize is None and self.scale is None and self.normalize is None:
+            return images_uint8.astype(np.float32), labels_int32
+
+        proc_path = self._proc_cache_path()
+        if self.cache and self.cache_preprocessed and proc_path.exists():
+            with np.load(proc_path) as npz:
+                images = npz["images"]
+                labels = npz["labels"]
+            return images, labels
+
+        from lucid.transforms import Compose, Resize, Normalize
+
+        class _Scale(lucid.nn.Module):
+            def __init__(self, factor: float) -> None:
+                super().__init__()
+                self.factor = factor
+
+            def forward(self, x: Tensor) -> Tensor:
+                return x * self.factor
+
+        transforms: list[lucid.nn.Module] = []
+        if self.resize is not None:
+            transforms.append(Resize(self.resize))
+        if self.scale is not None:
+            transforms.append(_Scale(self.scale))
+        if self.normalize is not None:
+            mean, std = self.normalize
+            transforms.append(Normalize(mean=mean, std=std))
+
+        transform = Compose(transforms)
+        n = images_uint8.shape[0]
+        out_h, out_w = self.resize if self.resize is not None else (32, 32)
+
+        out_dtype = np.float16 if self.preprocess_dtype == lucid.Float16 else np.float32
+        out_images = np.empty((n, 3, out_h, out_w), dtype=out_dtype)
+
+        for start in range(0, n, self.preprocess_chunk_size):
+            end = min(start + self.preprocess_chunk_size, n)
+            chunk = images_uint8[start:end].astype(np.float32)
+            x = lucid.to_tensor(chunk, dtype=lucid.Float32)
+            x = transform(x)
+            out_images[start:end] = x.numpy().astype(out_dtype, copy=False)
+
+        if self.cache and self.cache_preprocessed:
+            np.savez_compressed(proc_path, images=out_images, labels=labels_int32)
+
+        return out_images, labels_int32
+
+    def _load_data(self, split: str) -> Tuple[Tensor, Tensor]:
+        images, labels = self._ensure_raw_cache()
+        images, labels = self._maybe_preprocess_and_cache(images, labels)
+
+        train_size = int(math.floor(len(images) * (1 - self.test_size)))
+        if split == "train":
+            images, labels = images[:train_size], labels[:train_size]
+        else:
+            images, labels = images[train_size:], labels[train_size:]
+
+        if self.to_tensor:
+            images = lucid.to_tensor(images, dtype=lucid.Float32)
+            labels = lucid.to_tensor(labels, dtype=lucid.Int32)
+
+        return images, labels
+
+    def __getitem__(self, index: SupportsIndex) -> Tuple[Tensor, Tensor]:
+        image = self.data[index]
+        label = self.targets[index]
+
+        if self.transform:
+            image = self.transform(image)
+        if self.target_transform:
+            label = self.target_transform(label)
+
+        return image, label

{lucid_dl-2.11.5 → lucid_dl-2.12.0}/lucid/models/imgclf/vit.py

@@ -32,10 +32,12 @@ class ViT(nn.Module):
             in_channels, embedding_dim, kernel_size=patch_size, stride=patch_size
         )
 
-        self.cls_token = nn.Parameter(lucid.random.randn(1, 1, embedding_dim))
-        self.pos_emb = nn.Parameter(
-            lucid.random.randn(1, 1 + self.num_patches, embedding_dim)
-        )
+        self.cls_token = nn.Parameter(lucid.zeros(1, 1, embedding_dim))
+        self.pos_emb = nn.Parameter(lucid.zeros(1, 1 + self.num_patches, embedding_dim))
+
+        nn.init.normal(self.cls_token, std=0.02)
+        nn.init.normal(self.pos_emb, std=0.02)
+
         self.dropout = nn.Dropout(dropout_rate)
 
         encoder_layer = nn.TransformerEncoderLayer(

{lucid_dl-2.11.5 → lucid_dl-2.12.0}/lucid/nn/__init__.py

@@ -4,4 +4,4 @@ from lucid.nn.modules import *
 from lucid.nn.fused import *
 
 import lucid.nn.init as init
-import lucid.nn.util as util
+import lucid.nn.utils as utils

{lucid_dl-2.11.5 → lucid_dl-2.12.0}/lucid/nn/modules/rnn.py

@@ -5,6 +5,7 @@ import lucid.nn as nn
 import lucid.nn.functional as F
 
 from lucid._tensor import Tensor
+from lucid.nn.utils.rnn import PackedSequence
 from lucid.types import Numeric, _DeviceType
 
 from .activation import Tanh, ReLU
@@ -351,21 +352,47 @@ class RNNBase(nn.Module):
         )
 
     def forward(
-        self, input_: Tensor, hx: Tensor | tuple[Tensor, Tensor] | None = None
-    ) -> tuple[Tensor, Tensor] | tuple[Tensor, tuple[Tensor, Tensor]]:
-        if input_.ndim != 3:
-            raise ValueError(
-                f"RNNBase expected input with 3 dimensions, got {input_.ndim} dimensions"
-            )
+        self,
+        input_: Tensor | PackedSequence,
+        hx: Tensor | tuple[Tensor, Tensor] | None = None,
+    ) -> (
+        tuple[Tensor | PackedSequence, Tensor]
+        | tuple[Tensor | PackedSequence, tuple[Tensor, Tensor]]
+    ):
+        is_packed = isinstance(input_, PackedSequence)
+        if is_packed:
+            data = input_.data
+            batch_sizes = input_.batch_sizes
+            if data.ndim != 2:
+                raise ValueError(
+                    "RNNBase expected packed data with 2 dimensions, "
+                    f"got {data.ndim} dimensions"
+                )
+            if batch_sizes.ndim != 1 or batch_sizes.shape[0] == 0:
+                raise ValueError(
+                    "PackedSequence batch_sizes must be a non-empty 1D tensor"
+                )
 
-        if self.batch_first:
-            input_ = input_.swapaxes(0, 1)
+            batch_size = int(batch_sizes[0].item())
+            feat = data.shape[1]
+            if feat != self.input_size:
+                raise ValueError(
+                    f"RNNBase expected input with feature size {self.input_size}, got {feat}"
+                )
+        else:
+            if input_.ndim != 3:
+                raise ValueError(
+                    f"RNNBase expected input with 3 dimensions, got {input_.ndim} dimensions"
+                )
 
-        seq_len, batch_size, feat = input_.shape
-        if feat != self.input_size:
-            raise ValueError(
-                f"RNNBase expected input with feature size {self.input_size}, got {feat}"
-            )
+            if self.batch_first:
+                input_ = input_.swapaxes(0, 1)
+
+            seq_len, batch_size, feat = input_.shape
+            if feat != self.input_size:
+                raise ValueError(
+                    f"RNNBase expected input with feature size {self.input_size}, got {feat}"
+                )
 
         if self.is_lstm:
             if hx is None:
@@ -410,7 +437,7 @@ class RNNBase(nn.Module):
             if hx.shape[2] != self.hidden_size:
                 raise ValueError("Incorrect hidden size in hx")
 
-        layer_input = input_
+        layer_input = data if is_packed else input_
         h_n_list: list[Tensor] = []
         c_n_list: list[Tensor] | None = [] if self.is_lstm else None
 
@@ -420,33 +447,111 @@ class RNNBase(nn.Module):
                 c_t = hx_c[layer_idx]
             else:
                 h_t = hx[layer_idx]
+
             outputs = []
+            if is_packed:
+                final_h: list[Tensor] = []
+                final_c: list[Tensor] | None = [] if self.is_lstm else None
+                offset = 0
+
+                prev_bs: int | None = None
+                max_len = int(batch_sizes.shape[0])
+                for t in range(max_len):
+                    bs = int(batch_sizes[t].item())
+                    if bs == 0:
+                        break
+
+                    if prev_bs is None:
+                        prev_bs = bs
+                    if bs > prev_bs:
+                        raise ValueError(
+                            "PackedSequence batch_sizes must be non-increasing"
+                        )
+
+                    if bs < prev_bs:
+                        final_h.append(h_t[bs:prev_bs])
+                        if self.is_lstm and final_c is not None:
+                            final_c.append(c_t[bs:prev_bs])
+
+                        h_t = h_t[:bs]
+                        if self.is_lstm:
+                            c_t = c_t[:bs]
+
+                    step_input = layer_input[offset : offset + bs]
+                    offset += bs
+
+                    if self.is_lstm:
+                        h_t, c_t = cell(step_input, (h_t, c_t))
+                    else:
+                        h_t = cell(step_input, h_t)
+
+                    outputs.append(h_t)
+                    prev_bs = bs
+
+                final_h.append(h_t)
+                if self.is_lstm and final_c is not None:
+                    final_c.append(c_t)
+
+                h_n_list.append(
+                    lucid.concatenate(tuple(reversed(final_h)), axis=0).unsqueeze(
+                        axis=0
+                    )
+                )
+                if self.is_lstm and final_c is not None and c_n_list is not None:
+                    c_n_list.append(
+                        lucid.concatenate(tuple(reversed(final_c)), axis=0).unsqueeze(
+                            axis=0
+                        )
+                    )
+
+                layer_output = (
+                    lucid.concatenate(tuple(outputs), axis=0)
+                    if outputs
+                    else layer_input[:0]
+                )
 
-            for t in range(seq_len):
-                if self.is_lstm:
-                    h_t, c_t = cell(layer_input[t], (h_t, c_t))
-                    outputs.append(h_t.unsqueeze(axis=0))
-                else:
-                    h_t = cell(layer_input[t], h_t)
-                    outputs.append(h_t.unsqueeze(axis=0))
+            else:
+                for t in range(seq_len):
+                    if self.is_lstm:
+                        h_t, c_t = cell(layer_input[t], (h_t, c_t))
+                        outputs.append(h_t.unsqueeze(axis=0))
+                    else:
+                        h_t = cell(layer_input[t], h_t)
+                        outputs.append(h_t.unsqueeze(axis=0))
 
-            layer_output = lucid.concatenate(tuple(outputs), axis=0)
+                layer_output = lucid.concatenate(tuple(outputs), axis=0)
 
             if self.training and self.dropout > 0.0 and layer_idx < self.num_layers - 1:
                 layer_output = F.dropout(layer_output, p=self.dropout)
 
-            h_n_list.append(h_t.unsqueeze(axis=0))
-            if self.is_lstm and c_n_list is not None:
-                c_n_list.append(c_t.unsqueeze(axis=0))
+            if not is_packed:
+                h_n_list.append(h_t.unsqueeze(axis=0))
+                if self.is_lstm and c_n_list is not None:
+                    c_n_list.append(c_t.unsqueeze(axis=0))
             layer_input = layer_output
 
-        output = layer_input
+        if is_packed:
+            output = PackedSequence(
+                data=layer_input,
+                batch_sizes=batch_sizes,
+                sorted_indices=input_.sorted_indices,
+                unsorted_indices=input_.unsorted_indices,
+            )
+        else:
+            output = layer_input
+
         h_n = lucid.concatenate(tuple(h_n_list), axis=0)
         if self.is_lstm and c_n_list is not None:
             c_n = lucid.concatenate(tuple(c_n_list), axis=0)
 
-        if self.batch_first:
-            output = output.swapaxes(0, 1)
+        if is_packed:
+            if input_.unsorted_indices is not None:
+                h_n = h_n[:, input_.unsorted_indices]
+                if self.is_lstm and c_n_list is not None:
+                    c_n = c_n[:, input_.unsorted_indices]
+        else:
+            if self.batch_first:
+                output = output.swapaxes(0, 1)
 
         if self.is_lstm and c_n_list is not None:
             return output, (h_n, c_n)

lucid_dl-2.12.0/lucid/nn/utils/__init__.py

@@ -0,0 +1,2 @@
+from ._grad import *
+from . import rnn as rnn

lucid_dl-2.11.5/lucid/nn/util.py → lucid_dl-2.12.0/lucid/nn/utils/_grad.py

@@ -6,7 +6,7 @@ from lucid._tensor import Tensor
 from lucid.types import _Scalar
 
 
-__all__ = ["grad_norm", "clip_grad_norm", "clip_grad_value"]
+__all__ = ["grad_norm", "get_total_norm", "clip_grad_norm", "clip_grad_value"]
 
 
 def _as_iter(parameters: Iterable[Tensor] | Tensor) -> list[Tensor]:
@@ -32,6 +32,25 @@ def grad_norm(parameters: Iterable[Tensor] | Tensor, norm_type: int = 2) -> Tens
     return Tensor(total_norm, device=device)
 
 
+def get_total_norm(parameters: Iterable[Tensor] | Tensor, norm_type: int = 2) -> Tensor:
+    parameters = _as_iter(parameters)
+    if not parameters:
+        return Tensor(0.0)
+
+    device = parameters[0].device
+    grads: list[Tensor] = [p.grad for p in parameters if p.grad is not None]
+    if not grads:
+        return Tensor(0.0, device=device)
+
+    norm_pow_sum = 0.0
+    for g in grads:
+        grad_norm = lucid.linalg.norm(lucid.ravel(g), ord=norm_type).item()
+        norm_pow_sum += grad_norm**norm_type
+
+    total_norm = norm_pow_sum ** (1.0 / norm_type)
+    return Tensor(total_norm, device=device)
+
+
 def clip_grad_norm(
     parameters: Iterable[Tensor] | Tensor,
     max_norm: _Scalar,
@@ -39,7 +58,7 @@ def clip_grad_norm(
     eps: float = 1e-7,
 ) -> float:
     params: list[Tensor] = [p for p in _as_iter(parameters) if p.grad is not None]
-    total_norm = grad_norm(params, norm_type=norm_type)
+    total_norm = get_total_norm(params, norm_type=norm_type)
 
     clip_coef = float(max_norm) / (total_norm.item() + eps)
     if clip_coef < 1.0: