lucid-dl 2.7.9__py3-none-any.whl → 2.8.0__py3-none-any.whl

lucid/nn/modules/__init__.py

@@ -10,3 +10,4 @@ from .attention import *
 from .transformer import *
 from .sparse import *
 from .einops import *
+from .rnn import *

lucid/nn/modules/rnn.py

@@ -0,0 +1,97 @@
+from typing import Literal
+
+import lucid
+import lucid.nn as nn
+import lucid.nn.functional as F
+
+from lucid._tensor import Tensor
+
+from .activation import Tanh, ReLU
+
+
+__all__ = ["RNNCell"]
+
+
+def _get_activation(nonlinearity: str) -> type[nn.Module]:
+    if nonlinearity == "tanh":
+        return Tanh
+    elif nonlinearity == "relu":
+        return ReLU
+    else:
+        raise ValueError(
+            f"Invalid nonlinearity '{nonlinearity}'. "
+            "Supported nonlinearities are 'tanh' and 'relu'."
+        )
+
+
+class RNNCell(nn.Module):
+    def __init__(
+        self,
+        input_size: int,
+        hidden_size: int,
+        bias: bool = True,
+        nonlinearity: Literal["tanh", "relu"] = "tanh",
+    ) -> None:
+        super().__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.bias = bias
+        self.nonlinearity = _get_activation(nonlinearity)()
+
+        sqrt_k = 1.0 / (hidden_size**0.5)
+        self.weight_ih = nn.Parameter(
+            lucid.random.uniform(-sqrt_k, sqrt_k, (self.hidden_size, self.input_size))
+        )
+        self.weight_hh = nn.Parameter(
+            lucid.random.uniform(-sqrt_k, sqrt_k, (self.hidden_size, self.hidden_size))
+        )
+
+        if self.bias:
+            self.bias_ih = nn.Parameter(
+                lucid.random.uniform(-sqrt_k, sqrt_k, self.hidden_size)
+            )
+            self.bias_hh = nn.Parameter(
+                lucid.random.uniform(-sqrt_k, sqrt_k, self.hidden_size)
+            )
+        else:
+            self.bias_ih = None
+            self.bias_hh = None
+
+    def forward(self, input_: Tensor, hx: Tensor | None = None) -> Tensor:
+        if input_.ndim not in (1, 2):
+            raise ValueError(
+                "RNNCell expected input with 1 or 2 dimensions, "
+                f"got {input_.ndim} dimensions"
+            )
+
+        is_batched = input_.ndim == 2
+        if not is_batched:
+            input_ = input_.unsqueeze(axis=0)
+        batch_size = input_.shape[0]
+
+        if hx is None:
+            hx = lucid.zeros(
+                batch_size, self.hidden_size, dtype=input_.dtype, device=input_.device
+            )
+        else:
+            if hx.ndim not in (1, 2):
+                raise ValueError(
+                    "RNNCell expected hidden state with 1 or 2 dimensions, "
+                    f"got {hx.ndim} dimensions"
+                )
+            if hx.ndim == 1:
+                hx = hx.unsqueeze(axis=0)
+
+            if hx.shape[0] != batch_size or hx.shape[1] != self.hidden_size:
+                raise ValueError(
+                    "RNNCell expected hidden state with shape "
+                    f"({batch_size}, {self.hidden_size}), got {hx.shape}"
+                )
+
+        hy = F.linear(input_, self.weight_ih, self.bias_ih)
+        hy += F.linear(hx, self.weight_hh, self.bias_hh)
+        ret = self.nonlinearity(hy)
+
+        if not is_batched:
+            ret = ret.squeeze(axis=0)
+        return ret

lucid/optim/lr_scheduler/_schedulers.py

@@ -298,4 +298,6 @@ class NoamScheduler(LRScheduler):
         decay_term = step_num**-0.5
         lr_factor = scale * min(decay_term, warmup_term)
 
-        return [base_lr * lr_factor for base_lr in self.base_lrs]
+        # Noam's schedule computes the absolute learning rate, so we ignore
+        # the optimizer's initial lr (base_lr) when returning the new values.
+        return [lr_factor for _ in self.base_lrs]

{lucid_dl-2.7.9.dist-info → lucid_dl-2.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lucid-dl
-Version: 2.7.9
+Version: 2.8.0
 Summary: Lumerico's Comprehensive Interface for Deep Learning
 Home-page: https://github.com/ChanLumerico/lucid
 Author: ChanLumerico

{lucid_dl-2.7.9.dist-info → lucid_dl-2.8.0.dist-info}/RECORD

@@ -89,7 +89,7 @@ lucid/nn/functional/_spatial.py,sha256=lazoSvVMFcauBWRbMOqmkgixA5bDes6scGHVWCgVm
 lucid/nn/functional/_util.py,sha256=oaMbR76XuFrFtEjLCUEQBPgfFObP98WGnkGQLtFz2uk,4949
 lucid/nn/init/__init__.py,sha256=YFi-HD2TEglweJ-gyX3n4UVZYzd70gcUi1dBu6hnOAY,1533
 lucid/nn/init/_dist.py,sha256=zk4IoECjCvs-U4DfN17-6cqBu0-fpunAWYAKT4YJPRE,2023
-lucid/nn/modules/__init__.py,sha256=RSH073CXMrZDhkaLKp5KqxwNDBrUc9vP4cwq5-b95kk,266
+lucid/nn/modules/__init__.py,sha256=mol5Gfy-3ab5hBYZRxX0vjiI0w5VyKtBxVwj_vrOAZs,285
 lucid/nn/modules/activation.py,sha256=CpiKpzgZHoCp8UO5taCJ9BuwFz5mYUs0o1_TQcEwQbQ,2823
 lucid/nn/modules/attention.py,sha256=pZi7IGsNFu2xCmeLMuyWgveMyi2QXtaKRKQ70yAeE0c,4407
 lucid/nn/modules/conv.py,sha256=KbtInQgKSw3U_qXiqy7x53DZM9YAMUq7sFas1nV7NxY,13932
@@ -99,6 +99,7 @@ lucid/nn/modules/linear.py,sha256=87cuFWYct9JlmtVC3jGR-8eouxxzANaVA6cd7p9r2Ho,28
 lucid/nn/modules/loss.py,sha256=pjEMIruhtpTHhHFsNThS9LFz-aI_DAXLqMV8KRXydEg,3431
 lucid/nn/modules/norm.py,sha256=qaaVQ2vfOUkPRLTHT4hgsRNxxN1--kdEhlrKXJmE--w,6803
 lucid/nn/modules/pool.py,sha256=ymVnS2NZjh08Tw0VeOfkB6AVrMeLmCKvgxkmEO3KUuw,5044
+lucid/nn/modules/rnn.py,sha256=iMwWLtTC9i3k_h8pI6to-2rJfiqkXX9hzeDDJT3i7XU,2968
 lucid/nn/modules/sparse.py,sha256=EpjiviED2nI55wUjh1twFwa4Lvlrzw0TR6lpCDGeSbo,1147
 lucid/nn/modules/transformer.py,sha256=z56emF_eX18pxRELjfmmsY-7Bn9h2yjIdxCaxs6YDwA,11246
 lucid/nn/modules/vision.py,sha256=8xYasT7TNj4NXwMwwJIw1nbV1paeWEFg_ZohXn9kZBg,1579
@@ -110,7 +111,7 @@ lucid/optim/prop.py,sha256=CbsWmoBb_g_8z16M3T6dMoSR9c72hm8M375IT1UHjpw,4740
 lucid/optim/sgd.py,sha256=DBZ1ZXQ9TfKZCRECfNRMDH9mvqUWCOPdY5TobnVxpz8,4477
 lucid/optim/lr_scheduler/__init__.py,sha256=kUoyN2g9nwTtEAqEVij832WSRvzEpKZywSJdfD7MQvY,58
 lucid/optim/lr_scheduler/_base.py,sha256=NNJnjwmJpsRXathrbLtH4tjfBHtwOiJ5HwF1_S6Ym5c,3092
-lucid/optim/lr_scheduler/_schedulers.py,sha256=wxG6XvlTozz2TP57yXQL-krtSiO0hy2bySZq_sRDjh0,9227
+lucid/optim/lr_scheduler/_schedulers.py,sha256=x6naustFYJUD8SEwdwzP8Wv4pDXWg-yp1HD5scvf1ZY,9365
 lucid/random/__init__.py,sha256=s8EAaKhEiTKT_vYjP4IFHx0xQVa1jqc_qIyvMauUu7M,2727
 lucid/random/_func.py,sha256=1Lu4m-ciEK037chNDGqv_j00RgGGzQ7UfslSfYActUk,2232
 lucid/transforms/__init__.py,sha256=DGznMbqhXdU9FLDMKnJawScO4HCqu40Sf_j4vJGJrjc,90
@@ -120,8 +121,8 @@ lucid/visual/__init__.py,sha256=6TuFDfmXTwpLyHl7_KqBfdzW6zqHjGzIFvymjFPlvjI,21
 lucid/visual/graph.py,sha256=YjpIDM_lloZARw3sCBiXPl_hT5A2gTk2fEHvwvJWXTk,4599
 lucid/weights/__init__.py,sha256=z1AikA3rOEeckWGkYWlcZkxNlJo9Xwa39PL6ly3hWnc,8801
 lucid/weights/__init__.pyi,sha256=lFonYC3cUx2Idolf3AEPnjFcyqcn3UDU84oJlZafqLY,3013
-lucid_dl-2.7.9.dist-info/licenses/LICENSE,sha256=vxRFYnVD1IeYtsvw-KmoElfqrjxKHv1h9YTvsG54loQ,1065
-lucid_dl-2.7.9.dist-info/METADATA,sha256=-h9IwL5SzshStMIt4eYUB0rWJ7nTELdGWSD9FIIWM00,11519
-lucid_dl-2.7.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lucid_dl-2.7.9.dist-info/top_level.txt,sha256=uzP_qBx9iNWIHKJRlElYcBLYVqMpdm9Q1Ma63QPYbFc,6
-lucid_dl-2.7.9.dist-info/RECORD,,
+lucid_dl-2.8.0.dist-info/licenses/LICENSE,sha256=vxRFYnVD1IeYtsvw-KmoElfqrjxKHv1h9YTvsG54loQ,1065
+lucid_dl-2.8.0.dist-info/METADATA,sha256=zLT-u67ODSZMq4vod9l1GqZCQN-0XDl_lKXSTdVcSzY,11519
+lucid_dl-2.8.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lucid_dl-2.8.0.dist-info/top_level.txt,sha256=uzP_qBx9iNWIHKJRlElYcBLYVqMpdm9Q1Ma63QPYbFc,6
+lucid_dl-2.8.0.dist-info/RECORD,,