pytinytensor 0.1.5__tar.gz → 0.1.7__tar.gz

{pytinytensor-0.1.5/pytinytensor.egg-info → pytinytensor-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytinytensor
-Version: 0.1.5
+Version: 0.1.7
 Summary: мини ИИ фреймворк от IbrokimN ( github/IbrokhimN )
 Home-page: https://github.com/IbrokhimN/tinytensor
 Author: IbrokhimN
@@ -23,6 +23,8 @@ Dynamic: summary
 
 # tinytensor
 
+Новые доки напишу в след версии 
+
 Маленький самописный autograd на numpy. Тензоры, backprop, пара слоев,
 лоссы, оптимизаторы и даталоадер. По сути свой мини-pytorch, только без
 плюшек и без cuda (пока).

{pytinytensor-0.1.5 → pytinytensor-0.1.7}/README.md

@@ -1,5 +1,7 @@
 # tinytensor
 
+Новые доки напишу в след версии 
+
 Маленький самописный autograd на numpy. Тензоры, backprop, пара слоев,
 лоссы, оптимизаторы и даталоадер. По сути свой мини-pytorch, только без
 плюшек и без cuda (пока).
@@ -215,4 +217,4 @@ tinytensor/
 - [Loshchilov & Hutter - AdamW](https://arxiv.org/abs/1711.05101)
 - [Hendrycks & Gimpel - GELU](https://arxiv.org/abs/1606.08415)
 - [PyTorch - torch.nn.Module](https://pytorch.org/docs/stable/generated/torch.nn.Module.html)
-- [PyTorch - torch.utils.data.DataLoader](https://pytorch.org/docs/stable/data.html)
+- [PyTorch - torch.utils.data.DataLoader](https://pytorch.org/docs/stable/data.html)

{pytinytensor-0.1.5 → pytinytensor-0.1.7/pytinytensor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytinytensor
-Version: 0.1.5
+Version: 0.1.7
 Summary: мини ИИ фреймворк от IbrokimN ( github/IbrokhimN )
 Home-page: https://github.com/IbrokhimN/tinytensor
 Author: IbrokhimN
@@ -23,6 +23,8 @@ Dynamic: summary
 
 # tinytensor
 
+Новые доки напишу в след версии 
+
 Маленький самописный autograd на numpy. Тензоры, backprop, пара слоев,
 лоссы, оптимизаторы и даталоадер. По сути свой мини-pytorch, только без
 плюшек и без cuda (пока).

{pytinytensor-0.1.5 → pytinytensor-0.1.7}/pytinytensor.egg-info/SOURCES.txt

@@ -29,10 +29,17 @@ tinytensor/data/dataloader.py
 tinytensor/data/dataset.py
 tinytensor/nn/__init__.py
 tinytensor/nn/activations.py
+tinytensor/nn/batchnorm.py
+tinytensor/nn/conv.py
 tinytensor/nn/dropout.py
+tinytensor/nn/embedding.py
+tinytensor/nn/flatten.py
+tinytensor/nn/functional.py
 tinytensor/nn/linear.py
 tinytensor/nn/losses.py
 tinytensor/nn/modules.py
+tinytensor/nn/pooling.py
+tinytensor/nn/rnn.py
 tinytensor/nn/utils.py
 tinytensor/optim/__init__.py
 tinytensor/optim/lr_scheduler.py

{pytinytensor-0.1.5 → pytinytensor-0.1.7}/setup.py

@@ -112,7 +112,7 @@ else:
 
 setup(
     name="pytinytensor",
-    version="0.1.5",
+    version="0.1.7",
     description="мини ИИ фреймворк от IbrokimN ( github/IbrokhimN )",
     long_description=long_description,
     long_description_content_type="text/markdown",

{pytinytensor-0.1.5 → pytinytensor-0.1.7}/tinytensor/core/tensor.py

@@ -184,6 +184,20 @@ class Tensor:
             out._backward = _backward
         return out
 
+    def reshape(self, *shape):
+        out = Tensor(self.data.reshape(*shape), requires_grad=self.requires_grad, device=self.device)
+        if out.requires_grad:
+            out._prev = {self}
+
+            def _backward():
+                if self.requires_grad:
+                    if self.grad is None:
+                        self.grad = np.zeros_like(self.data, dtype=np.float32)
+                    self.grad += out.grad.reshape(self.data.shape)
+
+            out._backward = _backward
+        return out
+
     def relu(self):
         out = Tensor(
             np.maximum(0, self.data),

pytinytensor-0.1.7/tinytensor/nn/__init__.py

@@ -0,0 +1,19 @@
+from tinytensor.nn.modules import Module, Sequential
+from tinytensor.nn.linear import Linear
+from tinytensor.nn.conv import Conv2d
+from tinytensor.nn.pooling import MaxPool2d, AvgPool2d
+from tinytensor.nn.flatten import Flatten
+from tinytensor.nn.batchnorm import BatchNorm2d
+from tinytensor.nn.embedding import Embedding
+from tinytensor.nn.rnn import RNNCell, RNN  
+from tinytensor.nn.activations import ReLU, LeReLU, Sigmod, Tanh, GELU, Softmax
+from tinytensor.nn.dropout import Dropout
+from tinytensor.nn.losses import MSELoss, CrossEntropyLoss
+
+__all__ = [
+    "Module", "Sequential",
+    "Linear", "Conv2d", "MaxPool2d", "AvgPool2d", "Flatten", "BatchNorm2d",
+    "Embedding", "RNNCell", "RNN",
+    "ReLU", "LeReLU", "Sigmod", "Tanh", "GELU", "Softmax",
+    "Dropout", "MSELoss", "CrossEntropyLoss"
+]

pytinytensor-0.1.7/tinytensor/nn/activations.py

@@ -0,0 +1,68 @@
+import numpy as np 
+
+from tinytensor.core.tensor import Tensor 
+from tinytensor.nn.modules import Module
+
+class ReLU(Module):
+    def forward(self, x):
+        return x.relu()
+
+class LeReLU(Module):
+    # у нас есть тут альфа
+    def __init__(self, alpha=0.01):
+        super().__init__()
+        self.alpha = alpha
+    
+    def forward(self, x):
+        return x.leaky_relu(alpha=self.alpha)
+    
+class Sigmod(Module):
+    def forward(self, x):
+        return x.sigmoid()
+    
+class Tanh(Module):
+    def forward(self, x):
+        return x.tanh()
+    
+class GELU(Module):
+    def forward(self, x):
+        return x.gelu()
+
+class Softmax(Module):
+    def __init__(self, dim=1):
+        super().__init__()
+        self.dim = dim 
+
+    def forward(self, x):
+        max_x = np.max(x.data, axis=self.dim, keepdims=True)
+        exp_x = np.exp(x.data - max_x)
+
+        out_data = exp_x / np.sum(exp_x, axis=self.dim, keepdims=True)
+
+        out = Tensor(out_data, requires_grad=x.requires_grad, device=x.device)
+
+        if out.requires_grad:
+            out._prev = {x}
+
+            def _backward():
+                dy = out.grad
+                dot = np.sum(dy * out_data, axis=self.dim, keepdims=True)
+                dx = out_data * (dy - dot)
+
+                if x.grad is None:
+                    x.grad = np.zeros_like(x.data, dtype=np.float32)
+                x.grad += dx
+
+            out._backward = _backward
+
+        return out
+
+
+
+# ToDo:
+# ReLU       [x]
+# LeReLU     [x]
+# Sigmoid    [x]
+# Tanh       [x]
+# GELU       [x]
+# kama soska

pytinytensor-0.1.7/tinytensor/nn/batchnorm.py

@@ -0,0 +1,78 @@
+import numpy as np
+from tinytensor.core.tensor import Tensor
+from tinytensor.nn.modules import Module
+
+class BatchNorm2d(Module):
+    def __init__(self, num_features, eps=1e-5, momentum=0.1):
+        super().__init__()
+        self.num_features = num_features
+        self.eps = eps
+        self.momentum = momentum
+
+        self.gamma = Tensor(np.ones((1, num_features, 1, 1), dtype=np.float32), requires_grad=True)
+        self.beta = Tensor(np.zeros((1, num_features, 1, 1), dtype=np.float32), requires_grad=True)
+
+        self.running_mean = np.zeros((1, num_features, 1, 1), dtype=np.float32)
+        self.running_var = np.ones((1, num_features, 1, 1), dtype=np.float32)
+
+    def forward(self, x):
+        training = self.training
+        if training:
+            mean = np.mean(x.data, axis=(0, 2, 3), keepdims=True)
+            var = np.var(x.data, axis=(0, 2, 3), keepdims=True)
+
+            self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean
+            self.running_var = (1 - self.momentum) * self.running_var + self.momentum * var
+        else:
+            mean = self.running_mean
+            var = self.running_var
+
+        std_inv = 1.0 / np.sqrt(var + self.eps)
+        x_centered = x.data - mean
+        x_hat = x_centered * std_inv
+
+        out_data = self.gamma.data * x_hat + self.beta.data
+
+        out = Tensor(
+            out_data,
+            requires_grad=x.requires_grad or self.gamma.requires_grad or self.beta.requires_grad,
+            device=x.device,
+        )
+
+        if out.requires_grad:
+            out._prev = {x, self.gamma, self.beta}
+
+            def _backward():
+                dout = out.grad
+                N, C, H, W = x.data.shape
+                m = N * H * W
+
+                if self.gamma.requires_grad:
+                    if self.gamma.grad is None:
+                        self.gamma.grad = np.zeros_like(self.gamma.data, dtype=np.float32)
+                    self.gamma.grad += np.sum(dout * x_hat, axis=(0, 2, 3), keepdims=True)
+
+                if self.beta.requires_grad:
+                    if self.beta.grad is None:
+                        self.beta.grad = np.zeros_like(self.beta.data, dtype=np.float32)
+                    self.beta.grad += np.sum(dout, axis=(0, 2, 3), keepdims=True)
+
+                if x.requires_grad:
+                    if x.grad is None:
+                        x.grad = np.zeros_like(x.data, dtype=np.float32)
+
+                    if training:
+                        dx_hat = dout * self.gamma.data
+                        dvar = np.sum(dx_hat * x_centered * -0.5 * std_inv ** 3, axis=(0, 2, 3), keepdims=True)
+                        dmean = np.sum(dx_hat * -std_inv, axis=(0, 2, 3), keepdims=True) + \
+                            dvar * np.mean(-2.0 * x_centered, axis=(0, 2, 3), keepdims=True)
+                        dx = dx_hat * std_inv + dvar * 2.0 * x_centered / m + dmean / m
+                    else:
+                        dx = dout * self.gamma.data * std_inv
+
+                    x.grad += dx
+
+            out._backward = _backward
+
+        return out
+

pytinytensor-0.1.7/tinytensor/nn/conv.py

@@ -0,0 +1,84 @@
+import numpy as np 
+
+from tinytensor.core.tensor import Tensor
+from tinytensor.nn.modules import Module
+from tinytensor.nn.functional import im2col_indices, col2_im_indices
+
+
+class Conv2d(Module):
+    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):
+        super().__init__()
+
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+
+        # если как число
+        if isinstance(kernel_size, int):
+            self.kh = self.kw = kernel_size
+        #если кортеж
+        else:
+            self.kh, self.kw = kernel_size
+
+        self.stride = stride
+        self.padding = padding
+
+        n = in_channels * self.kh * self.kw
+        std = np.sqrt(2.0 / n)
+
+        w_data = np.random.randn(out_channels, in_channels, self.kh, self.kw).astype(np.float32) * std
+        self.weight = Tensor(w_data, requires_grad=True)
+
+        b_data = np.zeros((out_channels, 1), dtype=np.float32)
+        self.bias = Tensor(b_data, requires_grad=True)
+        
+    def forward(self, x):
+        N, C, H, W = x.data.shape
+        
+        x_col, out_h, out_w = im2col_indices(
+            x.data, self.kh, self.kw, padding=self.padding, stride=self.stride
+        )
+        
+        w_row = self.weight.data.reshape(self.out_channels, -1)
+
+        out_data = w_row @ x_col + self.bias.data
+
+        out_data = out_data.reshape(self.out_channels, out_h, out_w, N)
+        out_data = out_data.transpose(3, 0, 1, 2)
+
+        out = Tensor(
+            out_data,
+            requires_grad=x.requires_grad or self.weight.requires_grad or self.bias.requires_grad,
+            device=x.device,
+        )
+
+        if out.requires_grad:
+            out._prev = {x, self.weight, self.bias}
+
+            def _backward():
+                dout = out.grad
+                dout_reshaped = dout.transpose(1, 2, 3, 0).reshape(self.out_channels, -1)
+
+                if self.weight.requires_grad:
+                    if self.weight.grad is None:
+                        self.weight.grad = np.zeros_like(self.weight.data, dtype=np.float32)
+                    dW = dout_reshaped @ x_col.T
+                    self.weight.grad += dW.reshape(self.weight.data.shape)
+
+                if self.bias.requires_grad:
+                    if self.bias.grad is None:
+                        self.bias.grad = np.zeros_like(self.bias.data, dtype=np.float32)
+                    self.bias.grad += np.sum(dout_reshaped, axis=1, keepdims=True)
+
+                if x.requires_grad:
+                    if x.grad is None:
+                        x.grad = np.zeros_like(x.data, dtype=np.float32)
+                    dx_col = w_row.T @ dout_reshaped
+                    dx = col2_im_indices(
+                        dx_col, x.data.shape, self.kh, self.kw,
+                        padding=self.padding, stride=self.stride
+                    )
+                    x.grad += dx
+
+            out._backward = _backward
+
+        return out

pytinytensor-0.1.7/tinytensor/nn/embedding.py

@@ -0,0 +1,33 @@
+import numpy as np 
+
+from tinytensor.nn.modules import Module 
+from tinytensor.core.tensor import Tensor
+
+class Embedding(Module):
+    def __init__(self, num_embeddings: int, dim_embeddings:int ):
+        super().__init__()
+        self.num_embeddings = num_embeddings
+        self.dim_embeddings = dim_embeddings
+
+        weight_data = np.random.randn(num_embeddings, dim_embeddings).astype(np.float32) * 0.01
+        
+        self.weight = Tensor(weight_data, requires_grad=True)
+
+    def forward(self, x):
+        #x эт тензор с целыми числами
+        indices = x.data.astype(np.int64)
+        out_data = self.weight.data[indices]
+
+        out = Tensor(out_data, requires_grad=self.weight.requires_grad, device=self.weight.device)
+
+        if out.requires_grad:
+            out._prev = {self.weight}
+
+            def _backward():
+                if self.weight.grad is None:
+                    self.weight.grad = np.zeros_like(self.weight.data, dtype=np.float32)
+                np.add.at(self.weight.grad, indices, out.grad)
+
+            out._backward = _backward
+
+        return out

pytinytensor-0.1.7/tinytensor/nn/flatten.py

@@ -0,0 +1,9 @@
+from tinytensor.nn.modules import Module
+
+#делаем данные более пригодные для методов типо Linear
+
+class Flatten(Module):
+    def forward(self, x):
+        batch_size = x.data.shape[0]
+        return x.reshape(batch_size, -1)
+

pytinytensor-0.1.7/tinytensor/nn/functional.py

@@ -0,0 +1,54 @@
+import numpy as np 
+
+# вцелом идея im2col это то что изначально интуитивный метод перебора окна по изображению является просто написать 6 циклов для сдвигов и тд однако это ооочень медленный метод, и поэтому мы преобразовываем в 1 матрицу и умножаем и это все будет в разыыы быстрее чем наивный метод перебора
+
+def im2col_indices(x, kh, kw, padding=1, stride=1):
+    p = padding 
+    #делаем паддинг
+    x_padded = np.pad(x, ((0, 0), (0, 0), (p, p), (p, p)), mode='constant')
+    N, C, H, W = x.shape
+    out_h = int((H + 2 * p - kh) / stride + 1)
+    out_w = int((W + 2 * p - kw) / stride + 1)
+
+    i0 = np.repeat(np.arange(kh), kw)
+    i0 = np.tile(i0, C)
+    i1 = stride * np.repeat(np.arange(out_h), out_w)
+    j0 = np.tile(np.arange(kw), kh * C)
+    j1 = stride * np.tile(np.arange(out_w), out_h)
+    
+    i = i0.reshape(-1, 1) + i1.reshape(1, -1)
+    j = j0.reshape(-1, 1) + j1.reshape(1, -1)
+    k = np.repeat(np.arange(C), kh * kw).reshape(-1, 1)
+
+    cols = x_padded[:, k, i, j]
+    cols = cols.transpose(1, 2, 0).reshape(kh * kw * C, -1)
+    return cols, out_h, out_w
+
+def col2_im_indices(cols, x_shape, kh, kw, padding=1, stride=1):
+    N, C, H, W = x_shape
+    p = padding
+    H_padded, W_padded = H + 2 * p, W + 2 * p
+    x_padded = np.zeros((N, C, H_padded, W_padded), dtype=cols.dtype)
+    
+    out_h = int((H + 2 * p - kh) / stride + 1)
+    out_w = int((W + 2 * p - kw) / stride + 1)
+    
+    i0 = np.repeat(np.arange(kh), kw)
+    i0 = np.tile(i0, C)
+    i1 = stride * np.repeat(np.arange(out_h), out_w)
+    j0 = np.tile(np.arange(kw), kh * C)
+    j1 = stride * np.tile(np.arange(out_w), out_h)
+    
+    i = i0.reshape(-1, 1) + i1.reshape(1, -1)
+    j = j0.reshape(-1, 1) + j1.reshape(1, -1)
+    k = np.repeat(np.arange(C), kh * kw).reshape(-1, 1)
+
+    cols_reshaped = cols.reshape(C * kh * kw, -1, N)
+    cols_reshaped = cols_reshaped.transpose(2, 0, 1)
+    
+    np.add.at(x_padded, (slice(None), k, i, j), cols_reshaped)
+    
+    if p == 0:
+        return x_padded
+    return x_padded[:, :, p:-p, p:-p]
+

pytinytensor-0.1.7/tinytensor/nn/losses.py

@@ -0,0 +1,58 @@
+import numpy as np
+from tinytensor.core.tensor import Tensor
+from tinytensor.nn.modules import Module
+# 1/n * sum((n, i = 1), yi - y`i)^2
+class MSELoss(Module):
+    def forward(self, y_pred, y_true):
+        diff = y_pred - y_true
+        sq_diff = diff ** 2
+        return sq_diff.sum() * (1.0/y_pred.data.size)
+
+# -log((e-xtar - M)/(sum(exj-M)))=-(xt-M)+log sum(exj-M)
+class CrossEntropyLoss(Module):
+    def __init__(self, eps=1e-12):
+        super().__init__()
+        self.eps = eps 
+
+    def forward(self, logits, targets):
+        x = logits.data
+        batch_size = x.shape[0]
+
+        max_logits = np.max(x, axis=1, keepdims=True)
+        shifted_logits = x - max_logits
+        
+        #знаменатель софтмакс
+        exp_shifted = np.exp(shifted_logits)
+        sums_exp = np.sum(exp_shifted, axis=1, keepdims=True)
+        softmax = exp_shifted / sums_exp
+
+        #подсчет лог
+        #эпсилон чтоб на 0 не поделилось случайно
+        log_softmax = shifted_logits - np.log(sums_exp + self.eps)
+
+        if targets.data.ndim == 1 or targets.data.shape[1] == 1:
+            target_indices = targets.data.astype(np.int64).ravel()
+            correct_log_probs = log_softmax[np.arange(batch_size), target_indices]
+            one_hot = np.zeros_like(x)
+            one_hot[np.arange(batch_size), target_indices] = 1.0
+
+        else:
+            correct_log_probs = np.sum(log_softmax * targets.data, axis=1)
+            one_hot = targets.data
+
+        loss = -np.mean(correct_log_probs)
+
+        out = Tensor(loss, requires_grad=logits.requires_grad, device=logits.device)
+
+        if out.requires_grad:
+            out._prev = {logits}
+
+            def _backward():
+                if logits.grad is None:
+                    logits.grad = np.zeros_like(logits.data, dtype=np.float32)
+                logits.grad += (softmax - one_hot) / batch_size * out.grad
+
+            out._backward = _backward
+
+        return out
+

pytinytensor-0.1.7/tinytensor/nn/pooling.py

@@ -0,0 +1,113 @@
+import numpy as np
+from tinytensor.core.tensor import Tensor
+from tinytensor.nn.modules import Module
+from tinytensor.nn.functional import im2col_indices, col2_im_indices
+
+class MaxPool2d(Module):
+    def __init__(self, kernel_size=2, stride=2, padding=0):
+        super().__init__()
+        if isinstance(kernel_size, int):
+            self.kh = self.kw = kernel_size
+        # если кортедж
+        else:
+            self.kh, self.kw = kernel_size
+            
+        self.stride = stride
+        self.padding = padding
+
+    def forward(self,x):
+        N, C, H, W = x.data.shape
+        # меняем размерность
+        x_reshaped = x.data.reshape(N * C, 1, H, W)
+        x_col, out_h, out_w = im2col_indices(
+            x_reshaped, self.kh, self.kw, padding=self.padding, stride=self.stride
+        )
+        
+        #находим максимумы 
+        max_idx = np.argmax(x_col, axis=0)
+        out = x_col[max_idx, np.arange(x_col.shape[1])]
+        
+        #сново меняем размеры
+        out = out.reshape(out_h, out_w, N, C)
+        out = out.transpose(2, 3, 0, 1)
+
+        out_t = Tensor(out, requires_grad=x.requires_grad, device=x.device)
+
+        if out_t.requires_grad:
+            out_t._prev = {x}
+
+            def _backward():
+                dout = out_t.grad
+                dout_flat = dout.transpose(2, 3, 0, 1).reshape(-1)
+
+                dx_col = np.zeros_like(x_col)
+                dx_col[max_idx, np.arange(max_idx.size)] = dout_flat
+
+                dx_reshaped = col2_im_indices(
+                    dx_col, x_reshaped.shape, self.kh, self.kw,
+                    padding=self.padding, stride=self.stride
+                )
+                dx = dx_reshaped.reshape(N, C, H, W)
+
+                if x.grad is None:
+                    x.grad = np.zeros_like(x.data, dtype=np.float32)
+                x.grad += dx
+
+            out_t._backward = _backward
+
+        return out_t
+
+
+class AvgPool2d(Module):
+    def __init__(self, kernel_size=2, stride=2, padding=0):
+        super().__init__()
+        if isinstance(kernel_size, int):
+            self.kh = self.kw = kernel_size
+        # если кортедж
+        else:
+            self.kh, self.kw = kernel_size
+            
+        self.stride = stride
+        self.padding = padding
+
+    def forward(self,x):
+        N, C, H, W = x.data.shape
+        # меняем размерность
+        x_reshaped = x.data.reshape(N * C, 1, H, W)
+        x_col, out_h, out_w = im2col_indices(
+            x_reshaped, self.kh, self.kw, padding=self.padding, stride=self.stride
+        )
+        
+        #находим средние
+        out = np.mean(x_col, axis=0)
+        
+        #сново меняем размеры
+        out = out.reshape(out_h, out_w, N, C)
+        out = out.transpose(2, 3, 0, 1)
+
+        out_t = Tensor(out, requires_grad=x.requires_grad, device=x.device)
+
+        if out_t.requires_grad:
+            out_t._prev = {x}
+
+            def _backward():
+                dout = out_t.grad
+                dout_flat = dout.transpose(2, 3, 0, 1).reshape(-1)
+
+                dx_col = np.repeat((dout_flat / (self.kh * self.kw))[None, :], self.kh * self.kw, axis=0)
+
+                dx_reshaped = col2_im_indices(
+                    dx_col, x_reshaped.shape, self.kh, self.kw,
+                    padding=self.padding, stride=self.stride
+                )
+                dx = dx_reshaped.reshape(N, C, H, W)
+
+                if x.grad is None:
+                    x.grad = np.zeros_like(x.data, dtype=np.float32)
+                x.grad += dx
+
+            out_t._backward = _backward
+
+        return out_t
+
+

pytinytensor-0.1.7/tinytensor/nn/rnn.py

@@ -0,0 +1,140 @@
+import numpy as np 
+from tinytensor.core.tensor import Tensor
+from tinytensor.nn.modules import Module
+
+class RNNCell(Module):
+    def __init__(self,input_size: int, hidden_size: int):
+        super().__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+
+        std = 1.0 / np.sqrt(hidden_size)
+        w_ih = np.random.uniform(-std, std, (hidden_size, input_size)).astype(np.float32)
+        b_ih = np.zeros((hidden_size,), dtype=np.float32)
+        
+        #веса скрытог состояния
+        w_hh = np.random.uniform(-std, std, (hidden_size, hidden_size)).astype(np.float32)
+        b_hh = np.zeros((hidden_size,), dtype=np.float32)
+
+        self.weight_ih = Tensor(w_ih, requires_grad=True)
+        self.bias_ih = Tensor(b_ih, requires_grad=True)
+        
+        self.weight_hh = Tensor(w_hh, requires_grad=True)
+        self.bias_hh = Tensor(b_hh, requires_grad=True)
+
+    def forward(self, x, h=None):
+        batch_size = x.data.shape[0]
+
+        if h is None:
+            h = Tensor(np.zeros((batch_size, self.hidden_size), dtype=np.float32))
+
+        # h_n=tanh(x@W_ih.T+ b_ih+h@ W_hh.T+b_hh)
+        gate_x = x.data @ self.weight_ih.data.T + self.bias_ih.data
+        gate_h = h.data @ self.weight_hh.data.T + self.bias_hh.data
+
+        h_next_data = np.tanh(gate_x + gate_h)
+
+        out = Tensor(
+            h_next_data,
+            requires_grad=(
+                x.requires_grad or h.requires_grad
+                or self.weight_ih.requires_grad or self.bias_ih.requires_grad
+                or self.weight_hh.requires_grad or self.bias_hh.requires_grad
+            ),
+            device=x.device,
+        )
+
+        if out.requires_grad:
+            out._prev = {x, h, self.weight_ih, self.bias_ih, self.weight_hh, self.bias_hh}
+
+            def _backward():
+                dgate = out.grad * (1.0 - h_next_data ** 2)
+
+                if self.weight_ih.requires_grad:
+                    if self.weight_ih.grad is None:
+                        self.weight_ih.grad = np.zeros_like(self.weight_ih.data, dtype=np.float32)
+                    self.weight_ih.grad += dgate.T @ x.data
+
+                if self.bias_ih.requires_grad:
+                    if self.bias_ih.grad is None:
+                        self.bias_ih.grad = np.zeros_like(self.bias_ih.data, dtype=np.float32)
+                    self.bias_ih.grad += np.sum(dgate, axis=0)
+
+                if self.weight_hh.requires_grad:
+                    if self.weight_hh.grad is None:
+                        self.weight_hh.grad = np.zeros_like(self.weight_hh.data, dtype=np.float32)
+                    self.weight_hh.grad += dgate.T @ h.data
+
+                if self.bias_hh.requires_grad:
+                    if self.bias_hh.grad is None:
+                        self.bias_hh.grad = np.zeros_like(self.bias_hh.data, dtype=np.float32)
+                    self.bias_hh.grad += np.sum(dgate, axis=0)
+
+                if x.requires_grad:
+                    if x.grad is None:
+                        x.grad = np.zeros_like(x.data, dtype=np.float32)
+                    x.grad += dgate @ self.weight_ih.data
+
+                if h.requires_grad:
+                    if h.grad is None:
+                        h.grad = np.zeros_like(h.data, dtype=np.float32)
+                    h.grad += dgate @ self.weight_hh.data
+
+            out._backward = _backward
+
+        return out
+
+
+class RNN(Module):
+    def __init__(self, input_size: int, hidden_size: int):
+        super().__init__()
+        self.cell = RNNCell(input_size, hidden_size)
+    
+    def forward(self, x, h_0=None):
+        #x.shape->(batch sSize, Seq len,input size)
+        batch_size, seq_len, _ = x.data.shape
+
+        h = h_0
+        outputs = []
+
+        #по циклу по времени т
+        for t in range(seq_len):
+            x_t = Tensor(x.data[:, t, :], requires_grad=x.requires_grad)
+
+            if x.requires_grad:
+                x_t._prev = {x}
+
+                def make_backward(t, x_t):
+                    def _backward():
+                        if x.grad is None:
+                            x.grad = np.zeros_like(x.data, dtype=np.float32)
+                        x.grad[:, t, :] += x_t.grad
+                    return _backward
+
+                x_t._backward = make_backward(t, x_t)
+
+            h = self.cell(x_t, h)
+            outputs.append(h)
+
+        #сборка
+        out_data = np.stack([o.data for o in outputs], axis=1)
+
+        out = Tensor(
+            out_data,
+            requires_grad=any(o.requires_grad for o in outputs),
+            device=x.device,
+        )
+
+        if out.requires_grad:
+            out._prev = set(outputs)
+
+            def _backward():
+                for t, o in enumerate(outputs):
+                    if o.requires_grad:
+                        if o.grad is None:
+                            o.grad = np.zeros_like(o.data, dtype=np.float32)
+                        o.grad += out.grad[:, t, :]
+
+            out._backward = _backward
+
+        return out, h

pytinytensor-0.1.5/tinytensor/nn/__init__.py

@@ -1,14 +0,0 @@
-from tinytensor.nn.modules import Module, Sequential
-from tinytensor.nn.linear import Linear 
-from tinytensor.nn.activations import ReLU, LeReLU, Sigmod, Tanh, GELU
-from tinytensor.nn.dropout import Dropout
-from tinytensor.nn.losses import MSELoss
-
-__all__ = [
-    "Module",
-    "Sequential",
-    "Linear",
-    "ReLU", "LeReLU", "Sigmod", "Tanh", "GELU",
-    "Dropout",
-    "MSELoss"
-]

pytinytensor-0.1.5/tinytensor/nn/activations.py

@@ -1,37 +0,0 @@
-from tinytensor.nn.modules import Module
-
-class ReLU(Module):
-    def forward(self, x):
-        return x.relu()
-
-class LeReLU(Module):
-    # у нас есть тут альфа
-    def __init__(self, alpha=0.01):
-        super().__init__()
-        self.alpha = alpha
-    
-    def forward(self, x):
-        return x.leaky_relu(alpha=self.alpha)
-    
-class Sigmod(Module):
-    def forward(self, x):
-        return x.sigmoid()
-    
-class Tanh(Module):
-    def forward(self, x):
-        return x.tanh()
-    
-class GELU(Module):
-    def forward(self, x):
-        return x.gelu()
-
-
-
-
-# ToDo:
-# ReLU       [x]
-# LeReLU     [x]
-# Sigmoid    [x]
-# Tanh       [x]
-# GELU       [x]
-# kama soska

pytinytensor-0.1.5/tinytensor/nn/losses.py

@@ -1,12 +0,0 @@
-import numpy as np
-from tinytensor.nn.modules import Module
-# 1/n * sum((n, i = 1), yi - y`i)^2
-class MSELoss(Module):
-    def forward(self, y_pred, y_true):
-        diff = y_pred - y_true
-        sq_diff = diff ** 2
-        return sq_diff.sum() * (1.0/y_pred.data.size)
-
-
-
-# мб в будущем добавлю cross entropy loss