pytinytensor 0.1.6__tar.gz → 0.1.7__tar.gz

{pytinytensor-0.1.6/pytinytensor.egg-info → pytinytensor-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytinytensor
-Version: 0.1.6
+Version: 0.1.7
 Summary: мини ИИ фреймворк от IbrokimN ( github/IbrokhimN )
 Home-page: https://github.com/IbrokhimN/tinytensor
 Author: IbrokhimN
@@ -23,6 +23,8 @@ Dynamic: summary
 
 # tinytensor
 
+Новые доки напишу в след версии 
+
 Маленький самописный autograd на numpy. Тензоры, backprop, пара слоев,
 лоссы, оптимизаторы и даталоадер. По сути свой мини-pytorch, только без
 плюшек и без cuda (пока).

{pytinytensor-0.1.6 → pytinytensor-0.1.7}/README.md

@@ -1,5 +1,7 @@
 # tinytensor
 
+Новые доки напишу в след версии 
+
 Маленький самописный autograd на numpy. Тензоры, backprop, пара слоев,
 лоссы, оптимизаторы и даталоадер. По сути свой мини-pytorch, только без
 плюшек и без cuda (пока).
@@ -215,4 +217,4 @@ tinytensor/
 - [Loshchilov & Hutter - AdamW](https://arxiv.org/abs/1711.05101)
 - [Hendrycks & Gimpel - GELU](https://arxiv.org/abs/1606.08415)
 - [PyTorch - torch.nn.Module](https://pytorch.org/docs/stable/generated/torch.nn.Module.html)
-- [PyTorch - torch.utils.data.DataLoader](https://pytorch.org/docs/stable/data.html)
+- [PyTorch - torch.utils.data.DataLoader](https://pytorch.org/docs/stable/data.html)

{pytinytensor-0.1.6 → pytinytensor-0.1.7/pytinytensor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytinytensor
-Version: 0.1.6
+Version: 0.1.7
 Summary: мини ИИ фреймворк от IbrokimN ( github/IbrokhimN )
 Home-page: https://github.com/IbrokhimN/tinytensor
 Author: IbrokhimN
@@ -23,6 +23,8 @@ Dynamic: summary
 
 # tinytensor
 
+Новые доки напишу в след версии 
+
 Маленький самописный autograd на numpy. Тензоры, backprop, пара слоев,
 лоссы, оптимизаторы и даталоадер. По сути свой мини-pytorch, только без
 плюшек и без cuda (пока).

{pytinytensor-0.1.6 → pytinytensor-0.1.7}/pytinytensor.egg-info/SOURCES.txt

@@ -32,12 +32,14 @@ tinytensor/nn/activations.py
 tinytensor/nn/batchnorm.py
 tinytensor/nn/conv.py
 tinytensor/nn/dropout.py
+tinytensor/nn/embedding.py
 tinytensor/nn/flatten.py
 tinytensor/nn/functional.py
 tinytensor/nn/linear.py
 tinytensor/nn/losses.py
 tinytensor/nn/modules.py
 tinytensor/nn/pooling.py
+tinytensor/nn/rnn.py
 tinytensor/nn/utils.py
 tinytensor/optim/__init__.py
 tinytensor/optim/lr_scheduler.py

{pytinytensor-0.1.6 → pytinytensor-0.1.7}/setup.py

@@ -112,7 +112,7 @@ else:
 
 setup(
     name="pytinytensor",
-    version="0.1.6",
+    version="0.1.7",
     description="мини ИИ фреймворк от IbrokimN ( github/IbrokhimN )",
     long_description=long_description,
     long_description_content_type="text/markdown",

{pytinytensor-0.1.6 → pytinytensor-0.1.7}/tinytensor/nn/__init__.py

@@ -4,13 +4,16 @@ from tinytensor.nn.conv import Conv2d
 from tinytensor.nn.pooling import MaxPool2d, AvgPool2d
 from tinytensor.nn.flatten import Flatten
 from tinytensor.nn.batchnorm import BatchNorm2d
-from tinytensor.nn.activations import ReLU, LeReLU, Sigmod, Tanh, GELU
+from tinytensor.nn.embedding import Embedding
+from tinytensor.nn.rnn import RNNCell, RNN  
+from tinytensor.nn.activations import ReLU, LeReLU, Sigmod, Tanh, GELU, Softmax
 from tinytensor.nn.dropout import Dropout
-from tinytensor.nn.losses import MSELoss
+from tinytensor.nn.losses import MSELoss, CrossEntropyLoss
 
 __all__ = [
     "Module", "Sequential",
     "Linear", "Conv2d", "MaxPool2d", "AvgPool2d", "Flatten", "BatchNorm2d",
-    "ReLU", "LeReLU", "Sigmod", "Tanh", "GELU",
-    "Dropout", "MSELoss"
+    "Embedding", "RNNCell", "RNN",
+    "ReLU", "LeReLU", "Sigmod", "Tanh", "GELU", "Softmax",
+    "Dropout", "MSELoss", "CrossEntropyLoss"
 ]

pytinytensor-0.1.7/tinytensor/nn/activations.py

@@ -0,0 +1,68 @@
+import numpy as np 
+
+from tinytensor.core.tensor import Tensor 
+from tinytensor.nn.modules import Module
+
+class ReLU(Module):
+    def forward(self, x):
+        return x.relu()
+
+class LeReLU(Module):
+    # у нас есть тут альфа
+    def __init__(self, alpha=0.01):
+        super().__init__()
+        self.alpha = alpha
+    
+    def forward(self, x):
+        return x.leaky_relu(alpha=self.alpha)
+    
+class Sigmod(Module):
+    def forward(self, x):
+        return x.sigmoid()
+    
+class Tanh(Module):
+    def forward(self, x):
+        return x.tanh()
+    
+class GELU(Module):
+    def forward(self, x):
+        return x.gelu()
+
+class Softmax(Module):
+    def __init__(self, dim=1):
+        super().__init__()
+        self.dim = dim 
+
+    def forward(self, x):
+        max_x = np.max(x.data, axis=self.dim, keepdims=True)
+        exp_x = np.exp(x.data - max_x)
+
+        out_data = exp_x / np.sum(exp_x, axis=self.dim, keepdims=True)
+
+        out = Tensor(out_data, requires_grad=x.requires_grad, device=x.device)
+
+        if out.requires_grad:
+            out._prev = {x}
+
+            def _backward():
+                dy = out.grad
+                dot = np.sum(dy * out_data, axis=self.dim, keepdims=True)
+                dx = out_data * (dy - dot)
+
+                if x.grad is None:
+                    x.grad = np.zeros_like(x.data, dtype=np.float32)
+                x.grad += dx
+
+            out._backward = _backward
+
+        return out
+
+
+
+# ToDo:
+# ReLU       [x]
+# LeReLU     [x]
+# Sigmoid    [x]
+# Tanh       [x]
+# GELU       [x]
+# kama soska

pytinytensor-0.1.7/tinytensor/nn/embedding.py

@@ -0,0 +1,33 @@
+import numpy as np 
+
+from tinytensor.nn.modules import Module 
+from tinytensor.core.tensor import Tensor
+
+class Embedding(Module):
+    def __init__(self, num_embeddings: int, dim_embeddings:int ):
+        super().__init__()
+        self.num_embeddings = num_embeddings
+        self.dim_embeddings = dim_embeddings
+
+        weight_data = np.random.randn(num_embeddings, dim_embeddings).astype(np.float32) * 0.01
+        
+        self.weight = Tensor(weight_data, requires_grad=True)
+
+    def forward(self, x):
+        #x эт тензор с целыми числами
+        indices = x.data.astype(np.int64)
+        out_data = self.weight.data[indices]
+
+        out = Tensor(out_data, requires_grad=self.weight.requires_grad, device=self.weight.device)
+
+        if out.requires_grad:
+            out._prev = {self.weight}
+
+            def _backward():
+                if self.weight.grad is None:
+                    self.weight.grad = np.zeros_like(self.weight.data, dtype=np.float32)
+                np.add.at(self.weight.grad, indices, out.grad)
+
+            out._backward = _backward
+
+        return out

pytinytensor-0.1.7/tinytensor/nn/losses.py

@@ -0,0 +1,58 @@
+import numpy as np
+from tinytensor.core.tensor import Tensor
+from tinytensor.nn.modules import Module
+# 1/n * sum((n, i = 1), yi - y`i)^2
+class MSELoss(Module):
+    def forward(self, y_pred, y_true):
+        diff = y_pred - y_true
+        sq_diff = diff ** 2
+        return sq_diff.sum() * (1.0/y_pred.data.size)
+
+# -log((e-xtar - M)/(sum(exj-M)))=-(xt-M)+log sum(exj-M)
+class CrossEntropyLoss(Module):
+    def __init__(self, eps=1e-12):
+        super().__init__()
+        self.eps = eps 
+
+    def forward(self, logits, targets):
+        x = logits.data
+        batch_size = x.shape[0]
+
+        max_logits = np.max(x, axis=1, keepdims=True)
+        shifted_logits = x - max_logits
+        
+        #знаменатель софтмакс
+        exp_shifted = np.exp(shifted_logits)
+        sums_exp = np.sum(exp_shifted, axis=1, keepdims=True)
+        softmax = exp_shifted / sums_exp
+
+        #подсчет лог
+        #эпсилон чтоб на 0 не поделилось случайно
+        log_softmax = shifted_logits - np.log(sums_exp + self.eps)
+
+        if targets.data.ndim == 1 or targets.data.shape[1] == 1:
+            target_indices = targets.data.astype(np.int64).ravel()
+            correct_log_probs = log_softmax[np.arange(batch_size), target_indices]
+            one_hot = np.zeros_like(x)
+            one_hot[np.arange(batch_size), target_indices] = 1.0
+
+        else:
+            correct_log_probs = np.sum(log_softmax * targets.data, axis=1)
+            one_hot = targets.data
+
+        loss = -np.mean(correct_log_probs)
+
+        out = Tensor(loss, requires_grad=logits.requires_grad, device=logits.device)
+
+        if out.requires_grad:
+            out._prev = {logits}
+
+            def _backward():
+                if logits.grad is None:
+                    logits.grad = np.zeros_like(logits.data, dtype=np.float32)
+                logits.grad += (softmax - one_hot) / batch_size * out.grad
+
+            out._backward = _backward
+
+        return out
+

pytinytensor-0.1.7/tinytensor/nn/rnn.py

@@ -0,0 +1,140 @@
+import numpy as np 
+from tinytensor.core.tensor import Tensor
+from tinytensor.nn.modules import Module
+
+class RNNCell(Module):
+    def __init__(self,input_size: int, hidden_size: int):
+        super().__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+
+        std = 1.0 / np.sqrt(hidden_size)
+        w_ih = np.random.uniform(-std, std, (hidden_size, input_size)).astype(np.float32)
+        b_ih = np.zeros((hidden_size,), dtype=np.float32)
+        
+        #веса скрытог состояния
+        w_hh = np.random.uniform(-std, std, (hidden_size, hidden_size)).astype(np.float32)
+        b_hh = np.zeros((hidden_size,), dtype=np.float32)
+
+        self.weight_ih = Tensor(w_ih, requires_grad=True)
+        self.bias_ih = Tensor(b_ih, requires_grad=True)
+        
+        self.weight_hh = Tensor(w_hh, requires_grad=True)
+        self.bias_hh = Tensor(b_hh, requires_grad=True)
+
+    def forward(self, x, h=None):
+        batch_size = x.data.shape[0]
+
+        if h is None:
+            h = Tensor(np.zeros((batch_size, self.hidden_size), dtype=np.float32))
+
+        # h_n=tanh(x@W_ih.T+ b_ih+h@ W_hh.T+b_hh)
+        gate_x = x.data @ self.weight_ih.data.T + self.bias_ih.data
+        gate_h = h.data @ self.weight_hh.data.T + self.bias_hh.data
+
+        h_next_data = np.tanh(gate_x + gate_h)
+
+        out = Tensor(
+            h_next_data,
+            requires_grad=(
+                x.requires_grad or h.requires_grad
+                or self.weight_ih.requires_grad or self.bias_ih.requires_grad
+                or self.weight_hh.requires_grad or self.bias_hh.requires_grad
+            ),
+            device=x.device,
+        )
+
+        if out.requires_grad:
+            out._prev = {x, h, self.weight_ih, self.bias_ih, self.weight_hh, self.bias_hh}
+
+            def _backward():
+                dgate = out.grad * (1.0 - h_next_data ** 2)
+
+                if self.weight_ih.requires_grad:
+                    if self.weight_ih.grad is None:
+                        self.weight_ih.grad = np.zeros_like(self.weight_ih.data, dtype=np.float32)
+                    self.weight_ih.grad += dgate.T @ x.data
+
+                if self.bias_ih.requires_grad:
+                    if self.bias_ih.grad is None:
+                        self.bias_ih.grad = np.zeros_like(self.bias_ih.data, dtype=np.float32)
+                    self.bias_ih.grad += np.sum(dgate, axis=0)
+
+                if self.weight_hh.requires_grad:
+                    if self.weight_hh.grad is None:
+                        self.weight_hh.grad = np.zeros_like(self.weight_hh.data, dtype=np.float32)
+                    self.weight_hh.grad += dgate.T @ h.data
+
+                if self.bias_hh.requires_grad:
+                    if self.bias_hh.grad is None:
+                        self.bias_hh.grad = np.zeros_like(self.bias_hh.data, dtype=np.float32)
+                    self.bias_hh.grad += np.sum(dgate, axis=0)
+
+                if x.requires_grad:
+                    if x.grad is None:
+                        x.grad = np.zeros_like(x.data, dtype=np.float32)
+                    x.grad += dgate @ self.weight_ih.data
+
+                if h.requires_grad:
+                    if h.grad is None:
+                        h.grad = np.zeros_like(h.data, dtype=np.float32)
+                    h.grad += dgate @ self.weight_hh.data
+
+            out._backward = _backward
+
+        return out
+
+
+class RNN(Module):
+    def __init__(self, input_size: int, hidden_size: int):
+        super().__init__()
+        self.cell = RNNCell(input_size, hidden_size)
+    
+    def forward(self, x, h_0=None):
+        #x.shape->(batch sSize, Seq len,input size)
+        batch_size, seq_len, _ = x.data.shape
+
+        h = h_0
+        outputs = []
+
+        #по циклу по времени т
+        for t in range(seq_len):
+            x_t = Tensor(x.data[:, t, :], requires_grad=x.requires_grad)
+
+            if x.requires_grad:
+                x_t._prev = {x}
+
+                def make_backward(t, x_t):
+                    def _backward():
+                        if x.grad is None:
+                            x.grad = np.zeros_like(x.data, dtype=np.float32)
+                        x.grad[:, t, :] += x_t.grad
+                    return _backward
+
+                x_t._backward = make_backward(t, x_t)
+
+            h = self.cell(x_t, h)
+            outputs.append(h)
+
+        #сборка
+        out_data = np.stack([o.data for o in outputs], axis=1)
+
+        out = Tensor(
+            out_data,
+            requires_grad=any(o.requires_grad for o in outputs),
+            device=x.device,
+        )
+
+        if out.requires_grad:
+            out._prev = set(outputs)
+
+            def _backward():
+                for t, o in enumerate(outputs):
+                    if o.requires_grad:
+                        if o.grad is None:
+                            o.grad = np.zeros_like(o.data, dtype=np.float32)
+                        o.grad += out.grad[:, t, :]
+
+            out._backward = _backward
+
+        return out, h

pytinytensor-0.1.6/tinytensor/nn/activations.py

@@ -1,37 +0,0 @@
-from tinytensor.nn.modules import Module
-
-class ReLU(Module):
-    def forward(self, x):
-        return x.relu()
-
-class LeReLU(Module):
-    # у нас есть тут альфа
-    def __init__(self, alpha=0.01):
-        super().__init__()
-        self.alpha = alpha
-    
-    def forward(self, x):
-        return x.leaky_relu(alpha=self.alpha)
-    
-class Sigmod(Module):
-    def forward(self, x):
-        return x.sigmoid()
-    
-class Tanh(Module):
-    def forward(self, x):
-        return x.tanh()
-    
-class GELU(Module):
-    def forward(self, x):
-        return x.gelu()
-
-
-
-
-# ToDo:
-# ReLU       [x]
-# LeReLU     [x]
-# Sigmoid    [x]
-# Tanh       [x]
-# GELU       [x]
-# kama soska

pytinytensor-0.1.6/tinytensor/nn/losses.py

@@ -1,12 +0,0 @@
-import numpy as np
-from tinytensor.nn.modules import Module
-# 1/n * sum((n, i = 1), yi - y`i)^2
-class MSELoss(Module):
-    def forward(self, y_pred, y_true):
-        diff = y_pred - y_true
-        sq_diff = diff ** 2
-        return sq_diff.sum() * (1.0/y_pred.data.size)
-
-
-
-# мб в будущем добавлю cross entropy loss