pytinytensor 0.1.4__tar.gz → 0.1.6__tar.gz

{pytinytensor-0.1.4/pytinytensor.egg-info → pytinytensor-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytinytensor
-Version: 0.1.4
+Version: 0.1.6
 Summary: мини ИИ фреймворк от IbrokimN ( github/IbrokhimN )
 Home-page: https://github.com/IbrokhimN/tinytensor
 Author: IbrokhimN

{pytinytensor-0.1.4 → pytinytensor-0.1.6/pytinytensor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytinytensor
-Version: 0.1.4
+Version: 0.1.6
 Summary: мини ИИ фреймворк от IbrokimN ( github/IbrokhimN )
 Home-page: https://github.com/IbrokhimN/tinytensor
 Author: IbrokhimN

{pytinytensor-0.1.4 → pytinytensor-0.1.6}/pytinytensor.egg-info/SOURCES.txt

@@ -29,11 +29,18 @@ tinytensor/data/dataloader.py
 tinytensor/data/dataset.py
 tinytensor/nn/__init__.py
 tinytensor/nn/activations.py
+tinytensor/nn/batchnorm.py
+tinytensor/nn/conv.py
 tinytensor/nn/dropout.py
+tinytensor/nn/flatten.py
+tinytensor/nn/functional.py
 tinytensor/nn/linear.py
 tinytensor/nn/losses.py
 tinytensor/nn/modules.py
+tinytensor/nn/pooling.py
+tinytensor/nn/utils.py
 tinytensor/optim/__init__.py
+tinytensor/optim/lr_scheduler.py
 tinytensor/optim/optimizer.py
 tinytensor/utils/__init__.py
 tinytensor/utils/bar.py

{pytinytensor-0.1.4 → pytinytensor-0.1.6}/setup.py

@@ -35,7 +35,6 @@ def find_cuda_lib_dirs():
             found.append(lib_dir)
     return found
 
-
 def check_cuda():
     try:
         subprocess.check_output(["nvcc", "--version"])
@@ -113,7 +112,7 @@ else:
 
 setup(
     name="pytinytensor",
-    version="0.1.4",
+    version="0.1.6",
     description="мини ИИ фреймворк от IbrokimN ( github/IbrokhimN )",
     long_description=long_description,
     long_description_content_type="text/markdown",

{pytinytensor-0.1.4 → pytinytensor-0.1.6}/tinytensor/core/tensor.py

@@ -184,6 +184,20 @@ class Tensor:
             out._backward = _backward
         return out
 
+    def reshape(self, *shape):
+        out = Tensor(self.data.reshape(*shape), requires_grad=self.requires_grad, device=self.device)
+        if out.requires_grad:
+            out._prev = {self}
+
+            def _backward():
+                if self.requires_grad:
+                    if self.grad is None:
+                        self.grad = np.zeros_like(self.data, dtype=np.float32)
+                    self.grad += out.grad.reshape(self.data.shape)
+
+            out._backward = _backward
+        return out
+
     def relu(self):
         out = Tensor(
             np.maximum(0, self.data),

pytinytensor-0.1.6/tinytensor/nn/__init__.py

@@ -0,0 +1,16 @@
+from tinytensor.nn.modules import Module, Sequential
+from tinytensor.nn.linear import Linear
+from tinytensor.nn.conv import Conv2d
+from tinytensor.nn.pooling import MaxPool2d, AvgPool2d
+from tinytensor.nn.flatten import Flatten
+from tinytensor.nn.batchnorm import BatchNorm2d
+from tinytensor.nn.activations import ReLU, LeReLU, Sigmod, Tanh, GELU
+from tinytensor.nn.dropout import Dropout
+from tinytensor.nn.losses import MSELoss
+
+__all__ = [
+    "Module", "Sequential",
+    "Linear", "Conv2d", "MaxPool2d", "AvgPool2d", "Flatten", "BatchNorm2d",
+    "ReLU", "LeReLU", "Sigmod", "Tanh", "GELU",
+    "Dropout", "MSELoss"
+]

pytinytensor-0.1.6/tinytensor/nn/batchnorm.py

@@ -0,0 +1,78 @@
+import numpy as np
+from tinytensor.core.tensor import Tensor
+from tinytensor.nn.modules import Module
+
+class BatchNorm2d(Module):
+    def __init__(self, num_features, eps=1e-5, momentum=0.1):
+        super().__init__()
+        self.num_features = num_features
+        self.eps = eps
+        self.momentum = momentum
+
+        self.gamma = Tensor(np.ones((1, num_features, 1, 1), dtype=np.float32), requires_grad=True)
+        self.beta = Tensor(np.zeros((1, num_features, 1, 1), dtype=np.float32), requires_grad=True)
+
+        self.running_mean = np.zeros((1, num_features, 1, 1), dtype=np.float32)
+        self.running_var = np.ones((1, num_features, 1, 1), dtype=np.float32)
+
+    def forward(self, x):
+        training = self.training
+        if training:
+            mean = np.mean(x.data, axis=(0, 2, 3), keepdims=True)
+            var = np.var(x.data, axis=(0, 2, 3), keepdims=True)
+
+            self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean
+            self.running_var = (1 - self.momentum) * self.running_var + self.momentum * var
+        else:
+            mean = self.running_mean
+            var = self.running_var
+
+        std_inv = 1.0 / np.sqrt(var + self.eps)
+        x_centered = x.data - mean
+        x_hat = x_centered * std_inv
+
+        out_data = self.gamma.data * x_hat + self.beta.data
+
+        out = Tensor(
+            out_data,
+            requires_grad=x.requires_grad or self.gamma.requires_grad or self.beta.requires_grad,
+            device=x.device,
+        )
+
+        if out.requires_grad:
+            out._prev = {x, self.gamma, self.beta}
+
+            def _backward():
+                dout = out.grad
+                N, C, H, W = x.data.shape
+                m = N * H * W
+
+                if self.gamma.requires_grad:
+                    if self.gamma.grad is None:
+                        self.gamma.grad = np.zeros_like(self.gamma.data, dtype=np.float32)
+                    self.gamma.grad += np.sum(dout * x_hat, axis=(0, 2, 3), keepdims=True)
+
+                if self.beta.requires_grad:
+                    if self.beta.grad is None:
+                        self.beta.grad = np.zeros_like(self.beta.data, dtype=np.float32)
+                    self.beta.grad += np.sum(dout, axis=(0, 2, 3), keepdims=True)
+
+                if x.requires_grad:
+                    if x.grad is None:
+                        x.grad = np.zeros_like(x.data, dtype=np.float32)
+
+                    if training:
+                        dx_hat = dout * self.gamma.data
+                        dvar = np.sum(dx_hat * x_centered * -0.5 * std_inv ** 3, axis=(0, 2, 3), keepdims=True)
+                        dmean = np.sum(dx_hat * -std_inv, axis=(0, 2, 3), keepdims=True) + \
+                            dvar * np.mean(-2.0 * x_centered, axis=(0, 2, 3), keepdims=True)
+                        dx = dx_hat * std_inv + dvar * 2.0 * x_centered / m + dmean / m
+                    else:
+                        dx = dout * self.gamma.data * std_inv
+
+                    x.grad += dx
+
+            out._backward = _backward
+
+        return out
+

pytinytensor-0.1.6/tinytensor/nn/conv.py

@@ -0,0 +1,84 @@
+import numpy as np 
+
+from tinytensor.core.tensor import Tensor
+from tinytensor.nn.modules import Module
+from tinytensor.nn.functional import im2col_indices, col2_im_indices
+
+
+class Conv2d(Module):
+    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):
+        super().__init__()
+
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+
+        # если как число
+        if isinstance(kernel_size, int):
+            self.kh = self.kw = kernel_size
+        #если кортеж
+        else:
+            self.kh, self.kw = kernel_size
+
+        self.stride = stride
+        self.padding = padding
+
+        n = in_channels * self.kh * self.kw
+        std = np.sqrt(2.0 / n)
+
+        w_data = np.random.randn(out_channels, in_channels, self.kh, self.kw).astype(np.float32) * std
+        self.weight = Tensor(w_data, requires_grad=True)
+
+        b_data = np.zeros((out_channels, 1), dtype=np.float32)
+        self.bias = Tensor(b_data, requires_grad=True)
+        
+    def forward(self, x):
+        N, C, H, W = x.data.shape
+        
+        x_col, out_h, out_w = im2col_indices(
+            x.data, self.kh, self.kw, padding=self.padding, stride=self.stride
+        )
+        
+        w_row = self.weight.data.reshape(self.out_channels, -1)
+
+        out_data = w_row @ x_col + self.bias.data
+
+        out_data = out_data.reshape(self.out_channels, out_h, out_w, N)
+        out_data = out_data.transpose(3, 0, 1, 2)
+
+        out = Tensor(
+            out_data,
+            requires_grad=x.requires_grad or self.weight.requires_grad or self.bias.requires_grad,
+            device=x.device,
+        )
+
+        if out.requires_grad:
+            out._prev = {x, self.weight, self.bias}
+
+            def _backward():
+                dout = out.grad
+                dout_reshaped = dout.transpose(1, 2, 3, 0).reshape(self.out_channels, -1)
+
+                if self.weight.requires_grad:
+                    if self.weight.grad is None:
+                        self.weight.grad = np.zeros_like(self.weight.data, dtype=np.float32)
+                    dW = dout_reshaped @ x_col.T
+                    self.weight.grad += dW.reshape(self.weight.data.shape)
+
+                if self.bias.requires_grad:
+                    if self.bias.grad is None:
+                        self.bias.grad = np.zeros_like(self.bias.data, dtype=np.float32)
+                    self.bias.grad += np.sum(dout_reshaped, axis=1, keepdims=True)
+
+                if x.requires_grad:
+                    if x.grad is None:
+                        x.grad = np.zeros_like(x.data, dtype=np.float32)
+                    dx_col = w_row.T @ dout_reshaped
+                    dx = col2_im_indices(
+                        dx_col, x.data.shape, self.kh, self.kw,
+                        padding=self.padding, stride=self.stride
+                    )
+                    x.grad += dx
+
+            out._backward = _backward
+
+        return out

pytinytensor-0.1.6/tinytensor/nn/flatten.py

@@ -0,0 +1,9 @@
+from tinytensor.nn.modules import Module
+
+#делаем данные более пригодные для методов типо Linear
+
+class Flatten(Module):
+    def forward(self, x):
+        batch_size = x.data.shape[0]
+        return x.reshape(batch_size, -1)
+

pytinytensor-0.1.6/tinytensor/nn/functional.py

@@ -0,0 +1,54 @@
+import numpy as np 
+
+# вцелом идея im2col это то что изначально интуитивный метод перебора окна по изображению является просто написать 6 циклов для сдвигов и тд однако это ооочень медленный метод, и поэтому мы преобразовываем в 1 матрицу и умножаем и это все будет в разыыы быстрее чем наивный метод перебора
+
+def im2col_indices(x, kh, kw, padding=1, stride=1):
+    p = padding 
+    #делаем паддинг
+    x_padded = np.pad(x, ((0, 0), (0, 0), (p, p), (p, p)), mode='constant')
+    N, C, H, W = x.shape
+    out_h = int((H + 2 * p - kh) / stride + 1)
+    out_w = int((W + 2 * p - kw) / stride + 1)
+
+    i0 = np.repeat(np.arange(kh), kw)
+    i0 = np.tile(i0, C)
+    i1 = stride * np.repeat(np.arange(out_h), out_w)
+    j0 = np.tile(np.arange(kw), kh * C)
+    j1 = stride * np.tile(np.arange(out_w), out_h)
+    
+    i = i0.reshape(-1, 1) + i1.reshape(1, -1)
+    j = j0.reshape(-1, 1) + j1.reshape(1, -1)
+    k = np.repeat(np.arange(C), kh * kw).reshape(-1, 1)
+
+    cols = x_padded[:, k, i, j]
+    cols = cols.transpose(1, 2, 0).reshape(kh * kw * C, -1)
+    return cols, out_h, out_w
+
+def col2_im_indices(cols, x_shape, kh, kw, padding=1, stride=1):
+    N, C, H, W = x_shape
+    p = padding
+    H_padded, W_padded = H + 2 * p, W + 2 * p
+    x_padded = np.zeros((N, C, H_padded, W_padded), dtype=cols.dtype)
+    
+    out_h = int((H + 2 * p - kh) / stride + 1)
+    out_w = int((W + 2 * p - kw) / stride + 1)
+    
+    i0 = np.repeat(np.arange(kh), kw)
+    i0 = np.tile(i0, C)
+    i1 = stride * np.repeat(np.arange(out_h), out_w)
+    j0 = np.tile(np.arange(kw), kh * C)
+    j1 = stride * np.tile(np.arange(out_w), out_h)
+    
+    i = i0.reshape(-1, 1) + i1.reshape(1, -1)
+    j = j0.reshape(-1, 1) + j1.reshape(1, -1)
+    k = np.repeat(np.arange(C), kh * kw).reshape(-1, 1)
+
+    cols_reshaped = cols.reshape(C * kh * kw, -1, N)
+    cols_reshaped = cols_reshaped.transpose(2, 0, 1)
+    
+    np.add.at(x_padded, (slice(None), k, i, j), cols_reshaped)
+    
+    if p == 0:
+        return x_padded
+    return x_padded[:, :, p:-p, p:-p]
+

{pytinytensor-0.1.4 → pytinytensor-0.1.6}/tinytensor/nn/modules.py

@@ -37,7 +37,7 @@ class Module:
         sdict = {}
         for name, param in self._get_named_params().items():
             sdict[name] = param.data
-        return sdict
+        return sdict           
         
     def load_state_dict(self, sdict):
         #запихиваем обратно
@@ -63,3 +63,34 @@ class Module:
         with open(filepath, "rb") as f:
             sd = pickle.load(f)
             self.load_state_dict(sd)
+        
+
+class Sequential(Module):
+    def __init__(self, *args):
+        super().__init__()
+        self.layers = list(args)
+
+    def parameters(self):
+        params = []
+        for layer in self.layers:
+            params.extend(layer.parameters())
+        return params
+
+    def _get_named_params(self, prefix=""):
+        named = {}
+        for i, layer in enumerate(self.layers):
+            key = f"{prefix}.layers.{i}" if prefix else f"layers.{i}"
+            named.update(layer._get_named_params(prefix=key))
+        return named
+
+    def forward(self, x):
+        # закидываем x по слоям
+        for layer in self.layers:
+            x = layer(x)
+        return x
+
+    def __getitem__(self, idx):
+        return self.layers[idx]
+
+    def __len__(self):
+        return len(self.layers)

pytinytensor-0.1.6/tinytensor/nn/pooling.py

@@ -0,0 +1,113 @@
+import numpy as np
+from tinytensor.core.tensor import Tensor
+from tinytensor.nn.modules import Module
+from tinytensor.nn.functional import im2col_indices, col2_im_indices
+
+class MaxPool2d(Module):
+    def __init__(self, kernel_size=2, stride=2, padding=0):
+        super().__init__()
+        if isinstance(kernel_size, int):
+            self.kh = self.kw = kernel_size
+        # если кортедж
+        else:
+            self.kh, self.kw = kernel_size
+            
+        self.stride = stride
+        self.padding = padding
+
+    def forward(self,x):
+        N, C, H, W = x.data.shape
+        # меняем размерность
+        x_reshaped = x.data.reshape(N * C, 1, H, W)
+        x_col, out_h, out_w = im2col_indices(
+            x_reshaped, self.kh, self.kw, padding=self.padding, stride=self.stride
+        )
+        
+        #находим максимумы 
+        max_idx = np.argmax(x_col, axis=0)
+        out = x_col[max_idx, np.arange(x_col.shape[1])]
+        
+        #сново меняем размеры
+        out = out.reshape(out_h, out_w, N, C)
+        out = out.transpose(2, 3, 0, 1)
+
+        out_t = Tensor(out, requires_grad=x.requires_grad, device=x.device)
+
+        if out_t.requires_grad:
+            out_t._prev = {x}
+
+            def _backward():
+                dout = out_t.grad
+                dout_flat = dout.transpose(2, 3, 0, 1).reshape(-1)
+
+                dx_col = np.zeros_like(x_col)
+                dx_col[max_idx, np.arange(max_idx.size)] = dout_flat
+
+                dx_reshaped = col2_im_indices(
+                    dx_col, x_reshaped.shape, self.kh, self.kw,
+                    padding=self.padding, stride=self.stride
+                )
+                dx = dx_reshaped.reshape(N, C, H, W)
+
+                if x.grad is None:
+                    x.grad = np.zeros_like(x.data, dtype=np.float32)
+                x.grad += dx
+
+            out_t._backward = _backward
+
+        return out_t
+
+
+class AvgPool2d(Module):
+    def __init__(self, kernel_size=2, stride=2, padding=0):
+        super().__init__()
+        if isinstance(kernel_size, int):
+            self.kh = self.kw = kernel_size
+        # если кортедж
+        else:
+            self.kh, self.kw = kernel_size
+            
+        self.stride = stride
+        self.padding = padding
+
+    def forward(self,x):
+        N, C, H, W = x.data.shape
+        # меняем размерность
+        x_reshaped = x.data.reshape(N * C, 1, H, W)
+        x_col, out_h, out_w = im2col_indices(
+            x_reshaped, self.kh, self.kw, padding=self.padding, stride=self.stride
+        )
+        
+        #находим средние
+        out = np.mean(x_col, axis=0)
+        
+        #сново меняем размеры
+        out = out.reshape(out_h, out_w, N, C)
+        out = out.transpose(2, 3, 0, 1)
+
+        out_t = Tensor(out, requires_grad=x.requires_grad, device=x.device)
+
+        if out_t.requires_grad:
+            out_t._prev = {x}
+
+            def _backward():
+                dout = out_t.grad
+                dout_flat = dout.transpose(2, 3, 0, 1).reshape(-1)
+
+                dx_col = np.repeat((dout_flat / (self.kh * self.kw))[None, :], self.kh * self.kw, axis=0)
+
+                dx_reshaped = col2_im_indices(
+                    dx_col, x_reshaped.shape, self.kh, self.kw,
+                    padding=self.padding, stride=self.stride
+                )
+                dx = dx_reshaped.reshape(N, C, H, W)
+
+                if x.grad is None:
+                    x.grad = np.zeros_like(x.data, dtype=np.float32)
+                x.grad += dx
+
+            out_t._backward = _backward
+
+        return out_t
+
+

pytinytensor-0.1.6/tinytensor/nn/utils.py

@@ -0,0 +1,23 @@
+import numpy as np
+
+def clip_grad_norm_(parameters, max_norm=1.0):
+    if isinstance(parameters, Exception): # тп защита
+        return
+    
+    params = [p for p in parameters if p.grad is not None]
+    if len(params) == 0:
+        return 0.0
+    
+    total_norm_sq = 0.0
+    for p in params:
+        total_norm_sq += np.sum(p.grad ** 2)
+
+    total_norm = np.sqrt(total_norm_sq)
+
+    clip_coef = max_norm / (total_norm + 1e-6)
+
+    if clip_coef < 1.0:
+        for p in params:
+            p.grad *= clip_coef
+
+    return total_norm

{pytinytensor-0.1.4 → pytinytensor-0.1.6}/tinytensor/optim/__init__.py

@@ -1,6 +1,9 @@
 from tinytensor.optim.optimizer import Optimizer, SGD, AdamW
-
-__all__ = ["Optimizer", "SGD", "AdamW"]
+from tinytensor.optim.lr_scheduler import StepLR
+__all__ = [
+    "Optimizer", "SGD", "AdamW",
+    "StepLR"
+]
 
 #эт для удобства
 #чтоб можно было писать вот так:
@@ -8,4 +11,4 @@ __all__ = ["Optimizer", "SGD", "AdamW"]
 from tinytensor.optim import AdamW
 
 optimizer = AdamW(model.parameters(), lr=1e-3)
-"""
+"""

pytinytensor-0.1.6/tinytensor/optim/lr_scheduler.py

@@ -0,0 +1,13 @@
+#уменьшаем lr каждые n эпох чтоб модель обучилась более плавно
+class StepLR:
+    def __init__(self, optimizer, step_size: int, gamma: float = 0.1):
+        self.optimizer = optimizer
+        self.step_size = step_size
+        self.gamma = gamma
+        self.last_epoch = 0
+
+    def step(self):
+        self.last_epoch += 1 
+        if self.last_epoch % self.step_size == 0:
+            # уменьшаем условно в 10 раз
+            self.optimizer.lr *= self.gamma