froog 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl

froog/ops.py

@@ -9,6 +9,7 @@
 import numpy as np
 from froog.tensor import Function, register
 from froog.utils import im2col, col2im
+from froog.tensor import Tensor
 
 # *****************************************************
 #     ____  ___   _____ __________   ____  ____  _____
@@ -142,6 +143,29 @@ class Sigmoid(Function):
     return grad_input
 register("sigmoid", Sigmoid)
 
+# class Dropout(Function):
+#   """
+#   Randomly zeroes some of the elements of the input tensor with probability p during training.
+#   The elements to zero are randomized on every forward call.
+#   During inference, dropout is disabled and the input is scaled by (1-p) to maintain the expected value.
+#   """
+#   @staticmethod
+#   def forward(ctx, input, p=0.5, training=True):
+#     if training:
+#       # Create a binary mask with probability (1-p) of being 1
+#       mask = (np.random.random(input.shape) > p).astype(np.float32)
+#       ctx.save_for_backward(mask)
+#       return input * mask
+#     else:
+#       # during inference, scale the input by (1-p)
+#       return input * (1-p)
+
+#   @staticmethod
+#   def backward(ctx, grad_output):
+#     mask, = ctx.saved_tensors
+#     return grad_output * mask
+# register("dropout", Dropout)
+
 class Reshape(Function):
   @staticmethod
   def forward(ctx, x, shape):
@@ -358,3 +382,53 @@ class AvgPool2D(Function):
         ret[:, :, Y:my:py, X:mx:px] = grad_output / py / px   # divide by avg of pool, e.g. for 2x2 pool /= 4
     return ret
 register('avg_pool2d', AvgPool2D)
+
+# *************************************
+#     _   ___   __   ____  ____  _____
+#    / | / / | / /  / __ \/ __ \/ ___/
+#   /  |/ /  |/ /  / / / / /_/ /\__ \ 
+#  / /|  / /|  /  / /_/ / ____/___/ / 
+# /_/ |_/_/ |_/   \____/_/    /____/  
+#
+# ************* nn ops ************   
+
+def Linear(*x):
+  # random Glorot initialization
+  ret = np.random.uniform(-1., 1., size=x)/np.sqrt(np.prod(x))
+  return ret.astype(np.float32)
+
+def swish(x):
+  return x.mul(x.sigmoid())
+
+class BatchNorm2D:
+  """
+  __call__ follows the formula from the link below
+  pytorch version: https://pytorch.org/docs/stable/generated/torch.nn.BatchNorm2d.html
+
+  self.weight       = γ
+  self.bias         = β
+  self.running_mean = E[x] 
+  self.running_var  = Var[x]
+
+  the reshaping step ensures that each channel of the input has its 
+  own separate set of parameters (mean, variance, weight, and bias)
+
+  self.running_mean has shape [num_channels].
+  self.running_mean.reshape(shape=[1, -1, 1, 1]) reshapes it to [1, num_channels, 1, 1]
+  """
+  def __init__(self, sz, eps=0.001):
+    self.eps = eps
+    self.weight = Tensor.zeros(sz)
+    self.bias = Tensor.zeros(sz)
+
+    # TODO: need running_mean and running_var
+    self.running_mean = Tensor.zeros(sz)
+    self.running_var = Tensor.zeros(sz)
+    self.num_batches_tracked = Tensor.zeros(1)
+
+  def __call__(self, x):
+    x = x.sub(self.running_mean.reshape(shape=[1, -1, 1, 1]))
+    x = x.mul(self.weight.reshape(shape=[1, -1, 1, 1]))
+    x = x.div(self.running_var.add(Tensor([self.eps], gpu=x.gpu)).reshape(shape=[1, -1, 1, 1]).sqrt())
+    x = x.add(self.bias.reshape(shape=[1, -1, 1, 1]))
+    return x

froog/optim.py

@@ -57,7 +57,7 @@ class RMSprop(Optimizer):
   RMSprop divides the learning rate by an exponentially decaying average of squared gradients.
 
   Notes: 
-  The reason RPROP doesn’t work is that it violates the central idea behind stochastic gradient descent, 
+  The reason RPROP doesn't work is that it violates the central idea behind stochastic gradient descent, 
   which is when we have small enough learning rate, it averages the gradients over successive mini-batches.
   """
   def __init__(self, params, decay=0.9, lr=0.001, eps=1e-8):

{froog-0.4.0.dist-info → froog-0.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: froog
-Version: 0.4.0
+Version: 0.4.2
 Summary: a toy tensor library with opencl support
 Author: Kevin Buhler
 License: MIT
@@ -26,9 +26,10 @@ Requires-Dist: matplotlib
   <br/>
 </div>
 
-```froog``` is an easy-to-read tensor library (<a href="https://www.pepy.tech/projects/froog">25k pip installs!</a>) meant for those looking to get into machine learning and who want to understand how the underlying machine learning framework's code works before they are ultra-optimized (which all modern ml libraries are).
+```froog``` is an easy-to-read tensor library (<a href="https://www.pepy.tech/projects/froog">25k pip installs!</a>) with OpenCL support for GPU acceleration. Inspired by pytorch, tinygrad, and micrograd.
 
-```froog``` encapsulates everything from <a href="https://github.com/kevbuh/froog/blob/main/models/linear_regression.py">linear regression</a> to <a href="https://github.com/kevbuh/froog/blob/main/models/efficientnet.py">convolutional neural networks </a> in under 1000 lines.
+
+<!-- ```froog``` encapsulates everything from <a href="https://github.com/kevbuh/froog/blob/main/models/linear_regression.py">linear regression</a> to <a href="https://github.com/kevbuh/froog/blob/main/models/efficientnet.py">convolutional neural networks </a> in under 2000 lines. -->
 
 # Installation
 ```bash
@@ -191,9 +192,9 @@ So there are two quick examples to get you up and running. You might have notice
 - ```.max_pool2d()```
 - ```.avg_pool2d()```
 
-## GPU Support
+# GPU Support
 
-Have a GPU and need a speedup? You're in good luck because we have GPU support from for our operations defined in <a href="https://github.com/kevbuh/froog/blob/main/froog/ops_gpu.py">```ops_gpu.py```</a>. In order to do this we have a backend built on <a href="https://en.wikipedia.org/wiki/OpenCL">OpenCL</a> that invokes kernel functions that work on the GPU.
+Have a GPU and need a speedup? You're in good luck because we have GPU support via OpenCL for our operations defined in <a href="https://github.com/kevbuh/froog/blob/main/froog/ops_gpu.py">```ops_gpu.py```</a>.
 
 Here's how you can send data to the GPU during a forward pass and bring it back to the CPU.
 
@@ -204,75 +205,19 @@ if GPU:
   out = model.forward(Tensor(img).to_gpu()).cpu()
 ```
 
-## EfficientNet in froog!
+# EfficientNet in froog!
+
+<img src="assets/efficientnet_pug.png" alt="pug" height="300">
 
 We have a really cool finished implementation of EfficientNet built entirely in ```froog```!
 
 In order to run EfficientNet inference:
 
 ```bash
-VIZ=1 python models/efficientnet.py <https://put_your_image_url_here>
-```
-
-I would recommend checking out the <a href="https://github.com/kevbuh/froog/blob/main/models/efficientnet.py">code</a>, it's highly documented and pretty cool. Here's some of the documentation
-```
-Paper           : https://arxiv.org/abs/1905.11946
-PyTorch version : https://github.com/lukemelas/EfficientNet-PyTorch/blob/master/efficientnet_pytorch/model.py
-
-ConvNets are commonly developed at a fixed resource cost, and then scaled up in order to achieve better accuracy when more resources are made available
-The scaling method was found by performing a grid search to find the relationship between different scaling dimensions of the baseline network under a fixed resource constraint
-"SE" stands for "Squeeze-and-Excitation." Introduced by the "Squeeze-and-Excitation Networks" paper by Jie Hu, Li Shen, and Gang Sun (CVPR 2018).
-
-Environment Variables:
-  VIZ=1 --> plots processed image and output probabilities
-
-How to Run:
-  'VIZ=1 python models/efficientnet.py https://your_image_url'
-
-EfficientNet Hyper-Parameters and Weights:
-url_map = {
-    'efficientnet-b0': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b0-355c32eb.pth',
-    'efficientnet-b1': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b1-f1951068.pth',
-    'efficientnet-b2': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b2-8bb594d6.pth',
-    'efficientnet-b3': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b3-5fb5a3c3.pth',
-    'efficientnet-b4': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b4-6ed6700e.pth',
-    'efficientnet-b5': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b5-b6417697.pth',
-    'efficientnet-b6': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b6-c76e70fd.pth',
-    'efficientnet-b7': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b7-dcc49843.pth',
-}
-
-params_dict = {
-        # Coefficients:   width,depth,res,dropout
-        'efficientnet-b0': (1.0, 1.0, 224, 0.2),
-        'efficientnet-b1': (1.0, 1.1, 240, 0.2),
-        'efficientnet-b2': (1.1, 1.2, 260, 0.3),
-        'efficientnet-b3': (1.2, 1.4, 300, 0.3),
-        'efficientnet-b4': (1.4, 1.8, 380, 0.4),
-        'efficientnet-b5': (1.6, 2.2, 456, 0.4),
-        'efficientnet-b6': (1.8, 2.6, 528, 0.5),
-        'efficientnet-b7': (2.0, 3.1, 600, 0.5),
-        'efficientnet-b8': (2.2, 3.6, 672, 0.5),
-        'efficientnet-l2': (4.3, 5.3, 800, 0.5),
-    }
-
-blocks_args = [
-        'r1_k3_s11_e1_i32_o16_se0.25',
-        'r2_k3_s22_e6_i16_o24_se0.25',
-        'r2_k5_s22_e6_i24_o40_se0.25',
-        'r3_k3_s22_e6_i40_o80_se0.25',
-        'r3_k5_s11_e6_i80_o112_se0.25',
-        'r4_k5_s22_e6_i112_o192_se0.25',
-        'r1_k3_s11_e6_i192_o320_se0.25',
-    ]
+VIZ=1 python3 models/efficientnet.py <https://put_your_image_url_here>
 ```
 
-## Linear regression 
-
-Doing linear regression in ```froog``` is pretty easy, check out the entire <a href="https://github.com/kevbuh/froog/blob/main/models/linear_regression.py">code</a>.
-
-```bash
-VIZ=1 python3 linear_regression.py
-```
+I would recommend checking out the <a href="https://github.com/kevbuh/froog/blob/main/models/efficientnet.py">code</a>, it's highly documented and pretty cool.
 
 # Contributing
 <!-- THERES LOT OF STUFF TO WORK ON! VISIT THE <a href="https://github.com/kevbuh/froog/blob/main/docs/bounties.md">BOUNTY SHOP</a>  -->
@@ -282,12 +227,7 @@ Pull requests will be merged if they:
 * increase functionality
 * increase efficiency
 
-More info on <a href="https://github.com/kevbuh/froog/blob/main/docs/contributing.md">contributing</a>.
-
-# Documentation
-
-Need more information about how ```froog``` works? Visit the <a href="https://github.com/kevbuh/froog/tree/main/docs">documentation</a>.
-
-# Interested in more?
+More info on <a href="https://github.com/kevbuh/froog/blob/main/docs/contributing.md">contributing</a>. Make sure to run ```python -m pytest``` before creating a PR.
 
-If you thought ```froog``` was cool, check out the inspirations for this project: pytorch, tinygrad, and https://github.com/karpathy/micrograd/blob/master/micrograd/engine.py
+<!-- # Documentation
+Need more information about how ```froog``` works? Visit the <a href="https://github.com/kevbuh/froog/tree/main/docs">documentation</a>. -->

{froog-0.4.0.dist-info → froog-0.4.2.dist-info}/RECORD

@@ -1,13 +1,13 @@
 froog/__init__.py,sha256=Mzxgj9bA2G4kcmbmY8fY0KCKgimPucn3hTVRWBJ-5_Q,57
 froog/gradcheck.py,sha256=HlA0VDKE-c44o0E73QsUTIVoNs-w_C9FyKFlHfoagIQ,2415
 froog/nn.py,sha256=_5dzIoxz1L4yEnYfONVc8xIs8vqRpUBBwZwHLvBu9yY,2023
-froog/ops.py,sha256=t0P0OzzlhYBgAhM3urLsXLl9LJNff_7Yiyc_pYgP5B4,14388
+froog/ops.py,sha256=1JtzHJf9fMy9ccmVhNIHIbanvoxMYPyZ5WCUliyj8tU,16890
 froog/ops_gpu.py,sha256=ANDJiWS0e1ehcGCSDo_ZOOowaEPZrz2__FkX5z5uYf4,19367
-froog/optim.py,sha256=m8Q1xe3WwU41obGSMVjRMIs3rWqfqRWfhlbhF9oJyWA,2450
+froog/optim.py,sha256=BucVi-j-kphiG4ao7aCMbtxgF6PGcCHITWkgr7Ao0QU,2448
 froog/tensor.py,sha256=Wix4pE5-OIY8Pvv3bqNCSU_-c_wZV2HrmAtBwMPmAfE,7636
 froog/utils.py,sha256=vs9bmBOyfy0_NR8jPl2DMWBCAqIacJ6a75Lbso2MAKs,3347
-froog-0.4.0.dist-info/LICENSE,sha256=k_856uNmcNUoLC_HkI18c1WomqvQ1Ioqk6gwYfWQiaM,31
-froog-0.4.0.dist-info/METADATA,sha256=R87af4vXl_1TInaB-6XXD6y0b_OQPbZHBmgREJSc_RA,13782
-froog-0.4.0.dist-info/WHEEL,sha256=AtBG6SXL3KF_v0NxLf0ehyVOh0cold-JbJYXNGorC6Q,92
-froog-0.4.0.dist-info/top_level.txt,sha256=XPz35C_JWu20LlsVxIMdMZn8DD58Ak78LwgWFBGYZwY,6
-froog-0.4.0.dist-info/RECORD,,
+froog-0.4.2.dist-info/LICENSE,sha256=k_856uNmcNUoLC_HkI18c1WomqvQ1Ioqk6gwYfWQiaM,31
+froog-0.4.2.dist-info/METADATA,sha256=Z0U4MY_eWhxH2VXnR876fySyTJRRTjp4wKHWSwSVoRY,10442
+froog-0.4.2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+froog-0.4.2.dist-info/top_level.txt,sha256=XPz35C_JWu20LlsVxIMdMZn8DD58Ak78LwgWFBGYZwY,6
+froog-0.4.2.dist-info/RECORD,,

{froog-0.4.0.dist-info → froog-0.4.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.0)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py3-none-any