froog 0.3.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

froog/nn.py

@@ -10,8 +10,8 @@ from froog.tensor import Tensor
 import numpy as np
 
 def Linear(*x):
-  # TODO: why dividing by sqrt?
-  ret = np.random.uniform(-1., 1., size=x)/np.sqrt(np.prod(x)) # random init weights
+  # random Glorot initialization
+  ret = np.random.uniform(-1., 1., size=x)/np.sqrt(np.prod(x))
   return ret.astype(np.float32)
 
 def swish(x):
@@ -55,6 +55,6 @@ class BatchNorm2D:
   def __call__(self, x):
     x = x.sub(self.running_mean.reshape(shape=[1, -1, 1, 1]))
     x = x.mul(self.weight.reshape(shape=[1, -1, 1, 1]))
-    x = x.div(self.running_var.add(Tensor([self.eps], gpu=x.gpu)).reshape(shape=[1, -1, 1, 1]).sqrt()) # TODO: shouldn't div go first?
+    x = x.div(self.running_var.add(Tensor([self.eps], gpu=x.gpu)).reshape(shape=[1, -1, 1, 1]).sqrt())
     x = x.add(self.bias.reshape(shape=[1, -1, 1, 1]))
     return x

froog/ops.py

@@ -340,7 +340,6 @@ class MaxPool2D(Function):
                             *ctx.kernel_size)
 register('max_pool2d', MaxPool2D)
 
-
 class AvgPool2D(Function):
   @staticmethod
   def forward(ctx, x, kernel_size=(2,2)):
@@ -351,7 +350,7 @@ class AvgPool2D(Function):
   @staticmethod
   def backward(ctx, grad_output):
     s, = ctx.saved_tensors
-    py, px = ctx.kernel_size                                  # TODO: where does kernel_size come from?
+    py, px = ctx.kernel_size                                  # kernel_size passed from forward context
     my, mx = (s[2]//py)*py, (s[3]//px)*px
     ret = np.zeros(s, dtype=grad_output.dtype)
     for Y in range(py):

froog/ops_gpu.py

@@ -5,6 +5,8 @@
 # |    ___||    __  ||  |_|  ||  |_|  ||   ||  |
 # |   |    |   |  | ||       ||       ||   |_| |
 # |___|    |___|  |_||_______||_______||_______|
+#
+# OpenCL kernels
 
 import numpy as np
 from .tensor import Function, register
@@ -71,7 +73,6 @@ def unary_op(ctx, code, x):
   prg.unop(ctx.cl_queue, [np.prod(ret.shape)], None, x, ret)
   return ret
 
-# ???
 @functools.lru_cache
 def cl_pooling_krnl_build(cl_ctx, iter_op, result_op, init_val=0):
   prg = """
@@ -302,23 +303,42 @@ register('relu', ReLU, gpu=True)
 class LogSoftmax(Function):
   @staticmethod
   def forward(ctx, input):
+    # first find max values for numerical stability
+    max_vals = buffer_new(ctx, (input.shape[0],))
+    prg = clbuild(ctx.cl_ctx, """
+    __kernel void max_vals(
+        __global const float *a_g, int sz, __global float *res_g)
+    {
+      int gid = get_global_id(0);
+      int gidsz = gid*sz;
+      float max_val = -INFINITY;
+      for (int x = 0; x < sz; x++) {
+        max_val = max(max_val, a_g[gidsz+x]);
+      }
+      res_g[gid] = max_val;
+    }
+    """)
+    prg.max_vals(ctx.cl_queue, [input.shape[0]], None, input, np.int32(input.shape[1]), max_vals)
+
+    # compute exp(x - max) and sum
     lsum = buffer_new(ctx, (input.shape[0],))
     prg = clbuild(ctx.cl_ctx, """
     __kernel void logsoftmax(
-        __global const float *a_g, int sz, __global float *res_g)
+        __global const float *a_g, __global const float *max_vals, int sz, __global float *res_g)
     {
       int gid = get_global_id(0);
       int gidsz = gid*sz;
-      // TODO: stability with max
+      float max_val = max_vals[gid];
       float out = 0.0;
       for (int x = 0; x < sz; x++) {
-        out += exp(a_g[gidsz+x]);
+        out += exp(a_g[gidsz+x] - max_val);
       }
-      res_g[gid] = log(out);
+      res_g[gid] = log(out) + max_val;
     }
     """)
-    prg.logsoftmax(ctx.cl_queue, [input.shape[0]], None, input, np.int32(input.shape[1]), lsum)
+    prg.logsoftmax(ctx.cl_queue, [input.shape[0]], None, input, max_vals, np.int32(input.shape[1]), lsum)
 
+    # compute final output
     output = buffer_like(ctx, input)
     prg = clbuild(ctx.cl_ctx, """
     __kernel void lsmsub(
@@ -474,8 +494,38 @@ class AvgPool2D(Function):
 
   @staticmethod
   def backward(ctx, grad_output):
-    # TODO Finish this
-    pass
+    # for average pooling, we need to distribute the gradient evenly across all elements in the pooling window
+    input_shape = ctx.data.shape
+    N, C, Y, X = input_shape
+    py, px = ctx.kernel_size
+    ret = buffer_zeros(ctx, input_shape)
+    
+    prg = clbuild(ctx.cl_ctx, """
+    __kernel void avgpool_backward(
+      __global float *grad_input, __global const float *grad_output,
+      uint2 osize, uint2 isize, uint2 kernel_size, int nelem
+    ) {
+      int3 gid = (int3)(get_global_id(2), get_global_id(1), get_global_id(0));
+      int oid = gid.x + osize.x*(gid.y + osize.y*gid.z);
+      float grad = grad_output[oid] / (kernel_size.x * kernel_size.y);
+      
+      for (uint j=0; j<kernel_size.y; ++j) {
+        for (uint i=0; i<kernel_size.x; ++i) {
+          int iid = (gid.x*kernel_size.x+i) + isize.x*((gid.y*kernel_size.y+j) + isize.y*gid.z);
+          if (iid < nelem)
+            grad_input[iid] += grad;
+        }
+      }
+    }
+    """)
+    
+    osize = np.array((X//px, Y//py), dtype=cl.cltypes.uint2)
+    isize = np.array((X, Y), dtype=cl.cltypes.uint2)
+    ksize = np.array((px,py), dtype=cl.cltypes.uint2)
+    
+    prg.avgpool_backward(ctx.cl_queue, (N*C, Y//py, X//px), None, ret, grad_output, osize, isize, ksize, np.int32(input_shape.size))
+    
+    return ret
 register('avg_pool2d', AvgPool2D, gpu=True)
 
 class MaxPool2D(Function):
@@ -484,10 +534,65 @@ class MaxPool2D(Function):
     init_val = "FLT_MIN"
     iter_op = "group_res = max(group_res, input[iid])"
     result_op = "group_res"
-    return pooling_op(ctx, input, kernel_size, iter_op, result_op, init_val=init_val)
+    ret = pooling_op(ctx, input, kernel_size, iter_op, result_op, init_val=init_val)
+    
+    # save indices of max elements for backward pass
+    indices = buffer_new(ctx, ret.shape)
+    prg = clbuild(ctx.cl_ctx, """
+    __kernel void maxpool_indices(
+      __global const float *input, __global float *output, __global int *indices,
+      uint2 osize, uint2 isize, uint2 kernel_size, int nelem
+    ) {
+      int3 gid = (int3)(get_global_id(2), get_global_id(1), get_global_id(0));
+      int oid = gid.x + osize.x*(gid.y + osize.y*gid.z);
+      float max_val = -INFINITY;
+      int max_idx = 0;
+      
+      for (uint j=0; j<kernel_size.y; ++j) {
+        for (uint i=0; i<kernel_size.x; ++i) {
+          int iid = (gid.x*kernel_size.x+i) + isize.x*((gid.y*kernel_size.y+j) + isize.y*gid.z);
+          if (iid < nelem) {
+            float val = input[iid];
+            if (val > max_val) {
+              max_val = val;
+              max_idx = iid;
+            }
+          }
+        }
+      }
+      indices[oid] = max_idx;
+    }
+    """)
+    
+    N, C, Y, X = input.shape
+    py, px = kernel_size
+    osize = np.array((X//px, Y//py), dtype=cl.cltypes.uint2)
+    isize = np.array((X, Y), dtype=cl.cltypes.uint2)
+    ksize = np.array((px,py), dtype=cl.cltypes.uint2)
+    
+    prg.maxpool_indices(ctx.cl_queue, (N*C, Y//py, X//px), None, input, ret, indices, osize, isize, ksize, np.int32(input.size))
+    
+    ctx.save_for_backward(indices)
+    return ret
 
   @staticmethod
   def backward(ctx, grad_output):
-    # TODO Finish this
-    pass
+    indices, = ctx.saved_tensors
+    input_shape = ctx.data.shape
+    ret = buffer_zeros(ctx, input_shape)
+    prg = clbuild(ctx.cl_ctx, """
+    __kernel void maxpool_backward(
+      __global float *grad_input, __global const float *grad_output,
+      __global const int *indices, int nelem
+    ) {
+      int gid = get_global_id(0);
+      if (gid < nelem) {
+        int idx = indices[gid];
+        grad_input[idx] += grad_output[gid];
+      }
+    }
+    """)
+  
+    prg.maxpool_backward(ctx.cl_queue, [np.prod(grad_output.shape)], None, ret, grad_output, indices, np.int32(grad_output.size))
+    return ret
 register('max_pool2d', MaxPool2D, gpu=True)

froog/tensor.py

@@ -62,7 +62,7 @@ class Tensor:
       self.gpu = False
 
     self.data = data
-    self.grad = None # TODO: why self.grad.data instead of self.grad?
+    self.grad = None
 
     if gpu:
       self.gpu_()

froog/utils.py

@@ -67,7 +67,8 @@ def im2col(x, H, W):
   tx = x.reshape(bs, -1)[:, idx]
 
   # all the time is spent here
-  tx = tx.ravel() # TODO: whats the purpose of ravel ???
+  # np.ravel() flattens the array into a 1-dimensional shape
+  tx = tx.ravel()
   return tx.reshape(-1, cin*W*H)
 
 def col2im(tx, H, W, OY, OX):

{froog-0.3.1.dist-info → froog-0.4.0.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: froog
-Version: 0.3.1
-Summary: a beautifully simplistic tensor library
+Version: 0.4.0
+Summary: a toy tensor library with opencl support
 Author: Kevin Buhler
 License: MIT
 Classifier: Programming Language :: Python :: 3
@@ -12,7 +12,6 @@ License-File: LICENSE
 Requires-Dist: numpy
 Requires-Dist: requests
 Requires-Dist: matplotlib
-Requires-Dist: urllib
 
 # froog <img src="https://github.com/kevbuh/froog/actions/workflows/test.yml/badge.svg" alt="unit test badge" > <img src="https://static.pepy.tech/badge/froog" alt="num downloads badge">
 <div align="center" >
@@ -27,7 +26,7 @@ Requires-Dist: urllib
   <br/>
 </div>
 
-```froog``` is an easy-to-read tensor library (<a href="https://www.pepy.tech/projects/froog">16k pip installs!</a>) meant for those looking to get into machine learning and who want to understand how the underlying machine learning framework's code works before they are ultra-optimized (which all modern ml libraries are).
+```froog``` is an easy-to-read tensor library (<a href="https://www.pepy.tech/projects/froog">25k pip installs!</a>) meant for those looking to get into machine learning and who want to understand how the underlying machine learning framework's code works before they are ultra-optimized (which all modern ml libraries are).
 
 ```froog``` encapsulates everything from <a href="https://github.com/kevbuh/froog/blob/main/models/linear_regression.py">linear regression</a> to <a href="https://github.com/kevbuh/froog/blob/main/models/efficientnet.py">convolutional neural networks </a> in under 1000 lines.
 
@@ -85,7 +84,7 @@ from froog.tensor import Tensor
 my_tensor = Tensor([1,2,3])
 ```
 
-Notice how we had to import numpy. If you want to create a Tensor manually, make sure that it is a Numpy array!
+Notice how we had to import NumPy. If you want to create a Tensor manually, make sure that it is a NumPy array!
 
 <!-- Learn more about ```froog``` Tensors <a href="https://github.com/kevbuh/froog/blob/main/docs/tensors.md">here</a>. -->
 
@@ -95,13 +94,10 @@ Tensors are the fundamental datatype in froog, and one of the two main classes.
 
 - ```def __init__(self, data)```:
 
-  - Tensor takes in one param, which is the data. Since froog has a numpy backend, the input data into tensors has to be a numpy array.
-
+  - Tensor takes in one param, which is the data. Since ```froog``` has a NumPy backend, the input data into tensors has to be a NumPy array.
   - Tensor has a ```self.data``` state that it holds. this contains the data inside of the tensor.
-
   - In addition, it has ```self.grad```. this is to hold what the gradients of the tensor is. 
-
-  - Lastly, it has ```self._ctx```. theser are the internal vairables used for autograd graph construction. put more simply, this is where the backward gradient computations are saved. 
+  - Lastly, it has ```self._ctx```. These are the internal variables used for autograd graph construction. This is where the backward gradient computations are saved. 
 
 *Properties*
 
@@ -109,38 +105,34 @@ Tensors are the fundamental datatype in froog, and one of the two main classes.
 
 *Methods*
 - ```def zeros(*shape)```: this returns a tensor full of zeros with any shape that you pass in. Defaults to np.float32
-
 - ```def ones(*shape)```: this returns a tensor full of ones with any shape that you pass in. Defaults to np.float32
-
 - ```def randn(*shape):```: this returns a randomly initialized Tensor of *shape
 
 *Gradient calculations*
 
-- ```froog``` computes gradients automatically through a process called automatic differentiation. it has a variable ```_ctx```, which stores the chain of operations. it will take the current operation, lets say a dot product, and go to the dot product definition in ```froog/ops.py```, which contains a backward pass specfically for dot products. all methods, from add to 2x2 maxpools, have this backward pass implemented.
+- ```froog``` computes gradients automatically through a process called automatic differentiation. it has a variable ```_ctx```, which stores the chain of operations. It will take the current operation, let's say a dot product, and go to the dot product definition in ```froog/ops.py```, which contains a backward pass specifically for dot products. all methods, from add to 2x2 maxpools, have this backward pass implemented.
 
 *Functions*
 
 The other base class in froog is the class ```Function```. It keeps track of input tensors and tensors that need to be saved for backward passes
 
 - ```def __init__(self, *tensors)```: takes in an argument of tensors, which are then saved. 
-
 - ```def save_for_backward(self, *x)```: saves Tensors that are necessary to compute for the computation of gradients in the backward pass. 
-
-- ```def apply(self, arg, *x)```: This is what makes everything work. The apply() method takes care of the forward pass, applying the operation to the inputs.
+- ```def apply(self, arg, *x)```: takes care of the forward pass, applying the operation to the inputs.
 
 *Register*
 
-```def register(name, fxn)```: this function allows you to add a method to a Tensor. This allows you to chain any operations, e.g. x.dot(w).relu(), where w is a tensor
+- ```def register(name, fxn)```: allows you to add a method to a Tensor. This allows you to chain any operations, e.g. x.dot(w).relu(), where w is a tensor
 
 # Creating a model
 
 Okay cool, so now you know that ```froog```'s main datatype is a Tensor and uses NumPy in the background. How do I actually build a model? 
 
-Here's an example of how to create an MNIST multi-layer perceptron (MLP). We wanted to make it as simple as possible for you to do so so it resembles very basic python concepts like classes. There's really only two methods you need to define: 
+Here's an example of how to create an MNIST multi-layer perceptron (MLP). We wanted to make it as simple as possible for you to do so it resembles very basic Python concepts like classes. There are really only two methods you need to define: 
 1. ```__init__``` that defines layers of the model (here we use ```Linear```) 
 2. ```forward``` which defines how the input should flow through your model. We use a simple dot product with a ```Linear``` layer with a <a href="https://en.wikipedia.org/wiki/Rectifier_(neural_networks)">```ReLU```</a> activation.
 
-In order to create an instance of the ```mnistMLP``` model, do the same as you would in python: ```model = mnistMLP()``` . 
+To create an instance of the ```mnistMLP``` model, do the same as you would in Python: ```model = mnistMLP()```. 
 
 We support a few different optimizers, <a href="https://github.com/kevbuh/froog/blob/main/froog/optim.py">here</a> which include:
 - <a href="https://en.wikipedia.org/wiki/Stochastic_gradient_descent">Stochastic Gradient Descent (SGD)</a>
@@ -201,7 +193,7 @@ So there are two quick examples to get you up and running. You might have notice
 
 ## GPU Support
 
-Have a GPU and need a speedup? You're in good luck because we have GPU support from for our operations defined in <a href="https://github.com/kevbuh/froog/blob/main/froog/ops_gpu.py">```ops_gpu.py```</a>. In order to do this we have a backend built on <a href="https://en.wikipedia.org/wiki/OpenGL">OpenGL</a> that invokes kernel functions that work on the GPU.
+Have a GPU and need a speedup? You're in good luck because we have GPU support from for our operations defined in <a href="https://github.com/kevbuh/froog/blob/main/froog/ops_gpu.py">```ops_gpu.py```</a>. In order to do this we have a backend built on <a href="https://en.wikipedia.org/wiki/OpenCL">OpenCL</a> that invokes kernel functions that work on the GPU.
 
 Here's how you can send data to the GPU during a forward pass and bring it back to the CPU.
 

froog-0.4.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+froog/__init__.py,sha256=Mzxgj9bA2G4kcmbmY8fY0KCKgimPucn3hTVRWBJ-5_Q,57
+froog/gradcheck.py,sha256=HlA0VDKE-c44o0E73QsUTIVoNs-w_C9FyKFlHfoagIQ,2415
+froog/nn.py,sha256=_5dzIoxz1L4yEnYfONVc8xIs8vqRpUBBwZwHLvBu9yY,2023
+froog/ops.py,sha256=t0P0OzzlhYBgAhM3urLsXLl9LJNff_7Yiyc_pYgP5B4,14388
+froog/ops_gpu.py,sha256=ANDJiWS0e1ehcGCSDo_ZOOowaEPZrz2__FkX5z5uYf4,19367
+froog/optim.py,sha256=m8Q1xe3WwU41obGSMVjRMIs3rWqfqRWfhlbhF9oJyWA,2450
+froog/tensor.py,sha256=Wix4pE5-OIY8Pvv3bqNCSU_-c_wZV2HrmAtBwMPmAfE,7636
+froog/utils.py,sha256=vs9bmBOyfy0_NR8jPl2DMWBCAqIacJ6a75Lbso2MAKs,3347
+froog-0.4.0.dist-info/LICENSE,sha256=k_856uNmcNUoLC_HkI18c1WomqvQ1Ioqk6gwYfWQiaM,31
+froog-0.4.0.dist-info/METADATA,sha256=R87af4vXl_1TInaB-6XXD6y0b_OQPbZHBmgREJSc_RA,13782
+froog-0.4.0.dist-info/WHEEL,sha256=AtBG6SXL3KF_v0NxLf0ehyVOh0cold-JbJYXNGorC6Q,92
+froog-0.4.0.dist-info/top_level.txt,sha256=XPz35C_JWu20LlsVxIMdMZn8DD58Ak78LwgWFBGYZwY,6
+froog-0.4.0.dist-info/RECORD,,

{froog-0.3.1.dist-info → froog-0.4.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.41.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

froog-0.3.1.dist-info/RECORD

@@ -1,13 +0,0 @@
-froog/__init__.py,sha256=Mzxgj9bA2G4kcmbmY8fY0KCKgimPucn3hTVRWBJ-5_Q,57
-froog/gradcheck.py,sha256=HlA0VDKE-c44o0E73QsUTIVoNs-w_C9FyKFlHfoagIQ,2415
-froog/nn.py,sha256=VeUpmjrv1XrH786GEOc9ruzDAHULiVHKpDtn187EQgI,2076
-froog/ops.py,sha256=PEXZb3lNeYwFpRfcEvEs-KLB4rQqHHUdraR1qOgw5Zs,14389
-froog/ops_gpu.py,sha256=bQ2bzMqmvY2xLEaTjOIXN3KOb-7yZEDjmliV8tWlC6g,15469
-froog/optim.py,sha256=m8Q1xe3WwU41obGSMVjRMIs3rWqfqRWfhlbhF9oJyWA,2450
-froog/tensor.py,sha256=Dacd2UQVfl7RsjADjrhBO1rMVcDVaeJuI55sFfxonhk,7685
-froog/utils.py,sha256=zWquIwvOUBfCzaoA7eAOov7C5NbGxus5RXx9JPM1jeg,3325
-froog-0.3.1.dist-info/LICENSE,sha256=k_856uNmcNUoLC_HkI18c1WomqvQ1Ioqk6gwYfWQiaM,31
-froog-0.3.1.dist-info/METADATA,sha256=3whhO1sbqWZQpXBywz9ypN7iLpR-n82qpZPEjv0of_s,13898
-froog-0.3.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-froog-0.3.1.dist-info/top_level.txt,sha256=XPz35C_JWu20LlsVxIMdMZn8DD58Ak78LwgWFBGYZwY,6
-froog-0.3.1.dist-info/RECORD,,