rb-deeplearning-lib 0.0.1__py3-none-any.whl → 0.1.0__py3-none-any.whl

rb_deeplearning_lib/neural_net.py

@@ -1,3 +1,6 @@
+import optimizer
+import sequence
+
 class Layer:
   def __init__(self, input,out,activ="_",rangeW=(-1,1),rangeB=(-1,1)):
     self.weights = Values((rangeW[0]-rangeW[1])*np.random.rand(input,out)+rangeW[1])
@@ -17,13 +20,7 @@ class Layer:
 
   def params(self):
     return self.weights, self.bias
-
-  def updateParams(self, l_rate):
-    self.weights.vals = self.weights.vals - l_rate * self.weights.grad
-    self.bias.vals = self.bias.vals - l_rate * self.bias.grad
-    self.weights.grad = self.weights.grad * 0
-    self.bias.grad = self.bias.grad * 0
-
+  
 class Dense:
   def __init__(self, layNum, inL, midL, outL, activ="_",f_activ="_",rangeW=(-0.1,0.1),rangeB=(-0.1,0.1)):
     if layNum < 1:
@@ -46,8 +43,6 @@ class Dense:
 
   def params(self):
       return self.seq.params()
-  def updateParams(self, l_rate):
-    self.seq.updateParams(l_rate)
 
 class Dropout:
   def __init__(self, size, chance):
@@ -81,8 +76,15 @@ def mse_loss(y_true, y_pred):
 
 
 class Model:
-  def __init__(self, blocks, regu = "", train = True, loss_fn=None, pen_fn = None):
+  def __init__(self, blocks, regu = "", train = True, loss_fn=None, pen_fn = None, optimizer=None):
     self.blocks = Sequence(blocks)
+
+    # Handle optimizer instantiation
+    if optimizer is None:
+        self.optimizer = Optimizer()
+    else:
+        self.optimizer = optimizer
+
     self.regu = regu
     self.inTrain = train
     self.train_loss = []
@@ -103,7 +105,7 @@ class Model:
     x_ = x if isinstance(x, Values) else Values(x)
     return self.blocks(x_)
 
-  def train(self, epochs, x_t, y_t, x_v, y_v, val_run=1, l_rate=0.01, _lambda=0.1, batch_size = None):
+  def train(self, epochs, x_t, y_t, x_v, y_v, l_rate=0.01, val_run=1, _lambda=0.1, batch_size = None):
     x_trn = x_t if isinstance(x_t, Values) else Values(x_t)
     y_trn = y_t if isinstance(y_t, Values) else Values(y_t)
     x_vl = x_v if isinstance(x_v, Values) else Values(x_v)
@@ -156,7 +158,10 @@ class Model:
         penalized_loss = self.pen_fn(current_loss,self,_lambda)
         penalized_loss.grad = np.ones_like(penalized_loss.vals)
         penalized_loss.backward()
-        self.blocks.updateParams(l_rate)
+        # Use the optimizer to step and clear gradients
+        # Retrieve parameters within the training loop before each optimization step
+        all_params = self.blocks.params()
+        self.optimizer.step(all_params, l_rate)
       print("\r", end="")
 
     for l in self.blocks.arr:

rb_deeplearning_lib/optimizer.py

@@ -0,0 +1,100 @@
+import numpy as np
+import autogradient
+
+class Optimizer:
+  def __init__(self):
+    pass
+
+  def step(self, params, learning_rate):
+    for p in params:
+      p.vals = p.vals - learning_rate * p.grad
+      p.grad = np.zeros_like(p.grad)
+
+class Optim_SGD(Optimizer):
+  def __init__(self, finitters, fin_l_rate):
+    self.t = 0
+    self.finitter = finitters
+    self.fin_l_rate = fin_l_rate
+
+  def step(self, params, learning_rate):
+    self.t += 1
+    t = self.t
+    alpha = t/self.finitters
+    if(alpha < 1):
+      l_rate = learning_rate*(1-alpha) + alpha*self.fin_l_rate
+    else:
+      l_rate = self.fin_l_rate
+    for p in params:
+      p.vals = p.vals - l_rate * p.grad
+      p.grad = np.zeros_like(p.grad)
+
+class Optim_SGD_Momentum(Optimizer):
+  def __init__(self,mom_beta=0.9):
+    self.v = {}
+    self.beta = mom_beta
+
+  def step(self, params, learning_rate):
+    v = self.v
+    for p in params:
+      if p not in self.v:
+        v[p] = np.zeros_like(p.vals)
+      v[p] = self.beta*v[p] - learning_rate*p.grad
+      p.vals = p.vals + self.v[p]
+      p.grad = np.zeros_like(p.grad)
+    self.v = v
+
+class Optim_AdaGrad(Optimizer):
+  def __init__(self, gamma=0.0000001):
+    self.gamma = gamma
+    self.r = {}
+
+  def step(self, params, l_rate):
+    for p in params:
+      if p not in self.r:
+        self.r[p] = np.zeros_like(p.vals)
+      self.r[p] = self.r[p] + p.grad**2
+      p.vals = p.vals - l_rate * p.grad / (self.gamma + self.r[p]**0.5)
+      p.grad = np.zeros_like(p.grad)
+
+class Optim_RMSPropclass(Optimizer):
+  def __init__(self,decay_rate, gamma=0.000001):
+    self.decay_rate = decay_rate
+    self.gamma = gamma
+    self.r = {}
+  def step(self, params, l_rate):
+    dr = self.decay_rate
+    for p in params:
+      if p not in self.r:
+        self.r[p] = np.zeros_like(p.vals)
+      self.r[p] = dr*self.r[p] + (1-dr)*p.grad**2
+      p.vals = p.vals - l_rate*p.grad/(self.gamma + self.r[p]**0.5)
+      p.grad = np.zeros_like(p.grad)
+
+class Optim_Adam(Optimizer):
+  def __init__(self, beta1, beta2, gamma = 0.000001):
+    self.b1 = beta1
+    self.b2 = beta2
+    self.gamma = gamma
+    self.r = {}
+    self.s = {}
+    self.t = 0
+
+  def step(self, params, l_rate):
+    self.t += 1
+    t = self.t
+    beta1 = self.b1
+    beta2 = self.b2
+    for p in params:
+      if p not in self.r:
+        self.r[p] = np.zeros_like(p.vals)
+      if p not in self.s:
+        self.s[p] = np.zeros_like(p.vals)
+
+      self.s[p] = beta1*self.s[p] + (1-beta1)*p.grad
+      self.r[p] = beta2*self.r[p] + (1-beta2)*p.grad**2
+
+      s_hat = self.s[p]/(1-beta1**t)
+      r_hat = self.r[p]/(1-beta2**t)
+
+      p.vals = p.vals - l_rate*s_hat/(self.gamma + r_hat**0.5)
+      p.grad = np.zeros_like(p.grad)

{rb_deeplearning_lib-0.0.1.dist-info → rb_deeplearning_lib-0.1.0.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: rb-deeplearning-lib
-Version: 0.0.1
+Version: 0.1.0
 Summary: This is a machine learning--more specifically deep learning--library from my independent study on deep learning. This library is both a result of my learning and a tool for AI development.
 License-Expression: MIT
-Project-URL: Homepage, https://github.com/rylan-berry/DeepLearningIndependentStudy/deeplearning_package
+Project-URL: Homepage, https://github.com/rylan-berry/DeepLearningIndependentStudy/tree/main/deeplearning_package
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE

rb_deeplearning_lib-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+rb_deeplearning_lib/__init__.py,sha256=3pOhAKKTkdfXcPT1mxouH4ZOHfk419-Wnqv24oss6P4,54
+rb_deeplearning_lib/autogradient.py,sha256=woXGKi1EZ1QobvRbPMtj4WQqdkOCr7Xk_KeG8WE1d6k,9388
+rb_deeplearning_lib/neural_net.py,sha256=nwC4bdEYSYRtylqe7p68SFlRJKc1htBB-sMJGiFl8fk,6478
+rb_deeplearning_lib/optimizer.py,sha256=OI2x7xrna4ksbaprNgIx25jogVM10OGfNYKpwetRc7o,2690
+rb_deeplearning_lib/sequence.py,sha256=As8FiHVrcN2w0xXV8Vt_eHhNh1vDWDM3oIfrZMUkZ5g,417
+rb_deeplearning_lib-0.1.0.dist-info/licenses/LICENSE,sha256=RGN7cN89q7JPnZj-z1KSZT9M3zcJPwjzb-gEOt6SCCA,1069
+rb_deeplearning_lib-0.1.0.dist-info/METADATA,sha256=rFIR8_pNwAgckfor06KZlnkJdwAPPMohhpB4dmCp4S4,2274
+rb_deeplearning_lib-0.1.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+rb_deeplearning_lib-0.1.0.dist-info/top_level.txt,sha256=oK1ClPmIXDzAbwPHl69BE88PFjpsYta_a5vNEa5WJoA,20
+rb_deeplearning_lib-0.1.0.dist-info/RECORD,,

{rb_deeplearning_lib-0.0.1.dist-info → rb_deeplearning_lib-0.1.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

rb_deeplearning_lib-0.0.1.dist-info/RECORD

@@ -1,9 +0,0 @@
-rb_deeplearning_lib/__init__.py,sha256=3pOhAKKTkdfXcPT1mxouH4ZOHfk419-Wnqv24oss6P4,54
-rb_deeplearning_lib/autogradient.py,sha256=woXGKi1EZ1QobvRbPMtj4WQqdkOCr7Xk_KeG8WE1d6k,9388
-rb_deeplearning_lib/neural_net.py,sha256=tMGHeL1VVVTe3D4MzFMxQaz9pPdMA3-gbQbMqkPQzKs,6411
-rb_deeplearning_lib/sequence.py,sha256=As8FiHVrcN2w0xXV8Vt_eHhNh1vDWDM3oIfrZMUkZ5g,417
-rb_deeplearning_lib-0.0.1.dist-info/licenses/LICENSE,sha256=RGN7cN89q7JPnZj-z1KSZT9M3zcJPwjzb-gEOt6SCCA,1069
-rb_deeplearning_lib-0.0.1.dist-info/METADATA,sha256=LDVDxShZ5P5C_rHucP-lsx196KwVLZa9tX2E7k_7nLo,2264
-rb_deeplearning_lib-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rb_deeplearning_lib-0.0.1.dist-info/top_level.txt,sha256=oK1ClPmIXDzAbwPHl69BE88PFjpsYta_a5vNEa5WJoA,20
-rb_deeplearning_lib-0.0.1.dist-info/RECORD,,