cogforge-engine 1.0.4__tar.gz → 1.0.6__tar.gz

{cogforge_engine-1.0.4 → cogforge_engine-1.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cogforge-engine
-Version: 1.0.4
+Version: 1.0.6
 Summary: A custom autograd engine and Transformer block built from scratch.
 Project-URL: Homepage, https://github.com/avikmjd2/cogforge
 Author-email: Avik Majumder <avikmjd2@gmail.com>

{cogforge_engine-1.0.4 → cogforge_engine-1.0.6}/cogforge/app.py

@@ -387,10 +387,10 @@ class Tensor:
         flat, idx, rows = p.reshape(N, -1), target_ids.reshape(N), backend.np.arange(N)
         loss = cls(-backend.np.log(backend.np.clip(flat[rows, idx], 1e-15, 1.0)).sum() / N, children=(scores,))
         def _backward():
+            # g = p.reshape(N, -1).copy()
             g = p.reshape(N, -1)
-            g[rows, idx] -= 1.0
-            coef = numpy.float32(float(loss.grad) / N)      
-            _iadd_scaled(scores.grad, g.reshape(scores.shape), coef)
+            g[rows, idx] -= 1.0                      # (softmax - onehot)
+            scores.grad += (g.reshape(scores.shape) / N) * loss.grad
         loss._backwards = _backward
         return loss
     
@@ -403,7 +403,8 @@ class Tensor:
         def _backward():
             g = p.reshape(N, -1)
             g[rows, idx] -= 1.0
-            _iadd_scaled(scores.grad, g.reshape(scores.shape), backend.np.float32(loss.grad / N))
+            coef = numpy.float32(float(loss.grad) / N)      
+            _iadd_scaled(scores.grad, g.reshape(scores.shape), coef)
         loss._backwards = _backward
         return loss
     

{cogforge_engine-1.0.4 → cogforge_engine-1.0.6}/cogforge/models.py

@@ -1,5 +1,7 @@
 from . import backend
-from .app import Embedding, PositionalEncoding,Transformer,LayerNorm,Linear,RotatoryPositionalEncoding
+from .app import Embedding, PositionalEncoding,Transformer,LayerNorm,Linear,RotatoryPositionalEncoding,to_cpu
+import numpy
+
 
 class GPTV1:
     def __init__(self, vocab, d_model, n_heads, n_layers, max_len, d_ff=None):
@@ -26,21 +28,18 @@ class GPTV1:
         return ps
     
     def generate(self, idx, n_new, temperature=1.0, top_k=None):
-        """
-        idx: (B, T) int array — the prompt (B is usually 1)
-        returns: (B, T + n_new) int array
-        """
+        idx = numpy.asarray(idx)
         for _ in range(n_new):
-            cond = idx[:, -self.max_len:]                    # crop to context window
-            logits = self(cond).data[:, -1, :] / temperature # (B, V) — last position only
+            cond = idx[:, -self.max_len:]
+            logits = to_cpu(self(cond).data)[:, -1, :] / temperature
             if top_k is not None:
-                kth = backend.np.sort(logits, axis=-1)[:, -top_k][:, None]
-                logits = backend.np.where(logits < kth, -1e9, logits)  # keep only top-k choices
+                kth = numpy.sort(logits, axis=-1)[:, -top_k][:, None]
+                logits = numpy.where(logits < kth, -1e9, logits)
             z = logits - logits.max(-1, keepdims=True)
-            p = backend.np.exp(z).astype(backend.np.float64); p /= p.sum(-1, keepdims=True)     # softmax → probabilities
-            nxt = backend.np.array([[backend.np.random.choice(len(pr), p=pr)] for pr in p])  # sample
-            idx = backend.np.concatenate([idx, nxt], axis=1)         # append, feed back in
-        return idx                                            # <-- was a bare `return` 
+            p = numpy.exp(z).astype(numpy.float64); p /= p.sum(-1, keepdims=True)
+            nxt = numpy.array([[numpy.random.choice(len(pr), p=pr)] for pr in p])
+            idx = numpy.concatenate([idx, nxt], axis=1)
+        return idx                                          # <-- was a bare `return` 
         
         
         
@@ -71,18 +70,15 @@ class GPT2:
         return ps   
     
     def generate(self, idx, n_new, temperature=1.0, top_k=None):
-        """
-        idx: (B, T) int array — the prompt (B is usually 1)
-        returns: (B, T + n_new) int array
-        """
+        idx = numpy.asarray(idx)                                 # token ids live on host
         for _ in range(n_new):
-            cond = idx[:, -self.max_len:]                    # crop to context window
-            logits = self(cond).data[:, -1, :] / temperature # (B, V) — last position only
+            cond = idx[:, -self.max_len:]
+            logits = to_cpu(self(cond).data)[:, -1, :] / temperature   # forward on GPU, logits to host
             if top_k is not None:
-                kth = backend.np.sort(logits, axis=-1)[:, -top_k][:, None]
-                logits = backend.np.where(logits < kth, -1e9, logits)  # keep only top-k choices
+                kth = numpy.sort(logits, axis=-1)[:, -top_k][:, None]
+                logits = numpy.where(logits < kth, -1e9, logits)
             z = logits - logits.max(-1, keepdims=True)
-            p = backend.np.exp(z).astype(backend.np.float64); p /= p.sum(-1, keepdims=True)     # softmax → probabilities
-            nxt = backend.np.array([[backend.np.random.choice(len(pr), p=pr)] for pr in p])  # sample
-            idx = backend.np.concatenate([idx, nxt], axis=1)         # append, feed back in
-        return idx                               
+            p = numpy.exp(z).astype(numpy.float64); p /= p.sum(-1, keepdims=True)
+            nxt = numpy.array([[numpy.random.choice(len(pr), p=pr)] for pr in p])
+            idx = numpy.concatenate([idx, nxt], axis=1)
+        return idx                              

{cogforge_engine-1.0.4 → cogforge_engine-1.0.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cogforge-engine" 
-version = "1.0.4"
+version = "1.0.6"
 authors = [
   { name="Avik Majumder", email="avikmjd2@gmail.com" },
 ]