cogforge-engine 0.1.0__tar.gz → 0.2.0__tar.gz

{cogforge_engine-0.1.0 → cogforge_engine-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cogforge-engine
-Version: 0.1.0
+Version: 0.2.0
 Summary: A custom autograd engine and Transformer block built from scratch.
 Project-URL: Homepage, https://github.com/avikmjd2/cogforge
 Author-email: Avik Majumder <avikmjd2@gmail.com>

{cogforge_engine-0.1.0 → cogforge_engine-0.2.0}/cogforge/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 
 from .app import (
     Tensor,

{cogforge_engine-0.1.0 → cogforge_engine-0.2.0}/cogforge/app.py

@@ -1,14 +1,15 @@
 import numpy as np
 
 class Tensor:
-    def __init__(self, array,children=(),requires_grad = True):
-        self.data = np.asarray(array, dtype=float)
+    def __init__(self, array,children=(),requires_grad = True,typed="compressed"):
+        self.data = np.asarray(array, dtype=np.float32 if typed == "compressed" else np.float64)
         self.shape = self.data.shape
         self.requires_grad = requires_grad
         # if self.requires_grad:
-        self.grad = np.zeros(self.shape)
+        self.grad = np.zeros(self.shape,dtype=np.float32 if typed == "compressed" else np.float64)
         self._backwards = lambda:None
         self._children = set(children)
+        self.typed = typed
         
     def __getitem__(self, key):
         out_data = self.data[key]
@@ -221,7 +222,7 @@ class Tensor:
             
         topoSort(self)
         
-        self.grad = np.ones(self.shape)
+        self.grad = np.ones(self.shape,dtype=np.float32 if self.typed == "compressed" else np.float64)
         
         for node in reversed(topo):
             node._backwards()
@@ -244,7 +245,7 @@ class Tensor:
             for child in node._children:
                 stack.append((child,False))
                 
-        self.grad = np.ones(self.shape)
+        self.grad = np.ones(self.shape,dtype=np.float32 if self.typed == "compressed" else np.float64)
         
         for node in reversed(topo):
             node._backwards()
@@ -308,6 +309,22 @@ class Tensor:
 
         loss._backwards = _backward
         return loss
+
+    @classmethod
+    def sparse_softmax_cross_entropy(cls, scores, target_ids):
+        """ scores: (B, T, V) logits ; target_ids: (B, T) int """
+        z = scores.data - scores.data.max(axis=-1, keepdims=True)
+        e = np.exp(z)
+        p = e / e.sum(axis=-1, keepdims=True)
+        N = int(np.prod(scores.shape[:-1]))
+        flat, idx, rows = p.reshape(N, -1), target_ids.reshape(N), np.arange(N)
+        loss = cls(-np.log(np.clip(flat[rows, idx], 1e-15, 1.0)).sum() / N, children=(scores,))
+        def _backward():
+            g = p.reshape(N, -1).copy()
+            g[rows, idx] -= 1.0                      # (softmax - onehot)
+            scores.grad += (g.reshape(scores.shape) / N) * loss.grad
+        loss._backwards = _backward
+        return loss
     
     #Legacy
     @classmethod
@@ -860,5 +877,4 @@ class PositionalEncoding:
     
     def parameters(self):
         return []
-        
-        
+ 

cogforge_engine-0.2.0/cogforge/models.py

@@ -0,0 +1,44 @@
+import numpy as np
+from app import Embedding, PositionalEncoding,Transformer,LayerNorm,Linear
+
+class GPTV1:
+    def __init__(self, vocab, d_model, n_heads, n_layers, max_len, d_ff=None):
+        self.tok = Embedding(vocab, d_model)
+        self.pos    = PositionalEncoding(max_len, d_model)
+        self.blocks = [Transformer(dmodel=d_model,n=n_heads,dff=d_ff) for _ in range(n_layers)]
+        self.ln_f   = LayerNorm(d_model) 
+        self.head   = Linear(d_model, vocab)
+        self.max_len = max_len
+        
+    def __call__(self, idx):
+        T = idx.shape[1]
+        casual = np.triu((np.ones((T,T))),k=1).astype(bool)
+        x = self.pos(self.tok(idx))
+        for block in self.blocks:
+            x = block(x,mask=casual)
+            
+        x = self.ln_f(x)
+        return self.head(x)
+    
+    def parameters(self):
+        ps = self.tok.parameters() + self.ln_f.parameters() + self.head.parameters()
+        for b in self.blocks: ps += b.parameters()
+        return ps
+    
+    def generate(self, idx, n_new, temperature=1.0, top_k=None):
+        """
+        idx: (B, T) int array — the prompt (B is usually 1)
+        returns: (B, T + n_new) int array
+        """
+        for _ in range(n_new):
+            cond = idx[:, -self.max_len:]                    # crop to context window
+            logits = self(cond).data[:, -1, :] / temperature # (B, V) — last position only
+            if top_k is not None:
+                kth = np.sort(logits, axis=-1)[:, -top_k][:, None]
+                logits = np.where(logits < kth, -1e9, logits)  # keep only top-k choices
+            z = logits - logits.max(-1, keepdims=True)
+            p = np.exp(z).astype(np.float64); p /= p.sum(-1, keepdims=True)     # softmax → probabilities
+            nxt = np.array([[np.random.choice(len(pr), p=pr)] for pr in p])  # sample
+            idx = np.concatenate([idx, nxt], axis=1)         # append, feed back in
+        return idx                                            # <-- was a bare `return` 
+        

{cogforge_engine-0.1.0 → cogforge_engine-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cogforge-engine" 
-version = "0.1.0"
+version = "0.2.0"
 authors = [
   { name="Avik Majumder", email="avikmjd2@gmail.com" },
 ]