broccoli-ml 9.4.1__tar.gz → 9.5.0__tar.gz

{broccoli_ml-9.4.1 → broccoli_ml-9.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: broccoli-ml
-Version: 9.4.1
+Version: 9.5.0
 Summary: Some useful Pytorch models, circa 2025
 License: MIT
 Author: Nicholas Bailey

{broccoli_ml-9.4.1 → broccoli_ml-9.5.0}/broccoli/linear.py

@@ -202,7 +202,9 @@ class RecyclingLinear(nn.Module):
             idx_tensor = indices
 
         if idx_tensor.size(0):
+            value_indices = indices
             centred_value_weights = self._mean_value_weights()
+            centred_value_weights = centred_value_weights.expand(indices.size(0), -1)
             if self.xglu:
                 gate_indices = indices
                 value_indices = indices + (self.linear.out_features // 2)
@@ -226,11 +228,13 @@ class RecyclingLinear(nn.Module):
             idx_tensor = indices
 
         if idx_tensor.size(0):
-            zeros = torch.zeros(
-                (self.linear.weight.size(0), indices.size(0)),
-                device=self.linear.weight.device,
+            random_weights = self._random_weights(
+                self.linear.weight.size(0), indices.size(0)
             )
-            self._update_weights(indices, 1, zeros, self.optimisers)  # dim
+            random_weights *= (
+                0.01  # Make them quiet so they don't introduce loud noise!
+            )
+            self._update_weights(indices, 1, random_weights, self.optimisers)  # dim
         else:
             return
 
@@ -283,7 +287,10 @@ class RecyclingLinear(nn.Module):
         """
         weights = self.linear.weight.data
         rows = weights.size(0)
-        return self.linear.weight[int(rows / 2) :].data.mean(dim=0, keepdim=True)
+        if self.xglu:
+            return self.linear.weight[int(rows / 2) :].data.mean(dim=0, keepdim=True)
+        else:
+            return self.linear.weight.data.mean(dim=0, keepdim=True)
 
     def _mean_gate_weights(self):
         """

{broccoli_ml-9.4.1 → broccoli_ml-9.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "broccoli-ml"
-version = "9.4.1"
+version = "9.5.0"
 description = "Some useful Pytorch models, circa 2025"
 authors = [
     {name = "Nicholas Bailey"}