x-transformers 2.1.32__tar.gz → 2.1.35__tar.gz

{x_transformers-2.1.32 → x_transformers-2.1.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.1.32
+Version: 2.1.35
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers

{x_transformers-2.1.32 → x_transformers-2.1.35}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "x-transformers"
-version = "2.1.32"
+version = "2.1.35"
 description = "X-Transformers"
 authors = [
     { name = "Phil Wang", email = "lucidrains@gmail.com" }

{x_transformers-2.1.32 → x_transformers-2.1.35}/tests/test_x_transformers.py

@@ -743,6 +743,7 @@ def test_belief_state_wrapper(
         lens = torch.randint(4, 16, (2,))
 
     loss = model(seq, lens = lens) # backwards happen automatically
+    loss.backward()
 
     suffix = None
     if goal_suffix:

{x_transformers-2.1.32 → x_transformers-2.1.35}/train_belief_state.py

@@ -106,10 +106,8 @@ for i in tqdm.tqdm(range(NUM_BATCHES), mininterval = 10., desc = 'training'):
     model.train()
 
     for __ in range(GRADIENT_ACCUMULATE_EVERY):
-        loss = model(
-            next(train_loader),
-            loss_scale = 1./ GRADIENT_ACCUMULATE_EVERY
-        )
+        loss = model(next(train_loader))
+        (loss / GRADIENT_ACCUMULATE_EVERY).backward()
 
     print(f'training loss: {loss.item()}')
     torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)
@@ -119,7 +117,7 @@ for i in tqdm.tqdm(range(NUM_BATCHES), mininterval = 10., desc = 'training'):
     if i % VALIDATE_EVERY == 0:
         model.eval()
         with torch.no_grad():
-            loss = model(next(val_loader), return_loss_only = True)
+            loss = model(next(val_loader))
             print(f'validation loss: {loss.item()}')
 
     if i % GENERATE_EVERY == 0:

{x_transformers-2.1.32 → x_transformers-2.1.35}/x_transformers/belief_state_wrapper.py

@@ -5,6 +5,7 @@
 # https://www.youtube.com/watch?v=aqhbRtB2Fyg
 
 from __future__ import annotations
+from random import random
 
 import torch
 from torch.autograd import Function
@@ -53,6 +54,26 @@ def flip(x, dim = 1, lens = None):
 
     return x.gather(dim, flip_indices)
 
+# detach multiple tensors and backward the gradients once
+
+class DetachMultiple(Function):
+
+    @classmethod
+    def forward(self, ctx, *tensors):
+        detached_tensors = tuple(t.detach() for t in tensors)
+
+        for detached_tensor in detached_tensors:
+            detached_tensor.requires_grad_()
+
+        return detached_tensors
+
+    @classmethod
+    def backward(self, ctx, *grads):
+
+        return grads
+
+detach_multiple = DetachMultiple.apply
+
 # wrappers
 
 class BeliefStateWrapper(Module):
@@ -69,6 +90,8 @@ class BeliefStateWrapper(Module):
         backward_ar_loss_weight: float = 1., # can weigh the training of the backwards decoder differently, perhaps fwd/bwd have a shared backbone etc etc
         pred_distance = False,
         pred_distance_loss_weight: float = 1.,
+        cond_on_distance = False,
+        cond_on_distance_prob = 0.5,
         max_pred_distance = None
     ):
         super().__init__()
@@ -111,6 +134,21 @@ class BeliefStateWrapper(Module):
 
         self.pred_distance_loss_weight = pred_distance_loss_weight
 
+        # conditioning on distance
+
+        assert 0. < cond_on_distance_prob < 1.
+
+        self.cond_on_distance = cond_on_distance
+        self.cond_on_distance_prob = cond_on_distance_prob
+
+        if cond_on_distance:
+            self.to_distance_cond = nn.Sequential(
+                Rearrange('... -> ... 1'),
+                nn.Linear(1, dim),
+                nn.LeakyReLU(),
+                nn.Linear(dim, dim * 2),
+            )
+
         # the two decoders, one which is causal forward, the other causal backwards
 
         self.forward_decoder = forward_decoder
@@ -250,8 +288,6 @@ class BeliefStateWrapper(Module):
         self,
         seq,
         lens: Tensor | None = None, # Int['b']
-        return_loss_only = False,
-        loss_scale = 1.,
         loss_weight_by_fb_indices: callable | None = None
     ):
         batch, seq_len, device = *seq.shape, seq.device
@@ -284,11 +320,7 @@ class BeliefStateWrapper(Module):
 
         # trick to reduce memory on backwards pass
 
-        orig_forward_embeds, forward_embeds = forward_embeds, forward_embeds.detach()
-        orig_backward_embeds, backward_embeds = backward_embeds, backward_embeds.detach()
-
-        forward_embeds.requires_grad_()
-        backward_embeds.requires_grad_()
+        forward_embeds, backward_embeds = detach_multiple(forward_embeds, backward_embeds)
 
         # belief state objective
 
@@ -344,9 +376,19 @@ class BeliefStateWrapper(Module):
             ignore_index = -1
         )
 
-        # maybe predict terminal
+        # maybe condition on distance
+
+        cond_on_distance = self.cond_on_distance and (random() < self.cond_on_distance_prob)
+
+        if cond_on_distance:
+            distance = (bi - fi).float()
+            distance_cond = self.to_distance_cond(distance)
+
+            fb_embeds = fb_embeds * distance_cond
 
-        if exists(self.to_distance_logits):
+        # maybe predict distance
+
+        if exists(self.to_distance_logits) and not cond_on_distance:
             distance_logits = self.to_distance_logits(fb_embeds)
 
             distance_labels = (bi - fi).clamp(max = self.max_pred_distance - 1)
@@ -362,11 +404,6 @@ class BeliefStateWrapper(Module):
                 pred_dist_loss * self.pred_distance_loss_weight
             )
 
-        # maybe early return loss
-
-        if return_loss_only:
-            return loss
-
         # maybe loss weighting
 
         needs_loss_weight = default(self.needs_loss_weight, exists(loss_weight_by_fb_indices))
@@ -392,11 +429,4 @@ class BeliefStateWrapper(Module):
 
             loss = loss.mean()
 
-        # backwards
-
-        (loss * loss_scale).backward()
-
-        orig_forward_embeds.backward(forward_embeds.grad)
-        orig_backward_embeds.backward(backward_embeds.grad)
-
         return loss

{x_transformers-2.1.32 → x_transformers-2.1.35}/x_transformers/x_transformers.py

@@ -2055,7 +2055,7 @@ class AttentionLayers(Module):
         elif use_simple_rmsnorm:
             norm_class = SimpleRMSNorm
         elif use_dynamic_tanh:
-            assert pre_norm, 'only tested for pre-norm'
+            assert pre_norm, 'dynamic tanh norm only tested for pre-norm'
             norm_class = partial(DynamicTanh, init_alpha = dynamic_tanh_init_alpha)
         elif use_adaptive_layernorm:
             norm_need_condition = True