x-transformers 2.1.34__py3-none-any.whl → 2.1.36__py3-none-any.whl

x_transformers/belief_state_wrapper.py

@@ -54,6 +54,26 @@ def flip(x, dim = 1, lens = None):
 
     return x.gather(dim, flip_indices)
 
+# detach multiple tensors and backward the gradients once
+
+class DetachMultiple(Function):
+
+    @classmethod
+    def forward(self, ctx, *tensors):
+        detached_tensors = tuple(t.detach() for t in tensors)
+
+        for detached_tensor in detached_tensors:
+            detached_tensor.requires_grad_()
+
+        return detached_tensors
+
+    @classmethod
+    def backward(self, ctx, *grads):
+
+        return grads
+
+detach_multiple = DetachMultiple.apply
+
 # wrappers
 
 class BeliefStateWrapper(Module):
@@ -268,8 +288,6 @@ class BeliefStateWrapper(Module):
         self,
         seq,
         lens: Tensor | None = None, # Int['b']
-        return_loss_only = False,
-        loss_scale = 1.,
         loss_weight_by_fb_indices: callable | None = None
     ):
         batch, seq_len, device = *seq.shape, seq.device
@@ -302,11 +320,7 @@ class BeliefStateWrapper(Module):
 
         # trick to reduce memory on backwards pass
 
-        orig_forward_embeds, forward_embeds = forward_embeds, forward_embeds.detach()
-        orig_backward_embeds, backward_embeds = backward_embeds, backward_embeds.detach()
-
-        forward_embeds.requires_grad_()
-        backward_embeds.requires_grad_()
+        forward_embeds, backward_embeds = detach_multiple(forward_embeds, backward_embeds)
 
         # belief state objective
 
@@ -390,11 +404,6 @@ class BeliefStateWrapper(Module):
                 pred_dist_loss * self.pred_distance_loss_weight
             )
 
-        # maybe early return loss
-
-        if return_loss_only:
-            return loss
-
         # maybe loss weighting
 
         needs_loss_weight = default(self.needs_loss_weight, exists(loss_weight_by_fb_indices))
@@ -420,11 +429,4 @@ class BeliefStateWrapper(Module):
 
             loss = loss.mean()
 
-        # backwards
-
-        (loss * loss_scale).backward()
-
-        orig_forward_embeds.backward(forward_embeds.grad)
-        orig_backward_embeds.backward(backward_embeds.grad)
-
         return loss

x_transformers/x_transformers.py

@@ -864,14 +864,15 @@ class DynamicTanh(Module):
         self.gamma = nn.Parameter(torch.ones(dim))
         self.beta = nn.Parameter(torch.zeros(dim))
 
-        self.unit_offset = int(unit_offset)
+        self.pre_tanh_scale_offset = init_alpha if unit_offset else 0.
+        self.gamma_offset = float(unit_offset)
 
-        nn.init.constant_(self.pre_tanh_scale, 1. - float(unit_offset))
+        nn.init.constant_(self.pre_tanh_scale, 0 if unit_offset else init_alpha)
         nn.init.constant_(self.gamma, 1. - float(unit_offset))
 
     def forward(self, x):
-        pre_tanh_scale = self.pre_tanh_scale + self.unit_offset
-        gamma = self.gamma + self.unit_offset
+        pre_tanh_scale = self.pre_tanh_scale + self.pre_tanh_scale_offset
+        gamma = self.gamma + self.gamma_offset
         return (x * pre_tanh_scale).tanh() * gamma + self.beta
 
 # residual and residual gates

{x_transformers-2.1.34.dist-info → x_transformers-2.1.36.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.1.34
+Version: 2.1.36
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers

{x_transformers-2.1.34.dist-info → x_transformers-2.1.36.dist-info}/RECORD

@@ -1,16 +1,16 @@
 x_transformers/__init__.py,sha256=NDoiBivau559WQ0FvXG4ssU3Il9aoHmTIUFN_1juz0s,911
 x_transformers/attend.py,sha256=-5BWWhFsp7tvZTdN91Ay5SqOjyj9uOs-122vFvoO6b4,17253
 x_transformers/autoregressive_wrapper.py,sha256=reLCno9Z9pchVU79tBF8OMo21LwSZ67KAeB83jqkyAc,10505
-x_transformers/belief_state_wrapper.py,sha256=bx9AIyyYQDRfq6UxOQMBEEqSoqVm5cYwqawSmJe5bqk,13414
+x_transformers/belief_state_wrapper.py,sha256=YLUMk6t2MhFBEw5lHDDHJHcoCxTIkHvxTNY__GGZEKU,13374
 x_transformers/continuous.py,sha256=p0sCAiH1na236ygwgL1Yyhu36eZBf9cZvoW1JyP_fFE,7073
 x_transformers/dpo.py,sha256=xt4OuOWhU8pN3OKN2LZAaC2NC8iiEnchqqcrPWVqf0o,3521
 x_transformers/multi_input.py,sha256=tCh-fTJDj2ib4SMGtsa-AM8MxKzJAQSwqAXOu3HU2mg,9252
 x_transformers/neo_mlp.py,sha256=XCNnnop9WLarcxap1kGuYc1x8GHvwkZiDRnXOxSl3Po,3452
 x_transformers/nonautoregressive_wrapper.py,sha256=2NU58hYMgn-4Jzg3mie-mXb0XH_dCN7fjlzd3K1rLUY,10510
-x_transformers/x_transformers.py,sha256=oyYk31qhDyt6cCuDeaHBl4XhUo5bfIwmYE_z1E1OpXU,111502
+x_transformers/x_transformers.py,sha256=voN-uEBEKxpUu9K4MVcneSTrzdgJWnZGuQ1QRZQw4Q4,111596
 x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
 x_transformers/xval.py,sha256=7S00kCuab4tWQa-vf-z-XfzADjVj48MoFIr7VSIvttg,8575
-x_transformers-2.1.34.dist-info/METADATA,sha256=JXvbgbczvnKYCC0ccZxd5_pfT5U68nBBz8aXQRQtukw,88161
-x_transformers-2.1.34.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-x_transformers-2.1.34.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-2.1.34.dist-info/RECORD,,
+x_transformers-2.1.36.dist-info/METADATA,sha256=D0qdMRucK3PWwEi8WwdiJdZ8X_hGTm1r3_7bJzYiWSM,88161
+x_transformers-2.1.36.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+x_transformers-2.1.36.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-2.1.36.dist-info/RECORD,,