x-transformers 2.1.32__py3-none-any.whl → 2.1.34__py3-none-any.whl

x_transformers/belief_state_wrapper.py

@@ -5,6 +5,7 @@
 # https://www.youtube.com/watch?v=aqhbRtB2Fyg
 
 from __future__ import annotations
+from random import random
 
 import torch
 from torch.autograd import Function
@@ -69,6 +70,8 @@ class BeliefStateWrapper(Module):
         backward_ar_loss_weight: float = 1., # can weigh the training of the backwards decoder differently, perhaps fwd/bwd have a shared backbone etc etc
         pred_distance = False,
         pred_distance_loss_weight: float = 1.,
+        cond_on_distance = False,
+        cond_on_distance_prob = 0.5,
         max_pred_distance = None
     ):
         super().__init__()
@@ -111,6 +114,21 @@ class BeliefStateWrapper(Module):
 
         self.pred_distance_loss_weight = pred_distance_loss_weight
 
+        # conditioning on distance
+
+        assert 0. < cond_on_distance_prob < 1.
+
+        self.cond_on_distance = cond_on_distance
+        self.cond_on_distance_prob = cond_on_distance_prob
+
+        if cond_on_distance:
+            self.to_distance_cond = nn.Sequential(
+                Rearrange('... -> ... 1'),
+                nn.Linear(1, dim),
+                nn.LeakyReLU(),
+                nn.Linear(dim, dim * 2),
+            )
+
         # the two decoders, one which is causal forward, the other causal backwards
 
         self.forward_decoder = forward_decoder
@@ -344,9 +362,19 @@ class BeliefStateWrapper(Module):
             ignore_index = -1
         )
 
-        # maybe predict terminal
+        # maybe condition on distance
+
+        cond_on_distance = self.cond_on_distance and (random() < self.cond_on_distance_prob)
+
+        if cond_on_distance:
+            distance = (bi - fi).float()
+            distance_cond = self.to_distance_cond(distance)
+
+            fb_embeds = fb_embeds * distance_cond
+
+        # maybe predict distance
 
-        if exists(self.to_distance_logits):
+        if exists(self.to_distance_logits) and not cond_on_distance:
             distance_logits = self.to_distance_logits(fb_embeds)
 
             distance_labels = (bi - fi).clamp(max = self.max_pred_distance - 1)

x_transformers/x_transformers.py

@@ -2055,7 +2055,7 @@ class AttentionLayers(Module):
         elif use_simple_rmsnorm:
             norm_class = SimpleRMSNorm
         elif use_dynamic_tanh:
-            assert pre_norm, 'only tested for pre-norm'
+            assert pre_norm, 'dynamic tanh norm only tested for pre-norm'
             norm_class = partial(DynamicTanh, init_alpha = dynamic_tanh_init_alpha)
         elif use_adaptive_layernorm:
             norm_need_condition = True

{x_transformers-2.1.32.dist-info → x_transformers-2.1.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.1.32
+Version: 2.1.34
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers

{x_transformers-2.1.32.dist-info → x_transformers-2.1.34.dist-info}/RECORD

@@ -1,16 +1,16 @@
 x_transformers/__init__.py,sha256=NDoiBivau559WQ0FvXG4ssU3Il9aoHmTIUFN_1juz0s,911
 x_transformers/attend.py,sha256=-5BWWhFsp7tvZTdN91Ay5SqOjyj9uOs-122vFvoO6b4,17253
 x_transformers/autoregressive_wrapper.py,sha256=reLCno9Z9pchVU79tBF8OMo21LwSZ67KAeB83jqkyAc,10505
-x_transformers/belief_state_wrapper.py,sha256=PAGbBLU8xGmtiv_G6-RhX1Kb1GwxRmIxxkfHUI2l25U,12538
+x_transformers/belief_state_wrapper.py,sha256=bx9AIyyYQDRfq6UxOQMBEEqSoqVm5cYwqawSmJe5bqk,13414
 x_transformers/continuous.py,sha256=p0sCAiH1na236ygwgL1Yyhu36eZBf9cZvoW1JyP_fFE,7073
 x_transformers/dpo.py,sha256=xt4OuOWhU8pN3OKN2LZAaC2NC8iiEnchqqcrPWVqf0o,3521
 x_transformers/multi_input.py,sha256=tCh-fTJDj2ib4SMGtsa-AM8MxKzJAQSwqAXOu3HU2mg,9252
 x_transformers/neo_mlp.py,sha256=XCNnnop9WLarcxap1kGuYc1x8GHvwkZiDRnXOxSl3Po,3452
 x_transformers/nonautoregressive_wrapper.py,sha256=2NU58hYMgn-4Jzg3mie-mXb0XH_dCN7fjlzd3K1rLUY,10510
-x_transformers/x_transformers.py,sha256=YQODc4PDB_ddgm7vi0uktV5GGetgEuwADzt3CaIdAXs,111484
+x_transformers/x_transformers.py,sha256=oyYk31qhDyt6cCuDeaHBl4XhUo5bfIwmYE_z1E1OpXU,111502
 x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
 x_transformers/xval.py,sha256=7S00kCuab4tWQa-vf-z-XfzADjVj48MoFIr7VSIvttg,8575
-x_transformers-2.1.32.dist-info/METADATA,sha256=Jmgj9CByp1_kveLWVm5-QM_n55A9s3MFzUiV1ciD034,88161
-x_transformers-2.1.32.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-x_transformers-2.1.32.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-2.1.32.dist-info/RECORD,,
+x_transformers-2.1.34.dist-info/METADATA,sha256=JXvbgbczvnKYCC0ccZxd5_pfT5U68nBBz8aXQRQtukw,88161
+x_transformers-2.1.34.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+x_transformers-2.1.34.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-2.1.34.dist-info/RECORD,,