x-transformers 2.1.7__py3-none-any.whl → 2.1.10__py3-none-any.whl

x_transformers/belief_state_wrapper.py

@@ -1,8 +1,11 @@
+
 # Belief State Transformer
 
 # https://arxiv.org/abs/2410.23506
 # https://www.youtube.com/watch?v=aqhbRtB2Fyg
 
+from __future__ import annotations
+
 import torch
 from torch.autograd import Function
 from torch.nn import Module, ModuleList
@@ -35,11 +38,14 @@ class BeliefStateWrapper(Module):
     def __init__(
         self,
         forward_decoder: TransformerWrapper,
-        backward_decoder: TransformerWrapper,
+        backward_decoder: TransformerWrapper | None = None,
         train_frac_forward_backward_pairs: float = 1.,
+        text_head: Module | None = None,
         backward_ar_loss_weight: float = 1. # can weigh the training of the backwards decoder differently, perhaps fwd/bwd have a shared backbone etc etc
     ):
         super().__init__()
+        backward_decoder = default(backward_decoder, forward_decoder) # if backward decoder not set, use the same transformer, assume it knows how to switch gears based on suffix token
+
         assert forward_decoder.emb_dim == backward_decoder.emb_dim, 'forward and backwards model must have the same embedding dimension'
         assert forward_decoder.num_tokens == backward_decoder.num_tokens, 'forward and backwards model must have the same number of tokens'
 
@@ -53,11 +59,14 @@ class BeliefStateWrapper(Module):
 
         # the text prediction head, which predicts for the combinations of prefix and suffix the next and previous token for forwards and backward sequences
 
-        self.text_head = nn.Sequential(
-            nn.Linear(dim * 2, dim),
-            nn.LeakyReLU(),
-            nn.Linear(dim, num_tokens * 2),
-        )
+        if not exists(text_head):
+            text_head = nn.Sequential(
+                nn.Linear(dim * 2, dim),
+                nn.LeakyReLU(),
+                nn.Linear(dim, num_tokens * 2),
+            )
+
+        self.text_head = text_head
 
         # the two decoders, one which is causal forward, the other causal backwards
 

{x_transformers-2.1.7.dist-info → x_transformers-2.1.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.1.7
+Version: 2.1.10
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers

{x_transformers-2.1.7.dist-info → x_transformers-2.1.10.dist-info}/RECORD

@@ -1,7 +1,7 @@
 x_transformers/__init__.py,sha256=NDoiBivau559WQ0FvXG4ssU3Il9aoHmTIUFN_1juz0s,911
 x_transformers/attend.py,sha256=-5BWWhFsp7tvZTdN91Ay5SqOjyj9uOs-122vFvoO6b4,17253
 x_transformers/autoregressive_wrapper.py,sha256=reLCno9Z9pchVU79tBF8OMo21LwSZ67KAeB83jqkyAc,10505
-x_transformers/belief_state_wrapper.py,sha256=WoagJe_cHguWAWzhEAmeDD_TzUsQcMQkgP2Mt88wrAg,5827
+x_transformers/belief_state_wrapper.py,sha256=_jGwSjaNOP2ztFU6OR1B623PGyN-BYopYKQbV8b05do,6190
 x_transformers/continuous.py,sha256=p0sCAiH1na236ygwgL1Yyhu36eZBf9cZvoW1JyP_fFE,7073
 x_transformers/dpo.py,sha256=xt4OuOWhU8pN3OKN2LZAaC2NC8iiEnchqqcrPWVqf0o,3521
 x_transformers/multi_input.py,sha256=tCh-fTJDj2ib4SMGtsa-AM8MxKzJAQSwqAXOu3HU2mg,9252
@@ -10,7 +10,7 @@ x_transformers/nonautoregressive_wrapper.py,sha256=2NU58hYMgn-4Jzg3mie-mXb0XH_dC
 x_transformers/x_transformers.py,sha256=-80N4sqUr3sR51Ms4wCfc4jhxnPwf0ApNR4xfIsasfQ,110142
 x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
 x_transformers/xval.py,sha256=7S00kCuab4tWQa-vf-z-XfzADjVj48MoFIr7VSIvttg,8575
-x_transformers-2.1.7.dist-info/METADATA,sha256=D7enhdRYdhTddt4kGSS-rSsvtHgtFaLYQH-cVuYxUVM,87570
-x_transformers-2.1.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-x_transformers-2.1.7.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-2.1.7.dist-info/RECORD,,
+x_transformers-2.1.10.dist-info/METADATA,sha256=g71r5_pP-i2t9v5h5lkDpKVRwVE2SDTWskZAopLw0X8,87571
+x_transformers-2.1.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+x_transformers-2.1.10.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-2.1.10.dist-info/RECORD,,