x-transformers 1.25.10__py3-none-any.whl → 1.25.12__py3-none-any.whl

x_transformers/continuous.py

@@ -85,6 +85,7 @@ class ContinuousTransformerWrapper(nn.Module):
         mems = None,
         pos = None,
         prepend_embeds = None,
+        prepend_mask = None,
         **kwargs
     ):
         batch = x.shape[0]
@@ -112,6 +113,12 @@ class ContinuousTransformerWrapper(nn.Module):
 
             x = torch.cat((prepend_embeds, x), dim = -2)
 
+            if exists(prepend_mask) or exists(mask):
+                mask = default(mask, lambda: torch.ones((b, n), device = device, dtype = torch.bool))
+                prepend_mask = default(prepend_mask, lambda: torch.ones((b, prepend_seq), device = device, dtype = torch.bool))
+
+                mask = torch.cat((prepend_mask, mask), dim = -1)
+
         x = self.emb_dropout(x)
 
         # attention layers

x_transformers/x_transformers.py

@@ -1545,6 +1545,7 @@ class TransformerWrapper(nn.Module):
         mems = None,
         pos = None,
         prepend_embeds = None,
+        prepend_mask = None,
         sum_embeds = None,
         return_attn_z_loss = False,
         attn_z_loss_weight = 1e-4,
@@ -1578,6 +1579,12 @@ class TransformerWrapper(nn.Module):
 
             x = torch.cat((prepend_embeds, x), dim = -2)
 
+            if exists(prepend_mask) or exists(mask):
+                mask = default(mask, lambda: torch.ones((b, n), device = device, dtype = torch.bool))
+                prepend_mask = default(prepend_mask, lambda: torch.ones((b, prepend_seq), device = device, dtype = torch.bool))
+
+                mask = torch.cat((prepend_mask, mask), dim = -1)
+
         # whether to reduce the gradient going to the embedding, from cogview paper, corroborated by GLM-130B model
 
         if emb_frac_gradient < 1:
@@ -1712,11 +1719,11 @@ class XTransformer(nn.Module):
 
     def forward(self, src, tgt, mask = None, attn_mask = None, src_prepend_embeds = None):
 
+        enc = self.encoder(src, mask = mask, attn_mask = attn_mask, prepend_embeds = src_prepend_embeds, return_embeddings = True)
+
         if exists(src_prepend_embeds) and exists(mask):
             mask = pad_at_dim(mask, (src_prepend_embeds.shape[-2], 0), dim = -1, value = True)
 
-        enc = self.encoder(src, mask = mask, attn_mask = attn_mask, prepend_embeds = src_prepend_embeds, return_embeddings = True)
-
         if self.training and self.cross_attn_tokens_dropout > 0:
             enc, mask = dropout_seq(enc, mask, self.cross_attn_tokens_dropout)
 

{x_transformers-1.25.10.dist-info → x_transformers-1.25.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.25.10
+Version: 1.25.12
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.25.10.dist-info → x_transformers-1.25.12.dist-info}/RECORD

@@ -1,13 +1,13 @@
 x_transformers/__init__.py,sha256=pXc_U4M3ONUQcpNgZySDIlCF1rp7u4FFmcOYjc4WuXw,629
 x_transformers/attend.py,sha256=MFl_FbgPsm9mziZPTi_s8QbxASETwbGeciMH8sUIwT8,10188
 x_transformers/autoregressive_wrapper.py,sha256=2mzOq_rl_vevgrxCDncBlVJOAJGS-XGm-iBKJqMjj_c,9041
-x_transformers/continuous.py,sha256=7zo4lnYyIkIYvs_a_NCj86DUA_ZccU5ndjq-13UnEqg,5554
+x_transformers/continuous.py,sha256=s46BlvSHN7OL2Tya28dMomVF3xmpYxj5reerO6tUDoc,5933
 x_transformers/nonautoregressive_wrapper.py,sha256=AQLE4rA_Kh8VNoe9OzpwyeWson34sRkhks4dn4seNjI,10414
-x_transformers/x_transformers.py,sha256=SuGWhp9P-RDT3jnquILbPVeTVBUbXHB22QMFxh8YvxU,60435
+x_transformers/x_transformers.py,sha256=5T5fOdveqe4vwqGRrNZqJ053pmtw-dJ12AV0nNaWLRc,60814
 x_transformers/xl_autoregressive_wrapper.py,sha256=DCx4n0_c1tFai4nOqaWVnqx2p9eutsZsDMiMP1ckxNU,4117
 x_transformers/xval.py,sha256=lS9W_E_RskPQAqVZkPiUzbByoW1Ajsw_phsikA3JDAg,8139
-x_transformers-1.25.10.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-1.25.10.dist-info/METADATA,sha256=tIFJS0oIe-1oRS2-LfC1kd32-bFXL4ZrlvLP9cM1DNc,662
-x_transformers-1.25.10.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-x_transformers-1.25.10.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
-x_transformers-1.25.10.dist-info/RECORD,,
+x_transformers-1.25.12.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-1.25.12.dist-info/METADATA,sha256=0GzTD3LTld07_sgfvdFJHgNQhiE5bnPkyEzc9G12Klc,662
+x_transformers-1.25.12.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+x_transformers-1.25.12.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
+x_transformers-1.25.12.dist-info/RECORD,,

{x_transformers-1.25.10.dist-info → x_transformers-1.25.12.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.3)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any