x-transformers 2.11.0__py3-none-any.whl → 2.11.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of x-transformers might be problematic. Click here for more details.

Files changed (6) hide show
  1. x_transformers/free_transformer.py +1 -1
  2. x_transformers/gpt_vae.py +8 -2
  3. {x_transformers-2.11.0.dist-info → x_transformers-2.11.2.dist-info}/METADATA +1 -1
  4. {x_transformers-2.11.0.dist-info → x_transformers-2.11.2.dist-info}/RECORD +6 -6
  5. {x_transformers-2.11.0.dist-info → x_transformers-2.11.2.dist-info}/WHEEL +0 -0
  6. {x_transformers-2.11.0.dist-info → x_transformers-2.11.2.dist-info}/licenses/LICENSE +0 -0
x_transformers/free_transformer.py CHANGED
@@ -303,7 +303,7 @@ class FreeTransformer(Module):
303
303
 
304
304
  # decoder tail
305
305
 
306
- tokens = self.decoder_tail(tokens)
306
+ tokens = self.decoder_tail(tokens + condition)
307
307
 
308
308
  # cross entropy loss
309
309
 
x_transformers/gpt_vae.py CHANGED
@@ -44,6 +44,7 @@ class GPTVAE(Module):
44
44
  enc_kwargs: dict = dict(),
45
45
  dec_kwargs: dict = dict(),
46
46
  vae_kl_loss_weight = 1.,
47
+ vae_kl_div_floor = 0., # what was done in free transformer, which in turn came from Kingma 2016
47
48
  latents_dropout_prob = 0.5, # what percentage of the time to dropout the latents completely
48
49
  pad_id = -1,
49
50
  encoder: Module | None = None,
@@ -99,6 +100,7 @@ class GPTVAE(Module):
99
100
 
100
101
  # loss weights - vae kl loss
101
102
 
103
+ self.vae_kl_div_floor = vae_kl_div_floor
102
104
  self.vae_kl_loss_weight = vae_kl_loss_weight
103
105
 
104
106
  self.latents_dropout = nn.Dropout(latents_dropout_prob)
@@ -190,12 +192,16 @@ class GPTVAE(Module):
190
192
 
191
193
  # vae kl loss
192
194
 
193
- vae_kl_loss = (
195
+ vae_kl_loss = 0.5 * (
194
196
  latents_log_var.exp()
195
197
  + latents_mean.square()
196
198
  - latents_log_var
197
199
  - 1.
198
- ).sum(dim = -1).mean()
200
+ )
201
+
202
+ vae_kl_loss = F.relu(vae_kl_loss - self.vae_kl_div_floor)
203
+
204
+ vae_kl_loss = vae_kl_loss.sum(dim = -1).mean()
199
205
 
200
206
  # return losses
201
207
 
{x_transformers-2.11.0.dist-info → x_transformers-2.11.2.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: x-transformers
3
- Version: 2.11.0
3
+ Version: 2.11.2
4
4
  Summary: X-Transformers
5
5
  Project-URL: Homepage, https://pypi.org/project/x-transformers/
6
6
  Project-URL: Repository, https://github.com/lucidrains/x-transformers
{x_transformers-2.11.0.dist-info → x_transformers-2.11.2.dist-info}/RECORD RENAMED
@@ -5,8 +5,8 @@ x_transformers/belief_state_wrapper.py,sha256=YLUMk6t2MhFBEw5lHDDHJHcoCxTIkHvxTN
5
5
  x_transformers/continuous.py,sha256=WwpQCjyVY4PtuEAOFY68zqgklbF9I7AL5w6874YlDe8,13249
6
6
  x_transformers/dpo.py,sha256=xt4OuOWhU8pN3OKN2LZAaC2NC8iiEnchqqcrPWVqf0o,3521
7
7
  x_transformers/entropy_based_tokenizer.py,sha256=F2lO8-v3aLIcVDVNhu7RR-UtRdlmaaYJzBK9m7OnLE8,5018
8
- x_transformers/free_transformer.py,sha256=SvltfYD3K4kgy0hbdsy0S8NQ24x1wXVF1GvkF5O9GGU,8283
9
- x_transformers/gpt_vae.py,sha256=myYSgcx66V0M4zeEGKyhY1P2HlPDHcezhaZEoo_uMdo,5715
8
+ x_transformers/free_transformer.py,sha256=K9dX9xj0pJgiOi9jlOLCD9Nn9eYNNgTWj9YvQLhexHw,8295
9
+ x_transformers/gpt_vae.py,sha256=4QdznXZcU7pmMXUeEocAOKpcTkREYS-zDHktN5ADtNk,5981
10
10
  x_transformers/multi_input.py,sha256=tCh-fTJDj2ib4SMGtsa-AM8MxKzJAQSwqAXOu3HU2mg,9252
11
11
  x_transformers/neo_mlp.py,sha256=XCNnnop9WLarcxap1kGuYc1x8GHvwkZiDRnXOxSl3Po,3452
12
12
  x_transformers/nonautoregressive_wrapper.py,sha256=hMQqNimGtchNIe13cR5LZule1V7I1qM5LmY8VQfVdnA,11698
@@ -14,7 +14,7 @@ x_transformers/up_wrapper.py,sha256=YC2LN14_7Xx9Wtiek2rtEJ_qHqdfSmKlh3d7Cgxwd80,
14
14
  x_transformers/x_transformers.py,sha256=ADr83Fz2cehj_F7N1bMwxhAg-r48fGhlaZqw3hxoxMQ,125765
15
15
  x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
16
16
  x_transformers/xval.py,sha256=AwwYUm8yDAtKQyKJDIhYMsiLTJ_skh3scUFMjp5sda8,8597
17
- x_transformers-2.11.0.dist-info/METADATA,sha256=22nan4aatJ8_zRwYeORRCCA733rUqInhp3oATAABLMw,96011
18
- x_transformers-2.11.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
19
- x_transformers-2.11.0.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
20
- x_transformers-2.11.0.dist-info/RECORD,,
17
+ x_transformers-2.11.2.dist-info/METADATA,sha256=oDAvV32UGA_1ajVnH8qVo2lAt0tC9rMPI8ssKzA6IVE,96011
18
+ x_transformers-2.11.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
19
+ x_transformers-2.11.2.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
20
+ x_transformers-2.11.2.dist-info/RECORD,,