x-transformers 1.30.1__tar.gz → 1.30.2__tar.gz

{x_transformers-1.30.1/x_transformers.egg-info → x_transformers-1.30.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.30.1
+Version: 1.30.2
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.30.1 → x_transformers-1.30.2}/README.md

@@ -693,7 +693,7 @@ model = TransformerWrapper(
 )
 ```
 
-If you wish to do something more sophisticated, say 3 layers, with each layer recurrent 4 times before onto the next, that is possible as well. Be aware the `layers_execute_order` is 0-indexed
+If you wish to do something more sophisticated, say 3 layers, with each layer recurrent 4 times before onto the next (similar to <a href="https://arxiv.org/abs/2405.15071">this paper</a>), that is possible as well. Be aware the `layers_execute_order` is 0-indexed
 
 ```python
 import torch

{x_transformers-1.30.1 → x_transformers-1.30.2}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
   name = 'x-transformers',
   packages = find_packages(exclude=['examples']),
-  version = '1.30.1',
+  version = '1.30.2',
   license='MIT',
   description = 'X-Transformers - Pytorch',
   author = 'Phil Wang',

{x_transformers-1.30.1 → x_transformers-1.30.2}/x_transformers/x_transformers.py

@@ -468,7 +468,8 @@ def rotate_half(x):
 
 @autocast(enabled = False)
 def apply_rotary_pos_emb(t, freqs, scale = 1):
-    rot_dim, seq_len = freqs.shape[-1], t.shape[-2]
+    rot_dim, seq_len, orig_dtype = freqs.shape[-1], t.shape[-2], t.dtype
+
     freqs = freqs[-seq_len:, :]
     scale = scale[-seq_len:, :] if isinstance(scale, torch.Tensor) else scale
 
@@ -478,7 +479,9 @@ def apply_rotary_pos_emb(t, freqs, scale = 1):
     # partial rotary embeddings, Wang et al. GPT-J
     t, t_unrotated = t[..., :rot_dim], t[..., rot_dim:]
     t = (t * freqs.cos() * scale) + (rotate_half(t) * freqs.sin() * scale)
-    return torch.cat((t, t_unrotated), dim = -1)
+    out = torch.cat((t, t_unrotated), dim = -1)
+
+    return out.type(orig_dtype)
 
 # norms
 

{x_transformers-1.30.1 → x_transformers-1.30.2/x_transformers.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.30.1
+Version: 1.30.2
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang