x-transformers 2.4.4__py3-none-any.whl → 2.4.6__py3-none-any.whl

x_transformers/up_wrapper.py

@@ -6,7 +6,7 @@ from functools import partial
 from random import randrange, uniform
 
 import torch
-from torch import nn, cat, randperm
+from torch import nn, cat, tensor, randperm
 from torch.nn import LSTM, Module
 
 from x_transformers.x_transformers import (
@@ -76,6 +76,13 @@ class SyntheticDataGenerator(Module):
 
         self.apply(self.init_)
 
+    def reset_(self):
+        for m in self.modules():
+            if hasattr(m, 'reset_parameters'):
+                m.reset_parameters()
+
+        self.apply(self.init_)
+
     @torch.no_grad()
     def init_(self, m):
         if isinstance(m, nn.Linear):
@@ -133,12 +140,13 @@ class UniversalPretrainWrapper(Module):
     def __init__(
         self,
         model: TransformerWrapper,
-        data_generator: SyntheticDataGenerator | None = None,
+        data_generator: SyntheticDataGenerator | Module | None = None,
         buffer_size = None,
         num_reset = 20,
         batch_size = 32,
         seq_len = 512,
-        seed_length = 8
+        seed_length = 8,
+        reset_turing_machine_every = 0
     ):
         super().__init__()
 
@@ -157,6 +165,8 @@ class UniversalPretrainWrapper(Module):
                 max_seq_len = seq_len
             )
 
+        self.reset_turing_machine_every = reset_turing_machine_every
+
         self.seq_len = seq_len
         self.data_generator = data_generator
 
@@ -176,6 +186,7 @@ class UniversalPretrainWrapper(Module):
         init_data_buffer = self.random_sequences_fn(buffer_size // 2, buffer_size // 2)
 
         self.register_buffer('synth_data_buffer', init_data_buffer)
+        self.register_buffer('step', tensor(0))
 
     @property
     def device(self):
@@ -210,6 +221,13 @@ class UniversalPretrainWrapper(Module):
             seed = seeds
         )
 
+        self.step.add_(1)
+
+        # maybe reset turing machine
+
+        if self.reset_turing_machine_every > 0 and divisible_by(self.step.item(), self.reset_turing_machine_every):
+            self.data_generator.reset_()
+
         # reset
 
         if self.num_reset > 0:

{x_transformers-2.4.4.dist-info → x_transformers-2.4.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.4.4
+Version: 2.4.6
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers

{x_transformers-2.4.4.dist-info → x_transformers-2.4.6.dist-info}/RECORD

@@ -8,11 +8,11 @@ x_transformers/entropy_based_tokenizer.py,sha256=F2lO8-v3aLIcVDVNhu7RR-UtRdlmaaY
 x_transformers/multi_input.py,sha256=tCh-fTJDj2ib4SMGtsa-AM8MxKzJAQSwqAXOu3HU2mg,9252
 x_transformers/neo_mlp.py,sha256=XCNnnop9WLarcxap1kGuYc1x8GHvwkZiDRnXOxSl3Po,3452
 x_transformers/nonautoregressive_wrapper.py,sha256=2NU58hYMgn-4Jzg3mie-mXb0XH_dCN7fjlzd3K1rLUY,10510
-x_transformers/up_wrapper.py,sha256=g3QqhHwfII6i_tCNDG4GqnP55qEFpSuVYCHsGRDEpcs,6094
+x_transformers/up_wrapper.py,sha256=mUTbuTPrN_Z1j2qZ0NGX96djioANNGVdfCdIlg8nCsM,6664
 x_transformers/x_transformers.py,sha256=9Fi0HvzpeIJqM6HlAd2M6JqsfjhTN1zEH9iFIimyjS4,117608
 x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
 x_transformers/xval.py,sha256=AwwYUm8yDAtKQyKJDIhYMsiLTJ_skh3scUFMjp5sda8,8597
-x_transformers-2.4.4.dist-info/METADATA,sha256=WDrMULZIyiielXo44Pc4bicJV6ISYaIXcAIhZZ5Xwgw,90223
-x_transformers-2.4.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-x_transformers-2.4.4.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-2.4.4.dist-info/RECORD,,
+x_transformers-2.4.6.dist-info/METADATA,sha256=RlLYhhqc-kLKG4PRDHqbPeVmac4oVQjnqAANxq_nXrg,90223
+x_transformers-2.4.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+x_transformers-2.4.6.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-2.4.6.dist-info/RECORD,,