rxnn 0.1.71__py3-none-any.whl → 0.1.73__py3-none-any.whl

rxnn/training/dataset.py

@@ -32,11 +32,13 @@ class BaseDataset(Dataset):
     def __len__(self):
         return len(self.texts if not self.is_pre_tokenized else self.inputs)
 
-    def get_tokenized_text(self, idx: int):
+    def get_tokenized_text(self, idx: int, txt: str = None):
         if self.is_pre_tokenized:
             return self.inputs[idx]
         else:
-            if isinstance(self.texts, list):
+            if txt:
+                text = txt
+            elif isinstance(self.texts, list):
                 text = self.texts[idx]
             else:
                 text = self.texts[idx][self.hf_field]
@@ -75,16 +77,31 @@ class BaseDataset(Dataset):
         return self.__class__(subset, self.tokenizer, max_seq_len=self.max_seq_len, hf_field=self.hf_field, **kwargs)
 
     def pre_tokenize(self, verbose: bool = False, log_interval: int = 10_000):
+        """
+        Pre-tokenizes all the items in the dataset, for faster training. Training with pre-tokenized
+        dataset could be even 2x faster.
+
+        !! This method has extremely high memory usage, when used with HuggingFace datasets,
+        because of converting it to list. Additionally, for the most optimal performance,
+        pre-tokenized items are in reversed order - it shouldn't matter for training, as
+        items are shuffled then by DataLoader, but you should keep that in mind in case
+        of reproducibility.
+
+        :param(bool) verbose:
+        :param(int) log_interval: Interval of verbose logs
+        """
         if not self.is_pre_tokenized:
             num_texts = len(self.texts)
             txts = self.texts if isinstance(self.texts, list) else self.texts.to_list()
             del self.texts
+            self.texts = None
             for index in range(num_texts):
-                self.inputs.append(txts.pop())
+                item = txts.pop() if isinstance(txts, list) else txts.pop()[self.hf_field]
+                self.inputs.append(self.get_tokenized_text(index, txt=item))
                 if verbose and index % log_interval == 0:
                     print(f'Processed {index + 1}/{num_texts}')
             self.is_pre_tokenized = True
-            self.texts = None
+
 
     @classmethod
     def from_hf_hub(

{rxnn-0.1.71.dist-info → rxnn-0.1.73.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rxnn
-Version: 0.1.71
+Version: 0.1.73
 Summary: RxNN: Reactive Neural Networks Platform
 License: Apache-2.0
 Keywords: deep-learning,ai,machine-learning

{rxnn-0.1.71.dist-info → rxnn-0.1.73.dist-info}/RECORD

@@ -12,7 +12,7 @@ rxnn/training/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rxnn/training/base.py,sha256=xPMA2Bg9-oUZvSZg67ls2p7Gk9pZ9IHUiIJwUzSe2K8,11766
 rxnn/training/bml.py,sha256=S1ZaXTybzeJH7uVFamCr4TPl2bLyZ5xmn_lSsjThTiM,19162
 rxnn/training/callbacks.py,sha256=_YfMKY_eFdc-tubhO9nYH2PXDZDQwlSI74FVOoCXpQg,22108
-rxnn/training/dataset.py,sha256=XSA3WP5dhg-VhgkdyZ-3iueleWbqQhungIVWbXLkyeg,11644
+rxnn/training/dataset.py,sha256=Vak29XMVGOnu7okim_6Lo0E9Jz95U-scOPi-5d8alHg,12477
 rxnn/training/scheduler.py,sha256=ow6oALzWjWQmHSpcJEjv6tg4g4CDMvr73TypxfcefMc,712
 rxnn/training/tokenizer.py,sha256=umaLByMBx_NMrQElA45HLm9gkuzyKWDTFaKVd-CjXl0,8344
 rxnn/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -25,7 +25,7 @@ rxnn/transformers/moe.py,sha256=j6jEx6Ip0zttlUZKKn82azxo95lkLZs-H2GLSMD88hY,5859
 rxnn/transformers/positional.py,sha256=2l38RS0Dini3f6Z3LUHr3XwWzg1UK7fO2C6wazWDAYU,4292
 rxnn/transformers/sampler.py,sha256=poWBpxg1iuK5gEJtxHkk5VVfS9V48hs2Olqdhy_Gw8c,6548
 rxnn/utils.py,sha256=d5U8i5ukovgDyqiycc2AoxObTz_eF_bgo2MKvdtJ98s,467
-rxnn-0.1.71.dist-info/LICENSE,sha256=C8coDFIUYuOcke4JLPwTqahQUCyXyGq6WOaigOkx8tY,11275
-rxnn-0.1.71.dist-info/METADATA,sha256=e0Hw8eCtfl26grQfTffMWWOXJxMpcraPRuVAbIKwpqQ,16579
-rxnn-0.1.71.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-rxnn-0.1.71.dist-info/RECORD,,
+rxnn-0.1.73.dist-info/LICENSE,sha256=C8coDFIUYuOcke4JLPwTqahQUCyXyGq6WOaigOkx8tY,11275
+rxnn-0.1.73.dist-info/METADATA,sha256=lo-8t4WHBKlvoBFoEzqebdCdZUWhheLsPu-v-pW8CIU,16579
+rxnn-0.1.73.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+rxnn-0.1.73.dist-info/RECORD,,