rxnn 0.1.71__tar.gz → 0.1.72__tar.gz

{rxnn-0.1.71 → rxnn-0.1.72}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rxnn
-Version: 0.1.71
+Version: 0.1.72
 Summary: RxNN: Reactive Neural Networks Platform
 License: Apache-2.0
 Keywords: deep-learning,ai,machine-learning

{rxnn-0.1.71 → rxnn-0.1.72}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "rxnn"
-version = "0.1.71"
+version = "0.1.72"
 description = "RxNN: Reactive Neural Networks Platform"
 
 license = "Apache-2.0"

{rxnn-0.1.71 → rxnn-0.1.72}/src/rxnn/training/dataset.py

@@ -75,16 +75,30 @@ class BaseDataset(Dataset):
         return self.__class__(subset, self.tokenizer, max_seq_len=self.max_seq_len, hf_field=self.hf_field, **kwargs)
 
     def pre_tokenize(self, verbose: bool = False, log_interval: int = 10_000):
+        """
+        Pre-tokenizes all the items in the dataset, for faster training. Training with pre-tokenized
+        dataset could be even 2x faster.
+
+        !! This method has extremely high memory usage, when used with HuggingFace datasets,
+        because of converting it to list. Additionally, for the most optimal performance,
+        pre-tokenized items are in reversed order - it shouldn't matter for training, as
+        items are shuffled then by DataLoader, but you should keep that in mind in case
+        of reproducibility.
+
+        :param(bool) verbose:
+        :param(int) log_interval: Interval of verbose logs
+        """
         if not self.is_pre_tokenized:
             num_texts = len(self.texts)
             txts = self.texts if isinstance(self.texts, list) else self.texts.to_list()
             del self.texts
+            self.texts = None
             for index in range(num_texts):
-                self.inputs.append(txts.pop())
+                self.inputs.append(txts.pop() if isinstance(txts, list) else txts.pop()[self.hf_field])
                 if verbose and index % log_interval == 0:
                     print(f'Processed {index + 1}/{num_texts}')
             self.is_pre_tokenized = True
-            self.texts = None
+
 
     @classmethod
     def from_hf_hub(