rxnn 0.1.73__tar.gz → 0.1.75__tar.gz

{rxnn-0.1.73 → rxnn-0.1.75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rxnn
-Version: 0.1.73
+Version: 0.1.75
 Summary: RxNN: Reactive Neural Networks Platform
 License: Apache-2.0
 Keywords: deep-learning,ai,machine-learning

{rxnn-0.1.73 → rxnn-0.1.75}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "rxnn"
-version = "0.1.73"
+version = "0.1.75"
 description = "RxNN: Reactive Neural Networks Platform"
 
 license = "Apache-2.0"

{rxnn-0.1.73 → rxnn-0.1.75}/src/rxnn/training/dataset.py

@@ -76,7 +76,7 @@ class BaseDataset(Dataset):
             self.texts = self.texts[0:split_point] if not from_start else self.texts[split_point:-1]
         return self.__class__(subset, self.tokenizer, max_seq_len=self.max_seq_len, hf_field=self.hf_field, **kwargs)
 
-    def pre_tokenize(self, verbose: bool = False, log_interval: int = 10_000):
+    def pre_tokenize(self, verbose: bool = False, log_interval: int = 10_000, map_hf_ds_to_list: bool = True):
         """
         Pre-tokenizes all the items in the dataset, for faster training. Training with pre-tokenized
         dataset could be even 2x faster.
@@ -92,14 +92,22 @@ class BaseDataset(Dataset):
         """
         if not self.is_pre_tokenized:
             num_texts = len(self.texts)
-            txts = self.texts if isinstance(self.texts, list) else self.texts.to_list()
-            del self.texts
-            self.texts = None
-            for index in range(num_texts):
-                item = txts.pop() if isinstance(txts, list) else txts.pop()[self.hf_field]
-                self.inputs.append(self.get_tokenized_text(index, txt=item))
-                if verbose and index % log_interval == 0:
-                    print(f'Processed {index + 1}/{num_texts}')
+            is_txt_list = isinstance(self.texts, list)
+            if is_txt_list or map_hf_ds_to_list:
+                txts = self.texts if is_txt_list else self.texts.to_list()
+                del self.texts
+                self.texts = None
+                for index in range(num_texts):
+                    item = txts.pop() if is_txt_list else txts.pop()[self.hf_field]
+                    self.inputs.append(self.get_tokenized_text(index, txt=item))
+                    if verbose and index % log_interval == 0:
+                        print(f'Processed {index + 1}/{num_texts}')
+            else:
+                for index in range(num_texts):
+                    self.inputs.append(self.get_tokenized_text(index))
+                    del self.texts[index]
+                    if verbose and index % log_interval == 0:
+                        print(f'Processed {index + 1}/{num_texts}')
             self.is_pre_tokenized = True