rxnn 0.1.68__tar.gz → 0.1.70__tar.gz

{rxnn-0.1.68 → rxnn-0.1.70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rxnn
-Version: 0.1.68
+Version: 0.1.70
 Summary: RxNN: Reactive Neural Networks Platform
 License: Apache-2.0
 Keywords: deep-learning,ai,machine-learning

{rxnn-0.1.68 → rxnn-0.1.70}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "rxnn"
-version = "0.1.68"
+version = "0.1.70"
 description = "RxNN: Reactive Neural Networks Platform"
 
 license = "Apache-2.0"

{rxnn-0.1.68 → rxnn-0.1.70}/src/rxnn/training/dataset.py

@@ -27,7 +27,7 @@ class BaseDataset(Dataset):
         self.is_pre_tokenized = False
         self.cache_tokenized = cache_tokenized
         self.cache_remove_text = cache_remove_text
-        self.inputs = [] if self.cache_tokenized else None
+        self.inputs = []
 
     def __len__(self):
         return len(self.texts if not self.is_pre_tokenized else self.inputs)
@@ -74,13 +74,15 @@ class BaseDataset(Dataset):
             self.texts = self.texts[0:split_point] if not from_start else self.texts[split_point:-1]
         return self.__class__(subset, self.tokenizer, max_seq_len=self.max_seq_len, hf_field=self.hf_field, **kwargs)
 
-    def pre_tokenize(self, remove_texts: bool = True):
+    def pre_tokenize(self, verbose: bool = False, log_interval: int = 10_000):
         if not self.is_pre_tokenized:
-            self.inputs = list(map(lambda idx: self.get_tokenized_text(idx), range(len(self.texts))))
+            num_texts = len(self.texts)
+            for index in range(num_texts):
+                self.inputs.append(self.texts.pop())
+                if verbose and index % log_interval == 0:
+                    print(f'Processed {index + 1}/{num_texts}')
             self.is_pre_tokenized = True
-            if remove_texts:
-                del self.texts
-                self.texts = None
+            self.texts = None
 
     @classmethod
     def from_hf_hub(