rxnn 0.1.72__tar.gz → 0.1.73__tar.gz

{rxnn-0.1.72 → rxnn-0.1.73}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rxnn
-Version: 0.1.72
+Version: 0.1.73
 Summary: RxNN: Reactive Neural Networks Platform
 License: Apache-2.0
 Keywords: deep-learning,ai,machine-learning

{rxnn-0.1.72 → rxnn-0.1.73}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "rxnn"
-version = "0.1.72"
+version = "0.1.73"
 description = "RxNN: Reactive Neural Networks Platform"
 
 license = "Apache-2.0"

{rxnn-0.1.72 → rxnn-0.1.73}/src/rxnn/training/dataset.py

@@ -32,11 +32,13 @@ class BaseDataset(Dataset):
     def __len__(self):
         return len(self.texts if not self.is_pre_tokenized else self.inputs)
 
-    def get_tokenized_text(self, idx: int):
+    def get_tokenized_text(self, idx: int, txt: str = None):
         if self.is_pre_tokenized:
             return self.inputs[idx]
         else:
-            if isinstance(self.texts, list):
+            if txt:
+                text = txt
+            elif isinstance(self.texts, list):
                 text = self.texts[idx]
             else:
                 text = self.texts[idx][self.hf_field]
@@ -94,7 +96,8 @@ class BaseDataset(Dataset):
             del self.texts
             self.texts = None
             for index in range(num_texts):
-                self.inputs.append(txts.pop() if isinstance(txts, list) else txts.pop()[self.hf_field])
+                item = txts.pop() if isinstance(txts, list) else txts.pop()[self.hf_field]
+                self.inputs.append(self.get_tokenized_text(index, txt=item))
                 if verbose and index % log_interval == 0:
                     print(f'Processed {index + 1}/{num_texts}')
             self.is_pre_tokenized = True