rxnn 0.1.66__tar.gz → 0.1.68__tar.gz

{rxnn-0.1.66 → rxnn-0.1.68}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rxnn
-Version: 0.1.66
+Version: 0.1.68
 Summary: RxNN: Reactive Neural Networks Platform
 License: Apache-2.0
 Keywords: deep-learning,ai,machine-learning

{rxnn-0.1.66 → rxnn-0.1.68}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "rxnn"
-version = "0.1.66"
+version = "0.1.68"
 description = "RxNN: Reactive Neural Networks Platform"
 
 license = "Apache-2.0"

{rxnn-0.1.66 → rxnn-0.1.68}/src/rxnn/training/dataset.py

@@ -15,7 +15,7 @@ class BaseDataset(Dataset):
             max_seq_len: int = 1024,
             hf_field: str = 'text',
             cache_tokenized: bool = False,
-            cache_remove_text: bool = False,
+            cache_remove_text: bool = True,
             *args,
             **kwargs
     ):
@@ -29,6 +29,9 @@ class BaseDataset(Dataset):
         self.cache_remove_text = cache_remove_text
         self.inputs = [] if self.cache_tokenized else None
 
+    def __len__(self):
+        return len(self.texts if not self.is_pre_tokenized else self.inputs)
+
     def get_tokenized_text(self, idx: int):
         if self.is_pre_tokenized:
             return self.inputs[idx]
@@ -61,15 +64,15 @@ class BaseDataset(Dataset):
 
             return inputs
 
-    def get_subset(self, size: float, from_start: bool = False, use_hf_select: bool = False, **kwargs) -> "BaseDataset":
+    def get_subset(self, size: float, from_start: bool = False, **kwargs) -> "BaseDataset":
         split_point = int(len(self.texts) * ((1 - size) if not from_start else size))
-        if use_hf_select and not isinstance(self.texts, list):
+        if not isinstance(self.texts, list):
             subset = self.texts.select(range(split_point, len(self.texts)) if not from_start else range(split_point))
             self.texts = self.texts.select(range(split_point) if not from_start else range(split_point, len(self.texts)))
         else:
-            subset = self.texts[split_point:] if not from_start else self.texts[:split_point]
-            self.texts = self.texts[:split_point] if not from_start else self.texts[split_point:]
-        return self.__class__(subset, self.tokenizer, self.max_seq_len, self.hf_field, **kwargs)
+            subset = self.texts[split_point:-1] if not from_start else self.texts[0:split_point]
+            self.texts = self.texts[0:split_point] if not from_start else self.texts[split_point:-1]
+        return self.__class__(subset, self.tokenizer, max_seq_len=self.max_seq_len, hf_field=self.hf_field, **kwargs)
 
     def pre_tokenize(self, remove_texts: bool = True):
         if not self.is_pre_tokenized:
@@ -213,9 +216,6 @@ class JointLMDataset(BaseDataset):
             'attention_mask': attention_mask,
         }
 
-    def __len__(self):
-        return len(self.texts)
-
 
 class MaskedLMDataset(BaseDataset):
     def __init__(
@@ -253,9 +253,6 @@ class MaskedLMDataset(BaseDataset):
             'labels': labels
         }
 
-    def __len__(self):
-        return len(self.texts)
-
 
 class AutoregressiveLMDataset(BaseDataset):
     def __init__(
@@ -281,6 +278,3 @@ class AutoregressiveLMDataset(BaseDataset):
             'attention_mask': attention_mask,
             'targets': targets
         }
-
-    def __len__(self):
-        return len(self.texts)