wolof-translate 0.0.1__tar.gz → 0.0.2__tar.gz

{wolof_translate-0.0.1 → wolof_translate-0.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wolof_translate
-Version: 0.0.1
+Version: 0.0.2
 Summary: Contain function and classes to process corpora for making translation between wolof text and other languages.
 Author: Oumar Kane
 Author-email: oumar.kane@univ-thies.sn

{wolof_translate-0.0.1 → wolof_translate-0.0.2}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="wolof_translate",
-    version="0.0.1",
+    version="0.0.2",
     author="Oumar Kane",
     author_email="oumar.kane@univ-thies.sn",
     description="Contain function and classes to process corpora for making translation between wolof text and other languages.",

{wolof_translate-0.0.1 → wolof_translate-0.0.2}/wolof_translate/utils/bucket_iterator.py

@@ -1,84 +1,79 @@
-
 import torch
 import numpy as np
-from typing import *
+from typing import Optional, List, Iterator
 from torch.utils.data import Sampler
-from torch.nn.utils.rnn import pad_sequence
 from math import ceil
 
-class SequenceLengthBatchSampler(Sampler):
-    def __init__(self, dataset, boundaries, batch_sizes, input_key = None, label_key = None, drop_unique = True):
+class SequenceLengthBatchSampler(Sampler[List[int]]):
+    def __init__(
+        self,
+        dataset,
+        boundaries: List[int],
+        batch_sizes: List[int],
+        input_key: Optional[int] = None,
+        label_key: Optional[int] = None,
+        drop_unique: bool = True,
+    ):
         self.dataset = dataset
         self.boundaries = boundaries
         self.batch_sizes = batch_sizes
-        self.data_info = {}
         self.drop_unique = drop_unique
+        self.data_info = {}
 
-        # Initialize dictionary with indices and element lengths
+        # Extract lengths
         for i in range(len(dataset)):
             data = dataset[i]
-            length = (
-                max(len(data[0]), len(data[2]))
-                if (input_key is None and label_key is None)
-                else max(len(data[input_key]), len(data[label_key]))
-            )
+            if input_key is None or label_key is None:
+                length = max(len(data[0]), len(data[2]))
+            else:
+                length = max(len(data[input_key]), len(data[label_key]))
             self.data_info[i] = {"index": i, "length": length}
-            
+
         self.calculate_length()
 
     def calculate_length(self):
         self.batches = []
-
-        # Sort indices based on element length
         sorted_indices = sorted(self.data_info.keys(), key=lambda i: self.data_info[i]["length"])
-        
-        # Group indices into batches of sequences with the same length
+
+        prev_boundary = 0
         for boundary in self.boundaries:
-            batch = [i for i in sorted_indices if self.data_info[i]["length"] <= boundary]  # Filter indices based on length boundary
+            batch = [i for i in sorted_indices if prev_boundary < self.data_info[i]["length"] <= boundary]
             self.batches.append(batch)
-            sorted_indices = [i for i in sorted_indices if i not in batch]  # Remove processed indices
+            sorted_indices = [i for i in sorted_indices if i not in batch]
+            prev_boundary = boundary
 
-        # Add remaining indices to the last batch
+        # Remaining sequences > last boundary
         self.batches.append(sorted_indices)
 
-        # Calculate the total length of the data loader
-        self.length = sum(ceil(len(batch) / batch_size) for batch, batch_size in zip(self.batches, self.batch_sizes) if len(batch) % batch_size != 1 or not self.drop_unique)
-
-    def __iter__(self):
-#         indices = list(self.data_info.keys())  # Get indices from the data_info dictionary
-#         np.random.shuffle(indices)  # Shuffle the indices
+        total_batches = 0
+        for batch, batch_size in zip(self.batches, self.batch_sizes):
+            n_full_batches = len(batch) // batch_size
+            leftover = len(batch) % batch_size
+            total_batches += n_full_batches
+            if leftover > 0 and (leftover != 1 or not self.drop_unique):
+                total_batches += 1
+        self.length = total_batches
 
-        # Yield batches with the corresponding batch sizes
+    def __iter__(self) -> Iterator[List[int]]:
         for batch_indices, batch_size in zip(self.batches, self.batch_sizes):
             num_batches = len(batch_indices) // batch_size
 
             for i in range(num_batches):
-                # Recuperate the current bucket
                 current_bucket = batch_indices[i * batch_size: (i + 1) * batch_size]
-
-                # Shuffle the current bucket
                 np.random.shuffle(current_bucket)
+                yield [self.data_info[idx]["index"] for idx in current_bucket]
 
-                # Yield the current bucket
-                yield [self.data_info[i]["index"] for i in current_bucket]
-
-            remaining_indices = len(batch_indices) % batch_size
-    
-            if remaining_indices > 0 and remaining_indices != 1 or not self.drop_unique:
-                
-                # Recuperate the current bucket
-                current_bucket = batch_indices[-remaining_indices:]
-
-                # Shuffle the current bucket
+            remaining = len(batch_indices) % batch_size
+            if remaining > 0 and (remaining != 1 or not self.drop_unique):
+                current_bucket = batch_indices[-remaining:]
                 np.random.shuffle(current_bucket)
+                yield [self.data_info[idx]["index"] for idx in current_bucket]
 
-                # Yield the current bucket
-                yield [self.data_info[i]["index"] for i in batch_indices[-remaining_indices:]]
-
-    def __len__(self):
+    def __len__(self) -> int:
         return self.length
 
 
+
 class BucketSampler(Sampler):
     def __init__(self, dataset, batch_size, sort_key=lambda x, index_1, index_2: max(len(x[index_1]), len(x[index_2])), input_key: Union[str, int] = 0, label_key: Union[str, int] = 1):
         self.dataset = dataset

{wolof_translate-0.0.1 → wolof_translate-0.0.2}/wolof_translate.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wolof-translate
-Version: 0.0.1
+Version: 0.0.2
 Summary: Contain function and classes to process corpora for making translation between wolof text and other languages.
 Author: Oumar Kane
 Author-email: oumar.kane@univ-thies.sn