wolof-translate 0.0.2__py3-none-any.whl → 0.0.4__py3-none-any.whl

wolof_translate/utils/bucket_iterator.py

@@ -1,8 +1,10 @@
-import torch
 import numpy as np
-from typing import Optional, List, Iterator
+from typing import Optional, List, Iterator, Union
 from torch.utils.data import Sampler
 from math import ceil
+from tqdm import tqdm
+import time
+
 
 class SequenceLengthBatchSampler(Sampler[List[int]]):
     def __init__(
@@ -10,70 +12,173 @@ class SequenceLengthBatchSampler(Sampler[List[int]]):
         dataset,
         boundaries: List[int],
         batch_sizes: List[int],
-        input_key: Optional[int] = None,
-        label_key: Optional[int] = None,
+        input_key: Optional[Union[int, str]] = None,
+        label_key: Optional[Union[int, str]] = None,
         drop_unique: bool = True,
     ):
+        """
+        Sampler that batches sequences of similar lengths together to minimize padding.
+
+        Args:
+            dataset: Dataset to sample from.
+            boundaries: List of length boundaries to bucket sequences.
+            batch_sizes: List of batch sizes per bucket (length must be len(boundaries)+1).
+            input_key: Key or index to access input sequence in dataset item.
+            label_key: Key or index to access label sequence in dataset item.
+            drop_unique: Whether to drop batches with a single leftover element.
+        """
         self.dataset = dataset
         self.boundaries = boundaries
         self.batch_sizes = batch_sizes
         self.drop_unique = drop_unique
-        self.data_info = {}
 
-        # Extract lengths
-        for i in range(len(dataset)):
-            data = dataset[i]
-            if input_key is None or label_key is None:
-                length = max(len(data[0]), len(data[2]))
-            else:
-                length = max(len(data[input_key]), len(data[label_key]))
-            self.data_info[i] = {"index": i, "length": length}
+        assert len(batch_sizes) == len(boundaries) + 1, (
+            f"batch_sizes length ({len(batch_sizes)}) must be one more than boundaries length ({len(boundaries)})"
+        )
+
+        start_time = time.time()
+        tqdm.write("Computing sequence lengths...")
+
+        self.lengths = np.array([
+            self._get_length(data, input_key, label_key)
+            for data in tqdm(self.dataset, desc="Lengths", unit="seq")
+        ])
+
+        tqdm.write(f"Sequence lengths computed in {time.time() - start_time:.2f} seconds.")
 
-        self.calculate_length()
+        start_time = time.time()
+        tqdm.write("Assigning buckets...")
+
+        # Assign bucket ids (0-based)
+        self.bucket_ids = np.digitize(self.lengths, bins=self.boundaries, right=True)
+
+        # Create buckets of indices
+        self.buckets = [np.where(self.bucket_ids == i)[0] for i in range(len(boundaries) + 1)]
+
+        tqdm.write(f"Buckets assigned in {time.time() - start_time:.2f} seconds.")
+
+        start_time = time.time()
+        tqdm.write("Preparing batches...")
 
-    def calculate_length(self):
         self.batches = []
-        sorted_indices = sorted(self.data_info.keys(), key=lambda i: self.data_info[i]["length"])
-
-        prev_boundary = 0
-        for boundary in self.boundaries:
-            batch = [i for i in sorted_indices if prev_boundary < self.data_info[i]["length"] <= boundary]
-            self.batches.append(batch)
-            sorted_indices = [i for i in sorted_indices if i not in batch]
-            prev_boundary = boundary
-
-        # Remaining sequences > last boundary
-        self.batches.append(sorted_indices)
-
-        total_batches = 0
-        for batch, batch_size in zip(self.batches, self.batch_sizes):
-            n_full_batches = len(batch) // batch_size
-            leftover = len(batch) % batch_size
-            total_batches += n_full_batches
+        for bucket, batch_size in zip(self.buckets, self.batch_sizes):
+            bucket = bucket.copy()
+            np.random.shuffle(bucket)
+
+            n_full_batches = len(bucket) // batch_size
+            leftover = len(bucket) % batch_size
+
+            for i in range(n_full_batches):
+                batch = bucket[i * batch_size : (i + 1) * batch_size].tolist()
+                self.batches.append(batch)
+
             if leftover > 0 and (leftover != 1 or not self.drop_unique):
-                total_batches += 1
-        self.length = total_batches
+                batch = bucket[-leftover:].tolist()
+                self.batches.append(batch)
 
-    def __iter__(self) -> Iterator[List[int]]:
-        for batch_indices, batch_size in zip(self.batches, self.batch_sizes):
-            num_batches = len(batch_indices) // batch_size
+        self.length = len(self.batches)
+        tqdm.write(f"Batches prepared in {time.time() - start_time:.2f} seconds.")
 
-            for i in range(num_batches):
-                current_bucket = batch_indices[i * batch_size: (i + 1) * batch_size]
-                np.random.shuffle(current_bucket)
-                yield [self.data_info[idx]["index"] for idx in current_bucket]
+    def _get_length(self, data, input_key, label_key) -> int:
+        """
+        Helper to get the max length of input and label sequences in a dataset item.
 
-            remaining = len(batch_indices) % batch_size
-            if remaining > 0 and (remaining != 1 or not self.drop_unique):
-                current_bucket = batch_indices[-remaining:]
-                np.random.shuffle(current_bucket)
-                yield [self.data_info[idx]["index"] for idx in current_bucket]
+        Supports dict-like or tuple/list-like dataset items.
+        """
+        try:
+            if input_key is None or label_key is None:
+                # Assume tuple/list with input at 0, label at 2
+                input_seq = data[0]
+                label_seq = data[2]
+            else:
+                input_seq = data[input_key]
+                label_seq = data[label_key]
+            return max(len(input_seq), len(label_seq))
+        except Exception as e:
+            raise ValueError(f"Error accessing lengths with input_key={input_key}, label_key={label_key}: {e}")
+
+    def __iter__(self) -> Iterator[List[int]]:
+        # Shuffle batches globally for randomness
+        np.random.shuffle(self.batches)
+        for batch in self.batches:
+            yield batch
 
     def __len__(self) -> int:
         return self.length
 
 
 
+
+# class SequenceLengthBatchSampler(Sampler[List[int]]):
+#     def __init__(
+#         self,
+#         dataset,
+#         boundaries: List[int],
+#         batch_sizes: List[int],
+#         input_key: Optional[int] = None,
+#         label_key: Optional[int] = None,
+#         drop_unique: bool = True,
+#     ):
+#         self.dataset = dataset
+#         self.boundaries = boundaries
+#         self.batch_sizes = batch_sizes
+#         self.drop_unique = drop_unique
+#         self.data_info = {}
+
+#         # Extract lengths
+#         for i in range(len(dataset)):
+#             data = dataset[i]
+#             if input_key is None or label_key is None:
+#                 length = max(len(data[0]), len(data[2]))
+#             else:
+#                 length = max(len(data[input_key]), len(data[label_key]))
+#             self.data_info[i] = {"index": i, "length": length}
+
+#         self.calculate_length()
+
+#     def calculate_length(self):
+#         self.batches = []
+#         sorted_indices = sorted(self.data_info.keys(), key=lambda i: self.data_info[i]["length"])
+
+#         prev_boundary = 0
+#         for boundary in self.boundaries:
+#             batch = [i for i in sorted_indices if prev_boundary < self.data_info[i]["length"] <= boundary]
+#             self.batches.append(batch)
+#             sorted_indices = [i for i in sorted_indices if i not in batch]
+#             prev_boundary = boundary
+
+#         # Remaining sequences > last boundary
+#         self.batches.append(sorted_indices)
+
+#         total_batches = 0
+#         for batch, batch_size in zip(self.batches, self.batch_sizes):
+#             n_full_batches = len(batch) // batch_size
+#             leftover = len(batch) % batch_size
+#             total_batches += n_full_batches
+#             if leftover > 0 and (leftover != 1 or not self.drop_unique):
+#                 total_batches += 1
+#         self.length = total_batches
+
+#     def __iter__(self) -> Iterator[List[int]]:
+#         for batch_indices, batch_size in zip(self.batches, self.batch_sizes):
+#             num_batches = len(batch_indices) // batch_size
+
+#             for i in range(num_batches):
+#                 current_bucket = batch_indices[i * batch_size: (i + 1) * batch_size]
+#                 np.random.shuffle(current_bucket)
+#                 yield [self.data_info[idx]["index"] for idx in current_bucket]
+
+#             remaining = len(batch_indices) % batch_size
+#             if remaining > 0 and (remaining != 1 or not self.drop_unique):
+#                 current_bucket = batch_indices[-remaining:]
+#                 np.random.shuffle(current_bucket)
+#                 yield [self.data_info[idx]["index"] for idx in current_bucket]
+
+#     def __len__(self) -> int:
+#         return self.length
+
+
+
 class BucketSampler(Sampler):
     def __init__(self, dataset, batch_size, sort_key=lambda x, index_1, index_2: max(len(x[index_1]), len(x[index_2])), input_key: Union[str, int] = 0, label_key: Union[str, int] = 1):
         self.dataset = dataset

{wolof_translate-0.0.2.dist-info → wolof_translate-0.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wolof-translate
-Version: 0.0.2
+Version: 0.0.4
 Summary: Contain function and classes to process corpora for making translation between wolof text and other languages.
 Author: Oumar Kane
 Author-email: oumar.kane@univ-thies.sn

{wolof_translate-0.0.2.dist-info → wolof_translate-0.0.4.dist-info}/RECORD

@@ -22,7 +22,7 @@ wolof_translate/trainers/transformer_trainer_custom.py,sha256=hHUBcU4YK6wuRUMiwX
 wolof_translate/trainers/transformer_trainer_ml.py,sha256=WgggaugkVHSJlwIAZT-QwI90Fl-_zT8Clhb-7M0m8gM,33561
 wolof_translate/trainers/transformer_trainer_ml_.py,sha256=QaN9DB5pqhBxV4WlFmJCmUyfwlX-UyAzKRwL6rVEr4Q,38199
 wolof_translate/utils/__init__.py,sha256=Nl3300H-Xd3uTHDR8y-rYa-UUR9FqbqZPwUKJUpQOb4,64
-wolof_translate/utils/bucket_iterator.py,sha256=VLqmgZ9Z7PoXFS76R81McV3jKlWQ_A4LYLAZx_a0xfU,5800
+wolof_translate/utils/bucket_iterator.py,sha256=sGSBCGPn8NzZ32mfEKh0cfH1Z0WbNJWbDjghR8-u5tU,9847
 wolof_translate/utils/database_manager.py,sha256=7yhgBN1LvVFNEQikxCjSCva82h5nX44Nx2zh8cpFWyA,3543
 wolof_translate/utils/display_predictions.py,sha256=y5H5lfgIODl6E5Zfb1YIwiAxIlHUxRBoChfQR5kjh24,5145
 wolof_translate/utils/download_model.py,sha256=x92KpfVPvNK8Suen1qnOcPtZOlB4kXTfqWgoVuuMUEM,1241
@@ -43,7 +43,7 @@ wolof_translate/utils/training.py,sha256=5vPVuqHL6_gqLkh4PTxXqW4UvAJBWNWVDDXC9Fk
 wolof_translate/utils/trunc_hg_training.py,sha256=mMGrU7Mjr9vYd7eLc8nbFRhRXwSWMKyg35lGf0L6RtQ,6418
 wolof_translate/utils/improvements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wolof_translate/utils/improvements/end_marks.py,sha256=scmhMMYguZmrZTPozx1ZovizKrrPfPpMLXbU2-IOdGs,1194
-wolof_translate-0.0.2.dist-info/METADATA,sha256=l8TYjNCIujICJklQFsyVy9XINJNNAU0hPkkNt0kbKFM,818
-wolof_translate-0.0.2.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-wolof_translate-0.0.2.dist-info/top_level.txt,sha256=YG-kBnOwUZyQ7SofNvMxNYjzCreH2PVcW2UaEg1-Reg,16
-wolof_translate-0.0.2.dist-info/RECORD,,
+wolof_translate-0.0.4.dist-info/METADATA,sha256=Adyexcw1wpc80mDnoeLxFuaWe4Pg0ZAZroqx6eHnSz4,818
+wolof_translate-0.0.4.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+wolof_translate-0.0.4.dist-info/top_level.txt,sha256=YG-kBnOwUZyQ7SofNvMxNYjzCreH2PVcW2UaEg1-Reg,16
+wolof_translate-0.0.4.dist-info/RECORD,,