wolof-translate 0.0.3__tar.gz → 0.0.5__tar.gz

{wolof_translate-0.0.3 → wolof_translate-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wolof_translate
-Version: 0.0.3
+Version: 0.0.5
 Summary: Contain function and classes to process corpora for making translation between wolof text and other languages.
 Author: Oumar Kane
 Author-email: oumar.kane@univ-thies.sn

{wolof_translate-0.0.3 → wolof_translate-0.0.5}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="wolof_translate",
-    version="0.0.3",
+    version="0.0.5",
     author="Oumar Kane",
     author_email="oumar.kane@univ-thies.sn",
     description="Contain function and classes to process corpora for making translation between wolof text and other languages.",

{wolof_translate-0.0.3 → wolof_translate-0.0.5}/wolof_translate/utils/bucket_iterator.py

@@ -5,29 +5,43 @@ from math import ceil
 from tqdm import tqdm
 import time
 
+
 class SequenceLengthBatchSampler(Sampler[List[int]]):
     def __init__(
         self,
         dataset,
         boundaries: List[int],
         batch_sizes: List[int],
-        input_key: Optional[int] = None,
-        label_key: Optional[int] = None,
+        input_key: Optional[Union[int, str]] = None,
+        label_key: Optional[Union[int, str]] = None,
         drop_unique: bool = True,
     ):
+        """
+        Sampler that batches sequences of similar lengths together to minimize padding.
+
+        Args:
+            dataset: Dataset to sample from.
+            boundaries: List of length boundaries to bucket sequences.
+            batch_sizes: List of batch sizes per bucket (length must be len(boundaries)+1).
+            input_key: Key or index to access input sequence in dataset item.
+            label_key: Key or index to access label sequence in dataset item.
+            drop_unique: Whether to drop batches with a single leftover element.
+        """
         self.dataset = dataset
         self.boundaries = boundaries
         self.batch_sizes = batch_sizes
         self.drop_unique = drop_unique
 
+        assert len(batch_sizes) == len(boundaries) + 1, (
+            f"batch_sizes length ({len(batch_sizes)}) must be one more than boundaries length ({len(boundaries)})"
+        )
+
         start_time = time.time()
         tqdm.write("Computing sequence lengths...")
 
-        # Compute lengths with tqdm progress bar
         self.lengths = np.array([
-            max(len(data[0]), len(data[2])) if input_key is None or label_key is None
-            else max(len(data[input_key]), len(data[label_key]))
-            for data in tqdm(dataset, desc="Lengths", unit="seq")
+            self._get_length(data, input_key, label_key)
+            for data in tqdm(self.dataset, desc="Lengths", unit="seq")
         ])
 
         tqdm.write(f"Sequence lengths computed in {time.time() - start_time:.2f} seconds.")
@@ -35,7 +49,7 @@ class SequenceLengthBatchSampler(Sampler[List[int]]):
         start_time = time.time()
         tqdm.write("Assigning buckets...")
 
-        # Assign bucket ids using digitize (vectorized)
+        # Assign bucket ids (0-based)
         self.bucket_ids = np.digitize(self.lengths, bins=self.boundaries, right=True)
 
         # Create buckets of indices
@@ -46,7 +60,6 @@ class SequenceLengthBatchSampler(Sampler[List[int]]):
         start_time = time.time()
         tqdm.write("Preparing batches...")
 
-        # Prepare batches from buckets
         self.batches = []
         for bucket, batch_size in zip(self.buckets, self.batch_sizes):
             bucket = bucket.copy()
@@ -66,8 +79,26 @@ class SequenceLengthBatchSampler(Sampler[List[int]]):
         self.length = len(self.batches)
         tqdm.write(f"Batches prepared in {time.time() - start_time:.2f} seconds.")
 
+    def _get_length(self, data, input_key, label_key) -> int:
+        """
+        Helper to get the max length of input and label sequences in a dataset item.
+
+        Supports dict-like or tuple/list-like dataset items.
+        """
+        try:
+            if input_key is None or label_key is None:
+                # Assume tuple/list with input at 0, label at 2
+                input_seq = data[0]
+                label_seq = data[2]
+            else:
+                input_seq = data[input_key]
+                label_seq = data[label_key]
+            return max(len(input_seq), len(label_seq))
+        except Exception as e:
+            raise ValueError(f"Error accessing lengths with input_key={input_key}, label_key={label_key}: {e}")
+
     def __iter__(self) -> Iterator[List[int]]:
-        # Shuffle all batches globally to add randomness between buckets
+        # Shuffle batches globally for randomness
         np.random.shuffle(self.batches)
         for batch in self.batches:
             yield batch

{wolof_translate-0.0.3 → wolof_translate-0.0.5}/wolof_translate/utils/sent_corrections.py

@@ -13,6 +13,8 @@ def add_guillemet_space(sentences: Union[list, str]):
 
     if type(sentences) is str:
         sentences = [sentences]
+    elif type(sentences) is float:
+        sentences = [str(sentences)]
 
     for s in range(len(sentences)):
 
@@ -51,6 +53,8 @@ def delete_guillemet_space(sentences: Union[list, str]):
 
     if type(sentences) is str:
         sentences = [sentences]
+    elif type(sentences) is float:
+        sentences = [str(sentences)]
 
     for s in range(len(sentences)):
 
@@ -91,6 +95,8 @@ def add_mark_space(
 
     if type(sentences) is str:
         sentences = [sentences]
+    elif type(sentences) is float:
+        sentences = [str(sentences)]
 
     for s in range(len(sentences)):
 
@@ -123,6 +129,8 @@ def remove_mark_space(sentences: Union[list, str], marks: list = ["'", "-"]):
 
     if type(sentences) is str:
         sentences = [sentences]
+    elif type(sentences) is float:
+        sentences = [str(sentences)]
 
     for s in range(len(sentences)):
 
@@ -161,6 +169,8 @@ def delete_much_space(sentences: Union[list, str]):
 
     if type(sentences) is str:
         sentences = [sentences]
+    elif type(sentences) is float:
+        sentences = [str(sentences)]
 
     for i in range(len(sentences)):
 

{wolof_translate-0.0.3 → wolof_translate-0.0.5}/wolof_translate.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wolof-translate
-Version: 0.0.3
+Version: 0.0.5
 Summary: Contain function and classes to process corpora for making translation between wolof text and other languages.
 Author: Oumar Kane
 Author-email: oumar.kane@univ-thies.sn