translate-package 0.1.9__py3-none-any.whl → 0.2.1__py3-none-any.whl

translate_package/data/data_preparation.py

@@ -22,21 +22,24 @@ from translate_package import (
 
 # python translate_hyperparameter_tuning.py --model_generation "t5" --model_name "google-t5/t5-small" --tokenizer_name "sp" --use_bucketing --save_artifact
 
-def augment(examples, src_label, p_word = 0.12554160436087158, p_char = 0.8269672653838092, max_words = 21):
+def augment(examples, p_word = 0.12554160436087158, p_char = 0.8269672653838092, max_words = 21):
     
-    examples[src_label] = TransformerSequences(nac.RandomCharAug(action = 'swap', aug_word_p = p_word, aug_char_p = p_char, aug_word_max = max_words))(str(examples[src_label]))[0]
+    examples[examples["source"]] = TransformerSequences(nac.RandomCharAug(action = 'swap', aug_word_p = p_word, aug_char_p = p_char, aug_word_max = max_words))(examples[examples["source"]])[0]
     
     return examples
 
-def augment_(examples, src_label, tgt_label):
+def augment_(examples):
     
-    examples[src_label] = TransformerSequences(remove_mark_space, delete_guillemet_space)(str(examples[src_label]))[0]
+    examples[examples["source"]] = TransformerSequences(remove_mark_space, delete_guillemet_space)(examples[examples["source"]])[0]
     
-    examples[tgt_label] = TransformerSequences(remove_mark_space, delete_guillemet_space)(str(examples[tgt_label]))[0]
+    examples[examples["target"]] = TransformerSequences(remove_mark_space, delete_guillemet_space)(examples[examples["target"]])[0]
     
     return examples
 
-def tokenize(examples, tokenizer, src_label, tgt_label, model_generation):
+def tokenize(examples, tokenizer, model_generation, bidirectional = False):
+    
+    direction = f"{examples['source']}_{examples['target']}"
+    rev_direction = f"{examples['target']}_{examples['source']}"
     
     if model_generation in ["t5", "mt5", "nllb"]:
         
@@ -48,13 +51,21 @@ def tokenize(examples, tokenizer, src_label, tgt_label, model_generation):
         eos_token = tokenizer.eos_token 
         bos_token = tokenizer.bos_token
     
-    examples[src_label] = bos_token + examples[src_label] + eos_token
+    examples[examples["source"]] = bos_token + examples[examples["source"]] + eos_token
+
+    examples[examples["target"]] = bos_token + examples[examples["target"]] + eos_token
 
-    examples[tgt_label] = bos_token + examples[tgt_label] + eos_token
+    if bidirectional:
+        
+        examples.update({key: value[0] for key, value in tokenizer[direction](examples[examples["source"]], return_tensors = 'pt').items()})
 
-    examples.update({key: value[0] for key, value in tokenizer(examples[src_label], return_tensors = 'pt').items()})
+        examples.update({f'decoder_{key}': value[0] for key, value in tokenizer[rev_direction](examples[examples["target"]], return_tensors = 'pt').items()})
+    
+    else:
+        
+        examples.update({key: value[0] for key, value in tokenizer(examples[examples["source"]], return_tensors = 'pt').items()})
 
-    examples.update({f'decoder_{key}': value[0] for key, value in tokenizer(examples[tgt_label], return_tensors = 'pt').items()})
+        examples.update({f'decoder_{key}': value[0] for key, value in tokenizer(examples[examples["target"]], return_tensors = 'pt').items()})
 
     examples['labels'] = examples['decoder_input_ids']
 
@@ -76,7 +87,7 @@ def sequences(examples, functions):
 
 class SentenceDataset(Dataset):
     
-    def __init__(self, dataframe, transformers: Union[Callable, None] = None, source_column: str = 'WOLOF', target_column: str = 'FRENCH'):
+    def __init__(self, dataframe, transformers: Union[Callable, None] = None, source_column: str = 'WOLOF', target_column: str = 'FRENCH', bidirectional: bool = False):
         
         assert source_column in dataframe.columns.tolist() and target_column in dataframe.columns.tolist()
         
@@ -89,6 +100,8 @@ class SentenceDataset(Dataset):
         self.source_column = source_column
         
         self.target_column = target_column
+        
+        self.bidirectional = bidirectional
     
     def __getitem__(self, index):
         
@@ -96,10 +109,26 @@ class SentenceDataset(Dataset):
         
         target_sentence = self.target_sentences[index]
         
-        sentences = {
-            self.source_column: source_sentence,
-            self.target_column: target_sentence
-        }
+        if index > len(source_sentence):
+            
+            source_sentence = self.target_sentences[index]
+            
+            target_sentence = self.source_sentences[index]
+        
+            sentences = {
+                self.source_column: source_sentence,
+                self.target_column: target_sentence,
+                "source": self.source_column,
+                "target": self.target_column
+            }
+        
+        else:
+            sentences = {
+                self.source_column: source_sentence,
+                self.target_column: target_sentence,
+                "source": self.target_column,
+                "target": self.source_column
+            }
         
         if not self.transformers is None:
             
@@ -109,10 +138,14 @@ class SentenceDataset(Dataset):
     
     def __len__(self):
         
+        if self.bidirectional:
+        
+            return len(self.source_sentences) * 2
+        
         return len(self.source_sentences)
         
 
-def load_data(src_label, tgt_label, data_path, test_size, valid_size, seed):
+def load_data(src_label, tgt_label, data_path, test_size, valid_size, seed, bidirectional):
     
     # load the dataset with pandas
     dataset_ = pd.read_csv(data_path)
@@ -121,7 +154,7 @@ def load_data(src_label, tgt_label, data_path, test_size, valid_size, seed):
     if test_size == 1.0:
 
         dataset = {
-            "test": partial(SentenceDataset, dataframe = dataset_, source_column = src_label, target_column = tgt_label),
+            "test": partial(SentenceDataset, dataframe = dataset_, source_column = src_label, target_column = tgt_label, bidirectional=bidirectional),
         }
 
     else:
@@ -135,9 +168,9 @@ def load_data(src_label, tgt_label, data_path, test_size, valid_size, seed):
         )
 
         dataset = {
-            "train": partial(SentenceDataset, dataframe = train, source_column = src_label, target_column = tgt_label),
-            "val": partial(SentenceDataset, dataframe = valid, source_column = src_label, target_column = tgt_label),
-            "test": partial(SentenceDataset, dataframe = test, source_column = src_label, target_column = tgt_label),
+            "train": partial(SentenceDataset, dataframe = train, source_column = src_label, target_column = tgt_label, bidirectional=bidirectional),
+            "val": partial(SentenceDataset, dataframe = valid, source_column = src_label, target_column = tgt_label, bidirectional=bidirectional),
+            "test": partial(SentenceDataset, dataframe = test, source_column = src_label, target_column = tgt_label, bidirectional=bidirectional),
         }
 
     # The dataset actually contains 3 diff splits: train, validation, test.
@@ -243,25 +276,26 @@ def get_loaders(
     use_bucketing,
     use_truncation,
     batch_size,
+    bidirectional
 ):
 
     # get dataset
-    dataset = load_data(src_label, tgt_label, data_path, test_size, valid_size, seed)
+    dataset = load_data(src_label, tgt_label, data_path, test_size, valid_size, seed, bidirectional)
     
     # analysis transformations
     
     a_transformers = partial(sequences, 
                 functions = [
-                    partial(augment_, src_label = src_label, tgt_label = tgt_label),
-                    partial(tokenize, tokenizer = tokenizer, src_label = src_label, tgt_label = tgt_label, model_generation = model_generation)
+                    partial(augment_),
+                    partial(tokenize, tokenizer = tokenizer, model_generation = model_generation, bidirectional=bidirectional)
             ])
     
     # training transformations
     t_transformers = partial(sequences, 
                 functions = [
-                    partial(augment, src_label = src_label, p_word = p_word, p_char = p_char, max_words = max_words),
-                    partial(augment_, src_label = src_label, tgt_label = tgt_label),
-                    partial(tokenize, tokenizer = tokenizer, src_label = src_label, tgt_label = tgt_label, model_generation = model_generation)
+                    partial(augment, p_word = p_word, p_char = p_char, max_words = max_words),
+                    partial(augment_),
+                    partial(tokenize, tokenizer = tokenizer, model_generation = model_generation, bidirectional=bidirectional)
             ])
     
     if use_bucketing:

translate_package/tokenization/load_tokenizer.py

@@ -8,14 +8,23 @@ BCP_47_languages = {
     'wolof': 'wol_Latn',
 }
 
-def load_tokenizer(tokenizer_name, model, dir_path, file_name, model_name = None, src_lang = "french", tgt_lang = "wolof"):
+def load_tokenizer(tokenizer_name, model, dir_path, file_name, model_name = None, src_lang = "french", tgt_lang = "wolof", bidirectional: bool = False):
     
     if model == "nllb":
         
         if not model_name is None:
         
-            tokenizer = AutoTokenizer.from_pretrained(model_name, src_lang = BCP_47_languages[src_lang], tgt_lang = BCP_47_languages[tgt_lang])
+            if not bidirectional:
+                
+                tokenizer = AutoTokenizer.from_pretrained(model_name, src_lang = BCP_47_languages[src_lang], tgt_lang = BCP_47_languages[tgt_lang])
             
+            else:
+                
+                tokenizer = {
+                    f"{src_lang}_{tgt_lang}": AutoTokenizer.from_pretrained(model_name, src_lang = BCP_47_languages[src_lang], tgt_lang = BCP_47_languages[tgt_lang]),
+                    f"{tgt_lang}_{src_lang}": AutoTokenizer.from_pretrained(model_name, src_lang = BCP_47_languages[tgt_lang], tgt_lang = BCP_47_languages[src_lang])
+                }
+                
             print(f"The {model}'s tokenizer was successfully loaded")
         
         else:

translate_package/utils/upload_to_hughub.py

@@ -7,9 +7,10 @@ def upload_model(hub_token, directory = "my_model", username = "", repo_name = "
     
     login(token=hub_token)
     
-    create_repo(repo_id)
+    create_repo(repo_id, token = hub_token)
     
-    upload_folder(repo_id = repo_id, folder_path = directory, commit_message= commit_message)
+    upload_folder(repo_id = repo_id, folder_path = directory, commit_message= commit_message, token=hub_token)
     
     print(f"Model was successfully upload to {repo_id}.")
+    
     

{translate_package-0.1.9.dist-info → translate_package-0.2.1.dist-info}/METADATA

@@ -1,33 +1,37 @@
-Metadata-Version: 2.1
-Name: translate-package
-Version: 0.1.9
-Summary: Contain functions and classes to efficiently train a sequence to sequence to translate between two languages.
-Author: Oumar Kane
-Author-email: oumar.kane@univ-thies.sn
-Requires-Dist: accelerate
-Requires-Dist: torch (==2.7.0)
-Requires-Dist: torchvision
-Requires-Dist: spacy
-Requires-Dist: nltk
-Requires-Dist: gensim
-Requires-Dist: furo
-Requires-Dist: streamlit
-Requires-Dist: tokenizers
-Requires-Dist: tensorboard
-Requires-Dist: evaluate
-Requires-Dist: transformers
-Requires-Dist: pandas
-Requires-Dist: numpy
-Requires-Dist: scikit-learn
-Requires-Dist: matplotlib
-Requires-Dist: plotly
-Requires-Dist: sacrebleu
-Requires-Dist: nlpaug
-Requires-Dist: wandb
-Requires-Dist: pytorch-lightning
-Requires-Dist: selenium
-Requires-Dist: sentencepiece
-Requires-Dist: peft
-Requires-Dist: rouge-score
-Requires-Dist: wolof-translate
-
+Metadata-Version: 2.4
+Name: translate_package
+Version: 0.2.1
+Summary: Contain functions and classes to efficiently train a sequence to sequence to translate between two languages.
+Author: Oumar Kane
+Author-email: oumar.kane@univ-thies.sn
+Requires-Dist: accelerate
+Requires-Dist: torch==2.7.0
+Requires-Dist: torchvision
+Requires-Dist: spacy
+Requires-Dist: nltk
+Requires-Dist: gensim
+Requires-Dist: furo
+Requires-Dist: streamlit
+Requires-Dist: tokenizers
+Requires-Dist: tensorboard
+Requires-Dist: evaluate
+Requires-Dist: transformers
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: scikit-learn
+Requires-Dist: matplotlib
+Requires-Dist: plotly
+Requires-Dist: sacrebleu
+Requires-Dist: nlpaug
+Requires-Dist: wandb
+Requires-Dist: pytorch-lightning
+Requires-Dist: selenium
+Requires-Dist: sentencepiece
+Requires-Dist: peft
+Requires-Dist: rouge-score
+Requires-Dist: sacrebleu
+Requires-Dist: wolof-translate
+Dynamic: author
+Dynamic: author-email
+Dynamic: requires-dist
+Dynamic: summary

{translate_package-0.1.9.dist-info → translate_package-0.2.1.dist-info}/RECORD

@@ -1,18 +1,18 @@
 translate_package/__init__.py,sha256=miie3aAeUYHsVk2O-kd4T86fFksuCiY70Eo6RNeY1Oo,1312
 translate_package/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-translate_package/data/data_preparation.py,sha256=odTSpud2biQaxI8lW65FtS-hMuXbglD4mQCTYx2kNZs,14897
+translate_package/data/data_preparation.py,sha256=_fK2joza3FlzvrGPSk-KywhNj6R8IpK7TBBC0p6X940,16243
 translate_package/errors/__init__.py,sha256=gu6XjAIghG4lLkYo8x_7_yyLRtK2FIvmC-WcfJaeOlg,299
 translate_package/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 translate_package/models/gradient_observation.py,sha256=P91UA5i-RdkK46TqpPOJ54DsUYgTI9cRohgPS1Ch0Lc,294
 translate_package/models/lstm.py,sha256=OPkvvceowz5JqdGGH4cfPhH23kbP11z-29zIJn5d8ig,3273
 translate_package/models/machine_translation.py,sha256=1ot9Me6U1O7UHJMuJGvatx3DxoKY9TghzzHNzxdZa5g,11170
 translate_package/tokenization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-translate_package/tokenization/load_tokenizer.py,sha256=g8j5pDmimFhwjpeYNkWot0hXMzAqqURbtedcQK-1xYE,1543
+translate_package/tokenization/load_tokenizer.py,sha256=Q7ZFMCefs3vPe2CE9iWKkgGz3Wk4C9rbUTGmyjTioJQ,2069
 translate_package/tokenization/train_tokenizer.py,sha256=RkdT5DUx201OBNaswM6m54iqcrmCThd3ITLguQb_zVM,3347
 translate_package/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 translate_package/utils/checkpoint.py,sha256=GqymRvF8_QZgrQq9m79Ppj6Qr7NQm78kDARm3p_chC0,322
-translate_package/utils/upload_to_hughub.py,sha256=0qihZIAAUuJXfOZ23Njz0aWpDpe8twQNDGPplgrIfzA,480
-translate_package-0.1.9.dist-info/METADATA,sha256=Cm1FrRayiCWGmiL6-IOe57f2DQzghEpkgWrMK46Buzs,887
-translate_package-0.1.9.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-translate_package-0.1.9.dist-info/top_level.txt,sha256=8e2HIrGAMzoSukqu2q929dOJMV1zGYKI_BAFwl-P7XU,18
-translate_package-0.1.9.dist-info/RECORD,,
+translate_package/utils/upload_to_hughub.py,sha256=zWXJQfv1ZndFIeyGWO0JMLSCP7rj-B4RuiiY-TFkBnw,522
+translate_package-0.2.1.dist-info/METADATA,sha256=gf6SydqZxLYimXZsVsm8mHqYx3fz4kqgm68oYBQlf9A,1023
+translate_package-0.2.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+translate_package-0.2.1.dist-info/top_level.txt,sha256=8e2HIrGAMzoSukqu2q929dOJMV1zGYKI_BAFwl-P7XU,18
+translate_package-0.2.1.dist-info/RECORD,,

{translate_package-0.1.9.dist-info → translate_package-0.2.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.37.1)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any