translate-package 0.1.8__tar.gz → 0.2.0__tar.gz

translate_package-0.2.0/PKG-INFO

@@ -0,0 +1,37 @@
+Metadata-Version: 2.4
+Name: translate_package
+Version: 0.2.0
+Summary: Contain functions and classes to efficiently train a sequence to sequence to translate between two languages.
+Author: Oumar Kane
+Author-email: oumar.kane@univ-thies.sn
+Requires-Dist: accelerate
+Requires-Dist: torch==2.7.0
+Requires-Dist: torchvision
+Requires-Dist: spacy
+Requires-Dist: nltk
+Requires-Dist: gensim
+Requires-Dist: furo
+Requires-Dist: streamlit
+Requires-Dist: tokenizers
+Requires-Dist: tensorboard
+Requires-Dist: evaluate
+Requires-Dist: transformers
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: scikit-learn
+Requires-Dist: matplotlib
+Requires-Dist: plotly
+Requires-Dist: sacrebleu
+Requires-Dist: nlpaug
+Requires-Dist: wandb
+Requires-Dist: pytorch-lightning
+Requires-Dist: selenium
+Requires-Dist: sentencepiece
+Requires-Dist: peft
+Requires-Dist: rouge-score
+Requires-Dist: sacrebleu
+Requires-Dist: wolof-translate
+Dynamic: author
+Dynamic: author-email
+Dynamic: requires-dist
+Dynamic: summary

{translate_package-0.1.8 → translate_package-0.2.0}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="translate_package",
-    version="0.1.8",
+    version="0.2.0",
     author="Oumar Kane",
     packages=find_packages(),
     author_email="oumar.kane@univ-thies.sn",

{translate_package-0.1.8 → translate_package-0.2.0}/translate_package/data/data_preparation.py

@@ -22,21 +22,23 @@ from translate_package import (
 
 # python translate_hyperparameter_tuning.py --model_generation "t5" --model_name "google-t5/t5-small" --tokenizer_name "sp" --use_bucketing --save_artifact
 
-def augment(examples, src_label, p_word = 0.12554160436087158, p_char = 0.8269672653838092, max_words = 21):
+def augment(examples, p_word = 0.12554160436087158, p_char = 0.8269672653838092, max_words = 21):
     
-    examples[src_label] = TransformerSequences(nac.RandomCharAug(action = 'swap', aug_word_p = p_word, aug_char_p = p_char, aug_word_max = max_words))(examples[src_label])[0]
+    examples[examples["source"]] = TransformerSequences(nac.RandomCharAug(action = 'swap', aug_word_p = p_word, aug_char_p = p_char, aug_word_max = max_words))(examples[examples["source"]])[0]
     
     return examples
 
-def augment_(examples, src_label, tgt_label):
+def augment_(examples):
     
-    examples[src_label] = TransformerSequences(remove_mark_space, delete_guillemet_space)(examples[src_label])[0]
+    examples[examples["source"]] = TransformerSequences(remove_mark_space, delete_guillemet_space)(examples[examples["source"]])[0]
     
-    examples[tgt_label] = TransformerSequences(remove_mark_space, delete_guillemet_space)(examples[tgt_label])[0]
+    examples[examples["target"]] = TransformerSequences(remove_mark_space, delete_guillemet_space)(examples[examples["target"]])[0]
     
     return examples
 
-def tokenize(examples, tokenizer, src_label, tgt_label, model_generation):
+def tokenize(examples, tokenizer, model_generation, bidirectional = False):
+    
+    direction = f"{examples['source']}_{examples['target']}"
     
     if model_generation in ["t5", "mt5", "nllb"]:
         
@@ -48,13 +50,21 @@ def tokenize(examples, tokenizer, src_label, tgt_label, model_generation):
         eos_token = tokenizer.eos_token 
         bos_token = tokenizer.bos_token
     
-    examples[src_label] = bos_token + examples[src_label] + eos_token
+    examples[examples["source"]] = bos_token + examples[examples["source"]] + eos_token
+
+    examples[examples["target"]] = bos_token + examples[examples["target"]] + eos_token
 
-    examples[tgt_label] = bos_token + examples[tgt_label] + eos_token
+    if bidirectional:
+        
+        examples.update({key: value[0] for key, value in tokenizer[direction](examples[examples["source"]], return_tensors = 'pt').items()})
 
-    examples.update({key: value[0] for key, value in tokenizer(examples[src_label], return_tensors = 'pt').items()})
+        examples.update({f'decoder_{key}': value[0] for key, value in tokenizer[direction](examples[examples["target"]], return_tensors = 'pt').items()})
+    
+    else:
+        
+        examples.update({key: value[0] for key, value in tokenizer(examples[examples["source"]], return_tensors = 'pt').items()})
 
-    examples.update({f'decoder_{key}': value[0] for key, value in tokenizer(examples[tgt_label], return_tensors = 'pt').items()})
+        examples.update({f'decoder_{key}': value[0] for key, value in tokenizer(examples[examples["target"]], return_tensors = 'pt').items()})
 
     examples['labels'] = examples['decoder_input_ids']
 
@@ -76,7 +86,7 @@ def sequences(examples, functions):
 
 class SentenceDataset(Dataset):
     
-    def __init__(self, dataframe, transformers: Union[Callable, None] = None, source_column: str = 'WOLOF', target_column: str = 'FRENCH'):
+    def __init__(self, dataframe, transformers: Union[Callable, None] = None, source_column: str = 'WOLOF', target_column: str = 'FRENCH', bidirectional: bool = False):
         
         assert source_column in dataframe.columns.tolist() and target_column in dataframe.columns.tolist()
         
@@ -89,6 +99,8 @@ class SentenceDataset(Dataset):
         self.source_column = source_column
         
         self.target_column = target_column
+        
+        self.bidirectional = bidirectional
     
     def __getitem__(self, index):
         
@@ -96,10 +108,26 @@ class SentenceDataset(Dataset):
         
         target_sentence = self.target_sentences[index]
         
-        sentences = {
-            self.source_column: source_sentence,
-            self.target_column: target_sentence
-        }
+        if index > len(source_sentence):
+            
+            source_sentence = self.target_sentences[index]
+            
+            target_sentence = self.source_sentences[index]
+        
+            sentences = {
+                self.source_column: source_sentence,
+                self.target_column: target_sentence,
+                "source": self.source_column,
+                "target": self.target_column
+            }
+        
+        else:
+            sentences = {
+                self.source_column: source_sentence,
+                self.target_column: target_sentence,
+                "source": self.target_column,
+                "target": self.source_column
+            }
         
         if not self.transformers is None:
             
@@ -109,10 +137,14 @@ class SentenceDataset(Dataset):
     
     def __len__(self):
         
+        if self.bidirectional:
+        
+            return len(self.source_sentences) * 2
+        
         return len(self.source_sentences)
         
 
-def load_data(src_label, tgt_label, data_path, test_size, valid_size, seed):
+def load_data(src_label, tgt_label, data_path, test_size, valid_size, seed, bidirectional):
     
     # load the dataset with pandas
     dataset_ = pd.read_csv(data_path)
@@ -121,7 +153,7 @@ def load_data(src_label, tgt_label, data_path, test_size, valid_size, seed):
     if test_size == 1.0:
 
         dataset = {
-            "test": partial(SentenceDataset, dataframe = dataset_, source_column = src_label, target_column = tgt_label),
+            "test": partial(SentenceDataset, dataframe = dataset_, source_column = src_label, target_column = tgt_label, bidirectional=bidirectional),
         }
 
     else:
@@ -135,9 +167,9 @@ def load_data(src_label, tgt_label, data_path, test_size, valid_size, seed):
         )
 
         dataset = {
-            "train": partial(SentenceDataset, dataframe = train, source_column = src_label, target_column = tgt_label),
-            "val": partial(SentenceDataset, dataframe = valid, source_column = src_label, target_column = tgt_label),
-            "test": partial(SentenceDataset, dataframe = test, source_column = src_label, target_column = tgt_label),
+            "train": partial(SentenceDataset, dataframe = train, source_column = src_label, target_column = tgt_label, bidirectional=bidirectional),
+            "val": partial(SentenceDataset, dataframe = valid, source_column = src_label, target_column = tgt_label, bidirectional=bidirectional),
+            "test": partial(SentenceDataset, dataframe = test, source_column = src_label, target_column = tgt_label, bidirectional=bidirectional),
         }
 
     # The dataset actually contains 3 diff splits: train, validation, test.
@@ -243,25 +275,26 @@ def get_loaders(
     use_bucketing,
     use_truncation,
     batch_size,
+    bidirectional
 ):
 
     # get dataset
-    dataset = load_data(src_label, tgt_label, data_path, test_size, valid_size, seed)
+    dataset = load_data(src_label, tgt_label, data_path, test_size, valid_size, seed, bidirectional)
     
     # analysis transformations
     
     a_transformers = partial(sequences, 
                 functions = [
-                    partial(augment_, src_label = src_label, tgt_label = tgt_label),
-                    partial(tokenize, tokenizer = tokenizer, src_label = src_label, tgt_label = tgt_label, model_generation = model_generation)
+                    partial(augment_),
+                    partial(tokenize, tokenizer = tokenizer, model_generation = model_generation, bidirectional=bidirectional)
             ])
     
     # training transformations
     t_transformers = partial(sequences, 
                 functions = [
-                    partial(augment, src_label = src_label, p_word = p_word, p_char = p_char, max_words = max_words),
-                    partial(augment_, src_label = src_label, tgt_label = tgt_label),
-                    partial(tokenize, tokenizer = tokenizer, src_label = src_label, tgt_label = tgt_label, model_generation = model_generation)
+                    partial(augment, p_word = p_word, p_char = p_char, max_words = max_words),
+                    partial(augment_),
+                    partial(tokenize, tokenizer = tokenizer, model_generation = model_generation, bidirectional=bidirectional)
             ])
     
     if use_bucketing:

{translate_package-0.1.8 → translate_package-0.2.0}/translate_package/tokenization/load_tokenizer.py

@@ -8,14 +8,23 @@ BCP_47_languages = {
     'wolof': 'wol_Latn',
 }
 
-def load_tokenizer(tokenizer_name, model, dir_path, file_name, model_name = None, src_lang = "french", tgt_lang = "wolof"):
+def load_tokenizer(tokenizer_name, model, dir_path, file_name, model_name = None, src_lang = "french", tgt_lang = "wolof", bidirectional: bool = False):
     
     if model == "nllb":
         
         if not model_name is None:
         
-            tokenizer = AutoTokenizer.from_pretrained(model_name, src_lang = BCP_47_languages[src_lang], tgt_lang = BCP_47_languages[tgt_lang])
+            if not bidirectional:
+                
+                tokenizer = AutoTokenizer.from_pretrained(model_name, src_lang = BCP_47_languages[src_lang], tgt_lang = BCP_47_languages[tgt_lang])
             
+            else:
+                
+                tokenizer = {
+                    f"{src_lang}_{tgt_lang}": AutoTokenizer.from_pretrained(model_name, src_lang = BCP_47_languages[src_lang], tgt_lang = BCP_47_languages[tgt_lang]),
+                    f"{tgt_lang}_{src_lang}": AutoTokenizer.from_pretrained(model_name, src_lang = BCP_47_languages[tgt_lang], tgt_lang = BCP_47_languages[src_lang])
+                }
+                
             print(f"The {model}'s tokenizer was successfully loaded")
         
         else:

{translate_package-0.1.8 → translate_package-0.2.0}/translate_package/utils/upload_to_hughub.py

@@ -7,9 +7,10 @@ def upload_model(hub_token, directory = "my_model", username = "", repo_name = "
     
     login(token=hub_token)
     
-    create_repo(repo_id)
+    create_repo(repo_id, token = hub_token)
     
-    upload_folder(repo_id = repo_id, folder_path = directory, commit_message= commit_message)
+    upload_folder(repo_id = repo_id, folder_path = directory, commit_message= commit_message, token=hub_token)
     
     print(f"Model was successfully upload to {repo_id}.")
+    
     

translate_package-0.2.0/translate_package.egg-info/PKG-INFO

@@ -0,0 +1,37 @@
+Metadata-Version: 2.4
+Name: translate_package
+Version: 0.2.0
+Summary: Contain functions and classes to efficiently train a sequence to sequence to translate between two languages.
+Author: Oumar Kane
+Author-email: oumar.kane@univ-thies.sn
+Requires-Dist: accelerate
+Requires-Dist: torch==2.7.0
+Requires-Dist: torchvision
+Requires-Dist: spacy
+Requires-Dist: nltk
+Requires-Dist: gensim
+Requires-Dist: furo
+Requires-Dist: streamlit
+Requires-Dist: tokenizers
+Requires-Dist: tensorboard
+Requires-Dist: evaluate
+Requires-Dist: transformers
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: scikit-learn
+Requires-Dist: matplotlib
+Requires-Dist: plotly
+Requires-Dist: sacrebleu
+Requires-Dist: nlpaug
+Requires-Dist: wandb
+Requires-Dist: pytorch-lightning
+Requires-Dist: selenium
+Requires-Dist: sentencepiece
+Requires-Dist: peft
+Requires-Dist: rouge-score
+Requires-Dist: sacrebleu
+Requires-Dist: wolof-translate
+Dynamic: author
+Dynamic: author-email
+Dynamic: requires-dist
+Dynamic: summary

translate_package-0.1.8/PKG-INFO

@@ -1,6 +0,0 @@
-Metadata-Version: 2.1
-Name: translate_package
-Version: 0.1.8
-Summary: Contain functions and classes to efficiently train a sequence to sequence to translate between two languages.
-Author: Oumar Kane
-Author-email: oumar.kane@univ-thies.sn

translate_package-0.1.8/translate_package.egg-info/PKG-INFO

@@ -1,6 +0,0 @@
-Metadata-Version: 2.1
-Name: translate-package
-Version: 0.1.8
-Summary: Contain functions and classes to efficiently train a sequence to sequence to translate between two languages.
-Author: Oumar Kane
-Author-email: oumar.kane@univ-thies.sn