translate-package 0.0.7__tar.gz → 0.0.9__tar.gz

{translate_package-0.0.7 → translate_package-0.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: translate_package
-Version: 0.0.7
+Version: 0.0.9
 Summary: Contain functions and classes to efficiently train a sequence to sequence to translate between two languages.
 Author: Oumar Kane
 Author-email: oumar.kane@univ-thies.sn

{translate_package-0.0.7 → translate_package-0.0.9}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="translate_package",
-    version="0.0.7",
+    version="0.0.9",
     author="Oumar Kane",
     packages=find_packages(),
     author_email="oumar.kane@univ-thies.sn",

{translate_package-0.0.7 → translate_package-0.0.9}/translate_package/__init__.py

@@ -14,7 +14,7 @@ from transformers import (
     Trainer, AutoModelForSeq2SeqLM,
     get_linear_schedule_with_warmup,
     T5ForConditionalGeneration, Adafactor, BartForConditionalGeneration,
-    MT5ForConditionalGeneration, AdamWeightDecay
+    MT5ForConditionalGeneration, AdamWeightDecay, AutoTokenizer
 )
 from wolof_translate.utils.bucket_iterator import SequenceLengthBatchSampler, BucketSampler
 from wolof_translate.utils.sent_transformers import TransformerSequences

{translate_package-0.0.7 → translate_package-0.0.9}/translate_package/data/data_preparation.py

@@ -350,7 +350,7 @@ def get_loaders(
             )
             
             # add transformations
-            dataset = {s: dataset[s](transformers = t_transformers) for s in dataset}
+            dataset = {s: dataset[s](transformers = t_transformers) if s == 'train' else dataset[s](transformers = a_transformers) for s in dataset}
             
             # define data loaders
             train_loader = DataLoader(

{translate_package-0.0.7 → translate_package-0.0.9}/translate_package/models/machine_translation.py

@@ -11,6 +11,7 @@ from translate_package import (
     MT5ForConditionalGeneration,
     BartForConditionalGeneration,
     AutoModelForSeq2SeqLM,
+    AutoTokenizer,
     Adafactor,
     AdamWeightDecay
 )
@@ -57,7 +58,7 @@ class MachineTranslationTransformer(pl.LightningModule):
         hidden_size=128,
         dropout=0.1,
         bidirectional=False,
-        length_penalty=1.2
+        length_penalty=1.2,
     ):
 
         super().__init__()

{translate_package-0.0.7 → translate_package-0.0.9}/translate_package/tokenization/load_tokenizer.py

@@ -3,13 +3,18 @@ from transformers import T5TokenizerFast
 from transformers import AutoTokenizer
 import os
 
-def load_tokenizer(tokenizer_name, model, dir_path, file_name, model_name = None):
+BCP_47_languages = {
+    'french': 'fra_Latn',
+    'wolof': 'wol_Latn',
+}
+
+def load_tokenizer(tokenizer_name, model, dir_path, file_name, model_name = None, src_lang = "french", tgt_lang = "wolof"):
     
     if model == "nllb":
         
         if not model_name is None:
         
-            tokenizer = AutoTokenizer.from_pretrained(model_name)
+            tokenizer = AutoTokenizer.from_pretrained(model_name, src_lang = BCP_47_languages[src_lang], tgt_lang = BCP_47_languages[tgt_lang])
             
             print(f"The {model}'s tokenizer was successfully loaded")
         

{translate_package-0.0.7 → translate_package-0.0.9}/translate_package.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: translate-package
-Version: 0.0.7
+Version: 0.0.9
 Summary: Contain functions and classes to efficiently train a sequence to sequence to translate between two languages.
 Author: Oumar Kane
 Author-email: oumar.kane@univ-thies.sn