wolof-translate 0.0.1__py3-none-any.whl

wolof_translate/utils/training.py

@@ -0,0 +1,213 @@
+from wolof_translate import *
+import warnings
+
+
+def train(config: dict):
+
+    # ---------------------------------------
+    # add distribution if necessary (https://github.com/aws/amazon-sagemaker-examples/blob/main/sagemaker-python-sdk/pytorch_mnist/mnist.py)
+
+    logger = config["logger"]
+
+    is_distributed = len(config["hosts"]) > 1 and config["backend"] is not None
+
+    use_cuda = config["num_gpus"] > 0
+
+    config.update({"num_workers": 1, "pin_memory": True} if use_cuda else {})
+
+    if not logger is None:
+
+        logger.debug("Distributed training - {}".format(is_distributed))
+
+        logger.debug("Number of gpus available - {}".format(config["num_gpus"]))
+
+    if is_distributed:
+        # Initialize the distributed environment.
+        world_size = len(config["hosts"])
+
+        os.environ["WORLD_SIZE"] = str(world_size)
+
+        host_rank = config["hosts"].index(config["current_host"])
+
+        os.environ["RANK"] = str(host_rank)
+
+        dist.init_process_group(
+            backend=config["backend"], rank=host_rank, world_size=world_size
+        )
+
+        if not logger is None:
+            logger.info(
+                "Initialized the distributed environment: '{}' backend on {} nodes. ".format(
+                    config["backend"], dist.get_world_size()
+                )
+                + "Current host rank is {}. Number of gpus: {}".format(
+                    dist.get_rank(), config["num_gpus"]
+                )
+            )
+    # ---------------------------------------
+
+    # split the data
+    if config["include_split"]:
+        split_data(
+            config["random_state"], config["data_directory"], config["data_file"]
+        )
+
+    # recuperate the tokenizer
+    tokenizer = T5TokenizerFast(config["tokenizer_path"])
+
+    # recuperate train and test set
+    train_dataset, test_dataset = recuperate_datasets(
+        config["char_p"],
+        config["word_p"],
+        config["max_len"],
+        config["end_mark"],
+        tokenizer,
+        config["corpus_1"],
+        config["corpus_2"],
+        config["train_file"],
+        config["test_file"],
+    )
+
+    # initialize the evaluation object
+    evaluation = TranslationEvaluation(tokenizer, train_dataset.decode)
+
+    # let us initialize the trainer
+    trainer = ModelRunner(
+        model=Transformer,
+        version=config["version"],
+        seed=0,
+        evaluation=evaluation,
+        optimizer=Adafactor,
+    )
+
+    # initialize the encoder and the decoder layers
+    encoder_layer = nn.TransformerEncoderLayer(
+        config["d_model"],
+        config["n_head"],
+        config["dim_ff"],
+        config["drop_out_rate"],
+        batch_first=True,
+    )
+
+    decoder_layer = nn.TransformerDecoderLayer(
+        config["d_model"],
+        config["n_head"],
+        config["dim_ff"],
+        config["drop_out_rate"],
+        batch_first=True,
+    )
+
+    # let us initialize the encoder and the decoder
+    encoder = nn.TransformerEncoder(encoder_layer, config["n_encoders"])
+
+    decoder = nn.TransformerDecoder(decoder_layer, config["n_decoders"])
+
+    # -------------------------------------
+    # in the case when the linear learning rate scheduler with warmup is used
+
+    # let us calculate the appropriate warmup steps (let us take a max epoch of 100)
+    # length = len(train_dataset)
+
+    # n_steps = length // config['batch_size']
+
+    # num_steps = config['max_epoch'] * n_steps
+
+    # warmup_steps = (config['max_epoch'] * n_steps) * config['warmup_ratio']
+
+    # Initialize the scheduler parameters
+    # scheduler_args = {'num_warmup_steps': warmup_steps, 'num_training_steps': num_steps}
+    # -------------------------------------
+
+    # Initialize the transformer parameters
+    model_args = {
+        "vocab_size": len(tokenizer),
+        "encoder": encoder,
+        "decoder": decoder,
+        "class_criterion": nn.CrossEntropyLoss(
+            label_smoothing=config["label_smoothing"]
+        ),
+        "max_len": config["max_len"],
+    }
+
+    # Initialize the optimizer parameters
+    optimizer_args = {
+        "lr": config["learning_rate"],
+        "weight_decay": config["weight_decay"],
+        # 'betas': (0.9, 0.98),
+        "warmup_init": config["warmup_init"],
+        "relative_step": config["relative_step"],
+    }
+
+    # ----------------------------
+    # initialize the bucket samplers for distributed environment
+    boundaries = config["boundaries"]
+    batch_sizes = config["batch_sizes"]
+
+    train_sampler = SequenceLengthBatchSampler(
+        train_dataset, boundaries=boundaries, batch_sizes=batch_sizes
+    )
+
+    test_sampler = SequenceLengthBatchSampler(
+        test_dataset, boundaries=boundaries, batch_sizes=batch_sizes
+    )
+
+    # ------------------------------
+    # initialize a bucket sampler with fixed batch size in the case of single machine
+    # with parallelization on multiple gpus
+    # train_sampler = BucketSampler(train_dataset, config['batch_size'])
+
+    # test_sampler = BucketSampler(test_dataset, config['batch_size'])
+
+    # ------------------------------
+
+    # Initialize the loaders parameters
+    train_loader_args = {
+        "batch_sampler": train_sampler,
+        "collate_fn": collate_fn,
+        "num_workers": config["num_workers"],
+        "pin_memory": config["pin_memory"],
+    }
+
+    test_loader_args = {
+        "batch_sampler": test_sampler,
+        "collate_fn": collate_fn,
+        "num_workers": config["num_workers"],
+        "pin_memory": config["pin_memory"],
+    }
+
+    # Add the datasets and hyperparameters to trainer
+    trainer.compile(
+        train_dataset,
+        test_dataset,
+        tokenizer,
+        train_loader_args,
+        test_loader_args,
+        optimizer_kwargs=optimizer_args,
+        model_kwargs=model_args,
+        # lr_scheduler=get_linear_schedule_with_warmup,
+        # lr_scheduler_kwargs=scheduler_args,
+        predict_with_generate=True,
+        is_distributed=is_distributed,
+        logging_dir=config["logging_dir"],
+        dist=dist,
+    )
+
+    # load the model
+    trainer.load(config["model_dir"], load_best=not config["continue"])
+
+    # Train the model
+    trainer.train(
+        config["epochs"] - trainer.current_epoch,
+        auto_save=True,
+        log_step=config["log_step"],
+        saving_directory=config["new_model_dir"],
+        save_best=config["save_best"],
+        metric_for_best_model=config["metric_for_best_model"],
+        metric_objective=config["metric_objective"],
+    )
+
+    if config["return_trainer"]:
+
+        return trainer
+
+    return None

wolof_translate/utils/trunc_hg_training.py

@@ -0,0 +1,196 @@
+from wolof_translate import *
+import warnings
+
+
+def train(config: dict):
+
+    # ---------------------------------------
+    # add distribution if necessary (https://github.com/aws/amazon-sagemaker-examples/blob/main/sagemaker-python-sdk/pytorch_mnist/mnist.py)
+
+    logger = config["logger"]
+
+    is_distributed = len(config["hosts"]) > 1 and config["backend"] is not None
+
+    use_cuda = config["num_gpus"] > 0
+
+    config.update({"num_workers": 1, "pin_memory": True} if use_cuda else {})
+
+    if not logger is None:
+
+        logger.debug("Distributed training - {}".format(is_distributed))
+
+        logger.debug("Number of gpus available - {}".format(config["num_gpus"]))
+
+    if is_distributed:
+        # Initialize the distributed environment.
+        world_size = len(config["hosts"])
+
+        os.environ["WORLD_SIZE"] = str(world_size)
+
+        host_rank = config["hosts"].index(config["current_host"])
+
+        os.environ["RANK"] = str(host_rank)
+
+        dist.init_process_group(
+            backend=config["backend"], rank=host_rank, world_size=world_size
+        )
+
+        if not logger is None:
+            logger.info(
+                "Initialized the distributed environment: '{}' backend on {} nodes. ".format(
+                    config["backend"], dist.get_world_size()
+                )
+                + "Current host rank is {}. Number of gpus: {}".format(
+                    dist.get_rank(), config["num_gpus"]
+                )
+            )
+    # ---------------------------------------
+
+    # split the data
+    if config["include_split"]:
+        split_data(
+            config["random_state"], config["data_directory"], config["data_file"]
+        )
+
+    # recuperate the tokenizer
+    tokenizer = T5TokenizerFast(config["tokenizer_path"])
+
+    # Initialize the model name
+    model_name = "t5-small"
+
+    # import the model with its pre-trained weights
+    model = T5ForConditionalGeneration.from_pretrained(model_name)
+
+    # resize the token embeddings
+    model.resize_token_embeddings(len(tokenizer))
+
+    # recuperate train and test set
+    train_dataset, test_dataset = recuperate_datasets(
+        config["char_p"],
+        config["word_p"],
+        50,
+        config["end_mark"],
+        tokenizer,
+        config["corpus_1"],
+        config["corpus_2"],
+        config["train_file"],
+        config["test_file"],
+    )
+
+    # initialize the evaluation object
+    evaluation = TranslationEvaluation(tokenizer, train_dataset.decode)
+
+    # let us initialize the trainer
+    trainer = ModelRunner(
+        model=model,
+        version=config["version"],
+        seed=0,
+        evaluation=evaluation,
+        optimizer=Adafactor,
+    )
+
+    # -------------------------------------
+    # in the case when the linear learning rate scheduler with warmup is used
+
+    # let us calculate the appropriate warmup steps (let us take a max epoch of 100)
+    # length = len(train_dataset)
+
+    # n_steps = length // config['batch_size']
+
+    # num_steps = config['max_epoch'] * n_steps
+
+    # warmup_steps = (config['max_epoch'] * n_steps) * config['warmup_ratio']
+
+    # Initialize the scheduler parameters
+    # scheduler_args = {'num_warmup_steps': warmup_steps, 'num_training_steps': num_steps}
+    # -------------------------------------
+
+    # Initialize the optimizer parameters
+    optimizer_args = {
+        "lr": config["learning_rate"],
+        "weight_decay": config["weight_decay"],
+        # 'betas': (0.9, 0.98),
+        "warmup_init": config["warmup_init"],
+        "relative_step": config["relative_step"],
+    }
+
+    # ----------------------------
+    # initialize the bucket samplers for distributed environment
+    # boundaries = config['boundaries']
+    # batch_sizes = config['batch_sizes']
+
+    # train_sampler = SequenceLengthBatchSampler(train_dataset,
+    #                                             boundaries = boundaries,
+    #                                             batch_sizes = batch_sizes)
+
+    # test_sampler = SequenceLengthBatchSampler(test_dataset,
+    #                                             boundaries = boundaries,
+    #                                             batch_sizes = batch_sizes)
+
+    # ------------------------------
+    # initialize a bucket sampler with fixed batch size in the case of single machine
+    # with parallelization on multiple gpus
+    train_sampler = BucketSampler(train_dataset, config["batch_size"])
+
+    test_sampler = BucketSampler(test_dataset, config["batch_size"])
+
+    # ------------------------------
+
+    # Initialize the loaders parameters
+    train_loader_args = {
+        "batch_sampler": train_sampler,
+        "collate_fn": partial(
+            collate_fn_trunc,
+            max_len=train_dataset.max_len,
+            eos_token_id=tokenizer.eos_token_id,
+            pad_token_id=tokenizer.pad_token_id,
+        ),
+    }
+
+    test_loader_args = {
+        "batch_sampler": test_sampler,
+        "collate_fn": partial(
+            collate_fn_trunc,
+            max_len=train_dataset.max_len,
+            eos_token_id=tokenizer.eos_token_id,
+            pad_token_id=tokenizer.pad_token_id,
+        ),
+    }
+
+    # Add the datasets and hyperparameters to trainer
+    trainer.compile(
+        train_dataset,
+        test_dataset,
+        tokenizer,
+        train_loader_args,
+        test_loader_args,
+        optimizer_kwargs=optimizer_args,
+        # lr_scheduler=get_linear_schedule_with_warmup,
+        # lr_scheduler_kwargs=scheduler_args,
+        predict_with_generate=True,
+        stopping_patience=config["stopping_patience"],
+        hugging_face=True,
+        is_distributed=is_distributed,
+        logging_dir=config["logging_dir"],
+        dist=dist,
+    )
+
+    # load the model
+    trainer.load(config["model_dir"], load_best=not config["continue"])
+
+    # Train the model
+    trainer.train(
+        config["epochs"] - trainer.current_epoch,
+        auto_save=True,
+        log_step=config["log_step"],
+        saving_directory=config["new_model_dir"],
+        save_best=config["save_best"],
+        metric_for_best_model=config["metric_for_best_model"],
+        metric_objective=config["metric_objective"],
+    )
+
+    if config["return_trainer"]:
+
+        return trainer
+
+    return None

wolof_translate-0.0.1.dist-info/METADATA

@@ -0,0 +1,31 @@
+Metadata-Version: 2.1
+Name: wolof-translate
+Version: 0.0.1
+Summary: Contain function and classes to process corpora for making translation between wolof text and other languages.
+Author: Oumar Kane
+Author-email: oumar.kane@univ-thies.sn
+Requires-Dist: accelerate
+Requires-Dist: torch
+Requires-Dist: spacy
+Requires-Dist: nltk
+Requires-Dist: gensim
+Requires-Dist: furo
+Requires-Dist: streamlit
+Requires-Dist: tokenizers
+Requires-Dist: tensorboard
+Requires-Dist: evaluate
+Requires-Dist: transformers
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: scikit-learn
+Requires-Dist: matplotlib
+Requires-Dist: plotly
+Requires-Dist: sacrebleu
+Requires-Dist: nlpaug
+Requires-Dist: wandb
+Requires-Dist: pytorch-lightning
+Requires-Dist: selenium
+Requires-Dist: sentencepiece
+Requires-Dist: peft
+Requires-Dist: rouge-score
+

wolof_translate-0.0.1.dist-info/RECORD

@@ -0,0 +1,49 @@
+wolof_translate/__init__.py,sha256=qHFFSR2P3SpUh62FuZghk3KWNCeo2At_SJnRb3wRRpU,2509
+wolof_translate/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wolof_translate/data/dataset_v1.py,sha256=7BWW4jv8qR1Wau_-GT007hrZpnHxbPwnvDca8C0_eHU,4264
+wolof_translate/data/dataset_v2.py,sha256=nTYW88yP6mcky-T43mh-1CeYU30-Xlq4DxOpd3FZ0OQ,5613
+wolof_translate/data/dataset_v3.py,sha256=SD0VGr2oMl_4TUafD_5ZKqiZAnbwuFD44sajI2gGl2Y,5628
+wolof_translate/data/dataset_v3_2.py,sha256=-gq31O7dt41Zi196m6uHKV6PHtODAeZq95ICc9NJXRA,5628
+wolof_translate/data/dataset_v4.py,sha256=0c97Pkjb7TBC5G91-1bFvVzafLQpruX3lj_PDOzmaAE,6271
+wolof_translate/data/dataset_v5.py,sha256=JZMacAn4jdfCOJ3PH2tWnQ4qDN_OzYYquKHBTOe_ZdM,2050
+wolof_translate/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wolof_translate/models/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wolof_translate/models/transformers/main.py,sha256=Ua8AG0CRhkJ0At7Ma8nt7lcl9226DGg-PgrR8MOTgPo,30276
+wolof_translate/models/transformers/main_2.py,sha256=ApUMR5LxSYHfF48wOkAgfEgnUm-4fYD9JFrq_llx_0U,12015
+wolof_translate/models/transformers/optimization.py,sha256=2YJ66NUBJBM86CNNx8mCng1ryFAVbcS08fUK7f3OYOk,1401
+wolof_translate/models/transformers/position.py,sha256=44Z-qSfB-NGhqXOXKbaztC8Bpc17BHcOX69zMssjbeg,1672
+wolof_translate/models/transformers/size.py,sha256=UI4I30cwJMZ5RDs31X63HWW60_Zgy542FMC9sguQHqc,1317
+wolof_translate/pipe/__init__.py,sha256=n2k4pXK1y-xxVnX13E9H1-hkYtpHbCAJVs-LMARWlvI,81
+wolof_translate/pipe/nlp_pipeline.py,sha256=jmC5xXb1pAZ9uWXFepGCcYjIIzMk3Hpqu1cBMaMN2KE,15610
+wolof_translate/tokenizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wolof_translate/trainers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wolof_translate/trainers/transformer_trainer.py,sha256=yK6GtQUXRdcG9XxR5QeEOgId8T6Oh3WJtFVmiyFmI_U,26691
+wolof_translate/trainers/transformer_trainer_custom.py,sha256=hHUBcU4YK6wuRUMiwX5xG0oTaoDLt9bVSZbzIFRuces,31926
+wolof_translate/trainers/transformer_trainer_ml.py,sha256=WgggaugkVHSJlwIAZT-QwI90Fl-_zT8Clhb-7M0m8gM,33561
+wolof_translate/trainers/transformer_trainer_ml_.py,sha256=QaN9DB5pqhBxV4WlFmJCmUyfwlX-UyAzKRwL6rVEr4Q,38199
+wolof_translate/utils/__init__.py,sha256=Nl3300H-Xd3uTHDR8y-rYa-UUR9FqbqZPwUKJUpQOb4,64
+wolof_translate/utils/bucket_iterator.py,sha256=Hglii1Hj6H_K51JunTjUAxuLd4ehPb6LeeMVhsmhNxQ,6248
+wolof_translate/utils/database_manager.py,sha256=7yhgBN1LvVFNEQikxCjSCva82h5nX44Nx2zh8cpFWyA,3543
+wolof_translate/utils/display_predictions.py,sha256=y5H5lfgIODl6E5Zfb1YIwiAxIlHUxRBoChfQR5kjh24,5145
+wolof_translate/utils/download_model.py,sha256=x92KpfVPvNK8Suen1qnOcPtZOlB4kXTfqWgoVuuMUEM,1241
+wolof_translate/utils/evaluate_custom.py,sha256=cmcGfRAjhTuP9ekeJ0cioNoE1cQ7fQ7mZTh6_1IAaXM,3587
+wolof_translate/utils/evaluation.py,sha256=Taxv4UAgg5q3WxC73pp84srr_wX6Kw9Ub9MloYrUmLs,1838
+wolof_translate/utils/extract_new_sentences.py,sha256=li9UDgLa4nI6DSdB5oH0_m8xek3EEoVBL3CrVKTxGrc,22861
+wolof_translate/utils/extract_poems.py,sha256=9Pf1PluUq257vcS2iinGPi6azGjgmHU7Q57uwQkHfAs,1314
+wolof_translate/utils/extract_sentences.py,sha256=-PDBmceKUqiTdV9ieezSIITfADAnv_7OsNY8zdJi0To,15713
+wolof_translate/utils/recuperate_datasets.py,sha256=4yTNXPOIfTokon0Bke50SdB8MT_Ojmu1aTmYv_K_w64,2644
+wolof_translate/utils/recuperate_datasets_trunc.py,sha256=82T7mHbxruYJUw0L0ZUUoPHxO2Yr65rApakmIhe034M,2500
+wolof_translate/utils/send_model.py,sha256=v_dQJDDpk3ak_DutbhwSqKF8-Q_-Gx9zezZsTot6Onk,797
+wolof_translate/utils/sent_corrections.py,sha256=5iqdS4j78ayag0GxnCEl_dBUs4zbBAWAOac2h0ECv4c,3534
+wolof_translate/utils/sent_transformers.py,sha256=kbbc5H-zPkxSM1uOghGeZa9fCAcm2GwTSuiRHM0asgI,574
+wolof_translate/utils/sent_unification.py,sha256=UD9uZ--NREj5Z462n5hs-UjMPNhUN8Nr_6ZmR2w-B6Y,2104
+wolof_translate/utils/split_with_valid.py,sha256=7-e6EfvPbLpTYrZOXJVYYqm_nV7n6yUYOaWkn8hsJJw,2424
+wolof_translate/utils/tokenize_text.py,sha256=LZNsYmpchlkNsul00yb3HQToC-L7XSYuPHGCRCfbz9Y,1226
+wolof_translate/utils/training.py,sha256=5vPVuqHL6_gqLkh4PTxXqW4UvAJBWNWVDDXC9Fk7IQI,6732
+wolof_translate/utils/trunc_hg_training.py,sha256=mMGrU7Mjr9vYd7eLc8nbFRhRXwSWMKyg35lGf0L6RtQ,6418
+wolof_translate/utils/improvements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wolof_translate/utils/improvements/end_marks.py,sha256=scmhMMYguZmrZTPozx1ZovizKrrPfPpMLXbU2-IOdGs,1194
+wolof_translate-0.0.1.dist-info/METADATA,sha256=itQMCA-zGM3gSDiKco5dMtY5qfpPHvKNJbw9KtffxzI,818
+wolof_translate-0.0.1.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+wolof_translate-0.0.1.dist-info/top_level.txt,sha256=YG-kBnOwUZyQ7SofNvMxNYjzCreH2PVcW2UaEg1-Reg,16
+wolof_translate-0.0.1.dist-info/RECORD,,

wolof_translate-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.37.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

wolof_translate-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+wolof_translate