wolof-translate 0.0.1__tar.gz

wolof_translate-0.0.1/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.1
+Name: wolof_translate
+Version: 0.0.1
+Summary: Contain function and classes to process corpora for making translation between wolof text and other languages.
+Author: Oumar Kane
+Author-email: oumar.kane@univ-thies.sn

wolof_translate-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

wolof_translate-0.0.1/setup.py

@@ -0,0 +1,36 @@
+from setuptools import setup
+
+setup(
+    name="wolof_translate",
+    version="0.0.1",
+    author="Oumar Kane",
+    author_email="oumar.kane@univ-thies.sn",
+    description="Contain function and classes to process corpora for making translation between wolof text and other languages.",
+    install_requires=[
+        "accelerate",
+        "torch",
+        "spacy",
+        "nltk",
+        "gensim",
+        "furo",
+        "streamlit",
+        "tokenizers",
+        "tensorboard",
+        "evaluate",
+        "transformers",
+        "pandas",
+        "numpy",
+        "scikit-learn",
+        "matplotlib",
+        "plotly",
+        "sacrebleu",
+        "nlpaug",
+        "wandb",
+        "pytorch-lightning",
+        "selenium",
+        "sentencepiece",
+        "peft",
+        "rouge-score",
+        "sacrebleu"
+    ],
+)

wolof_translate-0.0.1/wolof_translate/__init__.py

@@ -0,0 +1,73 @@
+"""Script containing importation
+================================
+"""
+
+# let us import all necessary libraries
+from transformers import (
+    T5Model,
+    T5ForConditionalGeneration,
+    Seq2SeqTrainer,
+    T5TokenizerFast,
+    set_seed,
+    AdamWeightDecay,
+    get_linear_schedule_with_warmup,
+    get_linear_schedule_with_warmup,
+    get_cosine_schedule_with_warmup,
+    get_constant_schedule_with_warmup,
+    Adafactor,
+)
+from wolof_translate.utils.sent_transformers import TransformerSequences
+from wolof_translate.utils.improvements.end_marks import add_end_mark  # added
+from torch.nn import TransformerEncoderLayer, TransformerDecoderLayer
+from torch.utils.data import Dataset, DataLoader, random_split
+from wolof_translate.data.dataset_v4 import SentenceDataset  # v2 -> v3 -> v4
+from wolof_translate.utils.sent_corrections import *
+from sklearn.model_selection import train_test_split
+from torch.optim.lr_scheduler import _LRScheduler
+from torch.nn.utils.rnn import pad_sequence
+from plotly.subplots import make_subplots
+from nlpaug.augmenter import char as nac
+from torch.utils.data import DataLoader
+from torch.nn import functional as F
+import plotly.graph_objects as go
+from tokenizers import Tokenizer
+import torch.distributed as dist
+import matplotlib.pyplot as plt
+import pytorch_lightning as lt
+from tqdm import tqdm, trange
+from functools import partial
+from torch.nn import utils
+from copy import deepcopy
+from torch import optim
+from typing import *
+from torch import nn
+import pandas as pd
+import numpy as np
+import itertools
+import evaluate
+import random
+import string
+import shutil
+import wandb
+import torch
+import json
+import copy
+import os
+
+###-----------------------------------------------
+# Libraries imported from wolof translate
+from wolof_translate.utils.bucket_iterator import (
+    SequenceLengthBatchSampler,
+    BucketSampler,
+    collate_fn,
+    collate_fn_trunc,
+)
+from wolof_translate.trainers.transformer_trainer_custom import (
+    ModelRunner as CustomModelRunner,
+)
+from wolof_translate.models.transformers.optimization import TransformerScheduler
+from wolof_translate.utils.recuperate_datasets import recuperate_datasets
+from wolof_translate.trainers.transformer_trainer_ml_ import ModelRunner
+from wolof_translate.utils.evaluate_custom import TranslationEvaluation
+from wolof_translate.models.transformers.main import Transformer
+from wolof_translate.utils.split_with_valid import split_data

wolof_translate-0.0.1/wolof_translate/data/dataset_v1.py

@@ -0,0 +1,151 @@
+from wolof_translate.utils.sent_transformers import TransformerSequences
+from transformers import PreTrainedTokenizerFast
+from torch.utils.data import Dataset
+from tokenizers import Tokenizer
+from typing import *
+import pandas as pd
+import torch
+import re
+
+
+class SentenceDataset(Dataset):
+    def __init__(
+        self,
+        file_path: str,
+        corpus_1: str = "french_corpus",
+        corpus_2: str = "wolof_corpus",
+        tokenizer_path: str = "wolof-translate/wolof_translate/tokenizers/tokenizer_v1.json",
+        max_len: int = 379,
+        truncation: bool = False,
+        file_sep: str = ",",
+        cls_token: str = "<|endoftext|>",
+        sep_token: str = "<|translateto|>",
+        pad_token: str = "<|pad|>",
+        cp1_transformer: Union[TransformerSequences, None] = None,
+        cp2_transformer: Union[TransformerSequences, None] = None,
+        **kwargs,
+    ):
+
+        # let us recuperate the data frame
+        self.__sentences = pd.read_csv(file_path, sep=file_sep, **kwargs)
+
+        # let us recuperate the tokenizer
+        self.tokenizer = PreTrainedTokenizerFast(
+            tokenizer_file=tokenizer_path,
+            bos_token=cls_token,
+            eos_token=cls_token,
+            pad_token=pad_token,
+        )
+
+        # recuperate the first corpus' sentences
+        self.__sentences_1 = self.__sentences[corpus_1].to_list()
+
+        # recuperate the second corpus' sentences
+        self.__sentences_2 = self.__sentences[corpus_2].to_list()
+
+        # recuperate the special tokens
+        self.cls_token = cls_token
+
+        self.sep_token = sep_token
+
+        self.pad_token = pad_token
+
+        # recuperate the length
+        self.__length = len(self.__sentences_1)
+
+        # recuperate the max id
+        self.max_id = len(self.tokenizer) - 1
+
+        # let us recuperate the max len
+        self.max_len = max_len
+
+        # let us recuperate the truncate argument
+        self.truncation = truncation
+
+        # let us initialize the transformer
+        self.cp1_transformer = cp1_transformer
+
+        self.cp2_transformer = cp2_transformer
+
+    def __getitem__(self, index):
+
+        sentence_1 = self.__sentences_1[index]
+
+        sentence_2 = self.__sentences_2[index]
+
+        # apply transformers if necessary
+        if not self.cp1_transformer is None:
+
+            sentence_1 = self.cp1_transformer(sentence_1)
+
+        if not self.cp2_transformer is None:
+
+            sentence_2 = self.cp2_transformer(sentence_2)
+
+        # let us create the sentence with special tokens
+        sentence = (
+            f"{self.cls_token}{sentence_1}{self.sep_token}{sentence_2}{self.cls_token}"
+        )
+
+        # let us encode the sentence
+        encoding = self.tokenizer(
+            sentence,
+            truncation=self.truncation,
+            max_length=self.max_len,
+            padding="max_length",
+            return_tensors="pt",
+        )
+
+        return encoding.input_ids.squeeze(0), encoding.attention_mask.squeeze(0)
+
+    def __len__(self):
+
+        return self.__length
+
+    def decode(self, ids: torch.Tensor, for_prediction: bool = False):
+
+        if ids.ndim < 2:
+
+            ids = ids.unsqueeze(0)
+
+        ids = ids.tolist()
+
+        for id in ids:
+
+            sentence = self.tokenizer.decode(id)
+
+            if not for_prediction:
+
+                sentence = sentence.split(f"{self.sep_token}")
+
+            else:
+
+                try:
+
+                    while self.sep_token in sentence:
+
+                        sentence = re.findall(f"{self.sep_token}(.*)", sentence)[-1]
+
+                except:
+
+                    sentence = "None"
+
+            if for_prediction:
+
+                yield sentence.replace(f"{self.cls_token}", "").replace(
+                    f"{self.pad_token}", ""
+                )
+
+            else:
+
+                sents = []
+
+                for sent in sentence:
+
+                    sents.append(
+                        sent.replace(f"{self.cls_token}", "").replace(
+                            f"{self.pad_token}", ""
+                        )
+                    )
+
+                yield sents

wolof_translate-0.0.1/wolof_translate/data/dataset_v2.py

@@ -0,0 +1,187 @@
+from wolof_translate.utils.sent_transformers import TransformerSequences
+from transformers import PreTrainedTokenizerFast
+from torch.utils.data import Dataset
+from typing import *
+import pandas as pd
+import torch
+import re
+
+
+class T5SentenceDataset(Dataset):
+    def __init__(
+        self,
+        data_path: str,
+        tokenizer: PreTrainedTokenizerFast,
+        corpus_1: str = "french",
+        corpus_2: str = "wolof",
+        max_len: int = 51,
+        truncation: bool = False,
+        file_sep: str = ",",
+        cp1_transformer: Union[TransformerSequences, None] = None,
+        cp2_transformer: Union[TransformerSequences, None] = None,
+        **kwargs
+    ):
+
+        # let us recuperate the data frame
+        self.__sentences = pd.read_csv(data_path, sep=file_sep, **kwargs)
+
+        # let us recuperate the tokenizer
+        self.tokenizer = tokenizer
+
+        # recuperate the first corpus' sentences
+        self.sentences_1 = self.__sentences[corpus_1].to_list()
+
+        # recuperate the second corpus' sentences
+        self.sentences_2 = self.__sentences[corpus_2].to_list()
+
+        # recuperate the length
+        self.length = len(self.sentences_1)
+
+        # let us recuperate the max len
+        self.max_len = max_len
+
+        # let us recuperate the truncation argument
+        self.truncation = truncation
+
+        # let us initialize the transformer
+        self.cp1_transformer = cp1_transformer
+
+        self.cp2_transformer = cp2_transformer
+
+    def __getitem__(self, index):
+        """Recuperate ids and attention masks of sentences at index
+
+        Args:
+            index (int): The index of the sentences to recuperate
+
+        Returns:
+            tuple: The `sentence to translate' ids`, `the attention mask of the sentence to translate`
+            `the labels' ids`
+        """
+        sentence_1 = self.sentences_1[index]
+
+        sentence_2 = self.sentences_2[index]
+
+        # apply transformers if necessary
+        if not self.cp1_transformer is None:
+
+            sentence_1 = self.cp1_transformer(sentence_1)[0]
+
+        if not self.cp2_transformer is None:
+
+            sentence_2 = self.cp2_transformer(sentence_2)[0]
+
+        sentence_1 = sentence_1 + self.tokenizer.eos_token
+
+        sentence_2 = sentence_2 + self.tokenizer.eos_token
+
+        # let us encode the sentences (we provide the second sentence as labels to the tokenizer)
+        data = self.tokenizer(
+            sentence_1,
+            truncation=self.truncation,
+            max_length=self.max_len,
+            padding="max_length",
+            return_tensors="pt",
+            text_target=sentence_2,
+        )
+
+        return (
+            data.input_ids.squeeze(0),
+            data.attention_mask.squeeze(0),
+            data.labels.squeeze(0),
+        )
+
+    def __len__(self):
+
+        return self.length
+
+    def decode(self, labels: torch.Tensor):
+
+        if labels.ndim < 2:
+
+            labels = labels.unsqueeze(0)
+
+        sentences = self.tokenizer.batch_decode(labels, skip_special_tokens=True)
+
+        return sentences
+
+
+class SentenceDataset(T5SentenceDataset):
+    def __init__(
+        self,
+        data_path: str,
+        tokenizer: PreTrainedTokenizerFast,
+        corpus_1: str = "french",
+        corpus_2: str = "wolof",
+        max_len: int = 51,
+        truncation: bool = False,
+        file_sep: str = ",",
+        cp1_transformer: Union[TransformerSequences, None] = None,
+        cp2_transformer: Union[TransformerSequences, None] = None,
+        **kwargs
+    ):
+
+        super().__init__(
+            data_path,
+            tokenizer,
+            corpus_1,
+            corpus_2,
+            max_len,
+            truncation,
+            file_sep,
+            cp1_transformer,
+            cp2_transformer,
+            **kwargs
+        )
+
+    def __getitem__(self, index):
+        """Recuperate ids and attention masks of sentences at index
+
+        Args:
+            index (int): The index of the sentences to recuperate
+
+        Returns:
+            tuple: The `sentence to translate' ids`, `the attention mask of the sentence to translate`
+            `the labels' ids`
+        """
+        sentence_1 = self.sentences_1[index]
+
+        sentence_2 = self.sentences_2[index]
+
+        # apply transformers if necessary
+        if not self.cp1_transformer is None:
+
+            sentence_1 = self.cp1_transformer(sentence_1)[0]
+
+        if not self.cp2_transformer is None:
+
+            sentence_2 = self.cp2_transformer(sentence_2)[0]
+
+        sentence_1 = sentence_1 + self.tokenizer.eos_token
+
+        sentence_2 = sentence_2 + self.tokenizer.eos_token
+
+        # let us encode the sentences (we provide the second sentence as labels to the tokenizer)
+        data = self.tokenizer(
+            sentence_1,
+            truncation=self.truncation,
+            max_length=self.max_len,
+            padding="max_length",
+            return_tensors="pt",
+        )
+
+        # let us encode the sentences (we provide the second sentence as labels to the tokenizer)
+        labels = self.tokenizer(
+            sentence_2,
+            truncation=self.truncation,
+            max_length=self.max_len,
+            padding="max_length",
+            return_tensors="pt",
+        )
+
+        return (
+            data.input_ids.squeeze(0),
+            data.attention_mask.squeeze(0),
+            labels.input_ids.squeeze(0),
+            labels.attention_mask.squeeze(0),
+        )

wolof_translate-0.0.1/wolof_translate/data/dataset_v3.py

@@ -0,0 +1,187 @@
+from wolof_translate.utils.sent_transformers import TransformerSequences
+from transformers import PreTrainedTokenizerFast
+from torch.utils.data import Dataset
+from typing import *
+import pandas as pd
+import torch
+import re
+
+
+class T5SentenceDataset(Dataset):
+    def __init__(
+        self,
+        data_path: str,
+        tokenizer: PreTrainedTokenizerFast,
+        corpus_1: str = "french",
+        corpus_2: str = "wolof",
+        max_len: int = 60,
+        truncation: bool = False,
+        file_sep: str = ",",
+        cp1_transformer: Union[TransformerSequences, None] = None,
+        cp2_transformer: Union[TransformerSequences, None] = None,
+        **kwargs
+    ):
+
+        # let us recuperate the data frame
+        self.__sentences = pd.read_csv(data_path, sep=file_sep, **kwargs)
+
+        # let us recuperate the tokenizer
+        self.tokenizer = tokenizer
+
+        # recuperate the first corpus' sentences
+        self.sentences_1 = self.__sentences[corpus_1].to_list()
+
+        # recuperate the second corpus' sentences
+        self.sentences_2 = self.__sentences[corpus_2].to_list()
+
+        # recuperate the length
+        self.length = len(self.sentences_1)
+
+        # let us recuperate the max len
+        self.max_len = max_len + max_len // 6
+
+        # let us recuperate the truncation argument
+        self.truncation = truncation
+
+        # let us initialize the transformer
+        self.cp1_transformer = cp1_transformer
+
+        self.cp2_transformer = cp2_transformer
+
+    def __getitem__(self, index):
+        """Recuperate ids and attention masks of sentences at index
+
+        Args:
+            index (int): The index of the sentences to recuperate
+
+        Returns:
+            tuple: The `sentence to translate' ids`, `the attention mask of the sentence to translate`
+            `the labels' ids`
+        """
+        sentence_1 = self.sentences_1[index]
+
+        sentence_2 = self.sentences_2[index]
+
+        # apply transformers if necessary
+        if not self.cp1_transformer is None:
+
+            sentence_1 = self.cp1_transformer(sentence_1)[0]
+
+        if not self.cp2_transformer is None:
+
+            sentence_2 = self.cp2_transformer(sentence_2)[0]
+
+        sentence_1 = sentence_1 + self.tokenizer.eos_token
+
+        sentence_2 = sentence_2 + self.tokenizer.eos_token
+
+        # let us encode the sentences (we provide the second sentence as labels to the tokenizer)
+        data = self.tokenizer(
+            sentence_1,
+            truncation=self.truncation,
+            max_length=self.max_len,
+            padding="max_length",
+            return_tensors="pt",
+            text_target=sentence_2,
+        )
+
+        return (
+            data.input_ids.squeeze(0),
+            data.attention_mask.squeeze(0),
+            data.labels.squeeze(0),
+        )
+
+    def __len__(self):
+
+        return self.length
+
+    def decode(self, labels: torch.Tensor):
+
+        if labels.ndim < 2:
+
+            labels = labels.unsqueeze(0)
+
+        sentences = self.tokenizer.batch_decode(labels, skip_special_tokens=True)
+
+        return sentences
+
+
+class SentenceDataset(T5SentenceDataset):
+    def __init__(
+        self,
+        data_path: str,
+        tokenizer: PreTrainedTokenizerFast,
+        corpus_1: str = "french",
+        corpus_2: str = "wolof",
+        max_len: int = 42,
+        truncation: bool = False,
+        file_sep: str = ",",
+        cp1_transformer: Union[TransformerSequences, None] = None,
+        cp2_transformer: Union[TransformerSequences, None] = None,
+        **kwargs
+    ):
+
+        super().__init__(
+            data_path,
+            tokenizer,
+            corpus_1,
+            corpus_2,
+            max_len,
+            truncation,
+            file_sep,
+            cp1_transformer,
+            cp2_transformer,
+            **kwargs
+        )
+
+    def __getitem__(self, index):
+        """Recuperate ids and attention masks of sentences at index
+
+        Args:
+            index (int): The index of the sentences to recuperate
+
+        Returns:
+            tuple: The `sentence to translate' ids`, `the attention mask of the sentence to translate`
+            `the labels' ids`
+        """
+        sentence_1 = self.sentences_1[index]
+
+        sentence_2 = self.sentences_2[index]
+
+        # apply transformers if necessary
+        if not self.cp1_transformer is None:
+
+            sentence_1 = self.cp1_transformer(sentence_1)[0]
+
+        if not self.cp2_transformer is None:
+
+            sentence_2 = self.cp2_transformer(sentence_2)[0]
+
+        sentence_1 = sentence_1 + self.tokenizer.eos_token
+
+        sentence_2 = sentence_2 + self.tokenizer.eos_token
+
+        # let us encode the sentences (we provide the second sentence as labels to the tokenizer)
+        data = self.tokenizer(
+            sentence_1,
+            truncation=self.truncation,
+            max_length=self.max_len,
+            padding="max_length",
+            return_tensors="pt",
+        )
+
+        # let us encode the sentences (we provide the second sentence as labels to the tokenizer)
+        labels = self.tokenizer(
+            sentence_2,
+            truncation=self.truncation,
+            max_length=self.max_len,
+            padding="max_length",
+            return_tensors="pt",
+        )
+
+        return (
+            data.input_ids.squeeze(0),
+            data.attention_mask.squeeze(0),
+            labels.input_ids.squeeze(0),
+            labels.attention_mask.squeeze(0),
+        )