wolof-translate 0.0.1__py3-none-any.whl

wolof_translate/utils/recuperate_datasets.py

@@ -0,0 +1,94 @@
+from wolof_translate import *
+
+
+def recuperate_datasets(
+    char_p: float,
+    word_p: float,
+    max_len: int,
+    end_mark: int,
+    tokenizer: T5TokenizerFast,
+    corpus_1: str = "french",
+    corpus_2: str = "wolof",
+    train_file: str = "data/extractions/new_data/train_set.csv",
+    test_file: str = "data/extractions/new_data/test_file.csv",
+):
+
+    # Let us recuperate the end_mark adding option
+    if end_mark == 1:
+        # Create augmentation to add on French sentences
+        fr_augmentation_1 = TransformerSequences(
+            nac.RandomCharAug(
+                action="swap",
+                aug_char_p=char_p,
+                aug_word_p=word_p,
+                aug_word_max=max_len,
+            ),
+            remove_mark_space,
+            delete_guillemet_space,
+            add_mark_space,
+        )
+
+        fr_augmentation_2 = TransformerSequences(
+            remove_mark_space, delete_guillemet_space, add_mark_space
+        )
+
+    else:
+
+        if end_mark == 2:
+
+            end_mark_fn = partial(add_end_mark, end_mark_to_remove="!", replace=True)
+
+        elif end_mark == 3:
+
+            end_mark_fn = partial(add_end_mark)
+
+        elif end_mark == 4:
+
+            end_mark_fn = partial(add_end_mark, end_mark_to_remove="!")
+
+        else:
+
+            raise ValueError(f"No end mark number {end_mark}")
+
+        # Create augmentation to add on French sentences
+        fr_augmentation_1 = TransformerSequences(
+            nac.RandomCharAug(
+                action="swap",
+                aug_char_p=char_p,
+                aug_word_p=word_p,
+                aug_word_max=max_len,
+            ),
+            remove_mark_space,
+            delete_guillemet_space,
+            add_mark_space,
+            end_mark_fn,
+        )
+
+        fr_augmentation_2 = TransformerSequences(
+            remove_mark_space, delete_guillemet_space, add_mark_space, end_mark_fn
+        )
+
+    # Recuperate the train dataset
+    train_dataset_aug = SentenceDataset(
+        train_file,
+        tokenizer,
+        truncation=False,
+        cp1_transformer=fr_augmentation_1,
+        cp2_transformer=fr_augmentation_2,
+        corpus_1=corpus_1,
+        corpus_2=corpus_2,
+    )
+
+    # Recuperate the valid dataset
+    valid_dataset = SentenceDataset(
+        test_file,
+        tokenizer,
+        cp1_transformer=fr_augmentation_2,
+        cp2_transformer=fr_augmentation_2,
+        corpus_1=corpus_1,
+        corpus_2=corpus_2,
+        truncation=False,
+    )
+
+    # Return the datasets
+    return train_dataset_aug, valid_dataset

wolof_translate/utils/recuperate_datasets_trunc.py

@@ -0,0 +1,85 @@
+from wolof_translate import *
+
+
+def recuperate_datasets(
+    char_p: float,
+    word_p: float,
+    max_len: int,
+    end_mark: int,
+    tokenizer: T5TokenizerFast,
+    corpus_1: str = "french",
+    corpus_2: str = "wolof",
+    train_file: str = "data/extractions/new_data/train_set.csv",
+    test_file: str = "data/extractions/new_data/test_file.csv",
+    augmenter=partial(nac.RandomCharAug, action="swap"),
+):
+
+    # Let us recuperate the end_mark adding option
+    if end_mark == 1:
+        # Create augmentation to add on French sentences
+        fr_augmentation_1 = TransformerSequences(
+            augmenter(aug_char_p=fr_char_p, aug_word_p=fr_word_p, aug_word_max=max_len),
+            remove_mark_space,
+            delete_guillemet_space,
+            add_mark_space,
+        )
+
+        fr_augmentation_2 = TransformerSequences(
+            remove_mark_space, delete_guillemet_space, add_mark_space
+        )
+
+    else:
+
+        if end_mark == 2:
+
+            end_mark_fn = partial(add_end_mark, end_mark_to_remove="!", replace=True)
+
+        elif end_mark == 3:
+
+            end_mark_fn = partial(add_end_mark)
+
+        elif end_mark == 4:
+
+            end_mark_fn = partial(add_end_mark, end_mark_to_remove="!")
+
+        else:
+
+            raise ValueError(f"No end mark number {end_mark}")
+
+        # Create augmentation to add on French sentences
+        fr_augmentation_1 = TransformerSequences(
+            augmenter(aug_char_p=fr_char_p, aug_word_p=fr_word_p, aug_word_max=max_len),
+            remove_mark_space,
+            delete_guillemet_space,
+            add_mark_space,
+            end_mark_fn,
+        )
+
+        fr_augmentation_2 = TransformerSequences(
+            remove_mark_space, delete_guillemet_space, add_mark_space, end_mark_fn
+        )
+
+    # Recuperate the train dataset
+    train_dataset_aug = SentenceDataset(
+        train_file,
+        tokenizer,
+        truncation=False,
+        cp1_transformer=fr_augmentation_1,
+        cp2_transformer=fr_augmentation_2,
+        corpus_1=corpus_1,
+        corpus_2=corpus_2,
+    )
+
+    # Recuperate the valid dataset
+    valid_dataset = SentenceDataset(
+        test_file,
+        tokenizer,
+        cp1_transformer=fr_augmentation_2,
+        cp2_transformer=fr_augmentation_2,
+        corpus_1=corpus_1,
+        corpus_2=corpus_2,
+        truncation=False,
+    )
+
+    # Return the datasets
+    return train_dataset_aug, valid_dataset

wolof_translate/utils/send_model.py

@@ -0,0 +1,26 @@
+import wandb
+
+
+def add_directory(
+    directory: str,
+    artifact_name: str,
+    project: str = "fw_artifacts",
+    entity: str = "oumar-kane-team",
+):
+    """Initialize a project and add checkpoints as artifact to wandb
+
+    Args:
+        directory (str): The directory where are stored the checkpoints
+        artifact_name (_type_): The name of the artifact
+        project (str, optional): The project name. Defaults to 'fw_artifacts'.
+        entity (str, optional): The entity name. Defaults to 'oumar-kane-team'.
+    """
+
+    run = wandb.init(project=project, entity=entity)
+
+    # add a directory as artifact to wandb
+    artifact = wandb.Artifact(artifact_name, type="dataset")
+    artifact.add_dir(directory)
+    run.log_artifact(artifact)
+
+    wandb.finish()

wolof_translate/utils/sent_corrections.py

@@ -0,0 +1,169 @@
+from typing import *
+
+
+def add_guillemet_space(sentences: Union[list, str]):
+    """Adding space between a letter and guillemet in a sentence
+
+    Args:
+        sentence (Union[list, str]): The sentence that will be changed
+
+    Returns:
+        str: The modified sentence
+    """
+
+    if type(sentences) is str:
+        sentences = [sentences]
+
+    for s in range(len(sentences)):
+
+        sentence = sentences[s]
+
+        if "«" in sentence:
+
+            sentence = sentence.split()
+
+            for i in range(len(sentence)):
+
+                word = sentence[i]
+
+                if "«" in word and word != "«":
+
+                    word = word.split("«")
+
+                    word = "« ".join(word)
+
+                if "»" in word and word != "»":
+
+                    word = word.split("»")
+
+                    word = " »".join(word)
+
+                sentence[i] = word
+
+            sentence = " ".join(sentence)
+
+        sentences[s] = sentence
+
+    return sentences
+
+
+def delete_guillemet_space(sentences: Union[list, str]):
+
+    if type(sentences) is str:
+        sentences = [sentences]
+
+    for s in range(len(sentences)):
+
+        sentence = sentences[s]
+
+        letters = [sentence[0]]
+
+        for i in range(1, len(sentence)):
+
+            if sentence[i] == "”":
+
+                j = i - 1
+
+                while letters[j] == " ":
+
+                    letters[j] = ""
+
+                    j -= 1
+
+                letters.append(sentence[i])
+
+            elif letters[-1] == "“" and sentence[i] == " ":
+
+                letters.append("")
+
+            else:
+
+                letters.append(sentence[i])
+
+        sentences[s] = "".join(letters)
+
+    return sentences
+
+
+def add_mark_space(
+    sentences: Union[list, str], marks: list = ["?", "!", "–", ":", ";"]
+):
+
+    if type(sentences) is str:
+        sentences = [sentences]
+
+    for s in range(len(sentences)):
+
+        sentence = sentences[s]
+
+        letters = [sentence[0]]
+
+        for i in range(1, len(sentence)):
+
+            if sentence[i] in marks and letters[-1] != " ":
+
+                letters[-1] = letters[-1] + " "
+
+                letters.append(sentence[i])
+
+            elif letters[-1] in marks and sentence[i] != " ":
+
+                letters.append(sentence[i] + " ")
+
+            else:
+
+                letters.append(sentence[i])
+
+        sentences[s] = "".join(letters)
+
+    return sentences
+
+
+def remove_mark_space(sentences: Union[list, str], marks: list = ["'", "-"]):
+
+    if type(sentences) is str:
+        sentences = [sentences]
+
+    for s in range(len(sentences)):
+
+        sentence = sentences[s]
+
+        letters = [sentence[0]]
+
+        for i in range(1, len(sentence)):
+
+            if sentence[i] in marks:
+
+                j = i - 1
+
+                while letters[j] == " ":
+
+                    letters[j] = ""
+
+                    j -= 1
+
+                letters.append(sentence[i])
+
+            elif letters[-1] in marks and sentence[i] == " ":
+
+                letters.append("")
+
+            else:
+
+                letters.append(sentence[i])
+
+        sentences[s] = "".join(letters)
+
+    return sentences
+
+
+def delete_much_space(sentences: Union[list, str]):
+
+    if type(sentences) is str:
+        sentences = [sentences]
+
+    for i in range(len(sentences)):
+
+        sentences[i] = " ".join(sentences[i].split())
+
+    return sentences

wolof_translate/utils/sent_transformers.py

@@ -0,0 +1,27 @@
+from typing import *
+
+
+class TransformerSequences:
+    def __init__(self, *args, **kwargs):
+
+        self.transformers = []
+
+        self.transformers.extend(list(args))
+
+        self.transformers.extend(list(kwargs.values()))
+
+    def __call__(self, sentences: Union[List, str]):
+
+        output = sentences
+
+        for transformer in self.transformers:
+
+            if hasattr(transformer, "augment"):
+
+                output = transformer.augment(output)
+
+            else:
+
+                output = transformer(output)
+
+        return output

wolof_translate/utils/sent_unification.py

@@ -0,0 +1,97 @@
+from nlp_project import *
+
+
+def unify_correction(
+    sentences: list,
+    marks: List[Tuple] = [("«", "»", True), ("(", ")", False)],
+    unified_sentences_between_pos: List[Tuple] = [(925, 930)],
+):
+
+    corrected_sentences = []
+
+    only_end_mark = []
+
+    only_begin_mark = []
+
+    i = 0
+
+    while i < len(sentences):
+
+        for u in unified_sentences_between_pos:
+
+            if i >= u[0] - 1 and i < u[1]:
+
+                range_ = u[1] - u[0]
+
+                unification = sentences[u[0] - 1]
+
+                for j in range(u[0], u[0] + range_):
+
+                    unification += " " + sentences[j]
+
+                i += range_ + 1
+
+                corrected_sentences.append(unification)
+
+        unify_next = False
+
+        space = " "
+
+        if i != 0:
+
+            for mark in marks:
+
+                begin_mark = False
+
+                end_mark = False
+
+                for letter in corrected_sentences[-1]:
+
+                    if letter == mark[1]:
+
+                        begin_mark = False
+
+                    elif letter == mark[0]:
+
+                        begin_mark = True
+
+                for letter in sentences[i]:
+
+                    if letter == mark[1]:
+
+                        end_mark = True
+
+                        break
+
+                    else:
+
+                        break
+
+                if end_mark and not begin_mark:
+
+                    only_end_mark.append(sentences[i])
+
+                elif begin_mark and not end_mark:
+
+                    only_begin_mark.append(corrected_sentences[-1])
+
+                if end_mark and begin_mark:
+
+                    unify_next = True
+
+                    space = " " if mark[2] else ""
+
+        if unify_next:
+
+            corrected_sentences[-1] = corrected_sentences[-1] + space + sentences[i]
+
+        else:
+
+            corrected_sentences.append(sentences[i])
+
+        i += 1
+
+    return corrected_sentences, {
+        "begin_mark_only": only_begin_mark,
+        "end_mark_only": only_end_mark,
+    }

wolof_translate/utils/split_with_valid.py

@@ -0,0 +1,72 @@
+""" This module contains a function which split the data. It will consider adding the validation set
+"""
+from sklearn.model_selection import train_test_split
+import pandas as pd
+import os
+
+
+def split_data(
+    random_state: int = 50,
+    data_directory: str = "data/extractions/new_data",
+    csv_file: str = "sentences.csv",
+):
+    """Split data between train, validation and test sets
+
+    Args:
+      random_state (int): the seed of the splitting generator. Defaults to 50
+    """
+    # load the corpora and split into train and test sets
+    corpora = pd.read_csv(os.path.join(data_directory, csv_file))
+
+    train_set, test_valid_set = train_test_split(
+        corpora, test_size=0.2, random_state=random_state
+    )
+
+    # let us save the final training set when performing
+
+    test_set, valid_set = train_test_split(
+        test_valid_set, test_size=0.5, random_state=random_state
+    )
+
+    train_set.to_csv(os.path.join(data_directory, "final_train_set.csv"), index=False)
+
+    # let us save the sets
+    train_set.to_csv(os.path.join(data_directory, "train_set.csv"), index=False)
+
+    valid_set.to_csv(os.path.join(data_directory, "valid_set.csv"), index=False)
+
+    test_set.to_csv(os.path.join(data_directory, "test_set.csv"), index=False)
+
+
+def split_data_kaggle(
+    random_state: int,
+    data_directory: str,
+    split_directory: str = "kaggle/working/splits",
+    csv_file: str = "sentences.csv",
+):
+    """Split data between train, validation and test sets
+
+    Args:
+      random_state (int): the seed of the splitting generator. Defaults to 50
+    """
+    # load the corpora and split into train and test sets
+    corpora = pd.read_csv(os.path.join(data_directory, csv_file))
+
+    train_set, test_valid_set = train_test_split(
+        corpora, test_size=0.2, random_state=random_state
+    )
+
+    # let us save the final training set when performing
+
+    test_set, valid_set = train_test_split(
+        test_valid_set, test_size=0.5, random_state=random_state
+    )
+
+    train_set.to_csv(os.path.join(split_directory, "final_train_set.csv"), index=False)
+
+    # let us save the sets
+    train_set.to_csv(os.path.join(split_directory, "train_set.csv"), index=False)
+
+    valid_set.to_csv(os.path.join(split_directory, "valid_set.csv"), index=False)
+
+    test_set.to_csv(os.path.join(split_directory, "test_set.csv"), index=False)

wolof_translate/utils/tokenize_text.py

@@ -0,0 +1,46 @@
+import spacy
+from typing import *
+
+
+def tokenization(
+    nlp=spacy.load("fr_core_news_lg"),
+    corpus: Union[List[str], Tuple[str]] = [],
+    rm_spaces: bool = True,
+):
+    """Tokenize the text (keep each of the unique token both in the french and the wolof corpora)
+
+    Args:
+        nlp (_type_, optional): A spacy model. Defaults to spacy.load("fr_core_news_lg").
+        corpus (Union[List[str], Tuple[str]], optional): The list of documents. Defaults to [].
+        rm_spaces (bool, optional): Indicate if the too much spaces will be deleted. Defaults to True.
+
+    Returns:
+        List[List[str]]: The list of list of tokens
+    """
+
+    # Create a inner function to tokenize a given document
+    def transformation(doc):
+
+        tokens = []
+
+        for token in doc:
+
+            if not (rm_spaces and token.is_space):
+
+                tokens.append(token.text)
+
+        return tokens
+
+    # Let's create a pipeline with the nlp object
+    docs = nlp.pipe(corpus)
+
+    # Initialize the list of tokenized documents and the list of pos_tags
+    tokens = []
+
+    for doc in docs:
+
+        tokens_ = transformation(doc)
+
+        tokens.append(tokens_)
+
+    return tokens