SinaTools 0.1.26__py2.py3-none-any.whl → 0.1.28__py2.py3-none-any.whl

sinatools/ner/relation_extractor.py

@@ -0,0 +1,201 @@
+import torch
+import json
+from urllib.request import Request, urlopen
+from sinatools.ner.entity_extractor import extract
+from . import pipe
+
+
+# ============================ Extract entities and their types ========================
+def jsons_to_list_of_lists(json_list):
+    return [[d['token'], d['tags']] for d in json_list]
+
+def entities_and_types(sentence):
+    output_list = jsons_to_list_of_lists(extract(sentence))
+    json_short = distill_entities(output_list)
+
+    entities = {}
+    for entity in json_short:
+        name = entity[0]
+        entity_type = entity[1]
+        entities[name] = entity_type
+
+    return entities
+
+def distill_entities(entities):
+    # This is list that we put the output what we need
+    list_output = list()
+
+    # This line go to sort function and save the output to temp_entities
+    temp_entities = sortTags(entities)
+
+    # This list help us to make the output,
+    temp_list = list()
+
+    # initlize the temp_list
+    temp_list.append(["", "", 0, 0])
+    word_position = 0
+
+    # For each entity, convert ibo to distllir list.
+    for entity in temp_entities:
+        # This is counter tag of this entity
+        counter_tag = 0
+        # For each tag
+        for tag in str(entity[1]).split():
+            # If the counter tag greater than or equal to lenght of templist, if yes then we will append the empty value in templist
+            if counter_tag >= len(temp_list):
+                temp_list.append(["", "", 0, 0])
+
+            # If tag equal O and word postion of this tag is not equal zero then it will add all
+            # not empty eliment of temp list in output list
+            if "O" == tag and word_position != 0:
+                for j in range(0, len(temp_list)):
+                    if temp_list[j][1] != "":
+                        list_output.append([temp_list[j][0].strip(), temp_list[j][1], temp_list[j][2], temp_list[j][3]])
+                        temp_list[j][0] = ""
+                        temp_list[j][1] = ""
+                        temp_list[j][2] = word_position
+                        temp_list[j][3] = word_position
+            # if this tag not equal O, and split by '-' the tag and check the lenght equals two and if the first eliment
+            # of the split its B
+            elif "O" != tag and len(tag.split("-")) == 2 and tag.split("-")[0] == "B":
+                # if the temp_list of counter is not empty then it will append in output list and hten it will
+                # initilize by new string and tag in templist of counter
+                if temp_list[counter_tag][1] != "":
+                    list_output.append([temp_list[counter_tag][0].strip(), temp_list[counter_tag][1], temp_list[counter_tag][2], temp_list[counter_tag][3]])
+                temp_list[counter_tag][0] = str(entity[0]) + " "
+                temp_list[counter_tag][1] = str(tag).split("-")[1]
+                temp_list[counter_tag][2] = word_position
+                temp_list[counter_tag][3] = word_position
+
+            # if this tag not equal O, and split by '-' the tag and check the lenght equals two and if the first eliment
+            # of the split its O
+            elif "O" != tag and len(tag.split("-")) == 2 and tag.split("-")[0] == "I" and word_position != 0:
+                # For each of temp_list, check if in this counter tag of templist is same tag with this.tag
+                # then will complete if not it will save in output list and cheak another
+                for j in range(counter_tag,len(temp_list)):
+                    if temp_list[j][1] == tag[2:] and temp_list[j][3] != word_position:
+                        temp_list[j][0] += str(entity[0]) + " "
+                        temp_list[j][3] += 1
+                        break
+                    else:
+                        if temp_list[j][1] != "":
+                            list_output.append([temp_list[j][0].strip(), temp_list[j][1], temp_list[j][2], temp_list[j][3]])
+                            temp_list[j][0] = ""
+                            temp_list[j][1] = ""
+                            temp_list[j][2] = word_position
+                            temp_list[j][3] = word_position
+            counter_tag += 1
+        word_position += 1
+    # For each temp_list, at the end of the previous loop, there will be some
+    # values in this list, we should save it to the output list
+    for j in range(0, len(temp_list)):
+        if temp_list[j][1] != "":
+            list_output.append([temp_list[j][0].strip(), temp_list[j][1], temp_list[j][2], temp_list[j][3]])
+    return sorted(list_output, key=lambda x: (x[2]))
+
+def sortTags(entities):
+    temp_entities = entities
+    temp_counter = 0
+    # For each entity, this loop will sort each tag of entitiy, first it will check if the
+    # previous tags has same count of this tag, second will sort the tags and check if this tags is correct
+    for entity in temp_entities:
+        tags = entity[1].split()
+        for tag in tags:
+            # if the counter is not 0 then, will complete
+            if temp_counter != 0:
+                # Check if this tag is equal I-, if yes then it will count how many tag in this tags and
+                # count how many tag in previous tags
+                if "I-" == tag[0:2]:
+                    counter_of_this_tag = 0
+                    counter_of_previous_tag = 0
+                    for word in tags:
+                        if tag.split("-")[1] in word:
+                            counter_of_this_tag+=1
+                    for word in temp_entities[temp_counter-1][1].split():
+                        if tag.split("-")[1] in word:
+                            counter_of_previous_tag+=1
+                    # if the counter of previous tag is bigger than counter of this tag, then we
+                    # need to add I-tag in this tags
+                    if counter_of_previous_tag > counter_of_this_tag:
+                        tags.append("I-"+tag.split("-")[1])
+        # Sort the tags
+        tags.sort()
+        # Need to revers the tags because it should begins with I
+        tags.reverse()
+        # If the counter is not 0 then we can complete
+        if temp_counter != 0:
+            this_tags = tags
+            previous_tags = temp_entities[temp_counter - 1][1].split()
+            sorted_tags = list()
+
+            # Check if the this tag is not O and previous tags is not O, then will complete,
+            # if not then it will ignor this tag
+            if "O" not in this_tags and "O" not in previous_tags:
+                index = 0
+                #For each previous tags, need sort this tag by previous tags if its I, B we can ignor
+                for i in previous_tags:
+                    j = 0
+                    while this_tags and j < len(this_tags):
+                        if this_tags[j][0:2] == "I-" and this_tags[j][2:] == i[2:]:
+                            sorted_tags.insert(index, this_tags.pop(j))
+                            break
+                        elif this_tags[j][0:2] == "B-":
+                            break
+                        j += 1
+                    index += 1
+            sorted_tags += this_tags
+            tags = sorted_tags
+        str_tag = " "
+        str_tag = str_tag.join(tags)
+        str_tag = str_tag.strip()
+        temp_entities[temp_counter][1] = str_tag
+        temp_counter += 1
+    return temp_entities
+
+# ============= Prepare Templates and Catergorize Extracted Entities ================
+temp03={'location':'مكان حدوث','agent':'أحد المتأثرين في','happened at':'تاريخ حدوث'}
+categories = {
+    'agent': ['PERS', 'NORP', 'OCC', 'ORG'],
+    'location': ['LOC', 'FAC', 'GPE'],
+    'happened at': ['DATE', 'TIME']
+    }
+
+def get_entity_category(entity_type, categories):
+    for category, types in categories.items():
+        if entity_type in types:
+            return category
+    return None
+
+
+# ============ Extract entities, their types and categorize them ===============
+def relation_extraction(sentence):
+    #test_sentence="صورة إعتقال طفل فلسطيني خلال انتفاضة الأقصى ."
+    entities=entities_and_types(sentence)
+
+    event_indices = [i for i, (_, entity_type) in enumerate(entities.items()) if entity_type == 'EVENT']
+    arg_event_indices = [i for i, (_, entity_type) in enumerate(entities.items()) if entity_type != 'EVENT']
+
+    output_list=[]
+
+    for i in event_indices:
+        event_entity=list(entities.keys())[i]
+        for j in arg_event_indices:
+            arg_name= list(entities.keys())[j]
+            arg_type=entities[arg_name]
+            category = get_entity_category(arg_type, categories)
+            
+            if category in temp03:
+                relation_sentence=f"[CLS] {sentence} [SEP] {event_entity} {temp03[category]} {arg_name}"
+                predicted_relation=pipe(relation_sentence)
+                score = predicted_relation[0][0]['score']  
+                if score > 0.50:
+                    #print(f"Event:{event_entity} Relation:{category} Argument:{arg_name}\n")
+                    #output_list.append([{event_entity} ,{category}, {arg_name}])
+                    output_list.append(f"Event:{event_entity}, Relation:{category}, Argument:{arg_name}")
+
+                else:
+                    #print(f"Event:{event_entity} Relation:No relation Argument:{arg_name}\n")
+                    #output_list.append([{event_entity} ,'No relation', {arg_name}])
+                    output_list.append(f"Event:{event_entity}, Relation:No relation, Argument:{arg_name}")
+    
+    return output_list

sinatools/semantic_relatedness/compute_relatedness.py

@@ -4,6 +4,28 @@ from . import model
 
 #cosine using average embedding 
 def get_similarity_score(sentence1, sentence2):
+  """
+    Computes the degree of association between two sentences across various dimensions, meaning, underlying concepts, domain-specificity, topic overlap, viewpoint alignment. This method is described and implemented on this article.
+
+    Args:
+        sentence1 (:obj:`str`) – The Arabic sentence to find the semantic relatedness between it and the second sentence.
+        sentence2 (:obj:`int`) – The Arabic sentence to find the semantic relatedness between it and the first sentence.
+    
+    Returns:
+        :obj:`float`: An float number that represents the degree of relatedness between two provided sentences.
+
+    **Example:**
+
+    .. highlight:: python
+    .. code-block:: python
+
+    from sinatools.semantic_relatedness.compute_relatedness import get_similarity_score
+
+    sentence1 = "تبلغ سرعة دوران الأرض حول الشمس حوالي 110 كيلومتر في الساعة."
+    sentence2 = "تدور الأرض حول محورها بسرعة تصل تقريبا 1670 كيلومتر في الساعة."    
+    get_similarity_score(sentence1, sentence2)
+    Score = 0.90
+  """         
 
   # Tokenize and encode sentences
   inputs1 = tokenizer(sentence1, return_tensors="pt")

sinatools/synonyms/__init__.py

@@ -3,7 +3,7 @@ from sinatools.DataDownload import downloader
 import os 
 
 synonyms_level2_dict = {} 
-level2_dict = 'synonyms_level2.pkl'
+level2_dict = 'graph_l2.pkl'
 path = downloader.get_appdatadir()
 file_path = os.path.join(path, level2_dict)
 with open(file_path, 'rb') as f:
@@ -11,7 +11,7 @@ with open(file_path, 'rb') as f:
 
 
 synonyms_level3_dict = {}    
-level3_dict = 'synonyms_level3.pkl'
+level3_dict = 'graph_l3.pkl'
 path = downloader.get_appdatadir()
 file_path = os.path.join(path, level3_dict)
 with open(file_path, 'rb') as f:

sinatools/synonyms/synonyms_generator.py

@@ -76,7 +76,28 @@ def find_fuzzy_value_for_candidates(level, list_of_unique_synonyms, number_of_cy
 
 
 def extend_synonyms(synset, level):
-   
+    """
+    This method receives a set of one or more synonyms and a level number, then extends this set with additional synonyms. The more synonyms in the input, the more accurate in the results. Each synonym in the output is assigned a fuzzy value to indicate how much it is likely to be a synonymy. You can try the demo online. Read the article for more details.
+
+    Args:
+        synset (:obj:`str`) – A set of initial synonyms to be extended (string of synonyms seperated by |).
+        level (:obj:`int`) – The level number indicates the depth of the synonym graph that the method should explore. The level could be 2 or 3. The 3rd level is richer, but the 2nd is faster.
+    
+    Returns:
+        :obj:`list`: A list of lists, where each list could be contains:
+            synonym: Synonym related to the given synset (set of synonyms).
+            fuzzy_value: The synonyms strength as a percentage out of 100.
+
+    **Example:**
+
+    .. highlight:: python
+    .. code-block:: python
+
+        from sinatools.synonyms.synonyms_generator import extend_synonyms
+        extend_synonyms('ممر | طريق', 2)
+        [["مَسْلَك","61%"],["سبيل","61%"],["وَجْه","30%"],["نَهْج", "30%"],["نَمَطٌ","30%"],["مِنْهَج","30%"],["مِنهاج", "30%"],["مَوْر","30%"],["مَسَار","30%"],["مَرصَد", "30%"],["مَذْهَبٌ","30%"],["مَدْرَج","30%"],["مَجَاز","30%"]]
+
+    """         
     used_graph = {}
     if level == 2:
        used_graph = synonyms_level2_dict
@@ -119,6 +140,29 @@ def extend_synonyms(synset, level):
 
 def evaluate_synonyms(synset, level):
 
+   """
+    This method receives a set of synonyms and a level number, then evaluates how much each of these input synonyms is really a synonym (i.e., how much it belongs to the set). You can try the demo online.
+
+    Args:
+        synset (:obj:`str`) – A set of initial synonyms to be evaluated (string of synonyms seperated by |).
+        level (:obj:`int`) – The level number indicating the depth of synonym graph that the method will explore, which could be 2 or 3.
+    
+    Returns:
+        :obj:`list`: A list of lists, where each list could be contains:
+            synonym: Synonym related to the given synset (set of synonyms).
+            fuzzy_value: The synonyms strength as a percentage out of 100.
+
+    **Example:**
+
+    .. highlight:: python
+    .. code-block:: python
+
+    from sinatools.synonyms.synonyms_generator import evaluate_synonyms
+    
+    evaluate_synonyms('ممر | طريق | مَسْلَك | سبيل') 
+    [["مَسْلَك","61%"],["سبيل","60%"],["طريق","40%"],["ممر", "40%"]]
+   """         
+
    used_graph = {}
    if level == 2:
       used_graph = synonyms_level2_dict

sinatools/utils/jaccard.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 from sinatools.utils.parser import arStrip
-from sinatools.utils.implication import Implication
+from sinatools.utils.word_compare import Implication
 import argparse
 
 def normalize_word(word: str, ignore_all_diacritics_but_not_shadda: bool=True, ignore_shadda_diacritic: bool=True) -> str:

sinatools/utils/parser.py

@@ -4,16 +4,16 @@ import argparse
 def arStrip(text , diacs=True , small_diacs=True , shaddah=True , digit=True, alif=True , special_chars=True ):
     
     """
-    This method removes Arabic diacritics, small diacritcs, shaddah, Latin and Arabic digits, unify alif, remove special characters, extra spaces, underscore and Arabic tatwelah from the input text.
+    This method allows one to optionally remove (Arabic diacritics, small diacritics, shaddah, Latin and Arabic digits, unify alif, remove special characters, extra spaces, underscore and Arabic tatwelah) from the input text.
 
     Args:
         text (:obj:`str`): Arabic text to be processed.
-        diacs (:obj:`bool`): flag to remove Arabic diacretics [ ًٌٍَُِْ] (default is True).
-        small_diacs (:obj:`bool`): flag to remove small diacretics (default is True).
+        diacs (:obj:`bool`): flag to remove these 7 Arabic diacretics [ ٍ ِ ْ ٌ ُ َ ً] (default is True).
+        small_diacs (:obj:`bool`): flag to remove all Quranic annotation signs from this range [06D6-06ED] in addition to small alif. (default is True).
         shaddah (:obj:`bool`): flag to remove shaddah (default is True).
         digit (:obj:`bool`): flag to remove Latin and Arabic digits (default is True).
-        alif (:obj:`bool`): flag to unify alif (default is True).
-        special_chars (:obj:`bool`): flag to remove special characters (default is True).
+        alif (:obj:`bool`): flag to unify alif. Replace [ٱ أ إ آ] into [ا] (default is True).
+        special_chars (:obj:`bool`): flag to remove these special characters [?؟!@#$%] (default is True).
 
     Returns:
         :obj:`str`: stripped text.
@@ -30,10 +30,10 @@ def arStrip(text , diacs=True , small_diacs=True , shaddah=True , digit=True, al
         # output
         الجو جميل
 
-        output = parser.arStrip('أَلَمۡ یَأۡنِ لِلَّذِینَ ءَامَنُوۤا۟ أَن تَخۡشَعَ قُلُوبُهُمۡ لِذِكۡرِ ٱللَّهِ وَمَا نَزَلَ مِنَ ٱلۡحَقِّ وَلَا یَكُونُوا۟ كَٱلَّذِینَ أُوتُوا۟ ٱلۡكِتَـٰبَ مِن قَبۡلُ فَطَالَ عَلَیۡهِمُ ٱلۡأَمَدُ فَقَسَتۡ قُلُوبُهُمۡۖ وَكَثِیر مِّنۡهُمۡ فَـسِقُونَ' , True , True , True ,  True , True , True )
+        output =parser.arStrip('أَلَمۡ یَأۡنِ لِلَّذِینَ ءَامَنُوۤا۟ أَن تَخۡشَعَ قُلُوبُهُمۡ لِذِكۡرِ ٱللَّهِ وَمَا نَزَلَ مِنَ ٱلۡحَقِّ وَلَا یَكُونُوا۟ كَٱلَّذِینَ أُوتُوا۟ ٱلۡكِتَـٰبَ مِن قَبۡلُ فَطَالَ عَلَیۡهِمُ ٱلۡأَمَدُ فَقَسَتۡ قُلُوبُهُمۡۖ وَكَثِیر مِّنۡهُمۡ فَـسِقُونَ', True, True, True, True, False, False )
         print(output)
         #output
-        الم یان للذین ءامنوا ان تخشع قلوبهم لذكر الله وما نزل من الحق ولا یكونوا كالذین اوتوا الكتٰب من قبل فطال علیهم الامد فقست قلوبهم وكثیر منهم فسقون
+        ألم یأن للذین ءامنوا أن تخشع قلوبهم لذكر ٱلله وما نزل من ٱلحق ولا یكونوا كٱلذین أوتوا ٱلكتب من قبل فطال علیهم ٱلأمد فقست قلوبهم وكثیر منهم فسقون
     """
     try:
         if text: # if the input string is not empty do the following
@@ -67,13 +67,13 @@ def arStrip(text , diacs=True , small_diacs=True , shaddah=True , digit=True, al
     
 def remove_punctuation(text):
     """
-    Removes punctuation marks from the text.
+    Removes these arabic and english punctuation marks from the text [! " # $ % & ' ( ) * + , - . / : ; > = < ? @ [ \ ] ^ _ ` { | } ~ ، ؛ ؞ ؟ ـ ٓ ٬ ٪ ٫ ٭ ۔].
     
     Args:
       text (:obj:`str`): The input text.
     
     Returns:
-      :obj:`str`: The output text without punctuation marks.
+       :obj:`str`
 
     **Example:**
 
@@ -109,15 +109,12 @@ def remove_punctuation(text):
 
 def remove_latin(text):
     """
-    This method removes all Latin characters from the input text.
+    This method removes all Latin letters from the input text.
 
-    Args:
+    Parameters:
         text (:obj:`str`): The input text.
-
     Returns:
-         outputString (:obj:`str`): The text without Latin characters.
-    Note:
-        If an error occurs during processing, the original text is returned.
+        :obj:`str`
     **Example:**
 
     .. highlight:: python

sinatools/utils/similarity.py

@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+
+from sinatools.utils.parser import arStrip
+from sinatools.utils.word_compare import Implication
+import argparse
+
+def normalize_word(word: str, ignore_all_diacritics_but_not_shadda: bool=True, ignore_shadda_diacritic: bool=True) -> str:
+    if ignore_all_diacritics_but_not_shadda:
+        word = arStrip(word, True, True, False, False, False, False)
+        
+    if ignore_shadda_diacritic:
+        word = arStrip(word, False, False, True, False, False, False)
+    
+    return word
+
+    
+def get_preferred_word(word1, word2):
+    implication = Implication(word1, word2)
+    
+    direction = implication.get_direction()
+    
+    if direction in (0, 2):
+        return word1
+       
+    elif direction == 1:
+        return word2
+       
+    elif direction == 3:
+        if not word1.endswith("َ") and not word1.endswith("ُ"):
+            return word2
+        return word1
+     
+    
+def get_non_preferred_word(word1, word2):
+
+    implication = Implication(word1, word2)
+    if implication.get_distance() < 15:
+        direction = implication.get_direction()
+        if direction == 0 or direction == 1:
+            return word1
+        elif direction == 2:
+            return word2
+        elif direction == 3:
+            if not word1.endswith("َ") and not word1.endswith("ُ"):
+                return word1
+            return word2
+    return "#"
+
+def get_intersection(list1, list2, ignore_all_diacritics_but_not_shadda=False, ignore_shadda_diacritic=False):
+    """
+    Computes the intersection of two sets of Arabic words, considering the differences in their diacritization. The method provides two options for handling diacritics: (i) ignore all diacritics except for shadda, and (ii) ignore the shadda diacritic as well. You can try the demo online.
+    
+    Args:
+        list1 (:obj:`list`): The first list.
+        list2 (:obj:`bool`): The second list.
+        ignore_all_diacratics_but_not_shadda (:obj:`bool`, optional) – A flag to ignore all diacratics except for the shadda. Defaults to False.
+        ignore_shadda_diacritic (:obj:`bool`, optional) – A flag to ignore the shadda diacritic. Defaults to False. 
+    
+    Returns:
+        :obj:`list`: The intersection of the two lists, ignores diacritics if flags are true.
+    
+    **Example:**
+    
+    .. highlight:: python
+    .. code-block:: python
+    
+        from sinatools.utils.similarity import get_intersection
+        list1 = ["كتب","فَعل","فَعَلَ"]
+        list2 = ["كتب","فَعّل"]
+        print(get_intersection(list1, list2, False, True))
+        #output: ["كتب" ,"فعل"]        
+    """
+    list1 = [str(i) for i in list1 if i not in (None, ' ', '')]
+    list1 = [str(i.strip()) for i in list1]
+
+    list2 = [str(i) for i in list2 if i not in (None, ' ', '')]
+    list2 = [str(i.strip()) for i in list2]
+
+    interection_list = []
+
+    for list1_word in list1:
+        for list2_word in list2:
+            word1 = normalize_word(list1_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+            word2 = normalize_word(list2_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+
+            implication = Implication(word1, word2)
+            if implication.get_direction() >= 0 and implication.get_distance() < 15:
+                interection_list.append(get_preferred_word(word1, word2))
+
+    i = 0
+    while i < len(interection_list):
+        j = i + 1
+        while j < len(interection_list):
+            non_preferred_word = get_non_preferred_word(interection_list[i], interection_list[j])
+            if non_preferred_word != "#":
+                interection_list.remove(non_preferred_word)
+            j += 1
+        i += 1
+
+    return interection_list
+             
+
+
+def get_union(list1, list2, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic):
+    """
+    Computes the union of two sets of Arabic words, considering the differences in their diacritization. The method provides two options for handling diacritics: (i) ignore all diacritics except for shadda, and (ii) ignore the shadda diacritic as well. You can try the demo online.
+    
+    Args:
+        list1 (:obj:`list`): The first list.
+        list2 (:obj:`bool`): The second list.
+        ignore_all_diacratics_but_not_shadda (:obj:`bool`, optional) – A flag to ignore all diacratics except for the shadda. Defaults to False.
+        ignore_shadda_diacritic (:obj:`bool`, optional) – A flag to ignore the shadda diacritic. Defaults to False. 
+    
+    Returns:
+        :obj:`list`: The union of the two lists, ignoring diacritics if flags are true.
+    
+    **Example:**
+    
+    .. highlight:: python
+    .. code-block:: python
+    
+        from sinatools.utils.similarity import get_union
+        list1 = ["كتب","فَعل","فَعَلَ"]
+        list2 = ["كتب","فَعّل"]
+        print(get_union(list1, list2, False, True))
+        #output: ["كتب" ,"فَعل" ,"فَعَلَ"]
+    """
+    list1 = [str(i) for i in list1 if i not in (None, ' ', '')]
+
+    list2 = [str(i) for i in list2 if i not in (None, ' ', '')]
+
+    union_list = []
+
+    for list1_word in list1:
+        word1 = normalize_word(list1_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+        union_list.append(word1)
+
+    for list2_word in list2:
+        word2 = normalize_word(list2_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+        union_list.append(word2)
+
+    i = 0
+    while i < len(union_list):
+        j = i + 1
+        while j < len(union_list):
+            non_preferred_word = get_non_preferred_word(union_list[i], union_list[j])
+            if (non_preferred_word != "#"):
+                union_list.remove(non_preferred_word)
+            j = j + 1
+        i = i + 1
+
+    return union_list
+      
+
+
+def get_jaccard_similarity(list1: list, list2: list, ignore_all_diacritics_but_not_shadda: bool, ignore_shadda_diacritic: bool) -> float:
+    """
+    Calculates the Jaccard similarity coefficient between two lists of Arabic words, considering the differences in their diacritization. The method provides two options for handling diacritics: (i) ignore all diacritics except for shadda, and (ii) ignore the shadda diacritic as well. You can try the demo online.
+    
+    Args:
+        list1 (:obj:`list`): The first list.
+        list2 (:obj:`bool`): The second list.
+        ignore_all_diacratics_but_not_shadda (:obj:`bool`, optional) – A flag to ignore all diacratics except for the shadda. Defaults to False.
+        ignore_shadda_diacritic (:obj:`bool`, optional) – A flag to ignore the shadda diacritic. Defaults to False. 
+    
+    Returns:
+        :obj:`float`: The Jaccard similarity coefficient between the two lists, ignoring diacritics if flags are true.
+    
+    **Example:**
+    
+    .. highlight:: python
+    .. code-block:: python
+    
+        from sinatools.utils.similarity import get_jaccard_similarity
+        list1 = ["كتب","فَعل","فَعَلَ"]
+        list2 = ["كتب","فَعّل"]
+        print(get_jaccard_similarity(list1, list2, True, True))
+        #output: 0.67
+    """
+
+    intersection_list = get_intersection(list1, list2, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+    
+    union_list = get_union(list1, list2, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+    
+    return float(len(intersection_list)) / float(len(union_list))
+
+def get_jaccard(delimiter, str1, str2, selection, ignoreAllDiacriticsButNotShadda=True, ignoreShaddaDiacritic=True):
+    """
+    Calculates and returns the Jaccard similarity values (union, intersection, or Jaccard similarity) between two lists of Arabic words, considering the differences in their diacritization. The method provides two options for handling diacritics: (i) ignore all diacritics except for shadda, and (ii) ignore the shadda diacritic as well. You can try the demo online.
+    
+    Args:
+        delimiter (:obj:`str`): The delimiter used to split the input strings.
+        str1 (:obj:`str`): The first input string to compare.
+        str1 (:obj:`str`): The second input string to compare.
+        selection (:obj:`str`) – The desired operation to perform on the two sets of strings. Must be one of intersection, union, jaccardSimilarity, or jaccardAll.
+        ignore_all_diacratics_but_not_shadda (:obj:`bool`) – If True, ignore all diacratics except for the Shadda diacritic. (Default is True)
+        ignore_shadda_diacritic (:obj:`bool`) – If True, ignore the Shadda diacritic.(Default is True)
+    
+    Returns:
+        Three values (Jaccard similarity, union, or intersection) between the two lists of Arabic words depending on the parameter selection.
+    
+    **Example:**
+    
+    .. highlight:: python
+    .. code-block:: python
+    
+        from sinatools.utils.similarity import get_jaccard
+        str1 = "فَعَلَ | فَعل"
+        str2 = "فَعّل"
+        print(get_jaccard("|", "jaccardAll", str1, str2, True, True))
+        #output: ['intersection:', ['فعل'], 'union:', ['فعل', 'فعل'], 'similarity:', 0.5]
+    """
+    try:
+        list1 = str1.split(delimiter)
+        list2 = str2.split(delimiter)
+
+        if selection == "intersection":
+            intersection = get_intersection(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            return intersection
+        elif selection == "union":
+            union = get_union(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            return union
+        elif selection == "jaccardSimilarity":      
+            similarity = get_jaccard_similarity(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            return similarity
+        elif selection == "jaccardAll":    
+            intersection = get_intersection(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            union = get_union(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            similarity = get_jaccard_similarity(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            output_list = ["intersection:", intersection, "union:", union, "similarity:", similarity]
+            return output_list
+        else:
+            return 'Invalid selection option'
+
+    except AttributeError as ae:
+        print(f"Attribute error occurred: {str(ae)}")
+        return 'Invalid input type'
+    except Exception as e:
+        print(f"Error occurred: {str(e)}")
+        return 'An error has occurred'

sinatools/utils/text_dublication_detector.py

@@ -15,6 +15,28 @@ def validator(sentence, max_tokens=500):
 
 
 def removal(csv_file, columnName, finalFileName, deletedFileName, similarityThreshold=0.8):
+    """
+    This method is designed to identify dublicate text in a given corpora/text. It processes a CSV file of sentences to identify and remove duplicate sentences based on a specified threshold. We used cosine similarity to measure similarity between words and sentences. The method saves the filtered results and the identified duplicates to separate files.
+    
+    Args:
+        csv_file (:obj:`str`) – The CSV file contains Arabic text that needs to be cleaned.
+        column_name (:obj:`str`) – This is the name of the column containing the text that needs to be checked for duplicate removal.
+        final_file_name (:obj:`str`) – This is the name of the CSV file that will contain the data after duplicate removal.        
+        deleted_file_name (:obj:`str`) – This is the name of the file that will contain all the duplicate records that are deleted.        
+        similarity_threshold (:obj:`float`) – This is a floating-point number. The default value is 0.8, indicating the percentage of similarity that the function should use when deleting duplicates from the text column.    
+    
+    Returns:
+        csv files.
+    
+    **Example:**
+    
+    .. highlight:: python
+    .. code-block:: python
+    
+        from sinatools.utils.text_dublication_detector import removal
+        removal("/path/to/csv/file1", sentences, "/path/to/csv/file2", 0.8)
+    """
+
     # Read CSV file
     try:
         df = pd.read_csv(csv_file)

sinatools/utils/text_transliteration.py

@@ -165,7 +165,7 @@ bw2ar_map = {
 #It takes a text and the schema as input and return 2-values: the transliteration and a flag of whether all chars are transliterated or not
 def perform_transliteration(text , schema ):
     """
-    This method takes a text and a schema as input and returns a tuple of two values: the transliteration of the text based on the given schema and a flag indicating whether all characters in the text were transliterated or not.
+    This method takes a text and a schema as input and returns a tuple of two values: the transliteration of the text is based on the given schema and a flag indicating whether all characters in the text were transliterated or not.
 
     Args:
         text (:obj:`str`): The input text to be transliterated.