SinaTools 0.1.1__py2.py3-none-any.whl

nlptools/utils/jaccard.py

@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+
+from nlptools.utils.parser import arStrip
+from nlptools.utils.implication import Implication
+import argparse
+
+def normalize_word(word: str, ignore_all_diacritics_but_not_shadda: bool=True, ignore_shadda_diacritic: bool=True) -> str:
+    """
+    Normalize a given Arabic word by removing diacritics and/or shadda diacritic.
+
+    Args:
+        word (:obj:`str`): The input text.
+        ignore_all_diacritics_but_not_shadda (:obj:`bool`): A boolean flag indicating whether to remove all diacritics except shadda (default is True). 
+        ignore_shadda_diacritic (:obj:`bool`): A boolean flag indicating whether to remove shadda diacritic (default is True).
+
+    Returns:
+          :obj:`str` Normalized Arabic word.
+    """
+    if ignore_all_diacritics_but_not_shadda:
+        word = arStrip(word, True, True, False, False, False, False)
+        
+    if ignore_shadda_diacritic:
+        word = arStrip(word, False, False, True, False, False, False)
+    
+    return word
+
+    
+def get_preferred_word(word1, word2):
+    """
+    Returns the preferred word among two given words based on their implication.
+
+    Args:
+        word1 (:obj:`str`): The first word.
+        word2 (:obj:`str`): The second word.
+
+    Returns:
+        :obj:`str`: The preferred word among the two given words.
+    
+    """
+    implication = Implication(word1, word2)
+    
+    direction = implication.get_direction()
+    
+    if direction in (0, 2):
+        return word1
+       
+    elif direction == 1:
+        return word2
+       
+    elif direction == 3:
+        if not word1.endswith("َ") and not word1.endswith("ُ"):
+            return word2
+        return word1
+     
+    
+def get_non_preferred_word(word1, word2):
+    """
+    Returns the non-preferred word between the two input words.
+
+    Args:
+        word1 (:obj:`str`): The first word.
+        word2 (:obj:`str`): The second word.
+
+    Returns:
+        :obj:`str`: The non-preferred word. If there is no non-preferred word, The '#' is returned.
+
+    """
+
+    implication = Implication(word1, word2)
+    if implication.get_distance() < 15:
+        direction = implication.get_direction()
+        if direction == 0 or direction == 1:
+            return word1
+        elif direction == 2:
+            return word2
+        elif direction == 3:
+            if not word1.endswith("َ") and not word1.endswith("ُ"):
+                return word1
+            return word2
+    return "#"
+#@TBD
+def get_intersection(list1, list2, ignore_all_diacritics_but_not_shadda=False, ignore_shadda_diacritic=False):
+    """
+    Get the intersection of two lists after normalization and ignoring diacritics based on input flags.
+
+    Args:
+        list1 (:obj:`list`): The first list.
+        list2 (:obj:`list`): The second list.
+        ignore_all_diacritics_but_not_shadda (:obj:`bool`, optional): A flag to ignore all diacritics except for the shadda. Defaults to False.
+        ignore_shadda_diacritic (:obj:`bool`, optional): A flag to ignore the shadda diacritic. Defaults to False.
+
+    Returns:
+         :obj:`list`: The intersection of the two lists after normalization and ignoring diacritics.
+
+    """
+
+    # Remove all None and empty values from first list
+    list1 = [str(i) for i in list1 if i not in (None, ' ', '')]
+    list1 = [str(i.strip()) for i in list1]
+
+    # Remove all None and empty values from second list
+    list2 = [str(i) for i in list2 if i not in (None, ' ', '')]
+    list2 = [str(i.strip()) for i in list2]
+
+    interection_list = []
+
+    # Add all Common words between the two list1 and list2 to interectionList
+    for list1_word in list1:
+        for list2_word in list2:
+            word1 = normalize_word(list1_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+            word2 = normalize_word(list2_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+
+            implication = Implication(word1, word2)
+            if implication.get_direction() >= 0 and implication.get_distance() < 15:
+                interection_list.append(get_preferred_word(word1, word2))
+
+    i = 0
+    while i < len(interection_list):
+        j = i + 1
+        while j < len(interection_list):
+            non_preferred_word = get_non_preferred_word(interection_list[i], interection_list[j])
+            if non_preferred_word != "#":
+                interection_list.remove(non_preferred_word)
+            j += 1
+        i += 1
+
+    return interection_list
+             
+
+
+def get_union(list1, list2, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic):
+    """
+    Finds the union of two lists by removing duplicates and normalizing words.
+
+    Args:
+        list1 (:obj:`list`): The first list.
+        list2 (:obj:`list`): The second list.
+        ignore_all_diacritics_but_not_shadda (:obj:`bool`): Whether to ignore all diacritics except shadda or not.
+        ignore_shadda_diacritic (:obj:`bool`): Whether to ignore shadda diacritic or not.
+   Returns:
+         :obj:`list`: The union of the two lists after removing duplicates and normalizing words.
+    """
+
+    list1 = [str(i) for i in list1 if i not in (None, ' ', '')]
+
+    list2 = [str(i) for i in list2 if i not in (None, ' ', '')]
+
+    union_list = []
+
+    for list1_word in list1:
+        word1 = normalize_word(list1_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+        union_list.append(word1)
+
+    for list2_word in list2:
+        word2 = normalize_word(list2_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+        union_list.append(word2)
+
+    i = 0
+    while i < len(union_list):
+        j = i + 1
+        while j < len(union_list):
+            non_preferred_word = get_non_preferred_word(union_list[i], union_list[j])
+            if (non_preferred_word != "#"):
+                union_list.remove(non_preferred_word)
+            j = j + 1
+        i = i + 1
+
+    return union_list
+      
+
+
+def jaccard_similarity(list1: list, list2: list, ignore_all_diacritics_but_not_shadda: bool, ignore_shadda_diacritic: bool) -> float:
+    """
+    Calculates the Jaccard similarity coefficient between two lists.
+
+    Args:
+        list1 (:obj:`list`): The first list.
+        list2 (:obj:`list`): The second list.
+        ignore_all_diacritics_but_not_shadda (:obj:`bool`): A flag indicating whether to ignore all diacritics except for shadda.
+        ignore_shadda_diacritic (:obj:`bool`): A flag indicating whether to ignore the shadda diacritic.
+
+    Returns:
+         :obj:`float`: The Jaccard similarity coefficient between the two lists.
+    """
+    # Find the intersection between two sets
+    intersection_list = get_intersection(list1, list2, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+    
+    # Find the union between two sets
+    union_list = get_union(list1, list2, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+    
+    # Calculate the Jaccard similarity coefficient by dividing the length of the intersectionList by the length of the unionList
+    return float(len(intersection_list)) / float(len(union_list))
+   
+
+
+
+def jaccard(delimiter, str1, str2, selection, ignoreAllDiacriticsButNotShadda=True, ignoreShaddaDiacritic=True):
+    """
+    Compute the Jaccard similarity, union, or intersection of two sets of strings.
+
+    Args:
+        delimiter (:obj:`str`): The delimiter used to split the input strings.
+        str1 (:obj:`str`): The first input string to compare.
+        str2 (:obj:`str`): The second input string to compare.
+        selection (:obj:`str`): The desired operation to perform on the two sets of strings. 
+                         Must be one of *intersection*, *union*, *jaccardSimilarity*, or *jaccardAll*.
+        ignoreAllDiacriticsButNotShadda (:obj:`bool`): If True, ignore all diacritics except for the Shadda diacritic. (Defualt is True)
+        ignoreShaddaDiacritic (:obj:`bool`): If True, ignore the Shadda diacritic.(Default is True)
+
+    Returns:
+        The Jaccard similarity, union, or intersection of the two sets of strings, 
+        depending on the value of the `selection` argument.
+    
+    Note:
+        - If `selection` is *jaccardAll*, a list of the intersection, union, and Jaccard similarity 
+        of the two sets of strings will be returned.
+        - If an error occurs, the method will return the string "An error has occurred".
+          Online tool: https://sina.birzeit.edu/resources/jaccardFunction.html
+    """
+    try:
+        list1 = str1.split(delimiter)
+        list2 = str2.split(delimiter)
+
+        if selection == "intersection":
+            intersection = get_intersection(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            return intersection
+        elif selection == "union":
+            union = get_union(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            return union
+        elif selection == "jaccardSimilarity":      
+            similarity = jaccard_similarity(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            return similarity
+        elif selection == "jaccardAll":    
+            intersection = get_intersection(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            union = get_union(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            similarity = jaccard_similarity(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            output_list = ["intersection:", intersection, "union:", union, "similarity:", similarity]
+            return output_list
+        else:
+            return 'Invalid selection option'
+
+    except AttributeError as ae:
+        print(f"Attribute error occurred: {str(ae)}")
+        return 'Invalid input type'
+    except Exception as e:
+        print(f"Error occurred: {str(e)}")
+        return 'An error has occurred'

nlptools/utils/parser.py

@@ -0,0 +1,147 @@
+import re 
+import argparse
+
+def arStrip(text , diacs=True , smallDiacs=True , shaddah=True , digit=True, alif=True , specialChars=True ):
+    
+    """
+    This method removes Arabic diacritics, small diacritcs, shaddah, Latin and Arabic digits, and unify alif.
+    And remove special characters, spaces, underscore and Arabic tatwelah from the input text.
+
+    Args:
+        text (:obj:`str`): Arabic text to be processed.
+        diacs (:obj:`bool`): flag to remove Arabic diacretics [ ًٌٍَُِْ] (default is True).
+        smallDiacs (:obj:`bool`): flag to remove small diacretics (default is True).
+        shaddah (:obj:`bool`): flag to remove shaddah (default is True).
+        digit (:obj:`bool`): flag to remove Latin and Arabic digits (default is True).
+        alif (:obj:`bool`): flag to unify alif (default is True).
+        specialChars (:obj:`bool`): flag to remove special characters (default is True).
+
+    Returns:
+        :obj:`str`: stripped text.
+
+    **Example:**
+
+    .. highlight:: python
+    .. code-block:: python
+
+        from nlptools.utils import parser
+        processed_text =parser.arStrip('2023الجو جميلُ')
+        print(processed_text)
+
+        #putput
+        الجو جميل
+
+        name =parser.arStrip('أَلَمۡ یَأۡنِ لِلَّذِینَ ءَامَنُوۤا۟ أَن تَخۡشَعَ قُلُوبُهُمۡ لِذِكۡرِ ٱللَّهِ وَمَا نَزَلَ مِنَ ٱلۡحَقِّ وَلَا یَكُونُوا۟ كَٱلَّذِینَ أُوتُوا۟ ٱلۡكِتَـٰبَ مِن قَبۡلُ فَطَالَ عَلَیۡهِمُ ٱلۡأَمَدُ فَقَسَتۡ قُلُوبُهُمۡۖ وَكَثِیر مِّنۡهُمۡ فَـسِقُونَ' , True , True , True ,  True , True , True )
+        print(name)
+        #putput
+        الم یان للذین ءامنوا ان تخشع قلوبهم لذكر الله وما نزل من الحق ولا یكونوا كالذین اوتوا الكتٰب من قبل فطال علیهم الامد فقست قلوبهم وكثیر منهم فسقون
+
+
+    """
+    try:
+        if text: # if the input string is not empty do the following
+            #print("in if")
+            if diacs == True :
+                text = re.sub(r'[\u064B-\u0650]+', '',text) # Remove all Arabic diacretics [ ًٌٍَُِْ]
+                text = re.sub(r'[\u0652]+', '',text) # Remove SUKUN
+            if shaddah == True:
+                text = re.sub(r'[\u0651]+', '',text) # Remove shddah
+            if smallDiacs == True:
+                text = re.sub(r'[\u06D6-\u06ED]+', '',text) # Remove all small Quranic annotation signs
+            if digit == True:
+                text = re.sub('[0-9]+', ' ',text) # Remove English digits
+                text = re.sub('[٠-٩]+', ' ',text)# Remove Arabic digits
+            
+            if alif == True:                             # Unify alif with hamzah: 
+                text = re.sub('ٱ', 'ا',text);
+                text = re.sub('أ', 'ا',text);
+                text = re.sub('إ', 'ا',text);
+                text = re.sub('آ', 'ا',text);
+            if specialChars == True:
+                text = re.sub('[?؟!@#$%-]+' , '' , text) # Remove some of special chars 
+
+            text = re.sub('[\\s]+'," ",text) # Remove all spaces
+            text = text.replace("_" , '') #Remove underscore
+            text = text.replace("ـ" , '') # Remove Arabic tatwelah
+            text = text.strip() # Trim input string
+    except:
+        return text
+    return text
+    
+def remove_punctuation(text):
+    """
+    Removes punctuation marks from the text.
+    
+    Args:
+      text (:obj:`str`): The input text.
+    
+    Returns:
+      :obj:`str`: The output text without punctuation marks.
+
+    **Example:**
+
+    .. highlight:: python
+    .. code-block:: python
+    
+        from nlptools.utils import parser
+        return parser.remove_punctuation("te!@#،$%%؟st")
+
+        #output
+        test
+
+        output= parser.remove_punctuation(" {يَا أَيُّهَا الَّذِينَ آمَنُوا لِيَسْتَأْذِنْكُمُ ....}")
+        print(output)
+
+        #output
+        يَا أَيُّهَا الَّذِينَ آمَنُوا لِيَسْتَأْذِنْكُمُ  
+
+    """
+    try:
+        if text:
+            punctuation_marks = [r'[\u0021-\u002F]+', r'[U+060C]+', r'[\u003A-\u0040]+',
+                                 r'[\u005B-\u0060]+', r'[\u007B-\u007E]+', r'[\u060C]+',
+                                 r'[\u061B]+', r'[\u061E]+', r'[\u061F]+', r'[\u0640]+',
+                                 r'[\u0653]+', r'[\u065C]+', r'[\u066C]+', r'[\u066A]+',
+                                 r'["}"]+', r'["{"]+']
+            outputString = text
+            for punctuation in punctuation_marks:
+                outputString = re.sub(punctuation, '', outputString)
+    except:
+        return text
+    return outputString
+
+def remove_latin(text):
+    """
+    This method removes all Latin characters from the input text.
+
+    Args:
+        text (:obj:`str`): The input text.
+
+    Returns:
+         outputString (:obj:`str`): The text without Latin characters.
+    Note:
+        If an error occurs during processing, the original text is returned.
+    **Example:**
+
+    .. highlight:: python
+    .. code-block:: python
+
+        from nlptools.utils import parser
+        return parser.remove_latin("miojkdujhvaj1546545spkdpoqfoiehwv nWEQFGWERHERTJETAWIKUYFC")
+    
+        #output
+        1546545  
+
+        output = parser.remove_latin("أصل المسمى «تخطيط موارد المؤسسة» هو تعريب لمسمى التخطيط باللغة الإنجليزية Enterprise Resource Planning")
+        print(output)
+
+        #output
+        أصل المسمى «تخطيط موارد المؤسسة» هو تعريب لمسمى التخطيط باللغة الإنجليزية      
+    """
+    try:
+        if text:
+            text = re.sub('[a-zA-Z]+', ' ', text)
+    except:
+        return text
+    return text
+                 

nlptools/utils/readfile.py

@@ -0,0 +1,3 @@
+def read_file(file_path):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        return [line.strip() for line in file]

nlptools/utils/sentence_tokenizer.py

@@ -0,0 +1,53 @@
+def remove_empty_values(sentences):
+    return [value for value in sentences if value != '']
+
+
+def sent_tokenize(text, dot=True, new_line=True, question_mark=True, exclamation_mark=True):
+    """
+    This method tokenizes a text into a set of sentences based on the selected separators, including the dot, new line, question mark, and exclamation mark.
+
+    Args:
+        text (:obj:`str`): Arabic text to be tokenized.
+        dot (:obj:`str`): flag to split text based on Dot (default is True).
+        new_line (:obj:`str`): flag to split text based on new_line (default is True).
+        question_mark (:obj:`str`): flag to split text based on question_mark (default is True).
+        exclamation_mark (:obj:`str`): flag to split text based on exclamation_mark (default is True).
+
+    Returns:
+        :obj:`list`: list of sentences.
+
+    **Example:**
+
+    .. highlight:: python
+    .. code-block:: python
+
+        from nlptools.utils import sentence_tokenizer
+        sentences = sentence_tokenizer.sent_tokenize("مختبر سينا لحوسبة اللغة والذكاء الإصطناعي. في جامعة بيرزيت.", dot=True, new_line=True, question_mark=True, exclamation_mark=True)
+        print(sentences)
+
+        #output
+        ['مختبر سينا لحوسبة اللغة والذكاء الإصطناعي.', 'في جامعة بيرزيت.']
+    """
+    separators = []
+    split_text = [text]
+    if new_line==True:
+        separators.append('\n')
+    if dot==True:
+        separators.append('.')
+    if question_mark==True:
+        separators.append('?')
+        separators.append('؟')
+    if exclamation_mark==True:
+        separators.append('!')
+    
+    for sep in separators:
+        new_split_text = []
+        for part in split_text:
+            tokens = part.split(sep)
+            tokens_with_separator = [token + sep for token in tokens[:-1]]
+            tokens_with_separator.append(tokens[-1].strip())
+            new_split_text.extend(tokens_with_separator)
+        split_text = new_split_text
+    
+    split_text = remove_empty_values(split_text)    
+    return split_text

nlptools/utils/text_transliteration.py

@@ -0,0 +1,232 @@
+from collections import deque
+
+# This is a mapping dictionery of Arabic letters to BW 
+# dictionery contains: key -> Unicode to Arabic, value -> BW chars
+# It includes all BW mappings in addition to other special characters
+#  that are used in the SAMA database but not part of the BW character set such as numbers and Qur'anic diacritics
+ar2bw_map = {
+    '\u0621' : '\'' , # ء
+    '\u0622' : '|' , # آ
+    '\u0623' : '>' , # أ
+    '\u0624' : '&' , # ؤ
+    '\u0625' : '<' , # إ
+    '\u0626' : '}' , # ئ
+    '\u0627' : 'A' , # ا
+    '\u0628' : 'b' , # ب
+    '\u0629' : 'p' , # ة
+    '\u062A' : 't' , # ت
+    '\u062B' : 'v' , # ث
+    '\u062C' : 'j' , # ج
+    '\u062D' : 'H' , # ح
+    '\u062E' : 'x' , # خ
+    '\u062F' : 'd' , # د
+    '\u0630' : '*' , # ذ
+    '\u0631' : 'r' , # ر
+    '\u0632' : 'z' , # ز
+    '\u0633' : 's' , # س
+    '\u0634' : '$' , # ش
+    '\u0635' : 'S' , # ص
+    '\u0636' : 'D' , # ض
+    '\u0637' : 'T' , # ط
+    '\u0638' : 'Z' , # ظ
+    '\u0639' : 'E' , # ع
+    '\u063A' : 'g' , # غ
+    '\u0020' : ' ' , #Space is replaced by space 
+    '\u0640' : '_' , #ـ
+    '\u0641' : 'f' , # ف
+    '\u0642' : 'q' , # ق
+    '\u0643' : 'k' , # ك
+    '\u0644' : 'l' , # ل
+    '\u0645' : 'm' , # م
+    '\u0646' : 'n' , # ن
+    '\u0647' : 'h' , # ه
+    '\u0648' : 'w' , # و
+    '\u0649' : 'Y' , # ى
+    '\u064A' : 'y' , # ي
+    '\u064B' : 'F' , # TANWEEN FATH ً
+    '\u064C' : 'N' , #TANWEEN DHAM ٌ
+    '\u064D' : 'K' , #TANWEE KASR ٍ
+    '\u064E' : 'a' , #FATHA َ
+    '\u064F' : 'u' , #DHAMMA ُ
+    '\u0650' : 'i' , #KASRA ِ
+    '\u0651' : '~' , # SHADDAH ّ
+    '\u0652' : 'o' , #SUKON ْ
+    '\u0670' : '`' , # SHORT ALEF (dagger alif) ٰ
+    '\u0671' : '{' , #AL WITH HAMZA QATE'/WASEL MARK ٱ
+    '\u067E' : 'P' , # پ 
+    '\u0686' : 'J' , # چ 
+    '\u06A4' : 'V' , # ڤ 
+    '\u06AF' : 'G' , # گ 
+    '\u06EA' : '-' , # ۪ Not BW standerd
+    '\u0654' : '#' , # ٔ     Not BW standerd
+    '\u06DC ' : ':' , # short sen ۜ  Not BW standerd  
+    '\u06E0' : '"' , # small circle ۟  ,Not BW standerd
+    '\u06E2' : '[' , # short meem ۢ  , Not BW standerd
+    '\u06E3 ' : ';' , #small sen below letter  ۣ  , Not BW standerd
+    '\u06E5 ' : ',' , # samall wa ۥ  , Not BW standerd
+    '\u06E6' : '.' , # ۦ   Not BW standerd
+    '\u06E8 ' : '!' , # small noon ۨ  Not BW standerd
+    '\u06EA' : '-' , # special qura'nic arabic ( littel circule underneath), is not part of standerd BW , not in Buckwalter ۪  Not BW standerd
+    '\u06EC' : '%' , # small solid circle ۬  Not BW standerd
+    '\u06ED ' : ']',  # small meem ۭ  Not BW standerd
+    '0' : '0' , #
+    '1' : '1' , #
+    '2' : '2' , #
+    '3' : '3' , #
+    '4' : '4' , #
+    '5' : '5' , #
+    '6' : '6' , #
+    '7' : '7' , #
+    '8' : '8' , #
+    '9' : '9'  #
+}
+
+# This is a mapping dictionery of BW  letters to  Arabic
+# It includes all Arabic mappings in addition to other special characters
+#  that are used in the SAMA database but not part of the Arabic character set such as numbers and Qur'anic diacritics
+bw2ar_map = {
+    '\'' : '\u0621' , # ء
+    '|' : '\u0622' , # آ
+    '>' : '\u0623' , # أ
+    '&' : '\u0624' , # ؤ
+    '<' : '\u0625' , # إ
+    '}' : '\u0626' , # ئ
+    'A' : '\u0627' , # ا
+    'b' : '\u0628' , # ب
+    'p' : '\u0629' , # ة
+    't' : '\u062A' , # ت
+    'v' : '\u062B' , # ث
+    'j' : '\u062C' , # ج
+    'H' : '\u062D' , # ح
+    'x' : '\u062E' , # خ
+    'd' : '\u062F' , # د
+    '*' : '\u0630' , # ذ
+    'r' : '\u0631' , # ر
+    'z' : '\u0632' , # ز
+    's' : '\u0633' , # س
+    '$' : '\u0634' , # ش
+    'S' : '\u0635' , # ص
+    'D' : '\u0636' , # ض
+    'T' : '\u0637' , # ط
+    'Z' : '\u0638' , # ظ
+    'E' : '\u0639' , # ع
+    'g' : '\u063A' , # غ
+    ' ' : '\u0020' , #Space is replaced by space 
+    '_' : ' ' , #ـ    This is Temporary
+    # '_' : '\u0640' , #ـ
+    'f' : '\u0641' , # ف
+    'q' : '\u0642' , # ق
+    'k' : '\u0643' , # ك
+    'l' : '\u0644' , # ل
+    'm' : '\u0645' , # م
+    'n' : '\u0646' , # ن
+    'h' : '\u0647' , # ه
+    'w' : '\u0648' , # و
+    'Y' : '\u0649' , # ى
+    'y' : '\u064A' , # ي
+    'F' : '\u064B' , # TANWEEN FATH  ً
+    'N' : '\u064C' , #TANWEEN DHAM  ٌ
+    'K' : '\u064D' , #TANWEE KASR  ٍ
+    'a' : '\u064E' , #FATHA  َ
+    'u' : '\u064F' , #DHAMMA  ُ
+    'i' : '\u0650' , #KASRA  ِ
+    '~' : '\u0651' , # SHADDAH  ّ
+    'o' : '\u0652' , #SUKON  ْ
+    '`' : '\u0670' , # SHORT ALEF (dagger alif) ٰ
+    '{' : '\u0671' , #AL WITH HAMZA QATE'/WASEL MARK ٱ
+    'P' : '\u067E' , # پ 
+    'J' : '\u0686' , # چ 
+    'V' : '\u06A4' , # ڤ 
+    'G' : '\u06AF' , # گ 
+    '-' : '\u06EA' , # ۪ Not BW standerd
+    '#' : '\u0654' , # ٔ  ARABIC HAMZA ABOVE   Not BW standerd
+    ':' : '\u06DC ' , # short sen ۜ  Not BW standerd  
+    '"' : '\u06E0' , # small circle ۟  ,Not BW standerd
+    '[' : '\u06E2' , # short meem ۢ  , Not BW standerd
+    ';' : '\u06E3 ' , #small sen below letter  ۣ  , Not BW standerd
+    ',' : '\u06E5 ' , # samall wa ۥ  , Not BW standerd
+    '.' : '\u06E6' , # ۦ   Not BW standerd
+    '!' : '\u06E8 ' , # small noon ۨ  Not BW standerd
+    '-' : '\u06EA' , # special qura'nic arabic ( littel circule underneath), is not part of standerd BW , not in Buckwalter ۪  Not BW standerd
+    '%' : '\u06EC' , # small solid circle ۬  Not BW standerd
+    ']' : '\u06ED ',  # small meem ۭ  Not BW standerd
+    '0' : '0' , #
+    '1' : '1' , #
+    '2' : '2' , #
+    '3' : '3' , #
+    '4' : '4' , #
+    '5' : '5' , #
+    '6' : '6' , #
+    '7' : '7' , #
+    '8' : '8' , #
+    '9' : '9'  #
+}
+# A transliterate Function to transliterate Arabic letters and vice versa
+#It takes a text and the schema as input and return 2-values: the transliteration and a flag of whether all chars are transliterated or not
+def perform_transliteration(text , schema ):
+    """
+    This method takes a text and a schema as input and returns a tuple of two values: the transliteration of the text based on the given schema and a flag indicating whether all characters in the text were transliterated or not.
+
+    Args:
+        text (:obj:`str`): The input text to be transliterated.
+        schema (:obj:`str`): The transliteration schema to be used. Should be either `bw2ar` to transliterate Buckwalter-encoded Arabic text to Arabic script, or `ar2bw` to transliterate Arabic script to Buckwalter-encoded Arabic text.
+
+    Returns:
+        :obj:`tuple`: A tuple of two values:
+        - The transliterated text based on the given schema.
+        - A boolean flag indicating whether all characters in the input text were successfully transliterated or not.
+
+    **Example:**
+
+    .. highlight:: python
+    .. code-block:: python
+
+        from nlptools.utils import text_transliteration
+
+        print(text_transliteration.perform_transliteration("مُحَمَدٌ نَـشِيْطٌـ1"  , "ar2bw"))
+        print(text_transliteration.perform_transliteration("muHamadN"  , "bw2ar"))
+        print(text_transliteration.perform_transliteration("شَنُعُ۪ـ1"  , "ar2bw"))
+        print(text_transliteration.perform_transliteration("$aw~aH_2"  , "bw2ar"))
+
+        #output
+        ('muHamadN na_$iyoTN_1', True)
+        ('مُحَمَدٌ', True)
+        ('$anuE-u_1', True)
+        ('شَوَّح 2', True)
+    """
+    # to map BW into Arabic  
+    if schema == "bw2ar":
+        transliterated_text = deque() #Empty deque list
+        
+        is_all_mapped = True
+        for char in text:
+            # lockup every charecters in the dictionary, if not found return the original char 
+            char_value = bw2ar_map.get(char)
+   
+            if char_value is None:
+                is_all_mapped = False  # False if one cjars is not in map
+                transliterated_text.append(char)
+            else:
+                transliterated_text.append(char_value)
+
+        return ''.join(transliterated_text)  , is_all_mapped
+    # to map Arabic into BW
+    elif schema == "ar2bw":
+
+        transliterated_text = deque()
+   
+        is_all_mapped = True
+        for char in text:
+            # lockup evry charecters in the dictionary, if not found return the original char 
+            char_value = ar2bw_map.get(char)
+          
+            if char_value is None:
+                is_all_mapped = False   # False if one cjars is not in map
+                transliterated_text.append(char)
+            else:
+                transliterated_text.append(char_value)
+
+        return ''.join(transliterated_text)  , is_all_mapped
+
+    else:
+        raise ValueError("Schema must be either 'bw2ar' or 'ar2bw'.")

nlptools/utils/utils.py

@@ -0,0 +1,2 @@
+def hello():
+    return 'hello'