SinaTools 0.1.1__py2.py3-none-any.whl

nlptools/salma/views.py

@@ -0,0 +1,459 @@
+import json
+from nlptools.salma import settings 
+from nlptools.salma.wsd import normalizearabert
+from nlptools.salma.wsd import GlossPredictor
+from nlptools.utils.parser import arStrip
+from nlptools.morphology.tokenizers_words import simple_word_tokenize
+from nlptools.morphology.ALMA_multi_word import ALMA_multi_word
+from nlptools.morphology.morph_analyzer import analyze
+#from nlptools.arabiner.bin.infer import ner
+
+def delete_form_list(position, word_lemma):
+    #"""
+    #Remove specific elements from the word_lemma list based on the given position.
+    #
+    #Parameters:
+    #position (int): The current position in the input sentence.
+    #word_lemma (list): List of word lemma details.
+    #
+    #Returns:
+    #list: Updated word_lemma list with the specific elements removed.
+    #list: The list of removed elements.
+    #int: The new position in the input sentence.
+    #"""
+    tmp_word_lemma = [] 
+    output = []
+    for wordLemma in word_lemma:
+        if position == int(wordLemma[2]): # start 
+           word = wordLemma[0]
+           gloss = wordLemma[1]
+           position = int(wordLemma[3]) 
+           concept_count = int(wordLemma[4]) 
+           undiac_multi_word_lemma = wordLemma[5]
+           multi_word_lemma = wordLemma[6]
+           output.append([word, gloss, concept_count, undiac_multi_word_lemma, multi_word_lemma])# word
+        elif position < int(wordLemma[2]): 
+           tmp_word_lemma.append(wordLemma)
+    return tmp_word_lemma, output, position
+
+def find_two_word_lemma(input_sentence):
+    #"""
+    #Find two-word lemmas in the input sentence using the ALMA_multi_word function.
+    #
+    #Parameters:
+    #input_sentence (list): Tokenized input sentence.
+    #
+    #Returns:
+    #list: List of details of found two-word lemmas.
+    #"""
+    i = 0
+    output = []
+    length = len(input_sentence)
+    while i < length - 1:
+        two_grams = input_sentence[i] +" "+ input_sentence[i + 1] 
+        data = ALMA_multi_word(two_grams)
+        try :
+            glosses_list = []   
+            concept_count = 0
+            ids = data[0]["ids"]
+            for lemma_id in ids:
+               if lemma_id in settings.glosses_dic.keys(): 
+                  value = settings.glosses_dic[lemma_id]
+                  glosses_list.append(json.loads(value[1]))
+                  concept_count = concept_count + value[0]
+            
+            # found two_grams
+            #found_2Word_lemma = [two_grams,data[0]['glosses'], i, i + 1,data[0]['concept_count'], data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
+            found_2Word_lemma = [two_grams, glosses_list, i, i + 1, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
+            output.append(found_2Word_lemma) 
+            i = i + 1    
+        except: # no record found on this multi_lema
+            i = i + 1 
+    return output
+
+
+def find_three_word_lemma(input_sentence):
+    i = 0
+    output = []
+    length = len(input_sentence)
+    while i < length - 2:
+        three_grams = input_sentence[i] +" "+ input_sentence[i + 1] + " "+ input_sentence[i + 2]
+        data = ALMA_multi_word(three_grams)
+        try:
+           glosses_list = []   
+           concept_count = 0
+           ids = data[0]["ids"]
+           for lemma_id in ids:
+              if lemma_id in settings.glosses_dic.keys(): 
+                 value = settings.glosses_dic[lemma_id]
+                 glosses_list.append(json.loads(value[1]))
+                 concept_count = concept_count + value[0]
+                 
+           #found_3Word_lemma = [three_grams, data[0]['glosses'], i, i + 2,data[0]['concept_count'], data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
+           found_3Word_lemma = [three_grams, glosses_list, i, i + 2, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
+           output.append(found_3Word_lemma) 
+           i = i + 1    
+        except:  
+           i = i + 1 
+    return output
+
+def find_four_word_lemma(input_sentence):
+   i = 0
+   output = []
+   length = len(input_sentence)
+   while i < length - 3:
+      four_grams = input_sentence[i] +" "+ input_sentence[i + 1] + " "+ input_sentence[i + 2] + " "+ input_sentence[i + 3]
+      data = ALMA_multi_word(four_grams)
+      try:
+         glosses_list = []   
+         concept_count = 0
+         ids = data[0]["ids"]
+         for lemma_id in ids:
+            if lemma_id in settings.glosses_dic.keys(): 
+               value = settings.glosses_dic[lemma_id]
+               glosses_list.append(json.loads(value[1]))
+               concept_count = concept_count + value[0]
+         #found_4Word_lemma = [four_grams, data[0]['glosses'], i, i + 3,data[0]['concept_count'], data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
+         found_4Word_lemma = [four_grams, glosses_list, i, i + 3, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
+         output.append(found_4Word_lemma) 
+         i = i + 1    
+      except:  
+         i = i + 1 
+   return output
+
+
+def find_five_word_lemma(input_sentence):
+   i = 0
+   output = []
+   length = len(input_sentence)
+   while i < length - 4:
+      five_grams = input_sentence[i] +" "+ input_sentence[i + 1] + " "+ input_sentence[i + 2] + " "+ input_sentence[i + 3] + " "+ input_sentence[i + 4]
+      data = ALMA_multi_word(five_grams)
+      try:
+         glosses_list = []   
+         concept_count = 0
+         ids = data[0]["ids"]
+         for lemma_id in ids:
+            if lemma_id in settings.glosses_dic.keys(): 
+               value = settings.glosses_dic[lemma_id]
+               glosses_list.append(json.loads(value[1]))
+               concept_count = concept_count + value[0]
+         #found_5Word_lemma = [five_grams, data[0]['glosses'], i, i + 4,data[0]['concept_count'], data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
+         found_5Word_lemma = [five_grams, glosses_list, i, i + 4, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
+         output.append(found_5Word_lemma) 
+         i = i + 1    
+      except:  
+         i = i + 1 
+   return output
+
+def find_named_entities(string):
+   #"""
+   # Find named entities in the input string using a NER tool.
+   # 
+   # Parameters:
+   # string (str): Input string.
+   # 
+   # Returns:
+   # list: List of details of found named entities.
+   # """
+   found_entities = []
+   #entites = ner(string, "4")
+   entites = []
+   tag_gloss = {
+      "PERS": "اسم شخص",
+      "ORG": "اسم مؤسسة",
+      #"NORP": "مجموعة من الناس", 
+      #"OCC": "منصب/مسمى وظيفي",
+      "LOC": "اسم منطقة جغرافية",
+      "FAC": "اسم لمَعلَم",
+      #"EVENT": "حدث",
+      "DATE": "فترة زمنية تدل على تاريخ",
+      "UNIT": "وحدة قياس",
+      "CURR": "عملة",
+      "GPE": "اسم بلد، له حدود إدارية/جيوسياسية",
+      "TIME": "فترة زمنية تدل على الوقت",
+      "CARDINAL": "عدد يدل على معدود",
+      "ORDINAL": "رقم، لا يدل على معدود",
+      "PERCENT": "نسبة مئوية",
+      "QUANTITY": "كمية",
+      "MONEY": "مبلغ مالي",
+      "LANGUAGE": "اسم للغة طبيعية",
+      "PRODUCT": "اسم منتج",
+      "LAW": "قانون"
+   }
+
+   for entity in entites:
+      gloss_ner = ""
+      if entity[1] in tag_gloss.keys():
+         gloss_ner = tag_gloss[entity[1]]  
+
+      if gloss_ner != "":
+         gloss = [{'concept_id': '', 'resource_id': '', 'resource_name': '', 'gloss': gloss_ner}]   
+         entity = [entity[0],gloss,int(entity[2]), int(entity[3]),1,arStrip(entity[0],True,True,True,False,True,False),entity[0]]   
+         found_entities.append(entity)
+   return found_entities   
+
+
+def find_glosses_using_ALMA(word):
+
+   data = analyze(word)
+   Diac_lemma = ""
+   pos = ""
+   Undiac_lemma = ""
+   glosses = []
+   Diac_lemma = data[0][1]
+   pos = data[0][2]
+   Undiac_lemma = arStrip(Diac_lemma, True, True, True, True, True, False) # Remove diacs , smallDiacs , shaddah ,  digit , alif , specialChars
+   #"""
+   # Find glosses for the given word using the ALMA tool.
+   # 
+   # Parameters:
+   # word (str): Input word.
+   # 
+   # Returns:
+   # tuple: Details of the word including glosses, lemmas, and POS.
+   # """
+   ids = [] 
+   glosses_list = []   
+   concept_count = 0
+   for line in data:
+      lemma_id = line[3]
+      ids.append(lemma_id)
+
+   for lemma_id in ids:
+      if lemma_id in settings.glosses_dic.keys(): 
+         value = settings.glosses_dic[lemma_id]
+         glosses_list.append(json.loads(value[1]))
+         concept_count = concept_count + value[0]
+
+   #glosses = data[0][4]
+   #concept_count = data[0][3]
+   return word, Undiac_lemma, Diac_lemma, pos , concept_count, glosses
+   
+def disambiguate_glosses_using_SALMA(glosses, Diac_lemma, Undiac_lemma, word, sentence):
+   #"""
+   # Disambiguate glosses using the SALMA tool.
+   # 
+   # Parameters:
+   # glosses (list): List of glosses.
+   # Diac_lemma (str): Diacritic lemma of the word.
+   # Undiac_lemma (str): Undiacritic lemma of the word.
+   # word (str): The word being analyzed.
+   # sentence (str): The sentence containing the word.
+   # 
+   # Returns:
+   # dict: Disambiguated gloss details.
+   # """
+   word = normalizearabert(word)
+   glosses_dictionary = {}
+   if glosses != None:
+      for gloss in glosses:
+         glosses_dictionary.update({gloss['concept_id'] : gloss['gloss']})
+      concept_id, gloss = GlossPredictor(Diac_lemma, Undiac_lemma,word,sentence,glosses_dictionary)
+
+      my_json = {}    
+      my_json['Concept_id'] = concept_id
+      my_json['Gloss'] = gloss
+      my_json['word'] = word
+      my_json['Undiac_lemma'] = Undiac_lemma
+      my_json['Diac_lemma'] = Diac_lemma
+      return my_json
+   else:
+      my_json = {}    
+      my_json['word'] = word
+      my_json['Undiac_lemma'] = Undiac_lemma
+      my_json['Diac_lemma'] = Diac_lemma
+      return my_json
+
+
+def find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lemma, five_word_lemma, ner):
+      output_list = []
+      position = 0
+      while position < len(input_sentence):    
+         flag = "False"
+         output_from5word = delete_form_list(position, five_word_lemma)
+         five_word_lemma = output_from5word[0]
+         if output_from5word[1] != []: # output
+            position = output_from5word[2]  
+            flag = "True"
+            my_json = {}    
+            my_json['word'] = output_from5word[1][0][0]
+            my_json['concept_count'] = output_from5word[1][0][2]
+            my_json['glosses'] = output_from5word[1][0][1]
+            my_json['Diac_lemma'] = output_from5word[1][0][4]
+            my_json['Undiac_lemma'] = output_from5word[1][0][3]
+            output_list.append(my_json)
+            position = position + 1                
+
+
+
+         output_from4word = delete_form_list(position, four_word_lemma)
+         four_word_lemma = output_from4word[0]
+         if output_from4word[1] != []: # output
+            position = output_from4word[2]  
+            flag = "True"
+            my_json = {}    
+            my_json['word'] = output_from4word[1][0][0]
+            my_json['concept_count'] = output_from4word[1][0][2]
+            my_json['glosses'] = output_from4word[1][0][1]
+            my_json['Diac_lemma'] = output_from4word[1][0][4]
+            my_json['Undiac_lemma'] = output_from4word[1][0][3]
+            output_list.append(my_json)
+            position = position + 1                
+         
+         output_from3word = delete_form_list(position, three_word_lemma)
+         three_word_lemma = output_from3word[0]
+         if output_from3word[1] != []: # output
+            position = output_from3word[2]  
+            flag = "True"
+            my_json = {}    
+            my_json['word'] = output_from3word[1][0][0]
+            my_json['concept_count'] = output_from3word[1][0][2]
+            my_json['glosses'] = output_from3word[1][0][1]
+            my_json['Diac_lemma'] = output_from3word[1][0][4]
+            my_json['Undiac_lemma'] = output_from3word[1][0][3]
+            output_list.append(my_json)
+            position = position + 1                
+
+
+
+         output_from2Word = delete_form_list(position, two_word_lemma)
+         two_word_lemma = output_from2Word[0] 
+         if output_from2Word[1] != []:  
+            position = output_from2Word[2]
+            flag = "True"
+            my_json = {}    
+            word = output_from2Word[1][0][0]
+            my_json['word'] = word
+            my_json['concept_count'] = output_from2Word[1][0][2]
+            my_json['glosses'] = output_from2Word[1][0][1]
+            my_json['Diac_lemma'] = output_from2Word[1][0][4]
+            my_json['Undiac_lemma'] = output_from2Word[1][0][3] 
+            output_list.append(my_json)
+            position = position + 1                 
+               
+
+
+         output_from_ner = delete_form_list(position, ner)
+         ner = output_from_ner[0] 
+         if output_from_ner[1] != []:  
+            position = output_from_ner[2]
+            flag = "True"
+            my_json = {}    
+            word = output_from_ner[1][0][0]
+            my_json['word'] = word
+            my_json['concept_count'] = output_from_ner[1][0][2]
+            my_json['glosses'] = output_from_ner[1][0][1]
+            my_json['Diac_lemma'] = output_from_ner[1][0][4]
+            my_json['Undiac_lemma'] = output_from_ner[1][0][3] 
+            output_list.append(my_json)
+            position = position + 1                             
+         
+         if flag == "False": # Not found in ner or in multi_word_dictionary, ASK ALMA 
+            word = input_sentence[position]
+            word, Undiac_lemma, Diac_lemma, pos , concept_count, glosses = find_glosses_using_ALMA(word)   
+            my_json = {}    
+            my_json['word'] = word
+            my_json['concept_count'] = concept_count
+            my_json['glosses'] = glosses
+            my_json['Diac_lemma'] = Diac_lemma
+            my_json['Undiac_lemma'] = Undiac_lemma
+            output_list.append(my_json)
+            position = position + 1  
+      return output_list                    
+
+def disambiguate_glosses_main(word, sentence):
+   concept_count = word['concept_count']
+   if concept_count == 0:
+      my_json = {}    
+      my_json['word'] = word['word']
+      my_json['Diac_lemma'] = word['Diac_lemma']
+      my_json['Undiac_lemma'] = word['Undiac_lemma']
+      return my_json
+   elif concept_count == 1:
+      my_json = {}    
+      my_json['word'] = word['word']
+      glosses = word['glosses'][0]
+      my_json['Gloss'] = glosses['gloss']
+      my_json['Concept_id'] = glosses['concept_id']
+      my_json['Diac_lemma'] = word['Diac_lemma']
+      my_json['Undiac_lemma'] = word['Undiac_lemma']
+      return my_json
+   else:   
+      input_word = word['word']
+      concept_count = word['concept_count']
+      glosses = word['glosses']
+      Diac_lemma = word['Diac_lemma']
+      Undiac_lemma = word['Undiac_lemma']
+      return disambiguate_glosses_using_SALMA(glosses, Diac_lemma, Undiac_lemma, input_word, sentence)
+
+def WSD(sentence):
+   
+   input_sentence = simple_word_tokenize(sentence)
+   
+   five_word_lemma = find_five_word_lemma(input_sentence)
+   
+   four_word_lemma = find_four_word_lemma(input_sentence)
+   
+   three_word_lemma = find_three_word_lemma(input_sentence)
+   
+   two_word_lemma = find_two_word_lemma(input_sentence)
+   
+   ner = find_named_entities(" ".join(input_sentence))
+
+   output_list = find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lemma, five_word_lemma, ner)
+   
+   results = []
+   for word in output_list:
+      results.append(disambiguate_glosses_main(word, sentence))
+   return results
+
+
+def SALMA(sentence):
+    """
+    This method disambiguate words within a sentence.
+
+    Args:
+        sentence (:obj:`str`): The Arabic text to be disambiguated, it should be limited to less than 500 characters.
+
+    Returns:
+        :obj:`list`: The JSON output includes a list of words, with each word having a gloss if it exists or a lemma if no gloss is found.
+
+    **Example:**
+
+    .. highlight:: python
+    .. code-block:: python
+
+        from nlptools.salma.views import SALMA
+        JSON = SALMA("مختبر سينا لحوسبة اللغة والذكاء الإصطناعي. في جامعة بيرزيت.")
+        print(JSON["resp"])
+
+        #output
+         [
+             {
+                 "Concept_id": "303019218",
+                 "Gloss": "ذهَب إلى عملِه:- قصَده، توجَّه إليه \"ذهَب إلى الجامعة/ بيروت - اذهَب إلى أبيك والتمس منه الصفح - ذهَب إلى قول فلان أخذ به - <اذْهَبْ إِلَى فِرْعَوْنَ إِنَّهُ طَغَى> طه/ 24 \". ذهَب رأسًا إليه",
+                 "word": "ذهبت",
+                 "Undiac_lemma": "ذهب",
+                 "Diac_lemma": "ذَهَبَ۪ 1"
+             },
+             {
+                 "word": "إلى",
+                 "Diac_lemma": "إِلَى 1",
+                 "Undiac_lemma": "الى"
+             },
+             {
+                 "word": "جامعة بيرزيت",
+                 "Gloss": "جامعة فلسطينية تقع في بلدة بيرزيت، قرب مدينة رام الله، ويعود تاريخها إلى عام 1924 عندما تأسست كمدرسة ابتدائية ثم أصبحت جامعة عام 1975",
+                 "Concept_id": "334000099",
+                 "Diac_lemma": "جامِعَة بيرزَيت",
+                 "Undiac_lemma": "جامعة بيرزيت"
+             }
+         ]
+    """      
+    if len(sentence) > 500:
+       content = {"statusText":"Input is too long","statusCode":-7}
+       return content
+    else: 
+       results = WSD(sentence)
+       return {"resp": results, "statusText":"OK","statusCode":0}

nlptools/salma/wsd.py

@@ -0,0 +1,126 @@
+from nlptools.salma import settings 
+import re
+import warnings
+warnings.filterwarnings("ignore")
+import torch
+import numpy as np
+import pandas as pd
+from nlptools.arabert.preprocess import ArabertPreprocessor
+
+def normalizearabert(s):
+  model_name = 'aubmindlab/bert-base-arabertv02'
+  arabert_prep = ArabertPreprocessor(model_name.split("/")[-1])
+  return arabert_prep.preprocess(str(s))
+
+
+
+def glosses1(dfcand,target):
+# """
+# takes a dataframe 
+# return 
+	# 'none' if the maximum logistic regression score for TRUE class is less than -2 OR
+	# the predicted gloss having the maximum logistic regression score
+# """
+
+  wic_c = []
+  wic_c, _ = read_data(dfcand,normalizearabert,target)
+  tokenizedwic_c = np.array([settings.tokenizer.encode(x, max_length=512,padding='max_length',truncation='longest_first',add_special_tokens=True) for x in wic_c])
+  max_len = 512
+  segmentswic = torch.tensor([get_segments(settings.tokenizer.convert_ids_to_tokens(i),max_len) for i in tokenizedwic_c])
+  paddedwic = tokenizedwic_c
+  attention_maskwic = np.where(paddedwic != 0, 1, 0)
+  input_idswic = torch.tensor(paddedwic)  
+  attention_maskwic = torch.tensor(attention_maskwic)
+  settings.model = settings.model.eval()
+  wicpredictions , wictrue_labels = [], []
+  b_input_ids = input_idswic
+  b_input_mask =  attention_maskwic
+  b_input_seg = segmentswic
+
+  with torch.no_grad():
+    outputs = settings.model(b_input_ids,token_type_ids=b_input_seg,attention_mask=b_input_mask)
+
+  logits = outputs[0]
+  wicpredictions.append(logits)
+  wicflat_predictions = np.concatenate(wicpredictions, axis=0)
+
+  return dfcand['Concept_id'].to_list()[np.argmax(wicflat_predictions, axis=0).flatten()[1]],dfcand['Gloss'].to_list()[np.argmax(wicflat_predictions, axis=0).flatten()[1]]
+
+def read_data(data,normalize,target):
+  c = []
+  labels = []
+  for i,row in data.iterrows():
+      
+      example = normalize(row['Example'])
+      gloss = normalize(row['Gloss'])
+      label = row['Label']
+      
+      c.append('{} [SEP] {}: {}'.format(example,target,gloss))
+      if label == 1.0:
+          labels.append(1)
+      else:
+          labels.append(0)
+  return c,labels
+
+def inserttag1(sentence,tag,start,end):
+    before = sentence[:start]
+    after = sentence[end:]
+    target = sentence[start:end]
+    return before+tag+sentence[start:end]+tag+after
+
+def get_segments(tokens, max_seq_length):
+    if len(tokens)>max_seq_length:
+        raise IndexError("Token length more than max seq length!")
+    segments = []
+    current_segment_id = 0
+    for token in tokens:
+        segments.append(current_segment_id)
+        if token == "[SEP]":
+            current_segment_id = 1
+    return segments + [0] * (max_seq_length - len(tokens))
+
+def senttarget(target,example):
+  start = -1
+  try:
+    start = example.index(target)
+  except ValueError:
+    return -1
+  end = example.index(target)+len(target)
+  return inserttag1(example,"[UNUSED0]",start,end)
+
+
+def GlossPredictor(diac_lemma, undiac_lemma,target,example,glosses):
+# """ 
+# takes 
+	# a lemma
+	# corresponding target word 
+	# an example
+	# glosses as a dictionay, following an example:
+	#	glosses =	{"Concept_id1": "gloss1",  "Concept_id2": "gloss2",  "Concept_id3": "gloss3"}
+# returns 
+	# -1   if the example does not contain the target word  OR
+	# 'none' if no records in dftrue for the lemma and if the maximum logistic regression score for TRUE class is less than -2 OR
+	# the predicted gloss for the target word 
+	# 
+# """
+  example = senttarget(target,example)
+  if example == -1:
+    return -1,-1
+  
+  data = []
+  for g in glosses:
+      data.append([g,diac_lemma,undiac_lemma, glosses[g], target,example,0,1,'','',''])
+  dfcolumns = ['Concept_id', 'Diac_lemma', 'Undiac_lemma', 'Gloss', 'Target', 'Example', 'Is_training', 'Label', 'concept_id', 'lemma_id', 'POS']
+  dfcand = pd.DataFrame(data,columns=dfcolumns)
+  
+  
+  if len(dfcand) > 0:
+    dfcand['Example'] = dfcand['Example'].apply(lambda x: example)
+    dfcand['Target'] = dfcand['Target'].apply(lambda x: target)
+    dfcand = dfcand.drop_duplicates()
+  
+    dfcand['Example'] = dfcand['Example'].apply(lambda x: x.upper())
+    dfcand['Example'] = dfcand['Example'].apply(lambda x: re.sub(r'^((.?\[UNUSED0\].?){1})\[UNUSED0\]', r'\1[UNUSED1]', x) )
+    return glosses1(dfcand,target)
+  else:
+    return 'none','none'

nlptools/utils/corpus_tokenizer.py

@@ -0,0 +1,73 @@
+import os
+import csv
+from nlptools.utils.sentence_tokenizer import sent_tokenize
+from nlptools.morphology.tokenizers_words import simple_word_tokenize
+
+def corpus_tokenizer(dir_path, output_csv, row_id = 1, global_sentence_id = 1):
+    """
+    This method receives a directory and tokenizes all files within the input directory, as well as all files within subdirectories within the main directory. The results are then stored in a CSV file.
+
+    Args:
+        dir_path (:obj:`str`): The path of the directory containing multiple Arabic txt files.
+        output_csv (:obj:`str`): The name of the output CSV file, which will be generated in the current directory where this function is used.
+        row_id (:obj:`int`): Specifies the row_id you wish to start with; the default value is 1.
+        global_sentence_id (:obj:`int`): Specifies the global_sentence_id you wish to start with; the default value is 1.
+
+    Returns:
+        csv file (:obj:`str`): The CSV file contains the following fields: 
+            * Row_ID (primary key, unique for all records in outputfile)
+            * Docs_Sentence_Word_ID (DirectoryName_FileName_GlobalSentenceID_SentenceID_WordPosition)
+            * GlobalSentenceID (Integer, a unique identifier for each sentence in the entire file)
+            * SentenceID (Integer, a unique identifier for each file within the CSV file)
+            * Sentence (Generated text that forms a sentence)
+            * Word Position (Integer, the position of each word within the sentence)
+            * Word (Each row contains a word from the generated sentence).
+
+    **Example:**
+
+    .. highlight:: python
+    .. code-block:: python
+
+        from nlptools.utils.corpus_tokenizer import corpus_tokenizer
+        corpus_tokenizer(dir_path="History", output_csv="ouputFile.csv", row_id = 1, global_sentence_id = 1)
+
+        #output
+        # csv file called: ouputFile.csv 
+        # For example, if the 'History' directory contains 2 files named 'h1.txt' and 'h2.txt'. 
+        # The output file will contain: 
+        # Row_ID, Docs_Sentence_Word_ID, Global Sentence ID, Sentence ID, Sentence, Word Position, Word
+        # 1,History_h1_1_1_1,1,1,الطيور الضارة ومكافحتها,1,الطيور
+        # 2,History_h1_1_1_2,1,1,الطيور الضارة ومكافحتها,2,الضارة
+        # 3,History_h1_1_1_3,1,1,الطيور الضارة ومكافحتها,3,ومكافحتها
+        # 4,History_h2_2_1_1,1,1,بشكل عام,1,بشكل
+        # 5,History_h2_2_1_2,1,1,بشكل عام,2,عام
+    """    
+    row_id = row_id - 1
+    global_sentence_id = global_sentence_id - 1
+    with open(output_csv, 'w', newline='', encoding="utf-8") as csvfile:
+        fieldnames = ['Row_ID', 'Docs_Sentence_Word_ID', 'Global Sentence ID', 'Sentence ID', 'Sentence', 'Word Position', 'Word']
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        writer.writeheader()
+        
+        for root, dirs, files in os.walk(dir_path):
+            for file in files:
+                if file.endswith('.txt'):
+                    file_path = os.path.join(root, file)
+                    with open(file_path, 'r', encoding="utf-8") as f:
+                        content = f.read()
+                        sentences = sent_tokenize(content, dot=True, new_line=True, question_mark=False, exclamation_mark=False)
+                        for sentence_id, sentence in enumerate(sentences, start=1):
+                            words = simple_word_tokenize(sentence)
+                            global_sentence_id += 1
+                            for word_pos, word in enumerate(words, start=1):
+                                row_id += 1
+                                dir_name = os.path.basename(root)
+                                doc_sentence_filename = file.split(".txt")[0]
+                                docs_sentence_word_id = f"{dir_name}_{doc_sentence_filename}_{global_sentence_id}_{sentence_id}_{word_pos}"
+                                writer.writerow({'Row_ID': row_id,
+                                                 'Docs_Sentence_Word_ID': docs_sentence_word_id,
+                                                 'Global Sentence ID': global_sentence_id,
+                                                 'Sentence ID': sentence_id,
+                                                 'Sentence': sentence,
+                                                 'Word Position': word_pos,
+                                                 'Word': word})