SinaTools 0.1.19__tar.gz → 0.1.21__tar.gz

{SinaTools-0.1.19 → SinaTools-0.1.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.19
+Version: 0.1.21
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.19 → SinaTools-0.1.21}/SinaTools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.19
+Version: 0.1.21
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

SinaTools-0.1.21/sinatools/VERSION

@@ -0,0 +1 @@
+0.1.21

{SinaTools-0.1.19 → SinaTools-0.1.21}/sinatools/synonyms/__init__.py

@@ -11,8 +11,8 @@ with open(file_path, 'rb') as f:
 
 
 synonyms_level3_dict = {}    
-#level3_dict = 'synonyms_level3.pkl'
-#path = downloader.get_appdatadir()
-#file_path = os.path.join(path, level3_dict)
-#with open(file_path, 'rb') as f:
-#    synonyms_level3_dict = pickle.load(f, encoding='utf-8')
+level3_dict = 'synonyms_level3.pkl'
+path = downloader.get_appdatadir()
+file_path = os.path.join(path, level3_dict)
+with open(file_path, 'rb') as f:
+   synonyms_level3_dict = pickle.load(f, encoding='utf-8')

{SinaTools-0.1.19 → SinaTools-0.1.21}/sinatools/wsd/__init__.py

@@ -3,9 +3,9 @@ import pickle
 from sinatools.DataDownload import downloader
 import os 
 
-settings.glosses_dic = {}
+glosses_dic = {}
 filename = 'glosses_dic.pickle'
 path =downloader.get_appdatadir()
 file_path = os.path.join(path, filename)
 with open(file_path, 'rb') as f:
-    settings.glosses_dic = pickle.load(f)
+    glosses_dic = pickle.load(f)

{SinaTools-0.1.19 → SinaTools-0.1.21}/sinatools/wsd/disambiguator.py

@@ -7,6 +7,7 @@ from sinatools.utils.tokenizers_words import simple_word_tokenize
 from sinatools.morphology.ALMA_multi_word import ALMA_multi_word
 from sinatools.morphology.morph_analyzer import analyze
 from sinatools.ner.entity_extractor import extract
+from . import glosses_dic
 
 
 def distill_entities(entities):
@@ -135,12 +136,9 @@ def find_two_word_lemma(input_sentence):
             glosses_list = []   
             concept_count = 0
             ids = data[0]["ids"]
-            for lemma_id in ids:
-               if lemma_id in settings.glosses_dic.keys(): 
-                  value = settings.glosses_dic[lemma_id]
-                  glosses_list.append(json.loads(value[1]))
-                  concept_count = concept_count + value[0]
-            
+            for concepts in ids:
+               glosses_list.append(json.loads(concepts))
+            concept_count = concept_count + data[0]["POS"]
             found_2Word_lemma = [two_grams, glosses_list, i, i + 1, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
             output.append(found_2Word_lemma) 
             i = i + 1    
@@ -160,12 +158,9 @@ def find_three_word_lemma(input_sentence):
            glosses_list = []   
            concept_count = 0
            ids = data[0]["ids"]
-           for lemma_id in ids:
-              if lemma_id in settings.glosses_dic.keys(): 
-                 value = settings.glosses_dic[lemma_id]
-                 glosses_list.append(json.loads(value[1]))
-                 concept_count = concept_count + value[0]
-                 
+           for concepts in ids:
+              glosses_list.append(json.loads(concepts))
+           concept_count = concept_count + data[0]["POS"]
            found_3Word_lemma = [three_grams, glosses_list, i, i + 2, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
            output.append(found_3Word_lemma) 
            i = i + 1    
@@ -184,11 +179,9 @@ def find_four_word_lemma(input_sentence):
          glosses_list = []   
          concept_count = 0
          ids = data[0]["ids"]
-         for lemma_id in ids:
-            if lemma_id in settings.glosses_dic.keys(): 
-               value = settings.glosses_dic[lemma_id]
-               glosses_list.append(json.loads(value[1]))
-               concept_count = concept_count + value[0]
+         for concepts in ids:
+            glosses_list.append(json.loads(concepts))
+         concept_count = concept_count + data[0]["POS"] 
          found_4Word_lemma = [four_grams, glosses_list, i, i + 3, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
          output.append(found_4Word_lemma) 
          i = i + 1    
@@ -208,11 +201,9 @@ def find_five_word_lemma(input_sentence):
          glosses_list = []   
          concept_count = 0
          ids = data[0]["ids"]
-         for lemma_id in ids:
-            if lemma_id in settings.glosses_dic.keys(): 
-               value = settings.glosses_dic[lemma_id]
-               glosses_list.append(json.loads(value[1]))
-               concept_count = concept_count + value[0]
+         for concepts in ids:
+            glosses_list.append(json.loads(concepts))
+         concept_count = concept_count + data[0]["POS"]
          found_5Word_lemma = [five_grams, glosses_list, i, i + 4, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
          output.append(found_5Word_lemma) 
          i = i + 1    
@@ -276,16 +267,18 @@ def find_glosses_using_ALMA(word):
    pos = data[0]["pos"]
    Undiac_lemma = arStrip(Diac_lemma, True, True, True, True, True, False) # Remove diacs , smallDiacs , shaddah ,  digit , alif , specialChars
    ids = [] 
-   glosses_list = []   
+#    glosses_list = []   
    concept_count = 0
    lemma_id = data[0]["lemma_id"]
-   if lemma_id in settings.glosses_dic.keys(): 
-      value = settings.glosses_dic[lemma_id]
-      glosses_list.append(json.loads(value[1]))
-      concept_count = concept_count + value[0]
 
-   return word, Undiac_lemma, Diac_lemma, pos , concept_count, glosses
+   if lemma_id in glosses_dic.keys(): 
+      value = glosses_dic[lemma_id]
+      glosses= json.loads(value[1])
+    #   glosses_list.append(json.loads(value[1]))
+      concept_count = concept_count + value[0]
    
+   return word, Undiac_lemma, Diac_lemma, pos , concept_count, glosses
+
 def disambiguate_glosses_using_SALMA(glosses, Diac_lemma, Undiac_lemma, word, sentence):
    word = normalizearabert(word)
    glosses_dictionary = {}
@@ -309,7 +302,7 @@ def disambiguate_glosses_using_SALMA(glosses, Diac_lemma, Undiac_lemma, word, se
       return my_json
 
 
-def find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lemma, five_word_lemma, ner):
+def find_glosses(input_sentence, two_word_lemma, three_word_lemma,four_word_lemma, five_word_lemma, ner):
       output_list = []
       position = 0
       while position < len(input_sentence):    
@@ -376,7 +369,7 @@ def find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lem
             position = position + 1                 
                
 
-
+         
          output_from_ner = delete_form_list(position, ner)
          ner = output_from_ner[0] 
          if output_from_ner[1] != []:  
@@ -385,11 +378,13 @@ def find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lem
             my_json = {}    
             word = output_from_ner[1][0][0]
             my_json['word'] = word
-            my_json['concept_count'] = output_from_ner[1][0][2]
+            # my_json['concept_count'] = output_from_ner[1][0][2]
+            my_json['concept_count'] = '*'
             my_json['glosses'] = output_from_ner[1][0][1]
             my_json['Diac_lemma'] = output_from_ner[1][0][4]
             my_json['Undiac_lemma'] = output_from_ner[1][0][3] 
             output_list.append(my_json)
+            # print("output list: ", output_list)
             position = position + 1                             
          
          if flag == "False": # Not found in ner or in multi_word_dictionary, ASK ALMA 
@@ -417,6 +412,15 @@ def disambiguate_glosses_main(word, sentence):
       my_json = {}    
       my_json['word'] = word['word']
       glosses = word['glosses'][0]
+    #   my_json['Gloss'] = glosses['gloss']
+      my_json['Concept_id'] = glosses['concept_id']
+      my_json['Diac_lemma'] = word['Diac_lemma']
+      my_json['Undiac_lemma'] = word['Undiac_lemma']
+      return my_json
+   elif concept_count == '*':
+      my_json = {}    
+      my_json['word'] = word['word']
+      glosses = word['glosses'][0]
       my_json['Gloss'] = glosses['gloss']
       my_json['Concept_id'] = glosses['concept_id']
       my_json['Diac_lemma'] = word['Diac_lemma']
@@ -444,8 +448,7 @@ def WSD(sentence):
    
    ner = find_named_entities(" ".join(input_sentence))
 
-   output_list = find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lemma, five_word_lemma, ner)
-   
+   output_list = find_glosses(input_sentence, two_word_lemma, three_word_lemma, four_word_lemma, five_word_lemma, ner)
    results = []
    for word in output_list:
       results.append(disambiguate_glosses_main(word, sentence))

{SinaTools-0.1.19 → SinaTools-0.1.21}/sinatools/wsd/settings.py

@@ -9,7 +9,6 @@ import pandas as pd
 from sinatools.DataDownload import downloader
 import os 
 
-glosses_dic = {}
 
 model_file_name = "bert-base-arabertv02_22_May_2021_00h_allglosses_unused01"
 path =downloader.get_appdatadir()
@@ -21,11 +20,6 @@ tokenizer_file_path = os.path.join(path, tokenizer_file_name)
 
 dftrue = pd.DataFrame()
 
-# model = BertForSequenceClassification.from_pretrained('{}'.format("bert-base-arabertv02_22_May_2021_00h_allglosses_unused01"),
-                                                    #   output_hidden_states = True,
-                                                    #   num_labels=2
-                                                    #  )
-
 model = BertForSequenceClassification.from_pretrained(model_file_path, output_hidden_states=True, num_labels=2)
 
 tokenizer = BertTokenizer.from_pretrained('{}'.format(tokenizer_file_path))

SinaTools-0.1.19/sinatools/VERSION

@@ -1 +0,0 @@
-0.1.19