SinaTools 0.1.20__py2.py3-none-any.whl → 0.1.21__py2.py3-none-any.whl

{SinaTools-0.1.20.dist-info → SinaTools-0.1.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.20
+Version: 0.1.21
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.20.dist-info → SinaTools-0.1.21.dist-info}/RECORD

@@ -1,5 +1,5 @@
-SinaTools-0.1.20.data/data/sinatools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
-sinatools/VERSION,sha256=Cl3YqOjNQlou69HucXrHnlE2wQ5-6XQ2C-f24WemaN4,6
+SinaTools-0.1.21.data/data/sinatools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
+sinatools/VERSION,sha256=xAJpR_QiE53NSmTRvuvj7nJV3dmZVUy0LU-CE-qYKgs,6
 sinatools/__init__.py,sha256=bEosTU1o-FSpyytS6iVP_82BXHF2yHnzpJxPLYRbeII,135
 sinatools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
 sinatools/install_env.py,sha256=EODeeE0ZzfM_rz33_JSIruX03Nc4ghyVOM5BHVhsZaQ,404
@@ -109,14 +109,14 @@ sinatools/utils/text_dublication_detector.py,sha256=6yAOUtdw4TKiJkUPDDi3oK7CEoIu
 sinatools/utils/text_transliteration.py,sha256=NQoXrxI-h0UXnvVtDA3skNJduxIy0IW26r46N4tDxGk,8766
 sinatools/utils/tokenizer.py,sha256=QHyrVqJA_On4rKxexiWR2ovq4pI1-u6iZkdhRbK9tew,6676
 sinatools/utils/tokenizers_words.py,sha256=efNfOil9qDNVJ9yynk_8sqf65PsL-xtsHG7y2SZCkjQ,656
-sinatools/wsd/__init__.py,sha256=5Ondsp-Xe9YxVjRlTc4nLrxu6xiyML7B3bQ3EZ44uEM,327
-sinatools/wsd/disambiguator.py,sha256=7CDlcIM_O7J9wVIrHjauAHDhsKqnia-zLT7NLnDrLys,19999
-sinatools/wsd/settings.py,sha256=b_AqTxVWALuGXnsMd9KhnnwIo9-JEoWOTekB-7_xJCU,1111
+sinatools/wsd/__init__.py,sha256=yV-SQSCzSrjbNkciMbDCqzGZ_EESchL7rlJk56uibVI,309
+sinatools/wsd/disambiguator.py,sha256=8HrVAGpEQyrzwiuEreLX9X82WSL-U2Aeca0ttrtIw2Y,19998
+sinatools/wsd/settings.py,sha256=6XflVTFKD8SVySX9Wj7zYQtV26WDTcQ2-uW8-gDNHKE,747
 sinatools/wsd/wsd.py,sha256=gHIBUFXegoY1z3rRnIlK6TduhYq2BTa_dHakOjOlT4k,4434
-SinaTools-0.1.20.dist-info/AUTHORS.rst,sha256=aTWeWlIdfLi56iLJfIUAwIrmqDcgxXKLji75_Fjzjyg,174
-SinaTools-0.1.20.dist-info/LICENSE,sha256=uwsKYG4TayHXNANWdpfMN2lVW4dimxQjA_7vuCVhD70,1088
-SinaTools-0.1.20.dist-info/METADATA,sha256=mAAUGGq-SXM-psy8aOg0DOucVqbbJLkx7oDZGREno1Q,953
-SinaTools-0.1.20.dist-info/WHEEL,sha256=6T3TYZE4YFi2HTS1BeZHNXAi8N52OZT4O-dJ6-ome_4,116
-SinaTools-0.1.20.dist-info/entry_points.txt,sha256=ZwZLolnWog2fjdDrfaHNHob8SE_YtMbD6ayzsOzItxs,1234
-SinaTools-0.1.20.dist-info/top_level.txt,sha256=8tNdPTeJKw3TQCaua8IJIx6N6WpgZZmVekf1OdBNJpE,10
-SinaTools-0.1.20.dist-info/RECORD,,
+SinaTools-0.1.21.dist-info/AUTHORS.rst,sha256=aTWeWlIdfLi56iLJfIUAwIrmqDcgxXKLji75_Fjzjyg,174
+SinaTools-0.1.21.dist-info/LICENSE,sha256=uwsKYG4TayHXNANWdpfMN2lVW4dimxQjA_7vuCVhD70,1088
+SinaTools-0.1.21.dist-info/METADATA,sha256=BrzKHUWwumfn52P0RM00koZDTl0txyL-cfr1bho-8mQ,953
+SinaTools-0.1.21.dist-info/WHEEL,sha256=6T3TYZE4YFi2HTS1BeZHNXAi8N52OZT4O-dJ6-ome_4,116
+SinaTools-0.1.21.dist-info/entry_points.txt,sha256=ZwZLolnWog2fjdDrfaHNHob8SE_YtMbD6ayzsOzItxs,1234
+SinaTools-0.1.21.dist-info/top_level.txt,sha256=8tNdPTeJKw3TQCaua8IJIx6N6WpgZZmVekf1OdBNJpE,10
+SinaTools-0.1.21.dist-info/RECORD,,

sinatools/VERSION

@@ -1 +1 @@
-0.1.20
+0.1.21

sinatools/wsd/__init__.py

@@ -3,9 +3,9 @@ import pickle
 from sinatools.DataDownload import downloader
 import os 
 
-settings.glosses_dic = {}
+glosses_dic = {}
 filename = 'glosses_dic.pickle'
 path =downloader.get_appdatadir()
 file_path = os.path.join(path, filename)
 with open(file_path, 'rb') as f:
-    settings.glosses_dic = pickle.load(f)
+    glosses_dic = pickle.load(f)

sinatools/wsd/disambiguator.py

@@ -7,6 +7,7 @@ from sinatools.utils.tokenizers_words import simple_word_tokenize
 from sinatools.morphology.ALMA_multi_word import ALMA_multi_word
 from sinatools.morphology.morph_analyzer import analyze
 from sinatools.ner.entity_extractor import extract
+from . import glosses_dic
 
 
 def distill_entities(entities):
@@ -135,12 +136,9 @@ def find_two_word_lemma(input_sentence):
             glosses_list = []   
             concept_count = 0
             ids = data[0]["ids"]
-            for lemma_id in ids:
-               if lemma_id in settings.glosses_dic.keys(): 
-                  value = settings.glosses_dic[lemma_id]
-                  glosses_list.append(json.loads(value[1]))
-                  concept_count = concept_count + value[0]
-            
+            for concepts in ids:
+               glosses_list.append(json.loads(concepts))
+            concept_count = concept_count + data[0]["POS"]
             found_2Word_lemma = [two_grams, glosses_list, i, i + 1, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
             output.append(found_2Word_lemma) 
             i = i + 1    
@@ -160,12 +158,9 @@ def find_three_word_lemma(input_sentence):
            glosses_list = []   
            concept_count = 0
            ids = data[0]["ids"]
-           for lemma_id in ids:
-              if lemma_id in settings.glosses_dic.keys(): 
-                 value = settings.glosses_dic[lemma_id]
-                 glosses_list.append(json.loads(value[1]))
-                 concept_count = concept_count + value[0]
-                 
+           for concepts in ids:
+              glosses_list.append(json.loads(concepts))
+           concept_count = concept_count + data[0]["POS"]
            found_3Word_lemma = [three_grams, glosses_list, i, i + 2, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
            output.append(found_3Word_lemma) 
            i = i + 1    
@@ -184,11 +179,9 @@ def find_four_word_lemma(input_sentence):
          glosses_list = []   
          concept_count = 0
          ids = data[0]["ids"]
-         for lemma_id in ids:
-            if lemma_id in settings.glosses_dic.keys(): 
-               value = settings.glosses_dic[lemma_id]
-               glosses_list.append(json.loads(value[1]))
-               concept_count = concept_count + value[0]
+         for concepts in ids:
+            glosses_list.append(json.loads(concepts))
+         concept_count = concept_count + data[0]["POS"] 
          found_4Word_lemma = [four_grams, glosses_list, i, i + 3, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
          output.append(found_4Word_lemma) 
          i = i + 1    
@@ -208,11 +201,9 @@ def find_five_word_lemma(input_sentence):
          glosses_list = []   
          concept_count = 0
          ids = data[0]["ids"]
-         for lemma_id in ids:
-            if lemma_id in settings.glosses_dic.keys(): 
-               value = settings.glosses_dic[lemma_id]
-               glosses_list.append(json.loads(value[1]))
-               concept_count = concept_count + value[0]
+         for concepts in ids:
+            glosses_list.append(json.loads(concepts))
+         concept_count = concept_count + data[0]["POS"]
          found_5Word_lemma = [five_grams, glosses_list, i, i + 4, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
          output.append(found_5Word_lemma) 
          i = i + 1    
@@ -276,16 +267,18 @@ def find_glosses_using_ALMA(word):
    pos = data[0]["pos"]
    Undiac_lemma = arStrip(Diac_lemma, True, True, True, True, True, False) # Remove diacs , smallDiacs , shaddah ,  digit , alif , specialChars
    ids = [] 
-   glosses_list = []   
+#    glosses_list = []   
    concept_count = 0
    lemma_id = data[0]["lemma_id"]
-   if lemma_id in settings.glosses_dic.keys(): 
-      value = settings.glosses_dic[lemma_id]
-      glosses_list.append(json.loads(value[1]))
-      concept_count = concept_count + value[0]
 
-   return word, Undiac_lemma, Diac_lemma, pos , concept_count, glosses
+   if lemma_id in glosses_dic.keys(): 
+      value = glosses_dic[lemma_id]
+      glosses= json.loads(value[1])
+    #   glosses_list.append(json.loads(value[1]))
+      concept_count = concept_count + value[0]
    
+   return word, Undiac_lemma, Diac_lemma, pos , concept_count, glosses
+
 def disambiguate_glosses_using_SALMA(glosses, Diac_lemma, Undiac_lemma, word, sentence):
    word = normalizearabert(word)
    glosses_dictionary = {}
@@ -309,7 +302,7 @@ def disambiguate_glosses_using_SALMA(glosses, Diac_lemma, Undiac_lemma, word, se
       return my_json
 
 
-def find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lemma, five_word_lemma, ner):
+def find_glosses(input_sentence, two_word_lemma, three_word_lemma,four_word_lemma, five_word_lemma, ner):
       output_list = []
       position = 0
       while position < len(input_sentence):    
@@ -376,7 +369,7 @@ def find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lem
             position = position + 1                 
                
 
-
+         
          output_from_ner = delete_form_list(position, ner)
          ner = output_from_ner[0] 
          if output_from_ner[1] != []:  
@@ -385,11 +378,13 @@ def find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lem
             my_json = {}    
             word = output_from_ner[1][0][0]
             my_json['word'] = word
-            my_json['concept_count'] = output_from_ner[1][0][2]
+            # my_json['concept_count'] = output_from_ner[1][0][2]
+            my_json['concept_count'] = '*'
             my_json['glosses'] = output_from_ner[1][0][1]
             my_json['Diac_lemma'] = output_from_ner[1][0][4]
             my_json['Undiac_lemma'] = output_from_ner[1][0][3] 
             output_list.append(my_json)
+            # print("output list: ", output_list)
             position = position + 1                             
          
          if flag == "False": # Not found in ner or in multi_word_dictionary, ASK ALMA 
@@ -417,6 +412,15 @@ def disambiguate_glosses_main(word, sentence):
       my_json = {}    
       my_json['word'] = word['word']
       glosses = word['glosses'][0]
+    #   my_json['Gloss'] = glosses['gloss']
+      my_json['Concept_id'] = glosses['concept_id']
+      my_json['Diac_lemma'] = word['Diac_lemma']
+      my_json['Undiac_lemma'] = word['Undiac_lemma']
+      return my_json
+   elif concept_count == '*':
+      my_json = {}    
+      my_json['word'] = word['word']
+      glosses = word['glosses'][0]
       my_json['Gloss'] = glosses['gloss']
       my_json['Concept_id'] = glosses['concept_id']
       my_json['Diac_lemma'] = word['Diac_lemma']
@@ -444,8 +448,7 @@ def WSD(sentence):
    
    ner = find_named_entities(" ".join(input_sentence))
 
-   output_list = find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lemma, five_word_lemma, ner)
-   
+   output_list = find_glosses(input_sentence, two_word_lemma, three_word_lemma, four_word_lemma, five_word_lemma, ner)
    results = []
    for word in output_list:
       results.append(disambiguate_glosses_main(word, sentence))

sinatools/wsd/settings.py

@@ -9,7 +9,6 @@ import pandas as pd
 from sinatools.DataDownload import downloader
 import os 
 
-glosses_dic = {}
 
 model_file_name = "bert-base-arabertv02_22_May_2021_00h_allglosses_unused01"
 path =downloader.get_appdatadir()
@@ -21,11 +20,6 @@ tokenizer_file_path = os.path.join(path, tokenizer_file_name)
 
 dftrue = pd.DataFrame()
 
-# model = BertForSequenceClassification.from_pretrained('{}'.format("bert-base-arabertv02_22_May_2021_00h_allglosses_unused01"),
-                                                    #   output_hidden_states = True,
-                                                    #   num_labels=2
-                                                    #  )
-
 model = BertForSequenceClassification.from_pretrained(model_file_path, output_hidden_states=True, num_labels=2)
 
 tokenizer = BertTokenizer.from_pretrained('{}'.format(tokenizer_file_path))