SinaTools 0.1.20__tar.gz → 0.1.22__tar.gz

{SinaTools-0.1.20 → SinaTools-0.1.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.20
+Version: 0.1.22
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.20 → SinaTools-0.1.22}/SinaTools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.20
+Version: 0.1.22
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.20 → SinaTools-0.1.22}/sinatools/CLI/DataDownload/download_files.py

@@ -2,7 +2,7 @@
 About:
 ------
 
-The download_files tool is a command-line interface for downloading various NLP resources from pre-specified URLs. It is a part of the sinatools package and provides options to choose which files to download and to specify a download directory. The tool automatically handles file extraction for zip and tar.gz files.
+The download_files is a command-line interface for downloading various NLP resources from pre-specified URLs. It is a part of the sinatools package and provides options to choose which files to download and to specify a download directory. The tool automatically handles file extraction for zip and tar.gz files.
 
 Usage:
 ------
@@ -18,7 +18,7 @@ Below is the usage information that can be generated by running download_files -
 
         Options:
         -f, --files FILES
-            Names of the files to download. Available files are: ner, morph, salma_model, salma_tokenizer, glosses_dic, lemma_dic, five_grams, four_grams, three_grams, two_grams. 
+            Names of the files to download. Available files are: ner, morph, wsd_model, wsd_tokenizer, glosses_dic, five_grams, four_grams, three_grams, two_grams, synonyms_level2, synonyms_level3. 
             If no file is specified, all files will be downloaded.
 
 Examples:

{SinaTools-0.1.20 → SinaTools-0.1.22}/sinatools/DataDownload/downloader.py

@@ -6,16 +6,17 @@ import zipfile
 from tqdm import tqdm
 import tarfile
 urls = {
-    'morph': 'https://portal.sina.birzeit.edu/ALMA27012000.pickle',
+    'morph': 'https://portal.sina.birzeit.edu/lemmas_dic.pickle',
     'ner': 'https://portal.sina.birzeit.edu/Wj27012000.tar.gz',
-    'salma_model': 'https://portal.sina.birzeit.edu/bert-base-arabertv02_22_May_2021_00h_allglosses_unused01.zip',
-    'salma_tokenizer': 'https://portal.sina.birzeit.edu/bert-base-arabertv02.zip',
+    'wsd_model': 'https://portal.sina.birzeit.edu/bert-base-arabertv02_22_May_2021_00h_allglosses_unused01.zip',
+    'wsd_tokenizer': 'https://portal.sina.birzeit.edu/bert-base-arabertv02.zip',
     'glosses_dic': 'https://portal.sina.birzeit.edu/glosses_dic.pickle',
-    'lemma_dic': 'https://portal.sina.birzeit.edu/lemmas_dic.pickle',
     'five_grams': 'https://portal.sina.birzeit.edu/five_grams.pickle',
     'four_grams':'https://portal.sina.birzeit.edu/four_grams.pickle',
     'three_grams':'https://portal.sina.birzeit.edu/three_grams.pickle',
-    'two_grams':'https://portal.sina.birzeit.edu/two_grams.pickle'
+    'two_grams':'https://portal.sina.birzeit.edu/two_grams.pickle',
+    'synonyms_level2':'https://portal.sina.birzeit.edu/synonyms_level2.pkl',
+    'synonyms_level3':'https://portal.sina.birzeit.edu/synonyms_level3.pkl'
 }
 
 def get_appdatadir():
@@ -94,41 +95,37 @@ def download_file(url, dest_path=get_appdatadir()):
     print(filename)
     headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
 
-    try:
-        with requests.get(url, headers=headers, stream=True) as r:
-            r.raise_for_status()
-            with open(file_path, 'wb') as f:
-                total_size = int(r.headers.get('content-length', 0))
-                block_size = 8192
-                progress_bar = tqdm(total=total_size, unit='iB', unit_scale=True)
-                for chunk in r.iter_content(chunk_size=block_size):
-                    if chunk:
-                        f.write(chunk)
-                        progress_bar.update(len(chunk))
-                progress_bar.close()
-
-        # Check the file type and extract accordingly
-        file_extension = os.path.splitext(file_path)[1]
-        extracted_folder_name = os.path.splitext(file_path)[0]
-        
-        if file_extension == '.zip':
-            extract_zip(file_path, extracted_folder_name)
-        elif file_extension == '.gz':
-
-            extract_tar(file_path, extracted_folder_name)
-        elif file_extension =='.pickle':
-            print(f'Done: {file_extension}')
-
-        else:
-            print(f'Unsupported file type for extraction: {file_extension}')
-
-        return file_path
-
-    except requests.exceptions.HTTPError as e:
-        if e.response.status_code == 403:
-            print(f'Error 403: Forbidden. The requested file URL {url} could not be downloaded due to insufficient permissions. Please check the URL and try again.')
-        else:
-            print('An error occurred while downloading the file:', e)
+    # try:
+    with requests.get(url, headers=headers, stream=True) as r:
+        r.raise_for_status()
+        with open(file_path, 'wb') as f:
+            total_size = int(r.headers.get('content-length', 0))
+            block_size = 8192
+            progress_bar = tqdm(total=total_size, unit='iB', unit_scale=True)
+            for chunk in r.iter_content(chunk_size=block_size):
+                if chunk:
+                    f.write(chunk)
+                    progress_bar.update(len(chunk))
+            progress_bar.close()
+    # Check the file type and extract accordingly
+    file_extension = os.path.splitext(file_path)[1]
+    extracted_folder_name = os.path.splitext(file_path)[0]
+    
+    if file_extension == '.zip':
+        extract_zip(file_path, extracted_folder_name)
+    elif file_extension == '.gz':
+        extract_tar(file_path, extracted_folder_name)
+    elif file_extension =='.pickle':
+        print(f'Done: {file_extension}')
+    else:
+        print(f'Unsupported file type for extraction: {file_extension}')
+    return file_path
+
+    # except requests.exceptions.HTTPError as e:
+        # if e.response.status_code == 403:
+            # print(f'Error 403: Forbidden. The requested file URL {url} could not be downloaded due to insufficient permissions. Please check the URL and try again.')
+        # else:
+            # print('An error occurred while downloading the file:', e)
 
 def extract_zip(file_path, extracted_folder_name):
     """

SinaTools-0.1.22/sinatools/VERSION

@@ -0,0 +1 @@
+0.1.22

{SinaTools-0.1.20 → SinaTools-0.1.22}/sinatools/wsd/__init__.py

@@ -3,9 +3,9 @@ import pickle
 from sinatools.DataDownload import downloader
 import os 
 
-settings.glosses_dic = {}
+glosses_dic = {}
 filename = 'glosses_dic.pickle'
 path =downloader.get_appdatadir()
 file_path = os.path.join(path, filename)
 with open(file_path, 'rb') as f:
-    settings.glosses_dic = pickle.load(f)
+    glosses_dic = pickle.load(f)

{SinaTools-0.1.20 → SinaTools-0.1.22}/sinatools/wsd/disambiguator.py

@@ -7,6 +7,7 @@ from sinatools.utils.tokenizers_words import simple_word_tokenize
 from sinatools.morphology.ALMA_multi_word import ALMA_multi_word
 from sinatools.morphology.morph_analyzer import analyze
 from sinatools.ner.entity_extractor import extract
+from . import glosses_dic
 
 
 def distill_entities(entities):
@@ -135,12 +136,9 @@ def find_two_word_lemma(input_sentence):
             glosses_list = []   
             concept_count = 0
             ids = data[0]["ids"]
-            for lemma_id in ids:
-               if lemma_id in settings.glosses_dic.keys(): 
-                  value = settings.glosses_dic[lemma_id]
-                  glosses_list.append(json.loads(value[1]))
-                  concept_count = concept_count + value[0]
-            
+            for concepts in ids:
+               glosses_list.append(json.loads(concepts))
+            concept_count = concept_count + data[0]["POS"]
             found_2Word_lemma = [two_grams, glosses_list, i, i + 1, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
             output.append(found_2Word_lemma) 
             i = i + 1    
@@ -160,12 +158,9 @@ def find_three_word_lemma(input_sentence):
            glosses_list = []   
            concept_count = 0
            ids = data[0]["ids"]
-           for lemma_id in ids:
-              if lemma_id in settings.glosses_dic.keys(): 
-                 value = settings.glosses_dic[lemma_id]
-                 glosses_list.append(json.loads(value[1]))
-                 concept_count = concept_count + value[0]
-                 
+           for concepts in ids:
+              glosses_list.append(json.loads(concepts))
+           concept_count = concept_count + data[0]["POS"]
            found_3Word_lemma = [three_grams, glosses_list, i, i + 2, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
            output.append(found_3Word_lemma) 
            i = i + 1    
@@ -184,11 +179,9 @@ def find_four_word_lemma(input_sentence):
          glosses_list = []   
          concept_count = 0
          ids = data[0]["ids"]
-         for lemma_id in ids:
-            if lemma_id in settings.glosses_dic.keys(): 
-               value = settings.glosses_dic[lemma_id]
-               glosses_list.append(json.loads(value[1]))
-               concept_count = concept_count + value[0]
+         for concepts in ids:
+            glosses_list.append(json.loads(concepts))
+         concept_count = concept_count + data[0]["POS"] 
          found_4Word_lemma = [four_grams, glosses_list, i, i + 3, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
          output.append(found_4Word_lemma) 
          i = i + 1    
@@ -208,11 +201,9 @@ def find_five_word_lemma(input_sentence):
          glosses_list = []   
          concept_count = 0
          ids = data[0]["ids"]
-         for lemma_id in ids:
-            if lemma_id in settings.glosses_dic.keys(): 
-               value = settings.glosses_dic[lemma_id]
-               glosses_list.append(json.loads(value[1]))
-               concept_count = concept_count + value[0]
+         for concepts in ids:
+            glosses_list.append(json.loads(concepts))
+         concept_count = concept_count + data[0]["POS"]
          found_5Word_lemma = [five_grams, glosses_list, i, i + 4, concept_count, data[0]['undiac_multi_word_lemma'], data[0]['multi_word_lemma']]
          output.append(found_5Word_lemma) 
          i = i + 1    
@@ -276,16 +267,18 @@ def find_glosses_using_ALMA(word):
    pos = data[0]["pos"]
    Undiac_lemma = arStrip(Diac_lemma, True, True, True, True, True, False) # Remove diacs , smallDiacs , shaddah ,  digit , alif , specialChars
    ids = [] 
-   glosses_list = []   
+#    glosses_list = []   
    concept_count = 0
    lemma_id = data[0]["lemma_id"]
-   if lemma_id in settings.glosses_dic.keys(): 
-      value = settings.glosses_dic[lemma_id]
-      glosses_list.append(json.loads(value[1]))
-      concept_count = concept_count + value[0]
 
-   return word, Undiac_lemma, Diac_lemma, pos , concept_count, glosses
+   if lemma_id in glosses_dic.keys(): 
+      value = glosses_dic[lemma_id]
+      glosses= json.loads(value[1])
+    #   glosses_list.append(json.loads(value[1]))
+      concept_count = concept_count + value[0]
    
+   return word, Undiac_lemma, Diac_lemma, pos , concept_count, glosses
+
 def disambiguate_glosses_using_SALMA(glosses, Diac_lemma, Undiac_lemma, word, sentence):
    word = normalizearabert(word)
    glosses_dictionary = {}
@@ -309,7 +302,7 @@ def disambiguate_glosses_using_SALMA(glosses, Diac_lemma, Undiac_lemma, word, se
       return my_json
 
 
-def find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lemma, five_word_lemma, ner):
+def find_glosses(input_sentence, two_word_lemma, three_word_lemma,four_word_lemma, five_word_lemma, ner):
       output_list = []
       position = 0
       while position < len(input_sentence):    
@@ -376,7 +369,7 @@ def find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lem
             position = position + 1                 
                
 
-
+         
          output_from_ner = delete_form_list(position, ner)
          ner = output_from_ner[0] 
          if output_from_ner[1] != []:  
@@ -385,11 +378,13 @@ def find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lem
             my_json = {}    
             word = output_from_ner[1][0][0]
             my_json['word'] = word
-            my_json['concept_count'] = output_from_ner[1][0][2]
+            # my_json['concept_count'] = output_from_ner[1][0][2]
+            my_json['concept_count'] = '*'
             my_json['glosses'] = output_from_ner[1][0][1]
             my_json['Diac_lemma'] = output_from_ner[1][0][4]
             my_json['Undiac_lemma'] = output_from_ner[1][0][3] 
             output_list.append(my_json)
+            # print("output list: ", output_list)
             position = position + 1                             
          
          if flag == "False": # Not found in ner or in multi_word_dictionary, ASK ALMA 
@@ -417,6 +412,15 @@ def disambiguate_glosses_main(word, sentence):
       my_json = {}    
       my_json['word'] = word['word']
       glosses = word['glosses'][0]
+    #   my_json['Gloss'] = glosses['gloss']
+      my_json['Concept_id'] = glosses['concept_id']
+      my_json['Diac_lemma'] = word['Diac_lemma']
+      my_json['Undiac_lemma'] = word['Undiac_lemma']
+      return my_json
+   elif concept_count == '*':
+      my_json = {}    
+      my_json['word'] = word['word']
+      glosses = word['glosses'][0]
       my_json['Gloss'] = glosses['gloss']
       my_json['Concept_id'] = glosses['concept_id']
       my_json['Diac_lemma'] = word['Diac_lemma']
@@ -444,8 +448,7 @@ def WSD(sentence):
    
    ner = find_named_entities(" ".join(input_sentence))
 
-   output_list = find_glosses(input_sentence, three_word_lemma, two_word_lemma, four_word_lemma, five_word_lemma, ner)
-   
+   output_list = find_glosses(input_sentence, two_word_lemma, three_word_lemma, four_word_lemma, five_word_lemma, ner)
    results = []
    for word in output_list:
       results.append(disambiguate_glosses_main(word, sentence))

{SinaTools-0.1.20 → SinaTools-0.1.22}/sinatools/wsd/settings.py

@@ -9,7 +9,6 @@ import pandas as pd
 from sinatools.DataDownload import downloader
 import os 
 
-glosses_dic = {}
 
 model_file_name = "bert-base-arabertv02_22_May_2021_00h_allglosses_unused01"
 path =downloader.get_appdatadir()
@@ -21,11 +20,6 @@ tokenizer_file_path = os.path.join(path, tokenizer_file_name)
 
 dftrue = pd.DataFrame()
 
-# model = BertForSequenceClassification.from_pretrained('{}'.format("bert-base-arabertv02_22_May_2021_00h_allglosses_unused01"),
-                                                    #   output_hidden_states = True,
-                                                    #   num_labels=2
-                                                    #  )
-
 model = BertForSequenceClassification.from_pretrained(model_file_path, output_hidden_states=True, num_labels=2)
 
 tokenizer = BertTokenizer.from_pretrained('{}'.format(tokenizer_file_path))

SinaTools-0.1.20/sinatools/VERSION

@@ -1 +0,0 @@
-0.1.20