SinaTools 0.1.21__tar.gz → 0.1.22__tar.gz

{SinaTools-0.1.21 → SinaTools-0.1.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.21
+Version: 0.1.22
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.21 → SinaTools-0.1.22}/SinaTools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.21
+Version: 0.1.22
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.21 → SinaTools-0.1.22}/sinatools/CLI/DataDownload/download_files.py

@@ -2,7 +2,7 @@
 About:
 ------
 
-The download_files tool is a command-line interface for downloading various NLP resources from pre-specified URLs. It is a part of the sinatools package and provides options to choose which files to download and to specify a download directory. The tool automatically handles file extraction for zip and tar.gz files.
+The download_files is a command-line interface for downloading various NLP resources from pre-specified URLs. It is a part of the sinatools package and provides options to choose which files to download and to specify a download directory. The tool automatically handles file extraction for zip and tar.gz files.
 
 Usage:
 ------
@@ -18,7 +18,7 @@ Below is the usage information that can be generated by running download_files -
 
         Options:
         -f, --files FILES
-            Names of the files to download. Available files are: ner, morph, salma_model, salma_tokenizer, glosses_dic, lemma_dic, five_grams, four_grams, three_grams, two_grams. 
+            Names of the files to download. Available files are: ner, morph, wsd_model, wsd_tokenizer, glosses_dic, five_grams, four_grams, three_grams, two_grams, synonyms_level2, synonyms_level3. 
             If no file is specified, all files will be downloaded.
 
 Examples:

{SinaTools-0.1.21 → SinaTools-0.1.22}/sinatools/DataDownload/downloader.py

@@ -6,16 +6,17 @@ import zipfile
 from tqdm import tqdm
 import tarfile
 urls = {
-    'morph': 'https://portal.sina.birzeit.edu/ALMA27012000.pickle',
+    'morph': 'https://portal.sina.birzeit.edu/lemmas_dic.pickle',
     'ner': 'https://portal.sina.birzeit.edu/Wj27012000.tar.gz',
-    'salma_model': 'https://portal.sina.birzeit.edu/bert-base-arabertv02_22_May_2021_00h_allglosses_unused01.zip',
-    'salma_tokenizer': 'https://portal.sina.birzeit.edu/bert-base-arabertv02.zip',
+    'wsd_model': 'https://portal.sina.birzeit.edu/bert-base-arabertv02_22_May_2021_00h_allglosses_unused01.zip',
+    'wsd_tokenizer': 'https://portal.sina.birzeit.edu/bert-base-arabertv02.zip',
     'glosses_dic': 'https://portal.sina.birzeit.edu/glosses_dic.pickle',
-    'lemma_dic': 'https://portal.sina.birzeit.edu/lemmas_dic.pickle',
     'five_grams': 'https://portal.sina.birzeit.edu/five_grams.pickle',
     'four_grams':'https://portal.sina.birzeit.edu/four_grams.pickle',
     'three_grams':'https://portal.sina.birzeit.edu/three_grams.pickle',
-    'two_grams':'https://portal.sina.birzeit.edu/two_grams.pickle'
+    'two_grams':'https://portal.sina.birzeit.edu/two_grams.pickle',
+    'synonyms_level2':'https://portal.sina.birzeit.edu/synonyms_level2.pkl',
+    'synonyms_level3':'https://portal.sina.birzeit.edu/synonyms_level3.pkl'
 }
 
 def get_appdatadir():
@@ -94,41 +95,37 @@ def download_file(url, dest_path=get_appdatadir()):
     print(filename)
     headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
 
-    try:
-        with requests.get(url, headers=headers, stream=True) as r:
-            r.raise_for_status()
-            with open(file_path, 'wb') as f:
-                total_size = int(r.headers.get('content-length', 0))
-                block_size = 8192
-                progress_bar = tqdm(total=total_size, unit='iB', unit_scale=True)
-                for chunk in r.iter_content(chunk_size=block_size):
-                    if chunk:
-                        f.write(chunk)
-                        progress_bar.update(len(chunk))
-                progress_bar.close()
-
-        # Check the file type and extract accordingly
-        file_extension = os.path.splitext(file_path)[1]
-        extracted_folder_name = os.path.splitext(file_path)[0]
-        
-        if file_extension == '.zip':
-            extract_zip(file_path, extracted_folder_name)
-        elif file_extension == '.gz':
-
-            extract_tar(file_path, extracted_folder_name)
-        elif file_extension =='.pickle':
-            print(f'Done: {file_extension}')
-
-        else:
-            print(f'Unsupported file type for extraction: {file_extension}')
-
-        return file_path
-
-    except requests.exceptions.HTTPError as e:
-        if e.response.status_code == 403:
-            print(f'Error 403: Forbidden. The requested file URL {url} could not be downloaded due to insufficient permissions. Please check the URL and try again.')
-        else:
-            print('An error occurred while downloading the file:', e)
+    # try:
+    with requests.get(url, headers=headers, stream=True) as r:
+        r.raise_for_status()
+        with open(file_path, 'wb') as f:
+            total_size = int(r.headers.get('content-length', 0))
+            block_size = 8192
+            progress_bar = tqdm(total=total_size, unit='iB', unit_scale=True)
+            for chunk in r.iter_content(chunk_size=block_size):
+                if chunk:
+                    f.write(chunk)
+                    progress_bar.update(len(chunk))
+            progress_bar.close()
+    # Check the file type and extract accordingly
+    file_extension = os.path.splitext(file_path)[1]
+    extracted_folder_name = os.path.splitext(file_path)[0]
+    
+    if file_extension == '.zip':
+        extract_zip(file_path, extracted_folder_name)
+    elif file_extension == '.gz':
+        extract_tar(file_path, extracted_folder_name)
+    elif file_extension =='.pickle':
+        print(f'Done: {file_extension}')
+    else:
+        print(f'Unsupported file type for extraction: {file_extension}')
+    return file_path
+
+    # except requests.exceptions.HTTPError as e:
+        # if e.response.status_code == 403:
+            # print(f'Error 403: Forbidden. The requested file URL {url} could not be downloaded due to insufficient permissions. Please check the URL and try again.')
+        # else:
+            # print('An error occurred while downloading the file:', e)
 
 def extract_zip(file_path, extracted_folder_name):
     """

SinaTools-0.1.22/sinatools/VERSION

@@ -0,0 +1 @@
+0.1.22

SinaTools-0.1.21/sinatools/VERSION

@@ -1 +0,0 @@
-0.1.21