SinaTools 0.1.1__py2.py3-none-any.whl

nlptools/CLI/utils/latin_remove.py

@@ -0,0 +1,51 @@
+"""
+About:
+------
+The sina_remove_latin tool performs delete latin characters from the input text.
+
+Usage:
+------
+Below is the usage information that can be generated by running sina_remove_latin --help.
+
+.. code-block:: none
+
+    Usage:
+        sina_remove_latin --text=TEXT
+        sina_remove_latin --file "path/to/your/file.txt"
+
+Examples:
+---------
+
+.. code-block:: none
+
+    sina_remove_punctuation --text "123test"
+    
+    sina_remove_punctuation --file "path/to/your/file.txt"
+
+Note:
+-----
+
+.. code-block:: none
+
+    - This tool is specific to Arabic text, as it focuses on Arabic linguistic elements.
+    - Ensure that the text input is appropriately encoded in UTF-8 or compatible formats.
+    - This tool for latin characters, if the input text is an Arabic characters or numbers the output will be the same input 
+
+"""
+
+import argparse
+from nlptools.utils.parser import remove_latin
+
+
+def main():
+    parser = argparse.ArgumentParser(description='remove latin characters from the text')
+
+    parser.add_argument('--text', type=str, required=True, help='The input text')
+    args = parser.parse_args()
+    result = remove_latin(args.text)
+ 
+    print(result)
+    if __name__ == '__main__':
+        main()
+
+#sina_remove_latin --text "123test"

nlptools/CLI/utils/remove_Punc.py

@@ -0,0 +1,53 @@
+"""
+About:
+------
+The sina_remove_punctuation tool performs delete punctuation marks from the input text.
+
+Usage:
+------
+Below is the usage information that can be generated by running sina_remove_punctuation --help.
+
+.. code-block:: none
+
+    Usage:
+        sina_remove_punctuation --text=TEXT
+        sina_remove_punctuation --file "path/to/your/file.txt"
+
+Examples:
+---------
+
+.. code-block:: none
+
+    sina_remove_punctuation --text "te%s@t...!!?"
+    
+    sina_remove_punctuation --file "path/to/your/file.txt"
+
+Note:
+-----
+
+.. code-block:: none
+
+    - This tool is specific to Arabic text, as it focuses on Arabic linguistic elements.
+    - Ensure that the text input is appropriately encoded in UTF-8 or compatible formats.
+"""
+
+import argparse
+from nlptools.utils.parser import remove_punctuation
+#from nlptools.utils.parser import read_file
+#from nlptools.utils.parser import write_file
+
+
+def main():
+    parser = argparse.ArgumentParser(description='remove punctuation marks from the text')
+
+    parser.add_argument('--text',required=True,help="input text")
+   # parser.add_argument('myFile', type=argparse.FileType('r'),help='Input file csv')
+    args = parser.parse_args()
+    result = remove_punctuation(args.text)
+ 
+    print(result)
+    if __name__ == '__main__':
+        main()
+
+#sina_remove_punctuation --text "your text"
+

nlptools/CLI/utils/sentence_tokenizer.py

@@ -0,0 +1,90 @@
+"""
+About:
+------
+
+The sina_sentence_tokenize tool allows you to tokenize text into sentences using the SinaTools utility. It provides 
+flexibility in tokenizing at different punctuation marks, including dots, question marks, and exclamation marks. It also 
+allows tokenization at new lines.
+
+Usage:
+------
+Below is the usage information that can be generated by running sina_sentence_tokenize --help.
+
+.. code-block:: none
+
+    Usage:
+        sina_sentence_tokenize --text=TEXT [options]
+
+        sina_sentence_tokenize --file=FILE [options]
+        
+.. code-block:: none
+      
+    Options:
+      --text TEXT
+            Text to be tokenized into sentences.
+      --file FILE
+            File containing the text to be tokenized into sentences
+      --dot
+            Tokenize at dots.
+      --new_line
+            Tokenize at new lines.
+      --question_mark
+            Tokenize at question marks.
+      --exclamation_mark
+            Tokenize at exclamation marks.
+
+Examples:
+---------
+
+.. code-block:: none
+
+  sina_sentence_tokenize --text "Your text here. Does it work? Yes! Try with new lines." --dot --question_mark --exclamation_mark
+
+  sina_sentence_tokenize --file "path/to/your/file.txt" --dot --question_mark --exclamation_mark
+
+Note:
+-----
+
+.. code-block:: none
+
+    - The tokenization options allow for a customized experience. You can choose any combination of the options, or even none 
+    - of them, to achieve the desired sentence tokenization behavior. If no tokenization options are provided, the tool will 
+    - use default settings as implemented in the underlying `sent_tokenize` function of SinaTools.
+
+"""
+import argparse
+from nlptools.utils.sentence_tokenizer import sent_tokenize
+from nlptools.utils.readfile import read_file
+
+def main():
+    parser = argparse.ArgumentParser(description='Sentence Tokenization using SinaTools')
+    
+    # Adding arguments for the text, file, and tokenization options
+    parser.add_argument('--text', type=str, help='Text to be tokenized into sentences')
+    parser.add_argument('--file', type=str, help='File containing the text to be tokenized into sentences')
+    parser.add_argument('--dot', action='store_true', help='Tokenize at dots')
+    parser.add_argument('--new_line', action='store_true', help='Tokenize at new lines')
+    parser.add_argument('--question_mark', action='store_true', help='Tokenize at question marks')
+    parser.add_argument('--exclamation_mark', action='store_true', help='Tokenize at exclamation marks')
+
+    args = parser.parse_args()
+
+    # Check if either text or file is provided
+    if args.text is None and args.file is None:
+        print("Either --text or --file argument must be provided.")
+        return
+
+    text_content = args.text if args.text else read_file(args.file)
+
+    # Perform sentence tokenization
+    sentences = sent_tokenize(" ".join(text_content), dot=args.dot, new_line=args.new_line, 
+                              question_mark=args.question_mark, exclamation_mark=args.exclamation_mark)
+    
+    # Print each sentence in a new line
+    for sentence in sentences:
+        print(sentence)
+
+if __name__ == '__main__':
+    main()
+#sina_sentence_tokenize --text "Your text here. Does it work? Yes! Try with new lines." --dot --question_mark --exclamation_mark
+#sina_sentence_tokenize --file "path/to/your/file.txt" --dot --question_mark --exclamation_mark

nlptools/CLI/utils/text_transliteration.py

@@ -0,0 +1,77 @@
+"""
+About:
+------
+
+The sina_transliterate tool allows you to transliterate text using the SinaTools' utility. This command-line utility
+takes in a text and a desired schema, and outputs the transliterated text.
+
+Usage:
+------
+Below is the usage information that can be generated by running sina_transliterate --help.
+
+    Usage:
+    ------
+
+.. code-block:: none
+
+        sina_transliterate --text=TEXT --schema=SCHEMA
+
+        sina_transliterate --file=FILE --schema=SCHEMA
+
+Options:
+--------
+
+.. code-block:: none
+
+  --text TEXT
+        Text to be transliterated.
+  --schema SCHEMA
+        Transliteration schema to be used, which is bw2ar or ar2bw.
+
+Examples:
+---------
+
+.. code-block:: none
+
+    sina_transliterate --text "klmp" --schema "bw2ar"
+    sina_transliterate --file "path/to/your/file.txt" --schema "ar2bw"
+
+Note:
+-----
+
+.. code-block:: none
+
+    For available transliteration schemas and more details, please refer to the SinaTools' documentation or the source code 
+    of the function `perform_transliteration`.
+
+"""
+import argparse
+from nlptools.utils.text_transliteration import perform_transliteration
+from nlptools.utils.readfile import read_file
+
+def main():
+    parser = argparse.ArgumentParser(description='Perform text transliteration using SinaTools')
+    
+    # Adding arguments for the text, file, and schema
+    parser.add_argument('--text', type=str, help='Text to be transliterated')
+    parser.add_argument('--file', type=str, help='File containing the text to be transliterated')
+    parser.add_argument('--schema', type=str, required=True, help='Transliteration schema to be used')
+
+    args = parser.parse_args()
+
+    # Check if either text or file is provided
+    if args.text is None and args.file is None:
+        print("Either --text or --file argument must be provided.")
+        return
+
+    text_content = args.text if args.text else " ".join(read_file(args.file))
+    # Perform transliteration
+    result = perform_transliteration(text_content, args.schema)
+    
+    print(result)
+
+if __name__ == '__main__':
+    main()
+
+#sina_transliterate --text "example text" --schema "bw2ar"
+#sina_transliterate --file "path/to/your/file.txt" --schema "bw2ar"

nlptools/DataDownload/downloader.py

@@ -0,0 +1,185 @@
+import os
+import sys
+from pathlib import Path
+import requests  
+import zipfile
+from tqdm import tqdm
+import tarfile
+urls = {
+    'morph': 'https://portal.sina.birzeit.edu/ALMA27012000.pickle',
+    'ner': 'https://portal.sina.birzeit.edu/Wj27012000.tar.gz',
+    'salma_model': 'https://portal.sina.birzeit.edu/bert-base-arabertv02_22_May_2021_00h_allglosses_unused01.zip',
+    'salma_tokenizer': 'https://portal.sina.birzeit.edu/bert-base-arabertv02.zip',
+    'glosses_dic': 'https://portal.sina.birzeit.edu/glosses_dic.pickle',
+    'lemma_dic': 'https://portal.sina.birzeit.edu/lemmas_dic.pickle',
+    'five_grams': 'https://portal.sina.birzeit.edu/five_grams.pickle',
+    'four_grams':'https://portal.sina.birzeit.edu/four_grams.pickle',
+    'three_grams':'https://portal.sina.birzeit.edu/three_grams.pickle',
+    'two_grams':'https://portal.sina.birzeit.edu/two_grams.pickle'
+}
+
+def get_appdatadir():
+    """
+    This method checks if the directory exists and creates if it doesn't. And returns the path to the directory where the application data is stored.
+   
+    Returns:
+    --------
+    Path: A pathlib.Path object representing the path to the application data directory.
+
+    Raises:
+    -------
+    None.
+
+    **Example:**
+
+    .. highlight:: python
+    .. code-block:: python
+
+        from nlptools.DataDownload import downloader
+
+        path = downloader.get_appdatadir()
+
+        Windows: 'C:/Users/<Username>/AppData/Roaming/nlptools'
+        MacOS: '/Users/<Username>/Library/Application Support/nlptools'
+        Linux: '/home/<Username>/.nlptools'
+        Google Colab: '/content/nlptools'
+
+    """
+    home = str(Path.home())
+    if 'google.colab' in sys.modules:
+        path = Path('/content/nlptools')
+    elif sys.platform == 'win32':
+        path = Path(home, 'AppData/Roaming/nlptools')
+    elif sys.platform == 'darwin':
+        path = Path(home, 'Library/Application Support/nlptools')
+    else:
+        path = Path(home, '.nlptools')
+
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+    return path
+
+def download_file(url='https://portal.sina.birzeit.edu/Wj27012000.tar.gz', dest_path=get_appdatadir()):
+    """
+    Downloads a file from the specified URL and saves it to the specified destination path.
+
+    Args:
+        url (:obj:`str`): The URL of the file to be downloaded.
+        dest_path (:obj:`str`): The destination path to save the downloaded file to. Defaults
+            to the user's application data directory.
+
+
+    Returns:
+        :obj:`str`: The absolute path of the downloaded file.
+
+    Raises:
+        requests.exceptions.HTTPError: If there was an HTTP error during the request.
+
+    Note:
+        This method uses the `requests` and `tqdm` libraries. It also checks if the
+        compressed downloaded file type and extracts it.
+
+    **Example:**
+
+    .. highlight:: python
+    .. code-block:: python
+
+          download_file(url='https://example.com/data.zip', dest_path='data/')
+
+    """
+    filename = os.path.basename(url)
+    file_path = os.path.join(dest_path, filename)
+
+    print(filename)
+    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
+
+    try:
+        with requests.get(url, headers=headers, stream=True) as r:
+            r.raise_for_status()
+            with open(file_path, 'wb') as f:
+                total_size = int(r.headers.get('content-length', 0))
+                block_size = 8192
+                progress_bar = tqdm(total=total_size, unit='iB', unit_scale=True)
+                for chunk in r.iter_content(chunk_size=block_size):
+                    if chunk:
+                        f.write(chunk)
+                        progress_bar.update(len(chunk))
+                progress_bar.close()
+
+        # Check the file type and extract accordingly
+        file_extension = os.path.splitext(file_path)[1]
+        extracted_folder_name = os.path.splitext(file_path)[0]
+        
+        if file_extension == '.zip':
+            extract_zip(file_path, extracted_folder_name)
+        elif file_extension == '.gz':
+
+            extract_tar(file_path, extracted_folder_name)
+        elif file_extension =='.pickle':
+            print(f'Done: {file_extension}')
+
+        else:
+            print(f'Unsupported file type for extraction: {file_extension}')
+
+        return file_path
+
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code == 403:
+            print(f'Error 403: Forbidden. The requested file URL {url} could not be downloaded due to insufficient permissions. Please check the URL and try again.')
+        else:
+            print('An error occurred while downloading the file:', e)
+
+def extract_zip(file_path, extracted_folder_name):
+    """
+    Extracts the contents of a ZIP file to the specified folder.
+
+    Args:
+        file_path (str): The path to the ZIP file.
+        extracted_folder_name (str): The name of the folder where the contents will be extracted.
+
+    Returns:
+        None
+    """
+    with zipfile.ZipFile(file_path, 'r') as zip_file:
+        zip_file.extractall(extracted_folder_name)
+
+
+def extract_tar(file_path, dest_path):
+    """
+    Extracts the contents of a tar.gz file to the specified destination path.
+
+    Args:
+        file_path (str): The path to the tar.gz file.
+        dest_path (str): The destination path where the contents will be extracted.
+
+    Returns:
+        str: The path to the extracted folder if successful, or None if extraction failed.
+    """
+    try:
+        with tarfile.open(file_path, 'r:gz') as tar:
+            # Remove the extension from the file name
+            extracted_folder_name = os.path.splitext(os.path.basename(file_path))[0]
+            extracted_folder_path = os.path.join(dest_path, extracted_folder_name)
+
+            # Extract the contents to the destination path
+            tar.extractall(dest_path)
+        
+        # Remove the compressed file
+        os.remove(file_path)
+        
+        return extracted_folder_path
+
+    except tarfile.ReadError:
+        print(f'Failed to extract the file: {file_path}')
+        return None
+
+
+def download_files():
+    """
+    Downloads multiple files from a dictionary of URLs using the download_file() function.
+    Returns:
+        None
+    """
+    for url in urls.values():
+        download_file(url)

nlptools/VERSION

@@ -0,0 +1 @@
+0.1.1

nlptools/__init__.py

@@ -0,0 +1,5 @@
+"""Top-level package for nlptools."""
+
+__author__ = """SinaLab"""
+__email__ = 'sina.institute.bzu@gmail.com'
+__version__ = '0.8.5'

nlptools/arabert/__init__.py

@@ -0,0 +1 @@
+# coding=utf-8

nlptools/arabert/arabert/__init__.py

@@ -0,0 +1,14 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.