SinaTools 0.1.1__py2.py3-none-any.whl

nlptools/environment.yml

@@ -0,0 +1,227 @@
+name: arabicner
+channels:
+  - anaconda
+  - pytorch
+  - nvidia
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - abseil-cpp=20211102.0=h27087fc_1
+  - absl-py=1.3.0=pyhd8ed1ab_0
+  - aiohttp=3.8.1=py310h5764c6d_1
+  - aiosignal=1.2.0=pyhd8ed1ab_0
+  - arrow-cpp=8.0.0=py310h3098874_0
+  - async-timeout=4.0.2=py310h06a4308_0
+  - attrs=22.1.0=pyh71513ae_1
+  - aws-c-common=0.4.57=he6710b0_1
+  - aws-c-event-stream=0.1.6=h2531618_5
+  - aws-checksums=0.1.9=he6710b0_0
+  - aws-sdk-cpp=1.8.185=hce553d0_0
+  - blas=1.0=mkl
+  - blinker=1.5=pyhd8ed1ab_0
+  - boost-cpp=1.78.0=he72f1d9_0
+  - bottleneck=1.3.5=py310ha9d4c09_0
+  - brotli=1.0.9=h166bdaf_7
+  - brotli-bin=1.0.9=h166bdaf_7
+  - brotlipy=0.7.0=py310h7f8727e_1002
+  - bzip2=1.0.8=h7b6447c_0
+  - c-ares=1.18.1=h7f98852_0
+  - ca-certificates=2022.9.24=ha878542_0
+  - cachetools=5.2.0=pyhd8ed1ab_0
+  - certifi=2022.9.24=pyhd8ed1ab_0
+  - cffi=1.15.1=py310h74dc2b5_0
+  - charset-normalizer=2.0.4=pyhd3eb1b0_0
+  - click=8.1.3=unix_pyhd8ed1ab_2
+  - cryptography=38.0.1=py310h9ce1e76_0
+  - cuda=11.7.1=0
+  - cuda-cccl=11.7.91=0
+  - cuda-command-line-tools=11.7.1=0
+  - cuda-compiler=11.7.1=0
+  - cuda-cudart=11.7.99=0
+  - cuda-cudart-dev=11.7.99=0
+  - cuda-cuobjdump=11.7.91=0
+  - cuda-cupti=11.7.101=0
+  - cuda-cuxxfilt=11.7.91=0
+  - cuda-demo-suite=11.8.86=0
+  - cuda-documentation=11.8.86=0
+  - cuda-driver-dev=11.7.99=0
+  - cuda-gdb=11.8.86=0
+  - cuda-libraries=11.7.1=0
+  - cuda-libraries-dev=11.7.1=0
+  - cuda-memcheck=11.8.86=0
+  - cuda-nsight=11.8.86=0
+  - cuda-nsight-compute=11.8.0=0
+  - cuda-nvcc=11.7.99=0  
+  - cuda-nvdisasm=11.8.86=0
+  - cuda-nvml-dev=11.7.91=0
+  - cuda-nvprof=11.8.87=0
+  - cuda-nvprune=11.7.91=0
+  - cuda-nvrtc=11.7.99=0
+  - cuda-nvrtc-dev=11.7.99=0
+  - cuda-nvtx=11.7.91=0
+  - cuda-nvvp=11.8.87=0
+  - cuda-runtime=11.7.1=0
+  - cuda-sanitizer-api=11.8.86=0
+  - cuda-toolkit=11.7.1=0
+  - cuda-tools=11.7.1=0
+  - cuda-visual-tools=11.7.1=0
+  - dataclasses=0.8=pyhc8e2a94_3
+  - datasets=2.6.1=pyhd8ed1ab_0
+  - dill=0.3.5.1=pyhd8ed1ab_0
+  - ffmpeg=4.3=hf484d3e_0
+  - fftw=3.3.10=nompi_h77c792f_102
+  - filelock=3.8.0=pyhd8ed1ab_0
+  - freetype=2.12.1=h4a9f257_0
+  - frozenlist=1.2.0=py310h7f8727e_1
+  - fsspec=2022.10.0=pyhd8ed1ab_0
+  - gds-tools=1.4.0.31=0
+  - gflags=2.2.2=he1b5a44_1004
+  - giflib=5.2.1=h7b6447c_0
+  - glog=0.6.0=h6f12383_0
+  - gmp=6.2.1=h295c915_3
+  - gnutls=3.6.15=he1e5248_0
+  #- google-auth=2.14.0=pyh1a96a4e_0
+  #- google-auth-oauthlib=0.4.6=pyhd8ed1ab_0
+  - grpc-cpp=1.46.1=h33aed49_0
+  - grpcio=1.42.0=py310hce63b2e_0
+  - huggingface_hub=0.10.1=pyhd8ed1ab_0
+  - icu=70.1=h27087fc_0
+  - idna=3.4=py310h06a4308_0
+  - importlib-metadata=5.0.0=pyha770c72_1
+  - importlib_metadata=5.0.0=hd8ed1ab_1
+  - intel-openmp=2021.4.0=h06a4308_3561
+  - joblib=1.2.0=pyhd8ed1ab_0
+  - jpeg=9e=h7f8727e_0
+  - keyutils=1.6.1=h166bdaf_0
+  - krb5=1.19.3=h3790be6_0
+  - lame=3.100=h7b6447c_0
+  - lcms2=2.12=h3be6417_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - lerc=3.0=h295c915_0
+  - libbrotlicommon=1.0.9=h166bdaf_7
+  - libbrotlidec=1.0.9=h166bdaf_7
+  - libbrotlienc=1.0.9=h166bdaf_7
+  - libcublas=11.11.3.6=0
+  - libcublas-dev=11.11.3.6=0
+  - libcufft=10.9.0.58=0
+  - libcufft-dev=10.9.0.58=0
+  - libcufile=1.4.0.31=0
+  - libcufile-dev=1.4.0.31=0
+  - libcurand=10.3.0.86=0    
+  - libcurl=7.85.0=h91b91d3_0
+  - libcusolver=11.4.1.48=0
+  - libcusolver-dev=11.4.1.48=0
+  - libcusparse=11.7.5.86=0
+  - libcusparse-dev=11.7.5.86=0
+  - libdeflate=1.8=h7f8727e_5
+  - libedit=3.1.20191231=he28a2e2_2
+  - libev=4.33=h516909a_1
+  - libevent=2.1.10=h9b69904_4
+  - libffi=3.3=he6710b0_2
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgfortran-ng=12.2.0=h69a702a_19
+  - libgfortran5=12.2.0=h337968e_19
+  - libgomp=11.2.0=h1234567_1
+  - libiconv=1.16=h7f8727e_2
+  - libidn2=2.3.2=h7f8727e_0
+  - libnghttp2=1.46.0=hce63b2e_0
+  - libnpp=11.8.0.86=0
+  - libnpp-dev=11.8.0.86=0
+  - libnvjpeg=11.9.0.86=0
+  - libnvjpeg-dev=11.9.0.86=0
+  - libpng=1.6.37=hbc83047_0
+  - libprotobuf=3.20.1=h4ff587b_0
+  - libssh2=1.10.0=ha56f1ee_2
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - libtasn1=4.16.0=h27cfd23_0
+  - libthrift=0.15.0=he6d91bd_0
+  - libtiff=4.4.0=hecacb30_0  
+  - libunistring=0.9.10=h27cfd23_0
+  - libuuid=1.0.3=h7f8727e_2
+  - libwebp=1.2.4=h11a3e52_0
+  - libwebp-base=1.2.4=h5eee18b_0
+  - lz4-c=1.9.3=h295c915_1
+  - markdown=3.4.1=pyhd8ed1ab_0
+  - markupsafe=2.1.1=py310h5764c6d_1
+  - mkl=2021.4.0=h06a4308_640
+  - mkl-service=2.4.0=py310h7f8727e_0
+  - mkl_fft=1.3.1=py310hd6ae3a3_0
+  - mkl_random=1.2.2=py310h00e6091_0
+  - multidict=6.0.2=py310h5764c6d_1
+  - multiprocess=0.70.12.2=py310h5764c6d_2
+  - natsort=7.1.1=pyhd3eb1b0_0
+  - ncurses=6.3=h5eee18b_3
+  - nettle=3.7.3=hbbd107a_1
+  - nsight-compute=2022.3.0.22=0
+  - numexpr=2.8.3=py310hcea2de6_0
+  - numpy=1.23.3=py310hd5efca6_0
+  #- numpy-base=1.23.3=py310h8e6c178_0
+  - oauthlib=3.2.2=pyhd8ed1ab_0
+  - openh264=2.1.1=h4ff587b_0
+  - openssl=1.1.1s=h7f8727e_0
+  - orc=1.7.4=h07ed6aa_0
+  - packaging=21.3=pyhd8ed1ab_0
+  - pandas=1.4.4=py310h6a678d5_0
+  - pillow=9.2.0=py310hace64e9_1
+  - pip=22.2.2=py310h06a4308_0  
+  - protobuf=3.20.1=py310hd8f1fbe_0
+  - pyarrow=8.0.0=py310h468efa6_0
+  - pyasn1=0.4.8=py_0
+  - pyasn1-modules=0.2.7=py_0
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pyjwt=2.6.0=pyhd8ed1ab_0
+  - pyopenssl=22.0.0=pyhd3eb1b0_0
+  - pyparsing=3.0.9=pyhd8ed1ab_0
+  - pysocks=1.7.1=py310h06a4308_0
+  - python=3.10.6=haa1d7c7_1
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python-xxhash=3.0.0=py310h5764c6d_1
+  - python_abi=3.10=2_cp310
+  - pytorch=1.13.0=py3.10_cuda11.7_cudnn8.5.0_0
+  - pytorch-cuda=11.7=h67b0de4_0
+  - pytorch-mutex=1.0=cuda
+  - pytz=2022.6=pyhd8ed1ab_0
+  - pyu2f=0.1.5=pyhd8ed1ab_0
+  - pyyaml=6.0=py310h5764c6d_4
+  - re2=2022.04.01=h27087fc_0
+  - readline=8.2=h5eee18b_0
+  - regex=2022.7.9=py310h5eee18b_0
+  - requests=2.28.1=py310h06a4308_0
+  - requests-oauthlib=1.3.1=pyhd8ed1ab_0
+  - responses=0.18.0=pyhd8ed1ab_0
+  - rsa=4.9=pyhd8ed1ab_0
+  - sacremoses=0.0.53=pyhd8ed1ab_0
+  - scikit-learn=1.1.3=py310h6a678d5_0
+  - scipy=1.9.3=py310hd5efca6_0
+  - seqeval=1.2.2=pyhd3deb0d_0
+  - setuptools=65.4.0=py310h06a4308_0
+  - six=1.16.0=pyhd3eb1b0_1
+  - snappy=1.1.9=hbd366e4_1
+  - sqlite=3.39.3=h5082296_0
+  - tensorboard=2.10.1=pyhd8ed1ab_0
+  - tensorboard-data-server=0.6.0=py310h597c629_2
+  - tensorboard-plugin-wit=1.8.1=pyhd8ed1ab_0
+  - threadpoolctl=3.1.0=pyh8a188c0_0
+  - tk=8.6.12=h1ccaba5_0
+  - tokenizers=0.11.4=py310h3dcd8bd_1
+  - torchaudio=0.13.0=py310_cu117
+  - torchtext=0.14.0=py310
+  - torchvision=0.14.0=py310_cu117
+  - tqdm=4.64.1=py310h06a4308_0
+  - transformers=4.24.0=pyhd8ed1ab_0
+  - typing-extensions=4.3.0=py310h06a4308_0
+  - typing_extensions=4.3.0=py310h06a4308_0
+  - tzdata=2022e=h04d1e81_0
+  - urllib3=1.26.12=py310h06a4308_0
+  - utf8proc=2.6.1=h27cfd23_0
+  - werkzeug=2.2.2=pyhd8ed1ab_0
+  - wheel=0.37.1=pyhd3eb1b0_0
+  - xxhash=0.8.0=h7f98852_3
+  - xz=5.2.6=h5eee18b_0
+  - yaml=0.2.5=h7f98852_2
+  - yarl=1.7.2=py310h5764c6d_2    
+  - zipp=3.10.0=pyhd8ed1ab_0
+  - zlib=1.2.13=h5eee18b_0
+  - zstd=1.5.2=ha4553b6_0

nlptools/install_env.py

@@ -0,0 +1,13 @@
+import os
+import subprocess
+
+def main():
+    # Determine the path to the 'environment.yml' file within the package
+    package_dir = os.path.dirname(__file__)
+    env_file = os.path.join(package_dir, 'environment.yml')
+
+    # Create the conda environment using the 'environment.yml' file
+    subprocess.call(["conda", "env", "create", "-f", env_file])
+
+if __name__ == "__main__":
+    main()

nlptools/morphology/ALMA_multi_word.py

@@ -0,0 +1,34 @@
+from nlptools.morphology import settings 
+from nlptools.utils.parser import arStrip
+import json
+
+
+def ALMA_multi_word(multi_word):
+    undiac_multi_word = arStrip(multi_word, True, True, True, False, True, False)  # diacs , smallDiacs , shaddah ,  digit , alif , specialChars
+    result_word = []
+    if undiac_multi_word in settings.div_dic.keys():
+        result_word = settings.div_dic[undiac_multi_word]
+    
+    my_json = {}
+    glosses_list = []
+    output_list = []
+    concept_count = 0
+    my_json['multi_word_lemma'] = multi_word
+    my_json['undiac_multi_word_lemma'] = multi_word
+    ids = []
+    if result_word != []:
+        #my_json['concept_count'] = result_word[0][1] #concept_count
+        #my_json['POS'] = result_word[0][2] #POS
+        my_json['POS'] = result_word[0][1] #POS
+
+        for result in result_word: 
+           ids.append(result[3])
+           #if lemma_id in settings.glosses_dic.keys(): 
+           #   value = settings.glosses_dic[lemma_id]
+           #   glosses_list.append(json.loads(value[1]))
+           #   concept_count = concept_count + value[0]
+        my_json['ids'] = ids
+        #my_json['concept_count'] = concept_count
+        #my_json['glosses'] = glosses_list   
+        output_list.append(my_json)    
+    return output_list  

nlptools/morphology/__init__.py

@@ -0,0 +1,52 @@
+from nlptools.morphology import settings 
+import pickle
+from nlptools.DataDownload import downloader
+import os 
+
+#filename = 'ALMA27012000.pickle'
+#path =downloader.get_appdatadir()
+#file_path = os.path.join(path, filename)
+#with open(file_path, 'rb') as f:
+#    #Load the serialized data from the file
+#    settings.div_dic = pickle.load(f)
+
+
+filename = 'lemmas_dic.pickle'
+path =downloader.get_appdatadir()
+file_path = os.path.join(path, filename)
+with open(file_path, 'rb') as f:
+    #Load the serialized data from the file
+    settings.div_dic = pickle.load(f)
+    
+
+#filename_five = 'five_grams.pickle'
+#path =downloader.get_appdatadir()
+#file_path = os.path.join(path, filename_five)
+#with open(file_path, 'rb') as f:
+#    #Load the serialized data from the file
+#    settings.five_grams_dict = pickle.load(f, encoding='utf-8')
+#
+#
+#filename_four = 'four_grams.pickle'
+#path =downloader.get_appdatadir()
+#file_path = os.path.join(path, filename_four)
+#with open(file_path, 'rb') as f:
+#    #Load the serialized data from the file
+#    settings.four_grams_dict = pickle.load(f, encoding='utf-8')
+#    
+#       
+#filename_three = 'three_grams.pickle'
+#path =downloader.get_appdatadir()
+#file_path = os.path.join(path, filename_three)
+#with open(file_path, 'rb') as f:
+#    #Load the serialized data from the file
+#    settings.three_grams_dict = pickle.load(f, encoding='utf-8')
+#    
+#
+#filename_two = 'two_grams.pickle'
+#path =downloader.get_appdatadir()
+#file_path = os.path.join(path, filename_two)
+#with open(file_path, 'rb') as f:
+#    #Load the serialized data from the file
+#    settings.two_grams_dict = pickle.load(f, encoding='utf-8')
+#    

nlptools/morphology/charsets.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+# We acknoledge that this file  charsets.py is imported from Camel tools citation. url
+#
+
+import unicodedata
+
+from six import unichr
+
+
+UNICODE_PUNCT_CHARSET = frozenset(
+    [unichr(x) for x in range(65536) if unicodedata.category(
+        unichr(x))[0] == 'P'])
+UNICODE_SYMBOL_CHARSET = frozenset(
+    [unichr(x) for x in range(65536) if unicodedata.category(
+        unichr(x))[0] == 'S'])
+UNICODE_PUNCT_SYMBOL_CHARSET = UNICODE_PUNCT_CHARSET | UNICODE_SYMBOL_CHARSET
+
+UNICODE_LETTER_CHARSET = frozenset(
+    [unichr(x) for x in range(65536) if unicodedata.category(
+        unichr(x))[0] == 'L'])
+UNICODE_MARK_CHARSET = frozenset(
+    [unichr(x) for x in range(65536) if unicodedata.category(
+        unichr(x))[0] == 'M'])
+UNICODE_NUMBER_CHARSET = frozenset(
+    [unichr(x) for x in range(65536) if unicodedata.category(
+        unichr(x))[0] == 'N'])
+UNICODE_LETTER_MARK_NUMBER_CHARSET = (UNICODE_LETTER_CHARSET |
+                                      UNICODE_MARK_CHARSET |
+                                      UNICODE_NUMBER_CHARSET)
+
+AR_LETTERS_CHARSET = frozenset(u'\u0621\u0622\u0623\u0624\u0625\u0626\u0627'
+                               u'\u0628\u0629\u062a\u062b\u062c\u062d\u062e'
+                               u'\u062f\u0630\u0631\u0632\u0633\u0634\u0635'
+                               u'\u0636\u0637\u0638\u0639\u063a\u0640\u0641'
+                               u'\u0642\u0643\u0644\u0645\u0646\u0647\u0648'
+                               u'\u0649\u064a\u0671\u067e\u0686\u06a4\u06af')
+AR_DIAC_CHARSET = frozenset(u'\u064b\u064c\u064d\u064e\u064f\u0650\u0651\u0652'
+                            u'\u0670\u0640')
+AR_CHARSET = AR_LETTERS_CHARSET | AR_DIAC_CHARSET
+
+BW_LETTERS_CHARSET = frozenset(u'$&\'*<>ADEGHJPSTVYZ_bdfghjklmnpqrstvwxyz{|}')
+BW_DIAC_CHARSET = frozenset(u'FKN`aiou~_')
+BW_CHARSET = BW_LETTERS_CHARSET | BW_DIAC_CHARSET
+
+SAFEBW_LETTERS_CHARSET = frozenset(u'ABCDEGHIJLMOPQSTVWYZ_bcdefghjklmnpqrstvwx'
+                                   u'yz')
+SAFEBW_DIAC_CHARSET = frozenset(u'FKNaeiou~_')
+SAFEBW_CHARSET = SAFEBW_LETTERS_CHARSET | SAFEBW_DIAC_CHARSET
+
+XMLBW_LETTERS_CHARSET = frozenset(u'$\'*ABDEGHIJOPSTWYZ_bdfghjklmnpqrstvwxyz{|'
+                                  u'}')
+XMLBW_DIAC_CHARSET = frozenset(u'FKN`aiou~_')
+XMLBW_CHARSET = XMLBW_LETTERS_CHARSET | XMLBW_DIAC_CHARSET
+
+HSB_LETTERS_CHARSET = frozenset(u'\'ADHST_bcdfghjklmnpqrstvwxyz'
+                                u'\u00c2\u00c4\u00e1\u00f0\u00fd\u0100\u0102'
+                                u'\u010e\u0127\u0161\u0175\u0177\u03b3\u03b8'
+                                u'\u03c2')
+HSB_DIAC_CHARSET = frozenset(u'.aiu~\u00c4\u00e1\u00e3\u0129\u0169_')
+HSB_CHARSET = HSB_LETTERS_CHARSET | HSB_DIAC_CHARSET

nlptools/morphology/morph_analyzer.py

@@ -0,0 +1,170 @@
+
+from nlptools.morphology import settings 
+import re
+from nlptools.morphology.tokenizers_words import simple_word_tokenize
+from nlptools.utils.parser import arStrip
+from nlptools.morphology.charsets import AR_CHARSET, AR_DIAC_CHARSET
+
+_IS_AR_RE = re.compile(u'^[' + re.escape(u''.join(AR_CHARSET)) + u']+$')
+def find_solution(token, language, task):
+    """
+    Given a token, this method finds the morphological solution lemma and/or pos based on a spesific language and task.
+          
+    Args:
+        token (:obj:`str`): The Arabic token to be morphologcaly analyzed.
+        language (:obj:`str`): In the current version, `MSA` is only supported. 
+        task (:obj:`str`): The task to filter the results by [lemmatizer, pos, full]. The defualt task if not specified is `full`.
+
+    Returns:
+        list (:obj:`list`): A list of [token, lemma, pos], where:
+           token: the original input token
+           lemma: the lemma of the token 
+           pos: the part-of-speech of the token 
+    Note:
+        If no sloution is found for this token, an empty list is returned.
+    """
+    
+    if token in settings.div_dic.keys():
+        resulted_solutions = [] 
+        solutions = settings.div_dic[token]
+        for solution in solutions:
+            resulted_solutions.append([token, solution[0], solution[1], solution[3]])
+        return resulted_solutions
+    else:
+        return []
+
+def analyze(text, language ='MSA', task ='full'):
+   """
+    This method takes a text as input and returns a morphological solution for each token in this text, Based on the input language and task, such that,
+    if:
+        the task is lemmatizer, then the morphological soltuion is only the lemma.
+        the task is pos, then the morphological soltuion is only the pos.
+        the task is full, the the morphological soltuion is both the lemma and the pos.
+     
+    Args:
+        token (:obj:`str`): The Arabic token to be morphologcaly analyzed.
+        language (:obj:`str`): In the current version, `MSA` is only supported. 
+        task (:obj:`str`): The task to filter the results by [lemmatizer, pos, full]. The defualt task if not specified is `full`.
+         
+    Returns:
+        list (:obj:`list`): A list of [token, lemma, pos], based on the spesified task, where:
+           token: the original input token
+           lemma: the lemma of the token 
+           pos: the part-of-speech of the token 
+
+    **Example:**
+
+     .. highlight:: python
+     .. code-block:: python
+     
+          from nlptools.morphology import morph_analyzer
+     
+          #Return the morpological solution for each token in this text
+          #Example: task = full 
+          morph_analyzer.analyze('ذهب الولد الى المدرسة')
+           
+           [['ذهب', 'ذَهَبَ۪ 1', 'فعل'],
+           ['الولد', 'وَلَد 1', 'اسم'],
+           ['الى', 'إِلَى 1', 'كلمة وظيفية'],
+           ['المدرسة', 'مَدْرَسَة 1', 'اسم']]
+
+           #Exampel: task = pos
+           morph_analyzer.analyze('ذهب الولد الى المدرسة',task='pos')
+           #the output
+           [['ذهب', 'فعل'], ['الولد', 'اسم'], ['الى', 'كلمة وظيفية'], ['المدرسة', 'اسم']]
+
+           #Exampel: task = lemmatizer
+           morph_analyzer.analyze('طار العصور فوق الشجرة', task='lemmatizer')
+           #the output
+           [['طار', 'طارِ۪ 1'],
+            ['العصور', 'عَصْر 1'],
+            ['فوق', 'فَوْق 1'],
+            ['الشجرة', 'شَجَرَة 1']]
+     """
+ 
+   #@check if the init does not load data correctly, call load_alma inside
+   output_list = []
+
+   tokens = simple_word_tokenize(text)
+
+   for token in tokens:
+         result_token =[]
+         token = arStrip(token , False , True , False , False , False , False) 
+         token = re.sub('[ٱ]','ﺍ',token)
+         solution=[token, token+"_0","",0]
+
+         if token.isdigit():
+            solution[2] = "digit" #pos
+
+         elif not _is_ar(token):
+            solution[2] = "Foreign" #pos
+
+        #  elif re.match("^[a-zA-Z]*$", token): 
+        #     solution[2] = "Foreign" #pos
+
+         else:
+            result_token = find_solution(token,language, task)
+            
+            if result_token == []:
+               token_without_al = re.sub(r'^[ﻝ]','',re.sub(r'^[ﺍ]','',token))
+               if len(token_without_al) > 5  :
+                  result_token = find_solution(token_without_al, language, task)
+
+            if result_token == []:
+              # try with replace ﻩ with ﺓ
+               result_token = find_solution(re.sub(r'[ﻩ]$','ﺓ',token), language, task)
+               
+
+            if result_token == []:
+               # try with unify Alef
+               word_with_unify_alef = arStrip(token , False , False , False , False , True , False) # Unify Alef
+               result_token = find_solution(word_with_unify_alef, language, task)
+            
+            if result_token == []:
+               # try with remove diac
+               word_undiac = arStrip(token , True , False , True , True , False , False) # remove diacs, shaddah ,  digit
+               result_token = find_solution(word_undiac, language, task)
+
+            if result_token == []:
+               # try with remove diac and unify alef
+               word_undiac = arStrip(token , True , True , True , False, True , False) # diacs , smallDiacs , shaddah ,  alif
+               result_token = find_solution(word_undiac, language, task)
+
+         if result_token != []:
+               output_list.append(result_token)
+         else:
+            # if no solution is found
+            output_list.append([solution])
+        
+   return filter_results(task, output_list)
+
+
+def filter_results(task, lst):
+    if task == 'lemmatizer':
+        return remove_items_by_index(lst, [2])
+    elif task == 'pos':
+        return remove_items_by_index(lst, [1])
+    else: 
+        return lst
+
+
+def remove_items_by_index(lst, index_list):
+    for inner_list in lst:
+        for index in sorted(index_list, reverse=True):
+            if len(inner_list) > index:
+                inner_list.pop(index)
+    return lst
+
+
+def _is_ar(word):
+    return _IS_AR_RE.match(word) is not None       
+        
+
+  
+  
+  
+    
+    
+    
+    
+    

nlptools/morphology/settings.py

@@ -0,0 +1,8 @@
+flag = True
+div_dic = {}
+lemma_source = "DIC"
+
+#two_grams_dict =  {}
+#three_grams_dict = {}
+#four_grams_dict = {}
+#five_grams_dict = {}

nlptools/morphology/tokenizers_words.py

@@ -0,0 +1,19 @@
+# This code was taken from Camel tools without any change
+
+# -*- coding: utf-8 -*-
+
+
+import re
+from nlptools.morphology.charsets import UNICODE_PUNCT_SYMBOL_CHARSET
+from nlptools.morphology.charsets import UNICODE_LETTER_MARK_NUMBER_CHARSET
+
+
+_ALL_PUNCT = u''.join(UNICODE_PUNCT_SYMBOL_CHARSET)
+_ALL_LETTER_MARK_NUMBER = u''.join(UNICODE_LETTER_MARK_NUMBER_CHARSET)
+_TOKENIZE_RE = re.compile(r'[' + re.escape(_ALL_PUNCT) + r']|[' +
+                          re.escape(_ALL_LETTER_MARK_NUMBER) + r']+')
+
+
+def simple_word_tokenize(sentence):
+
+    return _TOKENIZE_RE.findall(sentence)

nlptools/nlptools.py

@@ -0,0 +1 @@
+"""Main module."""

nlptools/salma/__init__.py

@@ -0,0 +1,12 @@
+from nlptools.salma import settings 
+import pickle
+from nlptools.DataDownload import downloader
+import os 
+
+#filename = 'glosses_dic.pickle'
+#path =downloader.get_appdatadir()
+#file_path = os.path.join(path, filename)
+#with open(file_path, 'rb') as f:
+#    #Load the serialized data from the file
+#    settings.glosses_dic = pickle.load(f)
+settings.glosses_dic = {}

nlptools/salma/settings.py

@@ -0,0 +1,31 @@
+from transformers import BertTokenizer,BertForSequenceClassification
+import warnings
+warnings.filterwarnings("ignore")
+import pandas as pd
+
+
+
+
+from nlptools.DataDownload import downloader
+import os 
+
+glosses_dic = {}
+
+model_file_name = "bert-base-arabertv02_22_May_2021_00h_allglosses_unused01"
+path =downloader.get_appdatadir()
+model_file_path = os.path.join(path, model_file_name)
+
+tokenizer_file_name = "bert-base-arabertv02"
+path =downloader.get_appdatadir()
+tokenizer_file_path = os.path.join(path, tokenizer_file_name)
+
+dftrue = pd.DataFrame()
+
+# model = BertForSequenceClassification.from_pretrained('{}'.format("bert-base-arabertv02_22_May_2021_00h_allglosses_unused01"),
+                                                    #   output_hidden_states = True,
+                                                    #   num_labels=2
+                                                    #  )
+
+model = BertForSequenceClassification.from_pretrained(model_file_path, output_hidden_states=True, num_labels=2)
+
+tokenizer = BertTokenizer.from_pretrained('{}'.format(tokenizer_file_path))