SinaTools 0.1.29__py2.py3-none-any.whl → 0.1.30__py2.py3-none-any.whl

{SinaTools-0.1.29.dist-info → SinaTools-0.1.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.29
+Version: 0.1.30
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.29.dist-info → SinaTools-0.1.30.dist-info}/RECORD

@@ -1,5 +1,5 @@
-SinaTools-0.1.29.data/data/sinatools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
-sinatools/VERSION,sha256=3gAvtibHsL3Zih60tzJshU5QcbL40f0qBmAAXPGrB-Q,6
+SinaTools-0.1.30.data/data/sinatools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
+sinatools/VERSION,sha256=4IPaHhHWuxBZjZ0tYxwy4rdWjvuIZronPKqN26wZ7eE,6
 sinatools/__init__.py,sha256=bEosTU1o-FSpyytS6iVP_82BXHF2yHnzpJxPLYRbeII,135
 sinatools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
 sinatools/install_env.py,sha256=EODeeE0ZzfM_rz33_JSIruX03Nc4ghyVOM5BHVhsZaQ,404
@@ -20,7 +20,7 @@ sinatools/CLI/utils/sentence_tokenizer.py,sha256=Wli8eiDbWSd_Z8UKpu_JkaS8jImowa1
 sinatools/CLI/utils/text_dublication_detector.py,sha256=dW70O5O20GxeUDDF6zVYn52wWLmJF-HBZgvqIeVL2rQ,1661
 sinatools/CLI/utils/text_transliteration.py,sha256=vz-3kxWf8pNYVCqNAtBAiA6u_efrS5NtWT-ofN1NX6I,2014
 sinatools/DataDownload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sinatools/DataDownload/downloader.py,sha256=6xH55WlDhgtImPRFQ0AaeDFJjL8OMNU29x61PL8mZ2w,6468
+sinatools/DataDownload/downloader.py,sha256=3UkRRH4TLbut10V1BgWO3EqJQaHVBqr6pAj7Fn4AQZ8,6511
 sinatools/arabert/__init__.py,sha256=ely2PttjgSv7vKdzskuD1rtK_l_UOpmxJSz8isrveD0,16
 sinatools/arabert/preprocess.py,sha256=qI0FsuMTOzdRlYGCtLrjpXgikNElUZPv9bnjaKDZKJ4,33024
 sinatools/arabert/arabert/__init__.py,sha256=KbSAH-XqbRygn0y59m5-ZYOLXgpT1gSgE3F-qd4rKEc,627
@@ -75,8 +75,8 @@ sinatools/arabert/aragpt2/grover/train_tpu.py,sha256=qNgLI_j6-KYkTMJfVoFlh4NIKwe
 sinatools/arabert/aragpt2/grover/utils.py,sha256=V5wMUxK03r5g_pb7R3_uGLOPqQJfbIB0VaJ8ZDM4XAo,8473
 sinatools/morphology/ALMA_multi_word.py,sha256=hj_-8ojrYYHnfCGk8WKtJdUR8mauzQdma4WUm-okDps,1346
 sinatools/morphology/__init__.py,sha256=I4wVBh8BhyNl-CySVdiI_nUSn6gj1j-gmLKP300RpE0,1216
-sinatools/morphology/morph_analyzer.py,sha256=3B-ewxFg_If83oYlk1bDdVS1clb-mgyAF4WgAMqcAVI,7009
-sinatools/ner/__init__.py,sha256=CLPaqUcvPGAA4lU-6hjAqjNfKJ5WtwRfsma6QkYZHEk,1379
+sinatools/morphology/morph_analyzer.py,sha256=XrLkFqI89GmQuRyZB5X7GNIpfedfGNnQwHzrz5bDu5A,7190
+sinatools/ner/__init__.py,sha256=isVSWoFZNiWpDCiT4hNKY5C2eVupN2SvCqYbie8oN2k,1289
 sinatools/ner/data.py,sha256=lvOW86dXse8SC75Q0supQaE0rrRffoxNjIA0Qbv5WZY,4354
 sinatools/ner/data_format.py,sha256=7Yt0aOicOn9_YuuyCkM_IYi_rgjGYxR9bCuUaNGM73o,4341
 sinatools/ner/datasets.py,sha256=mG1iwqSm3lXCFHLqE-b4wNi176cpuzNBz8tKaBU6z6M,5059
@@ -116,10 +116,10 @@ sinatools/wsd/__init__.py,sha256=mwmCUurOV42rsNRpIUP3luG0oEzeTfEx3oeDl93Oif8,306
 sinatools/wsd/disambiguator.py,sha256=h-3idc5rPPbMDSE_QVJAsEVkDHwzYY3L2SEPNXIdOcc,20104
 sinatools/wsd/settings.py,sha256=6XflVTFKD8SVySX9Wj7zYQtV26WDTcQ2-uW8-gDNHKE,747
 sinatools/wsd/wsd.py,sha256=gHIBUFXegoY1z3rRnIlK6TduhYq2BTa_dHakOjOlT4k,4434
-SinaTools-0.1.29.dist-info/AUTHORS.rst,sha256=aTWeWlIdfLi56iLJfIUAwIrmqDcgxXKLji75_Fjzjyg,174
-SinaTools-0.1.29.dist-info/LICENSE,sha256=uwsKYG4TayHXNANWdpfMN2lVW4dimxQjA_7vuCVhD70,1088
-SinaTools-0.1.29.dist-info/METADATA,sha256=IorbBd2klVKi0amBxKMKIEgyJHjRxcpJNxAQBgyNn04,3267
-SinaTools-0.1.29.dist-info/WHEEL,sha256=6T3TYZE4YFi2HTS1BeZHNXAi8N52OZT4O-dJ6-ome_4,116
-SinaTools-0.1.29.dist-info/entry_points.txt,sha256=ZwZLolnWog2fjdDrfaHNHob8SE_YtMbD6ayzsOzItxs,1234
-SinaTools-0.1.29.dist-info/top_level.txt,sha256=8tNdPTeJKw3TQCaua8IJIx6N6WpgZZmVekf1OdBNJpE,10
-SinaTools-0.1.29.dist-info/RECORD,,
+SinaTools-0.1.30.dist-info/AUTHORS.rst,sha256=aTWeWlIdfLi56iLJfIUAwIrmqDcgxXKLji75_Fjzjyg,174
+SinaTools-0.1.30.dist-info/LICENSE,sha256=uwsKYG4TayHXNANWdpfMN2lVW4dimxQjA_7vuCVhD70,1088
+SinaTools-0.1.30.dist-info/METADATA,sha256=Pl7OjoUAbjqPtzJuGqvXeb7GVCx2t-7cxW4-APUKLIk,3267
+SinaTools-0.1.30.dist-info/WHEEL,sha256=6T3TYZE4YFi2HTS1BeZHNXAi8N52OZT4O-dJ6-ome_4,116
+SinaTools-0.1.30.dist-info/entry_points.txt,sha256=ZwZLolnWog2fjdDrfaHNHob8SE_YtMbD6ayzsOzItxs,1234
+SinaTools-0.1.30.dist-info/top_level.txt,sha256=8tNdPTeJKw3TQCaua8IJIx6N6WpgZZmVekf1OdBNJpE,10
+SinaTools-0.1.30.dist-info/RECORD,,

sinatools/DataDownload/downloader.py

@@ -10,13 +10,14 @@ urls = {
     'ner': 'https://sina.birzeit.edu/Wj27012000.tar.gz',
     'wsd_model': 'https://sina.birzeit.edu/bert-base-arabertv02_22_May_2021_00h_allglosses_unused01.zip',
     'wsd_tokenizer': 'https://sina.birzeit.edu/bert-base-arabertv02.zip',
-    'glosses_dic': 'https://sina.birzeit.edu/glosses_dic.pickle',
+    'one_gram': 'https://sina.birzeit.edu/one_gram.pickle',
     'five_grams': 'https://sina.birzeit.edu/five_grams.pickle',
     'four_grams':'https://sina.birzeit.edu/four_grams.pickle',
     'three_grams':'https://sina.birzeit.edu/three_grams.pickle',
     'two_grams':'https://sina.birzeit.edu/two_grams.pickle',
-    'synonyms_level2':'https://sina.birzeit.edu/graph_l2.pkl',
-    'synonyms_level3':'https://sina.birzeit.edu/graph_l3.pkl'
+    'graph_l2':'https://sina.birzeit.edu/graph_l2.pkl',
+    'graph_l3':'https://sina.birzeit.edu/graph_l3.pkl',
+    'relation':'https://sina.birzeit.edu/relation_model.zip'
 }
 
 def get_appdatadir():

sinatools/VERSION

@@ -1 +1 @@
-0.1.29
+0.1.30

sinatools/morphology/morph_analyzer.py

@@ -3,6 +3,7 @@ from sinatools.utils.tokenizers_words import simple_word_tokenize
 from sinatools.utils.parser import arStrip
 from sinatools.utils.charsets import AR_CHARSET, AR_DIAC_CHARSET
 from sinatools.DataDownload.downloader import get_appdatadir
+from sinatools.morphology.morph_analyzer import remove_punctuation
 from . import dictionary
 
 _IS_AR_RE = re.compile(u'^[' + re.escape(u''.join(AR_CHARSET)) + u']+$')
@@ -98,13 +99,16 @@ def analyze(text, language ='MSA', task ='full', flag="1"):
          token = arStrip(token , False , True , False , False , False , False) 
          token = re.sub('[ٱ]','ﺍ',token)
          # token, freq, lemma, lemma_id, root, pos
-         solution = [token, 0, token+"_0", 0, token, ""]
+         solution = [token, 0, token, 0, token, ""]
 
          if token.isdigit():
-            solution[5] = "digit" #pos
+            solution[5] = "رقم" #pos
+
+         elif remove_punctuation(token).strip() == "":
+            solution[5] = "علامة ترقيم" #pos
 
          elif not _is_ar(token):
-            solution[5] = "Foreign" #pos
+            solution[5] = "أجنبي" #pos
 
          else:
             result_token = find_solution(token,language,flag)

sinatools/ner/__init__.py

@@ -39,5 +39,4 @@ train_config.trainer_config["kwargs"]["model"] = model
 tagger = load_object(train_config.trainer_config["fn"], train_config.trainer_config["kwargs"])
 tagger.load(os.path.join(model_path,"checkpoints"))
 
-pipe = pipeline("sentiment-analysis", model= os.path.join(path, "best_model"), return_all_scores =True, max_length=128, truncation=True)
-#pipe = AutoModelForSequenceClassification.from_pretrained(os.path.join(path, "best_model"))
+pipe = pipeline("sentiment-analysis", model= os.path.join(path, "relation_model"), return_all_scores =True, max_length=128, truncation=True)