SinaTools 0.1.29__tar.gz → 0.1.30__tar.gz

{SinaTools-0.1.29 → SinaTools-0.1.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.29
+Version: 0.1.30
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.29 → SinaTools-0.1.30}/SinaTools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.29
+Version: 0.1.30
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.29 → SinaTools-0.1.30}/sinatools/DataDownload/downloader.py

@@ -16,7 +16,8 @@ urls = {
     'three_grams':'https://sina.birzeit.edu/three_grams.pickle',
     'two_grams':'https://sina.birzeit.edu/two_grams.pickle',
     'graph_l2':'https://sina.birzeit.edu/graph_l2.pkl',
-    'graph_l3':'https://sina.birzeit.edu/graph_l3.pkl'
+    'graph_l3':'https://sina.birzeit.edu/graph_l3.pkl',
+    'relation':'https://sina.birzeit.edu/relation_model.zip'
 }
 
 def get_appdatadir():

SinaTools-0.1.30/sinatools/VERSION

@@ -0,0 +1 @@
+0.1.30

{SinaTools-0.1.29 → SinaTools-0.1.30}/sinatools/morphology/morph_analyzer.py

@@ -3,6 +3,7 @@ from sinatools.utils.tokenizers_words import simple_word_tokenize
 from sinatools.utils.parser import arStrip
 from sinatools.utils.charsets import AR_CHARSET, AR_DIAC_CHARSET
 from sinatools.DataDownload.downloader import get_appdatadir
+from sinatools.morphology.morph_analyzer import remove_punctuation
 from . import dictionary
 
 _IS_AR_RE = re.compile(u'^[' + re.escape(u''.join(AR_CHARSET)) + u']+$')
@@ -98,13 +99,16 @@ def analyze(text, language ='MSA', task ='full', flag="1"):
          token = arStrip(token , False , True , False , False , False , False) 
          token = re.sub('[ٱ]','ﺍ',token)
          # token, freq, lemma, lemma_id, root, pos
-         solution = [token, 0, token+"_0", 0, token, ""]
+         solution = [token, 0, token, 0, token, ""]
 
          if token.isdigit():
-            solution[5] = "digit" #pos
+            solution[5] = "رقم" #pos
+
+         elif remove_punctuation(token).strip() == "":
+            solution[5] = "علامة ترقيم" #pos
 
          elif not _is_ar(token):
-            solution[5] = "Foreign" #pos
+            solution[5] = "أجنبي" #pos
 
          else:
             result_token = find_solution(token,language,flag)

{SinaTools-0.1.29 → SinaTools-0.1.30}/sinatools/ner/__init__.py

@@ -39,5 +39,4 @@ train_config.trainer_config["kwargs"]["model"] = model
 tagger = load_object(train_config.trainer_config["fn"], train_config.trainer_config["kwargs"])
 tagger.load(os.path.join(model_path,"checkpoints"))
 
-pipe = pipeline("sentiment-analysis", model= os.path.join(path, "best_model"), return_all_scores =True, max_length=128, truncation=True)
-#pipe = AutoModelForSequenceClassification.from_pretrained(os.path.join(path, "best_model"))
+pipe = pipeline("sentiment-analysis", model= os.path.join(path, "relation_model"), return_all_scores =True, max_length=128, truncation=True)

SinaTools-0.1.29/sinatools/VERSION

@@ -1 +0,0 @@
-0.1.29