SinaTools 0.1.29__tar.gz → 0.1.31__tar.gz

{SinaTools-0.1.29 → SinaTools-0.1.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.29
+Version: 0.1.31
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.29 → SinaTools-0.1.31}/SinaTools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.29
+Version: 0.1.31
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.29 → SinaTools-0.1.31}/SinaTools.egg-info/SOURCES.txt

@@ -155,7 +155,6 @@ sinatools/ner/datasets.py
 sinatools/ner/entity_extractor.py
 sinatools/ner/helpers.py
 sinatools/ner/metrics.py
-sinatools/ner/relation_extractor.py
 sinatools/ner/transforms.py
 sinatools/ner/data/__init__.py
 sinatools/ner/data/datasets.py
@@ -168,6 +167,8 @@ sinatools/ner/trainers/BaseTrainer.py
 sinatools/ner/trainers/BertNestedTrainer.py
 sinatools/ner/trainers/BertTrainer.py
 sinatools/ner/trainers/__init__.py
+sinatools/relations/__init__.py
+sinatools/relations/relation_extractor.py
 sinatools/semantic_relatedness/__init__.py
 sinatools/semantic_relatedness/compute_relatedness.py
 sinatools/synonyms/__init__.py

{SinaTools-0.1.29 → SinaTools-0.1.31}/sinatools/DataDownload/downloader.py

@@ -16,7 +16,8 @@ urls = {
     'three_grams':'https://sina.birzeit.edu/three_grams.pickle',
     'two_grams':'https://sina.birzeit.edu/two_grams.pickle',
     'graph_l2':'https://sina.birzeit.edu/graph_l2.pkl',
-    'graph_l3':'https://sina.birzeit.edu/graph_l3.pkl'
+    'graph_l3':'https://sina.birzeit.edu/graph_l3.pkl',
+    'relation':'https://sina.birzeit.edu/relation_model.zip'
 }
 
 def get_appdatadir():

SinaTools-0.1.31/sinatools/VERSION

@@ -0,0 +1 @@
+0.1.31

{SinaTools-0.1.29 → SinaTools-0.1.31}/sinatools/morphology/morph_analyzer.py

@@ -3,6 +3,7 @@ from sinatools.utils.tokenizers_words import simple_word_tokenize
 from sinatools.utils.parser import arStrip
 from sinatools.utils.charsets import AR_CHARSET, AR_DIAC_CHARSET
 from sinatools.DataDownload.downloader import get_appdatadir
+from sinatools.morphology.morph_analyzer import remove_punctuation
 from . import dictionary
 
 _IS_AR_RE = re.compile(u'^[' + re.escape(u''.join(AR_CHARSET)) + u']+$')
@@ -98,13 +99,16 @@ def analyze(text, language ='MSA', task ='full', flag="1"):
          token = arStrip(token , False , True , False , False , False , False) 
          token = re.sub('[ٱ]','ﺍ',token)
          # token, freq, lemma, lemma_id, root, pos
-         solution = [token, 0, token+"_0", 0, token, ""]
+         solution = [token, 0, token, 0, token, ""]
 
          if token.isdigit():
-            solution[5] = "digit" #pos
+            solution[5] = "رقم" #pos
+
+         elif remove_punctuation(token).strip() == "":
+            solution[5] = "علامة ترقيم" #pos
 
          elif not _is_ar(token):
-            solution[5] = "Foreign" #pos
+            solution[5] = "أجنبي" #pos
 
          else:
             result_token = find_solution(token,language,flag)

{SinaTools-0.1.29 → SinaTools-0.1.31}/sinatools/ner/__init__.py

@@ -7,8 +7,6 @@ import torch
 import pickle
 import json
 from argparse import Namespace
-from transformers import pipeline
-#from transformers import AutoModelForSequenceClassification
 
 tagger = None
 tag_vocab = None
@@ -38,6 +36,3 @@ if torch.cuda.is_available():
 train_config.trainer_config["kwargs"]["model"] = model
 tagger = load_object(train_config.trainer_config["fn"], train_config.trainer_config["kwargs"])
 tagger.load(os.path.join(model_path,"checkpoints"))
-
-pipe = pipeline("sentiment-analysis", model= os.path.join(path, "best_model"), return_all_scores =True, max_length=128, truncation=True)
-#pipe = AutoModelForSequenceClassification.from_pretrained(os.path.join(path, "best_model"))

SinaTools-0.1.31/sinatools/relations/__init__.py

@@ -0,0 +1,7 @@
+from sinatools.DataDownload import downloader
+import os
+from transformers import pipeline
+
+path =downloader.get_appdatadir()
+
+pipe = pipeline("sentiment-analysis", model= os.path.join(path, "relation_model"), return_all_scores =True, max_length=128, truncation=True)

{SinaTools-0.1.29/sinatools/ner → SinaTools-0.1.31/sinatools/relations}/relation_extractor.py

@@ -1,10 +1,8 @@
-import torch
-import json
 from urllib.request import Request, urlopen
 from sinatools.ner.entity_extractor import extract
+from sinatools.utils.tokenizer import sentence_tokenizer
 from . import pipe
 
-
 # ============================ Extract entities and their types ========================
 def jsons_to_list_of_lists(json_list):
     return [[d['token'], d['tags']] for d in json_list]
@@ -168,34 +166,34 @@ def get_entity_category(entity_type, categories):
 
 
 # ============ Extract entities, their types and categorize them ===============
-def relation_extraction(sentence):
-    #test_sentence="صورة إعتقال طفل فلسطيني خلال انتفاضة الأقصى ."
-    entities=entities_and_types(sentence)
-
-    event_indices = [i for i, (_, entity_type) in enumerate(entities.items()) if entity_type == 'EVENT']
-    arg_event_indices = [i for i, (_, entity_type) in enumerate(entities.items()) if entity_type != 'EVENT']
+def event_argument_relation_extraction(documnet):
 
+    sentences=sentence_tokenizer(documnet)
     output_list=[]
-
-    for i in event_indices:
-        event_entity=list(entities.keys())[i]
-        for j in arg_event_indices:
-            arg_name= list(entities.keys())[j]
-            arg_type=entities[arg_name]
-            category = get_entity_category(arg_type, categories)
-            
-            if category in temp03:
-                relation_sentence=f"[CLS] {sentence} [SEP] {event_entity} {temp03[category]} {arg_name}"
-                predicted_relation=pipe(relation_sentence)
-                score = predicted_relation[0][0]['score']  
-                if score > 0.50:
-                    #print(f"Event:{event_entity} Relation:{category} Argument:{arg_name}\n")
-                    #output_list.append([{event_entity} ,{category}, {arg_name}])
-                    output_list.append(f"Event:{event_entity}, Relation:{category}, Argument:{arg_name}")
-
-                else:
-                    #print(f"Event:{event_entity} Relation:No relation Argument:{arg_name}\n")
-                    #output_list.append([{event_entity} ,'No relation', {arg_name}])
-                    output_list.append(f"Event:{event_entity}, Relation:No relation, Argument:{arg_name}")
+    relation={}
+    triple_id=0
+    for sentence in sentences:
+        entities=entities_and_types(sentence)
+        entity_identifier={entity:i for entity, i in zip(entities,range(1,len(entities)+1))}
+
+        event_indices = [i for i, (_, entity_type) in enumerate(entities.items()) if entity_type == 'EVENT']
+        arg_event_indices = [i for i, (_, entity_type) in enumerate(entities.items()) if entity_type != 'EVENT']
+
+
+        for i in event_indices:
+            event_entity=list(entities.keys())[i]
+            for j in arg_event_indices:
+                arg_name= list(entities.keys())[j]
+                arg_type=entities[arg_name]
+                category = get_entity_category(arg_type, categories)
+                
+                if category in temp03:
+                    relation_sentence=f"[CLS] {sentence} [SEP] {event_entity} {temp03[category]} {arg_name}"
+                    predicted_relation=pipe(relation_sentence)
+                    score = predicted_relation[0][0]['score']  
+                    if score > 0.50:
+                        triple_id+=1
+                        relation={"TripleID":triple_id,"Subject":{"ID":entity_identifier[event_entity],"Type": entities[event_entity], "Label":event_entity}, "Relation": category, "Object":{"ID":entity_identifier[arg_name],"Type": entities[arg_name], "Label":arg_name,}}
+                        output_list.append(relation)
     
     return output_list

SinaTools-0.1.29/sinatools/VERSION

@@ -1 +0,0 @@
-0.1.29