SinaTools 0.1.26__py2.py3-none-any.whl → 0.1.27__py2.py3-none-any.whl

SinaTools-0.1.27.dist-info/METADATA

@@ -0,0 +1,64 @@
+Metadata-Version: 2.1
+Name: SinaTools
+Version: 0.1.27
+Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
+Home-page: https://github.com/SinaLab/sinatools
+License: MIT license
+Keywords: sinatools
+Platform: UNKNOWN
+Description-Content-Type: text/markdown
+Requires-Dist: six
+Requires-Dist: farasapy
+Requires-Dist: tqdm
+Requires-Dist: requests
+Requires-Dist: regex
+Requires-Dist: pathlib
+Requires-Dist: torch (==1.13.0)
+Requires-Dist: transformers (==4.24.0)
+Requires-Dist: torchtext (==0.14.0)
+Requires-Dist: torchvision (==0.14.0)
+Requires-Dist: seqeval (==1.2.2)
+Requires-Dist: natsort (==7.1.1)
+
+SinaTools
+======================
+Open Source Toolkit for Arabic NLP and NLU developed by [SinaLab](http://sina.birzeit.edu/) at Birzeit University. SinaTools is available through Python APIs, command lines, colabs, and online demos.
+
+See the full list of [Available Packages](https://sina.birzeit.edu/sinatools/), which include: (1) [Morphology Tagging](https://sina.birzeit.edu/sinatools/index.html#morph), (2) [Named Entity Recognition (NER)](https://sina.birzeit.edu/sinatools/index.html#ner), (3) [Word Sense Disambiguation (WSD)](https://sina.birzeit.edu/sinatools/index.html#wsd), (4) [Semantic Relatedness](https://sina.birzeit.edu/sinatools/index.html#sr), (5) [Synonymy Extraction and Evaluation](https://sina.birzeit.edu/sinatools/index.html#se), (6) [Relation Extraction](https://sina.birzeit.edu/sinatools/index.html), (7) [Utilities](https://sina.birzeit.edu/sinatools/index.html#u) (diacritic-based word matching, Jaccard similarly, parser, tokenizers, corpora processing, transliteration, etc).
+
+See [Demo Pages](https://sina.birzeit.edu/sinatools/).
+
+See the [benchmarking](https://www.jarrar.info/publications/HJK24.pdf), which shows that SinaTools outperformed all related toolkits. 
+
+Installation 
+--------
+To install SinaTools, ensure you are using Python version 3.10.8, then clone the [GitHub](git://github.com/SinaLab/SinaTools) repository.
+
+Alternatively, you can execute the following command:
+
+```bash
+pip install sinatools
+```
+
+Installing Models and Data Files
+--------
+Some modules in SinaTools require some data files and fine-tuned models to be downloaded. To download these models, please consult the [DataDownload](https://sina.birzeit.edu/sinatools/documentation/cli_tools/DataDownload/DataDownload.html).
+
+Documentation
+--------
+For information, please refer to the [main page](https://sina.birzeit.edu/sinatools) or the [online domuementation](https://sina.birzeit.edu/sinatools/documentation). 
+
+Citation
+-------
+Tymaa Hammouda, Mustafa Jarrar, Mohammed Khalilia: [SinaTools: Open Source Toolkit for Arabic Natural Language Understanding](http://www.jarrar.info/publications/HJK24.pdf). In Proceedings of the 2024 AI in Computational Linguistics (ACLing 2024), Procedia Computer Science, Dubai. ELSEVIER.
+
+License
+--------
+SinaTools is available under the MIT License. See the [LICENSE](https://github.com/SinaLab/sinatools/blob/main/LICENSE) file for more information.
+
+Reporting Issues
+--------
+To report any issues or bugs, please contact us at "sina.institute.bzu@gmail.com" or visit [SinaTools Issues](https://github.com/SinaLab/sinatools/issues).
+
+
+

{SinaTools-0.1.26.dist-info → SinaTools-0.1.27.dist-info}/RECORD

@@ -1,10 +1,10 @@
-SinaTools-0.1.26.data/data/sinatools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
-sinatools/VERSION,sha256=5E6i4X07Go6cKsVD3uEZkX9jXfyE05s7HlzVXSisTX8,6
+SinaTools-0.1.27.data/data/sinatools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
+sinatools/VERSION,sha256=W9ZpgLhiag6deRMB5DzdHO6EDLvNbZPVIRIUv6nHT3U,6
 sinatools/__init__.py,sha256=bEosTU1o-FSpyytS6iVP_82BXHF2yHnzpJxPLYRbeII,135
 sinatools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
 sinatools/install_env.py,sha256=EODeeE0ZzfM_rz33_JSIruX03Nc4ghyVOM5BHVhsZaQ,404
 sinatools/sinatools.py,sha256=vR5AaF0iel21LvsdcqwheoBz0SIj9K9I_Ub8M8oA98Y,20
-sinatools/CLI/DataDownload/download_files.py,sha256=VunXU_vAweKs7aS0FNM84N_2lhYT5T94Y8B3NWmGksg,2630
+sinatools/CLI/DataDownload/download_files.py,sha256=2jLohYd5Vjtz7gVd8zePhK6A_r0svpoV3ykbO5rwR8s,2613
 sinatools/CLI/morphology/ALMA_multi_word.py,sha256=ZImJ1vtcpSHydI1BjJmK3KcMJbGBZX16kO4L6rxvBvA,2086
 sinatools/CLI/morphology/morph_analyzer.py,sha256=ieIM47QK9Nct3MtCS9uq3h2rZN5r4qNhsLmlVeE6wiE,3503
 sinatools/CLI/ner/corpus_entity_extractor.py,sha256=Da-DHFrqT6if7w6WnodB4TBE5ze3DJYjb2Mmju_Qd7g,4034
@@ -13,14 +13,14 @@ sinatools/CLI/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 sinatools/CLI/utils/arStrip.py,sha256=NLyp8vOu2xv80tL9jiKRvyptmbkRZVg-wcAr-9YyvNY,3264
 sinatools/CLI/utils/corpus_tokenizer.py,sha256=nH0T4h6urr_0Qy6-wN3PquOtnwybj0REde5Ts_OE4U8,1650
 sinatools/CLI/utils/implication.py,sha256=nvoiI5UHHaJdd6MICql0pB_-h3L0icYwP1WgJi2h7p0,2854
-sinatools/CLI/utils/jaccard.py,sha256=NoKbWAq6dHDtQ56mAc1kdAnROm8NXEjZ1ecVZ7EYm6Y,4205
+sinatools/CLI/utils/jaccard.py,sha256=w56N_cNEFJ0A7WtunmY_xtms4srFagKBzrW_0YhH2DE,4216
 sinatools/CLI/utils/remove_latin.py,sha256=NOaTm2RHxt5IQrV98ySTmD8rTXTmcqSmfbPAwTyaXqU,848
 sinatools/CLI/utils/remove_punctuation.py,sha256=vJAZlEn7WGftZAFVFYnddkRrxdJ_rMmKB9vFZkY-jN4,1097
 sinatools/CLI/utils/sentence_tokenizer.py,sha256=Wli8eiDbWSd_Z8UKpu_JkaS8jImowa1vnRL0oYCSfqw,2823
 sinatools/CLI/utils/text_dublication_detector.py,sha256=dW70O5O20GxeUDDF6zVYn52wWLmJF-HBZgvqIeVL2rQ,1661
 sinatools/CLI/utils/text_transliteration.py,sha256=vz-3kxWf8pNYVCqNAtBAiA6u_efrS5NtWT-ofN1NX6I,2014
 sinatools/DataDownload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sinatools/DataDownload/downloader.py,sha256=F-SV-0mbYMYFSNCx8FoAYXhn0X1j0dF37PTLU0nUBVg,6482
+sinatools/DataDownload/downloader.py,sha256=6xH55WlDhgtImPRFQ0AaeDFJjL8OMNU29x61PL8mZ2w,6468
 sinatools/arabert/__init__.py,sha256=ely2PttjgSv7vKdzskuD1rtK_l_UOpmxJSz8isrveD0,16
 sinatools/arabert/preprocess.py,sha256=qI0FsuMTOzdRlYGCtLrjpXgikNElUZPv9bnjaKDZKJ4,33024
 sinatools/arabert/arabert/__init__.py,sha256=KbSAH-XqbRygn0y59m5-ZYOLXgpT1gSgE3F-qd4rKEc,627
@@ -76,13 +76,14 @@ sinatools/arabert/aragpt2/grover/utils.py,sha256=V5wMUxK03r5g_pb7R3_uGLOPqQJfbIB
 sinatools/morphology/ALMA_multi_word.py,sha256=hj_-8ojrYYHnfCGk8WKtJdUR8mauzQdma4WUm-okDps,1346
 sinatools/morphology/__init__.py,sha256=I4wVBh8BhyNl-CySVdiI_nUSn6gj1j-gmLKP300RpE0,1216
 sinatools/morphology/morph_analyzer.py,sha256=tA78gWg6iaE_G1c2xqxZoXZWNbvHBJLrTSxPyir5Xn8,6941
-sinatools/ner/__init__.py,sha256=gSs0x6veWJ8j3_iOs79tynBd_hJP0t44CGpJ0xzoiW4,1048
+sinatools/ner/__init__.py,sha256=CLPaqUcvPGAA4lU-6hjAqjNfKJ5WtwRfsma6QkYZHEk,1379
 sinatools/ner/data.py,sha256=lvOW86dXse8SC75Q0supQaE0rrRffoxNjIA0Qbv5WZY,4354
 sinatools/ner/data_format.py,sha256=7Yt0aOicOn9_YuuyCkM_IYi_rgjGYxR9bCuUaNGM73o,4341
 sinatools/ner/datasets.py,sha256=mG1iwqSm3lXCFHLqE-b4wNi176cpuzNBz8tKaBU6z6M,5059
-sinatools/ner/entity_extractor.py,sha256=yQnfayT03qAnQ4FBdBFhvl8M2pgIttrdWSWE9wgO2LI,1876
+sinatools/ner/entity_extractor.py,sha256=bDInJYLC1z_kze5jtQFgQVLZ76tRzUK_FJxqETfVw70,1885
 sinatools/ner/helpers.py,sha256=dnOoDY5JMyOLTUWVIZLMt8mBn2IbWlVaqHhQyjs1voo,2343
 sinatools/ner/metrics.py,sha256=Irz6SsIvpOzGIA2lWxrEV86xnTnm0TzKm9SUVT4SXUU,2734
+sinatools/ner/relation_extractor.py,sha256=a85xGX6V72fDpJk0GKmmtlWf8S8ezY-2pm5oGc9_ESY,9750
 sinatools/ner/transforms.py,sha256=vti3mDdi-IRP8i0aTQ37QqpPlP9hdMmJ6_bAMa0uL-s,4871
 sinatools/ner/data/__init__.py,sha256=W0C1ge_XxTfmdEGz0hkclz57aLI5VFS5t6BjByCfkFk,57
 sinatools/ner/data/datasets.py,sha256=lcdDDenFMEKIGYQmfww2dk_9WKWrJO9HtKptaAEsRmY,5064
@@ -97,7 +98,7 @@ sinatools/ner/trainers/BertTrainer.py,sha256=B_uVtUwfv_eFwMMPsKQvZgW_ZNLy6XEsX5e
 sinatools/ner/trainers/__init__.py,sha256=UDok8pDDpYOpwRBBKVLKaOgSUlmqqb-zHZI1p0xPxzI,188
 sinatools/semantic_relatedness/__init__.py,sha256=S0xrmqtl72L02N56nbNMudPoebnYQgsaIyyX-587DsU,830
 sinatools/semantic_relatedness/compute_relatedness.py,sha256=JvI0cXgukKtuMpmAygMnlocCsPeAJ98LD1jZCP_6SyQ,1110
-sinatools/synonyms/__init__.py,sha256=BN1f99w4yqnuT9PrOBsbeOMepPJPi-Fh1hEMvxqfMdM,562
+sinatools/synonyms/__init__.py,sha256=yMuphNZrm5XLOR2T0weOHcUysJm-JKHUmVLoLQO8390,548
 sinatools/synonyms/synonyms_generator.py,sha256=FgAiuduSFyM6vJobWZKHg4KNWIQz8T6MGBPVIuVuw-8,6506
 sinatools/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sinatools/utils/charsets.py,sha256=rs82oZJqRqosZdTKXfFAJfJ5t4PxjMM_oAPsiWSWuwU,2817
@@ -105,18 +106,19 @@ sinatools/utils/implication.py,sha256=MsbI6S1LNY-fCxGMxFTuaV639r3QijkkdcfH48rvY7
 sinatools/utils/jaccard.py,sha256=S7OgvaMqkN5HFgTZkKhMCNAuAnQ0LhRyXPN79jAzmKM,10113
 sinatools/utils/parser.py,sha256=CPPtCrsbxUqsjhY5C9wTOgkAs6iw0k_WvMUxLEPM1IU,6168
 sinatools/utils/readfile.py,sha256=xE4LEaCqXJIk9v37QUSSmWb-aY3UnCFUNb7uVdx3cpM,133
+sinatools/utils/similarity.py,sha256=oEdpcn3hm8v3qvNvr1USQ7eQlK0zU-7a7W090rCIFyw,5688
 sinatools/utils/text_dublication_detector.py,sha256=6yAOUtdw4TKiJkUPDDi3oK7CEoIuBDbliJ4PU7kapfo,4249
 sinatools/utils/text_transliteration.py,sha256=NQoXrxI-h0UXnvVtDA3skNJduxIy0IW26r46N4tDxGk,8766
 sinatools/utils/tokenizer.py,sha256=QHyrVqJA_On4rKxexiWR2ovq4pI1-u6iZkdhRbK9tew,6676
 sinatools/utils/tokenizers_words.py,sha256=efNfOil9qDNVJ9yynk_8sqf65PsL-xtsHG7y2SZCkjQ,656
-sinatools/wsd/__init__.py,sha256=yV-SQSCzSrjbNkciMbDCqzGZ_EESchL7rlJk56uibVI,309
+sinatools/wsd/__init__.py,sha256=mwmCUurOV42rsNRpIUP3luG0oEzeTfEx3oeDl93Oif8,306
 sinatools/wsd/disambiguator.py,sha256=43Iq7NTZsiYWGFg-NUDrQuJKO1NT9QOnfBPB10IOJNs,19828
 sinatools/wsd/settings.py,sha256=6XflVTFKD8SVySX9Wj7zYQtV26WDTcQ2-uW8-gDNHKE,747
 sinatools/wsd/wsd.py,sha256=gHIBUFXegoY1z3rRnIlK6TduhYq2BTa_dHakOjOlT4k,4434
-SinaTools-0.1.26.dist-info/AUTHORS.rst,sha256=aTWeWlIdfLi56iLJfIUAwIrmqDcgxXKLji75_Fjzjyg,174
-SinaTools-0.1.26.dist-info/LICENSE,sha256=uwsKYG4TayHXNANWdpfMN2lVW4dimxQjA_7vuCVhD70,1088
-SinaTools-0.1.26.dist-info/METADATA,sha256=jqsARSXI1Z0hT9-ev6ewzZeNH_H350lv_c2oav_SKWg,953
-SinaTools-0.1.26.dist-info/WHEEL,sha256=6T3TYZE4YFi2HTS1BeZHNXAi8N52OZT4O-dJ6-ome_4,116
-SinaTools-0.1.26.dist-info/entry_points.txt,sha256=ZwZLolnWog2fjdDrfaHNHob8SE_YtMbD6ayzsOzItxs,1234
-SinaTools-0.1.26.dist-info/top_level.txt,sha256=8tNdPTeJKw3TQCaua8IJIx6N6WpgZZmVekf1OdBNJpE,10
-SinaTools-0.1.26.dist-info/RECORD,,
+SinaTools-0.1.27.dist-info/AUTHORS.rst,sha256=aTWeWlIdfLi56iLJfIUAwIrmqDcgxXKLji75_Fjzjyg,174
+SinaTools-0.1.27.dist-info/LICENSE,sha256=uwsKYG4TayHXNANWdpfMN2lVW4dimxQjA_7vuCVhD70,1088
+SinaTools-0.1.27.dist-info/METADATA,sha256=jcBDhXCwjYOsgNkcEjQLG7l_kdWSHkEUpSsoMt7uI1c,3264
+SinaTools-0.1.27.dist-info/WHEEL,sha256=6T3TYZE4YFi2HTS1BeZHNXAi8N52OZT4O-dJ6-ome_4,116
+SinaTools-0.1.27.dist-info/entry_points.txt,sha256=ZwZLolnWog2fjdDrfaHNHob8SE_YtMbD6ayzsOzItxs,1234
+SinaTools-0.1.27.dist-info/top_level.txt,sha256=8tNdPTeJKw3TQCaua8IJIx6N6WpgZZmVekf1OdBNJpE,10
+SinaTools-0.1.27.dist-info/RECORD,,

sinatools/CLI/DataDownload/download_files.py

@@ -56,14 +56,14 @@ def main():
                 download_file(urls["ner"])
                 download_file(urls["wsd_model"])
                 download_file(urls["wsd_tokenizer"])
-                download_file(urls["glosses_dic"])
+                download_file(urls["one_gram"])
                 download_file(urls["five_grams"])
                 download_file(urls["four_grams"])
                 download_file(urls["three_grams"])
                 download_file(urls["two_grams"])
             elif file == "synonyms":
-                download_file(urls["synonyms_level2"])
-                download_file(urls["synonyms_level3"])
+                download_file(urls["graph_l2"])
+                download_file(urls["graph_l3"])
             else:
                url = urls[file]
                download_file(url)

sinatools/CLI/utils/jaccard.py

@@ -46,7 +46,7 @@ Examples:
 """
 
 import argparse
-from sinatools.utils.jaccard import jaccard
+from sinatools.utils.similarity import get_jaccard
 from sinatools.utils.readfile import read_file
 
 
@@ -76,7 +76,7 @@ def main():
         print("Either --file1 and --file2 arguments or both --set1 and --set2 arguments must be provided.")
         return
 
-    similarity = jaccard(args.delimiter, set1, set2, args.selection, args.ignoreAllDiacriticsButNotShadda, args.ignoreShaddaDiacritic)
+    similarity = get_jaccard(args.delimiter, set1, set2, args.selection, args.ignoreAllDiacriticsButNotShadda, args.ignoreShaddaDiacritic)
     
     print("Jaccard Result:", similarity)
 

sinatools/DataDownload/downloader.py

@@ -15,8 +15,8 @@ urls = {
     'four_grams':'https://sina.birzeit.edu/four_grams.pickle',
     'three_grams':'https://sina.birzeit.edu/three_grams.pickle',
     'two_grams':'https://sina.birzeit.edu/two_grams.pickle',
-    'synonyms_level2':'https://sina.birzeit.edu/synonyms_level2.pkl',
-    'synonyms_level3':'https://sina.birzeit.edu/synonyms_level3.pkl'
+    'synonyms_level2':'https://sina.birzeit.edu/graph_l2.pkl',
+    'synonyms_level3':'https://sina.birzeit.edu/graph_l3.pkl'
 }
 
 def get_appdatadir():

sinatools/VERSION

@@ -1 +1 @@
-0.1.26
+0.1.27

sinatools/ner/__init__.py

@@ -7,6 +7,8 @@ import torch
 import pickle
 import json
 from argparse import Namespace
+from transformers import pipeline
+#from transformers import AutoModelForSequenceClassification
 
 tagger = None
 tag_vocab = None
@@ -35,4 +37,7 @@ if torch.cuda.is_available():
 
 train_config.trainer_config["kwargs"]["model"] = model
 tagger = load_object(train_config.trainer_config["fn"], train_config.trainer_config["kwargs"])
-tagger.load(os.path.join(model_path,"checkpoints"))
+tagger.load(os.path.join(model_path,"checkpoints"))
+
+pipe = pipeline("sentiment-analysis", model= os.path.join(path, "best_model"), return_all_scores =True, max_length=128, truncation=True)
+#pipe = AutoModelForSequenceClassification.from_pretrained(os.path.join(path, "best_model"))

sinatools/ner/entity_extractor.py

@@ -27,7 +27,7 @@ def convert_nested_to_flat(nested_tags):
     
     return flat_tags
 
-def extract(text, ner_method):
+def extract(text, ner_method="nested"):
     
     dataset, token_vocab = text2segments(text)
 

sinatools/ner/relation_extractor.py

@@ -0,0 +1,201 @@
+import torch
+import json
+from urllib.request import Request, urlopen
+from sinatools.ner.entity_extractor import extract
+from . import pipe
+
+
+# ============================ Extract entities and their types ========================
+def jsons_to_list_of_lists(json_list):
+    return [[d['token'], d['tags']] for d in json_list]
+
+def entities_and_types(sentence):
+    output_list = jsons_to_list_of_lists(extract(sentence))
+    json_short = distill_entities(output_list)
+
+    entities = {}
+    for entity in json_short:
+        name = entity[0]
+        entity_type = entity[1]
+        entities[name] = entity_type
+
+    return entities
+
+def distill_entities(entities):
+    # This is list that we put the output what we need
+    list_output = list()
+
+    # This line go to sort function and save the output to temp_entities
+    temp_entities = sortTags(entities)
+
+    # This list help us to make the output,
+    temp_list = list()
+
+    # initlize the temp_list
+    temp_list.append(["", "", 0, 0])
+    word_position = 0
+
+    # For each entity, convert ibo to distllir list.
+    for entity in temp_entities:
+        # This is counter tag of this entity
+        counter_tag = 0
+        # For each tag
+        for tag in str(entity[1]).split():
+            # If the counter tag greater than or equal to lenght of templist, if yes then we will append the empty value in templist
+            if counter_tag >= len(temp_list):
+                temp_list.append(["", "", 0, 0])
+
+            # If tag equal O and word postion of this tag is not equal zero then it will add all
+            # not empty eliment of temp list in output list
+            if "O" == tag and word_position != 0:
+                for j in range(0, len(temp_list)):
+                    if temp_list[j][1] != "":
+                        list_output.append([temp_list[j][0].strip(), temp_list[j][1], temp_list[j][2], temp_list[j][3]])
+                        temp_list[j][0] = ""
+                        temp_list[j][1] = ""
+                        temp_list[j][2] = word_position
+                        temp_list[j][3] = word_position
+            # if this tag not equal O, and split by '-' the tag and check the lenght equals two and if the first eliment
+            # of the split its B
+            elif "O" != tag and len(tag.split("-")) == 2 and tag.split("-")[0] == "B":
+                # if the temp_list of counter is not empty then it will append in output list and hten it will
+                # initilize by new string and tag in templist of counter
+                if temp_list[counter_tag][1] != "":
+                    list_output.append([temp_list[counter_tag][0].strip(), temp_list[counter_tag][1], temp_list[counter_tag][2], temp_list[counter_tag][3]])
+                temp_list[counter_tag][0] = str(entity[0]) + " "
+                temp_list[counter_tag][1] = str(tag).split("-")[1]
+                temp_list[counter_tag][2] = word_position
+                temp_list[counter_tag][3] = word_position
+
+            # if this tag not equal O, and split by '-' the tag and check the lenght equals two and if the first eliment
+            # of the split its O
+            elif "O" != tag and len(tag.split("-")) == 2 and tag.split("-")[0] == "I" and word_position != 0:
+                # For each of temp_list, check if in this counter tag of templist is same tag with this.tag
+                # then will complete if not it will save in output list and cheak another
+                for j in range(counter_tag,len(temp_list)):
+                    if temp_list[j][1] == tag[2:] and temp_list[j][3] != word_position:
+                        temp_list[j][0] += str(entity[0]) + " "
+                        temp_list[j][3] += 1
+                        break
+                    else:
+                        if temp_list[j][1] != "":
+                            list_output.append([temp_list[j][0].strip(), temp_list[j][1], temp_list[j][2], temp_list[j][3]])
+                            temp_list[j][0] = ""
+                            temp_list[j][1] = ""
+                            temp_list[j][2] = word_position
+                            temp_list[j][3] = word_position
+            counter_tag += 1
+        word_position += 1
+    # For each temp_list, at the end of the previous loop, there will be some
+    # values in this list, we should save it to the output list
+    for j in range(0, len(temp_list)):
+        if temp_list[j][1] != "":
+            list_output.append([temp_list[j][0].strip(), temp_list[j][1], temp_list[j][2], temp_list[j][3]])
+    return sorted(list_output, key=lambda x: (x[2]))
+
+def sortTags(entities):
+    temp_entities = entities
+    temp_counter = 0
+    # For each entity, this loop will sort each tag of entitiy, first it will check if the
+    # previous tags has same count of this tag, second will sort the tags and check if this tags is correct
+    for entity in temp_entities:
+        tags = entity[1].split()
+        for tag in tags:
+            # if the counter is not 0 then, will complete
+            if temp_counter != 0:
+                # Check if this tag is equal I-, if yes then it will count how many tag in this tags and
+                # count how many tag in previous tags
+                if "I-" == tag[0:2]:
+                    counter_of_this_tag = 0
+                    counter_of_previous_tag = 0
+                    for word in tags:
+                        if tag.split("-")[1] in word:
+                            counter_of_this_tag+=1
+                    for word in temp_entities[temp_counter-1][1].split():
+                        if tag.split("-")[1] in word:
+                            counter_of_previous_tag+=1
+                    # if the counter of previous tag is bigger than counter of this tag, then we
+                    # need to add I-tag in this tags
+                    if counter_of_previous_tag > counter_of_this_tag:
+                        tags.append("I-"+tag.split("-")[1])
+        # Sort the tags
+        tags.sort()
+        # Need to revers the tags because it should begins with I
+        tags.reverse()
+        # If the counter is not 0 then we can complete
+        if temp_counter != 0:
+            this_tags = tags
+            previous_tags = temp_entities[temp_counter - 1][1].split()
+            sorted_tags = list()
+
+            # Check if the this tag is not O and previous tags is not O, then will complete,
+            # if not then it will ignor this tag
+            if "O" not in this_tags and "O" not in previous_tags:
+                index = 0
+                #For each previous tags, need sort this tag by previous tags if its I, B we can ignor
+                for i in previous_tags:
+                    j = 0
+                    while this_tags and j < len(this_tags):
+                        if this_tags[j][0:2] == "I-" and this_tags[j][2:] == i[2:]:
+                            sorted_tags.insert(index, this_tags.pop(j))
+                            break
+                        elif this_tags[j][0:2] == "B-":
+                            break
+                        j += 1
+                    index += 1
+            sorted_tags += this_tags
+            tags = sorted_tags
+        str_tag = " "
+        str_tag = str_tag.join(tags)
+        str_tag = str_tag.strip()
+        temp_entities[temp_counter][1] = str_tag
+        temp_counter += 1
+    return temp_entities
+
+# ============= Prepare Templates and Catergorize Extracted Entities ================
+temp03={'location':'مكان حدوث','agent':'أحد المتأثرين في','happened at':'تاريخ حدوث'}
+categories = {
+    'agent': ['PERS', 'NORP', 'OCC', 'ORG'],
+    'location': ['LOC', 'FAC', 'GPE'],
+    'happened at': ['DATE', 'TIME']
+    }
+
+def get_entity_category(entity_type, categories):
+    for category, types in categories.items():
+        if entity_type in types:
+            return category
+    return None
+
+
+# ============ Extract entities, their types and categorize them ===============
+def relation_extraction(sentence):
+    #test_sentence="صورة إعتقال طفل فلسطيني خلال انتفاضة الأقصى ."
+    entities=entities_and_types(sentence)
+
+    event_indices = [i for i, (_, entity_type) in enumerate(entities.items()) if entity_type == 'EVENT']
+    arg_event_indices = [i for i, (_, entity_type) in enumerate(entities.items()) if entity_type != 'EVENT']
+
+    output_list=[]
+
+    for i in event_indices:
+        event_entity=list(entities.keys())[i]
+        for j in arg_event_indices:
+            arg_name= list(entities.keys())[j]
+            arg_type=entities[arg_name]
+            category = get_entity_category(arg_type, categories)
+            
+            if category in temp03:
+                relation_sentence=f"[CLS] {sentence} [SEP] {event_entity} {temp03[category]} {arg_name}"
+                predicted_relation=pipe(relation_sentence)
+                score = predicted_relation[0][0]['score']  
+                if score > 0.50:
+                    #print(f"Event:{event_entity} Relation:{category} Argument:{arg_name}\n")
+                    #output_list.append([{event_entity} ,{category}, {arg_name}])
+                    output_list.append(f"Event:{event_entity}, Relation:{category}, Argument:{arg_name}")
+
+                else:
+                    #print(f"Event:{event_entity} Relation:No relation Argument:{arg_name}\n")
+                    #output_list.append([{event_entity} ,'No relation', {arg_name}])
+                    output_list.append(f"Event:{event_entity}, Relation:No relation, Argument:{arg_name}")
+    
+    return output_list

sinatools/synonyms/__init__.py

@@ -3,7 +3,7 @@ from sinatools.DataDownload import downloader
 import os 
 
 synonyms_level2_dict = {} 
-level2_dict = 'synonyms_level2.pkl'
+level2_dict = 'graph_l2.pkl'
 path = downloader.get_appdatadir()
 file_path = os.path.join(path, level2_dict)
 with open(file_path, 'rb') as f:
@@ -11,7 +11,7 @@ with open(file_path, 'rb') as f:
 
 
 synonyms_level3_dict = {}    
-level3_dict = 'synonyms_level3.pkl'
+level3_dict = 'graph_l3.pkl'
 path = downloader.get_appdatadir()
 file_path = os.path.join(path, level3_dict)
 with open(file_path, 'rb') as f:

sinatools/utils/similarity.py

@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+
+from sinatools.utils.parser import arStrip
+from sinatools.utils.implication import Implication
+import argparse
+
+def normalize_word(word: str, ignore_all_diacritics_but_not_shadda: bool=True, ignore_shadda_diacritic: bool=True) -> str:
+    if ignore_all_diacritics_but_not_shadda:
+        word = arStrip(word, True, True, False, False, False, False)
+        
+    if ignore_shadda_diacritic:
+        word = arStrip(word, False, False, True, False, False, False)
+    
+    return word
+
+    
+def get_preferred_word(word1, word2):
+    implication = Implication(word1, word2)
+    
+    direction = implication.get_direction()
+    
+    if direction in (0, 2):
+        return word1
+       
+    elif direction == 1:
+        return word2
+       
+    elif direction == 3:
+        if not word1.endswith("َ") and not word1.endswith("ُ"):
+            return word2
+        return word1
+     
+    
+def get_non_preferred_word(word1, word2):
+
+    implication = Implication(word1, word2)
+    if implication.get_distance() < 15:
+        direction = implication.get_direction()
+        if direction == 0 or direction == 1:
+            return word1
+        elif direction == 2:
+            return word2
+        elif direction == 3:
+            if not word1.endswith("َ") and not word1.endswith("ُ"):
+                return word1
+            return word2
+    return "#"
+
+def get_intersection(list1, list2, ignore_all_diacritics_but_not_shadda=False, ignore_shadda_diacritic=False):
+
+    list1 = [str(i) for i in list1 if i not in (None, ' ', '')]
+    list1 = [str(i.strip()) for i in list1]
+
+    list2 = [str(i) for i in list2 if i not in (None, ' ', '')]
+    list2 = [str(i.strip()) for i in list2]
+
+    interection_list = []
+
+    for list1_word in list1:
+        for list2_word in list2:
+            word1 = normalize_word(list1_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+            word2 = normalize_word(list2_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+
+            implication = Implication(word1, word2)
+            if implication.get_direction() >= 0 and implication.get_distance() < 15:
+                interection_list.append(get_preferred_word(word1, word2))
+
+    i = 0
+    while i < len(interection_list):
+        j = i + 1
+        while j < len(interection_list):
+            non_preferred_word = get_non_preferred_word(interection_list[i], interection_list[j])
+            if non_preferred_word != "#":
+                interection_list.remove(non_preferred_word)
+            j += 1
+        i += 1
+
+    return interection_list
+             
+
+
+def get_union(list1, list2, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic):
+
+    list1 = [str(i) for i in list1 if i not in (None, ' ', '')]
+
+    list2 = [str(i) for i in list2 if i not in (None, ' ', '')]
+
+    union_list = []
+
+    for list1_word in list1:
+        word1 = normalize_word(list1_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+        union_list.append(word1)
+
+    for list2_word in list2:
+        word2 = normalize_word(list2_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+        union_list.append(word2)
+
+    i = 0
+    while i < len(union_list):
+        j = i + 1
+        while j < len(union_list):
+            non_preferred_word = get_non_preferred_word(union_list[i], union_list[j])
+            if (non_preferred_word != "#"):
+                union_list.remove(non_preferred_word)
+            j = j + 1
+        i = i + 1
+
+    return union_list
+      
+
+
+def get_jaccard_similarity(list1: list, list2: list, ignore_all_diacritics_but_not_shadda: bool, ignore_shadda_diacritic: bool) -> float:
+    
+    intersection_list = get_intersection(list1, list2, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+    
+    union_list = get_union(list1, list2, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+    
+    return float(len(intersection_list)) / float(len(union_list))
+
+def get_jaccard(delimiter, str1, str2, selection, ignoreAllDiacriticsButNotShadda=True, ignoreShaddaDiacritic=True):
+
+    try:
+        list1 = str1.split(delimiter)
+        list2 = str2.split(delimiter)
+
+        if selection == "intersection":
+            intersection = get_intersection(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            return intersection
+        elif selection == "union":
+            union = get_union(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            return union
+        elif selection == "jaccardSimilarity":      
+            similarity = get_jaccard_similarity(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            return similarity
+        elif selection == "jaccardAll":    
+            intersection = get_intersection(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            union = get_union(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            similarity = get_jaccard_similarity(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            output_list = ["intersection:", intersection, "union:", union, "similarity:", similarity]
+            return output_list
+        else:
+            return 'Invalid selection option'
+
+    except AttributeError as ae:
+        print(f"Attribute error occurred: {str(ae)}")
+        return 'Invalid input type'
+    except Exception as e:
+        print(f"Error occurred: {str(e)}")
+        return 'An error has occurred'

sinatools/wsd/__init__.py

@@ -4,7 +4,7 @@ from sinatools.DataDownload import downloader
 import os 
 
 glosses_dic = {}
-filename = 'glosses_dic.pickle'
+filename = 'one_gram.pickle'
 path =downloader.get_appdatadir()
 file_path = os.path.join(path, filename)
 with open(file_path, 'rb') as f:

SinaTools-0.1.26.dist-info/METADATA

@@ -1,34 +0,0 @@
-Metadata-Version: 2.1
-Name: SinaTools
-Version: 0.1.26
-Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
-Home-page: https://github.com/SinaLab/sinatools
-License: MIT license
-Keywords: sinatools
-Platform: UNKNOWN
-Description-Content-Type: text/markdown
-Requires-Dist: six
-Requires-Dist: farasapy
-Requires-Dist: tqdm
-Requires-Dist: requests
-Requires-Dist: regex
-Requires-Dist: pathlib
-Requires-Dist: torch (==1.13.0)
-Requires-Dist: transformers (==4.24.0)
-Requires-Dist: torchtext (==0.14.0)
-Requires-Dist: torchvision (==0.14.0)
-Requires-Dist: seqeval (==1.2.2)
-Requires-Dist: natsort (==7.1.1)
-
-SinaTools
----------
-
-Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
-
-Python APIs, command lines, colabs, and online demos.
-
-* Free software: MIT license
-* Documentation: https://sina.birzeit.edu/sinatools/
-
-
-