SinaTools 0.1.25__py2.py3-none-any.whl → 0.1.27__py2.py3-none-any.whl

SinaTools-0.1.27.dist-info/METADATA

@@ -0,0 +1,64 @@
+Metadata-Version: 2.1
+Name: SinaTools
+Version: 0.1.27
+Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
+Home-page: https://github.com/SinaLab/sinatools
+License: MIT license
+Keywords: sinatools
+Platform: UNKNOWN
+Description-Content-Type: text/markdown
+Requires-Dist: six
+Requires-Dist: farasapy
+Requires-Dist: tqdm
+Requires-Dist: requests
+Requires-Dist: regex
+Requires-Dist: pathlib
+Requires-Dist: torch (==1.13.0)
+Requires-Dist: transformers (==4.24.0)
+Requires-Dist: torchtext (==0.14.0)
+Requires-Dist: torchvision (==0.14.0)
+Requires-Dist: seqeval (==1.2.2)
+Requires-Dist: natsort (==7.1.1)
+
+SinaTools
+======================
+Open Source Toolkit for Arabic NLP and NLU developed by [SinaLab](http://sina.birzeit.edu/) at Birzeit University. SinaTools is available through Python APIs, command lines, colabs, and online demos.
+
+See the full list of [Available Packages](https://sina.birzeit.edu/sinatools/), which include: (1) [Morphology Tagging](https://sina.birzeit.edu/sinatools/index.html#morph), (2) [Named Entity Recognition (NER)](https://sina.birzeit.edu/sinatools/index.html#ner), (3) [Word Sense Disambiguation (WSD)](https://sina.birzeit.edu/sinatools/index.html#wsd), (4) [Semantic Relatedness](https://sina.birzeit.edu/sinatools/index.html#sr), (5) [Synonymy Extraction and Evaluation](https://sina.birzeit.edu/sinatools/index.html#se), (6) [Relation Extraction](https://sina.birzeit.edu/sinatools/index.html), (7) [Utilities](https://sina.birzeit.edu/sinatools/index.html#u) (diacritic-based word matching, Jaccard similarly, parser, tokenizers, corpora processing, transliteration, etc).
+
+See [Demo Pages](https://sina.birzeit.edu/sinatools/).
+
+See the [benchmarking](https://www.jarrar.info/publications/HJK24.pdf), which shows that SinaTools outperformed all related toolkits. 
+
+Installation 
+--------
+To install SinaTools, ensure you are using Python version 3.10.8, then clone the [GitHub](git://github.com/SinaLab/SinaTools) repository.
+
+Alternatively, you can execute the following command:
+
+```bash
+pip install sinatools
+```
+
+Installing Models and Data Files
+--------
+Some modules in SinaTools require some data files and fine-tuned models to be downloaded. To download these models, please consult the [DataDownload](https://sina.birzeit.edu/sinatools/documentation/cli_tools/DataDownload/DataDownload.html).
+
+Documentation
+--------
+For information, please refer to the [main page](https://sina.birzeit.edu/sinatools) or the [online domuementation](https://sina.birzeit.edu/sinatools/documentation). 
+
+Citation
+-------
+Tymaa Hammouda, Mustafa Jarrar, Mohammed Khalilia: [SinaTools: Open Source Toolkit for Arabic Natural Language Understanding](http://www.jarrar.info/publications/HJK24.pdf). In Proceedings of the 2024 AI in Computational Linguistics (ACLing 2024), Procedia Computer Science, Dubai. ELSEVIER.
+
+License
+--------
+SinaTools is available under the MIT License. See the [LICENSE](https://github.com/SinaLab/sinatools/blob/main/LICENSE) file for more information.
+
+Reporting Issues
+--------
+To report any issues or bugs, please contact us at "sina.institute.bzu@gmail.com" or visit [SinaTools Issues](https://github.com/SinaLab/sinatools/issues).
+
+
+

{SinaTools-0.1.25.dist-info → SinaTools-0.1.27.dist-info}/RECORD

@@ -1,26 +1,26 @@
-SinaTools-0.1.25.data/data/sinatools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
-sinatools/VERSION,sha256=ZC9GuKFeLfVc4onMU4koq0Nk_oc7Tx3zDdE0H4bcUKU,6
+SinaTools-0.1.27.data/data/sinatools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
+sinatools/VERSION,sha256=W9ZpgLhiag6deRMB5DzdHO6EDLvNbZPVIRIUv6nHT3U,6
 sinatools/__init__.py,sha256=bEosTU1o-FSpyytS6iVP_82BXHF2yHnzpJxPLYRbeII,135
 sinatools/environment.yml,sha256=OzilhLjZbo_3nU93EQNUFX-6G5O3newiSWrwxvMH2Os,7231
 sinatools/install_env.py,sha256=EODeeE0ZzfM_rz33_JSIruX03Nc4ghyVOM5BHVhsZaQ,404
 sinatools/sinatools.py,sha256=vR5AaF0iel21LvsdcqwheoBz0SIj9K9I_Ub8M8oA98Y,20
-sinatools/CLI/DataDownload/download_files.py,sha256=KG9W-Y5kJG_9yLUyo-cA33B5uO3avdZ5sSYUeW3wM6s,1960
+sinatools/CLI/DataDownload/download_files.py,sha256=2jLohYd5Vjtz7gVd8zePhK6A_r0svpoV3ykbO5rwR8s,2613
 sinatools/CLI/morphology/ALMA_multi_word.py,sha256=ZImJ1vtcpSHydI1BjJmK3KcMJbGBZX16kO4L6rxvBvA,2086
 sinatools/CLI/morphology/morph_analyzer.py,sha256=ieIM47QK9Nct3MtCS9uq3h2rZN5r4qNhsLmlVeE6wiE,3503
-sinatools/CLI/ner/corpus_entity_extractor.py,sha256=_o0frMSgpsFVXPoztS3mQTK7LjHsgzUv9gfs6iJL424,4024
-sinatools/CLI/ner/entity_extractor.py,sha256=zn0Jd37BEDE1wHE5HOAK0_N2tURAznFNj7WDd6WGLIw,2932
+sinatools/CLI/ner/corpus_entity_extractor.py,sha256=Da-DHFrqT6if7w6WnodB4TBE5ze3DJYjb2Mmju_Qd7g,4034
+sinatools/CLI/ner/entity_extractor.py,sha256=IiTioe0px0aJ1E58FrDVa2yNgM8Ie4uS2LZKK_z2Qn4,2942
 sinatools/CLI/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sinatools/CLI/utils/arStrip.py,sha256=NLyp8vOu2xv80tL9jiKRvyptmbkRZVg-wcAr-9YyvNY,3264
 sinatools/CLI/utils/corpus_tokenizer.py,sha256=nH0T4h6urr_0Qy6-wN3PquOtnwybj0REde5Ts_OE4U8,1650
 sinatools/CLI/utils/implication.py,sha256=nvoiI5UHHaJdd6MICql0pB_-h3L0icYwP1WgJi2h7p0,2854
-sinatools/CLI/utils/jaccard.py,sha256=NoKbWAq6dHDtQ56mAc1kdAnROm8NXEjZ1ecVZ7EYm6Y,4205
+sinatools/CLI/utils/jaccard.py,sha256=w56N_cNEFJ0A7WtunmY_xtms4srFagKBzrW_0YhH2DE,4216
 sinatools/CLI/utils/remove_latin.py,sha256=NOaTm2RHxt5IQrV98ySTmD8rTXTmcqSmfbPAwTyaXqU,848
 sinatools/CLI/utils/remove_punctuation.py,sha256=vJAZlEn7WGftZAFVFYnddkRrxdJ_rMmKB9vFZkY-jN4,1097
 sinatools/CLI/utils/sentence_tokenizer.py,sha256=Wli8eiDbWSd_Z8UKpu_JkaS8jImowa1vnRL0oYCSfqw,2823
 sinatools/CLI/utils/text_dublication_detector.py,sha256=dW70O5O20GxeUDDF6zVYn52wWLmJF-HBZgvqIeVL2rQ,1661
 sinatools/CLI/utils/text_transliteration.py,sha256=vz-3kxWf8pNYVCqNAtBAiA6u_efrS5NtWT-ofN1NX6I,2014
 sinatools/DataDownload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sinatools/DataDownload/downloader.py,sha256=F-SV-0mbYMYFSNCx8FoAYXhn0X1j0dF37PTLU0nUBVg,6482
+sinatools/DataDownload/downloader.py,sha256=6xH55WlDhgtImPRFQ0AaeDFJjL8OMNU29x61PL8mZ2w,6468
 sinatools/arabert/__init__.py,sha256=ely2PttjgSv7vKdzskuD1rtK_l_UOpmxJSz8isrveD0,16
 sinatools/arabert/preprocess.py,sha256=qI0FsuMTOzdRlYGCtLrjpXgikNElUZPv9bnjaKDZKJ4,33024
 sinatools/arabert/arabert/__init__.py,sha256=KbSAH-XqbRygn0y59m5-ZYOLXgpT1gSgE3F-qd4rKEc,627
@@ -76,13 +76,14 @@ sinatools/arabert/aragpt2/grover/utils.py,sha256=V5wMUxK03r5g_pb7R3_uGLOPqQJfbIB
 sinatools/morphology/ALMA_multi_word.py,sha256=hj_-8ojrYYHnfCGk8WKtJdUR8mauzQdma4WUm-okDps,1346
 sinatools/morphology/__init__.py,sha256=I4wVBh8BhyNl-CySVdiI_nUSn6gj1j-gmLKP300RpE0,1216
 sinatools/morphology/morph_analyzer.py,sha256=tA78gWg6iaE_G1c2xqxZoXZWNbvHBJLrTSxPyir5Xn8,6941
-sinatools/ner/__init__.py,sha256=gSs0x6veWJ8j3_iOs79tynBd_hJP0t44CGpJ0xzoiW4,1048
+sinatools/ner/__init__.py,sha256=CLPaqUcvPGAA4lU-6hjAqjNfKJ5WtwRfsma6QkYZHEk,1379
 sinatools/ner/data.py,sha256=lvOW86dXse8SC75Q0supQaE0rrRffoxNjIA0Qbv5WZY,4354
 sinatools/ner/data_format.py,sha256=7Yt0aOicOn9_YuuyCkM_IYi_rgjGYxR9bCuUaNGM73o,4341
 sinatools/ner/datasets.py,sha256=mG1iwqSm3lXCFHLqE-b4wNi176cpuzNBz8tKaBU6z6M,5059
-sinatools/ner/entity_extractor.py,sha256=k0Yvvg_aknINkFSdqOgG1KulS0UIo-W0qycv9J2MtNo,2273
+sinatools/ner/entity_extractor.py,sha256=bDInJYLC1z_kze5jtQFgQVLZ76tRzUK_FJxqETfVw70,1885
 sinatools/ner/helpers.py,sha256=dnOoDY5JMyOLTUWVIZLMt8mBn2IbWlVaqHhQyjs1voo,2343
 sinatools/ner/metrics.py,sha256=Irz6SsIvpOzGIA2lWxrEV86xnTnm0TzKm9SUVT4SXUU,2734
+sinatools/ner/relation_extractor.py,sha256=a85xGX6V72fDpJk0GKmmtlWf8S8ezY-2pm5oGc9_ESY,9750
 sinatools/ner/transforms.py,sha256=vti3mDdi-IRP8i0aTQ37QqpPlP9hdMmJ6_bAMa0uL-s,4871
 sinatools/ner/data/__init__.py,sha256=W0C1ge_XxTfmdEGz0hkclz57aLI5VFS5t6BjByCfkFk,57
 sinatools/ner/data/datasets.py,sha256=lcdDDenFMEKIGYQmfww2dk_9WKWrJO9HtKptaAEsRmY,5064
@@ -91,13 +92,13 @@ sinatools/ner/nn/BaseModel.py,sha256=3GmujQasTZZunOBuFXpY2p1W8W256iI_Uu4hxhOY2Z0
 sinatools/ner/nn/BertNestedTagger.py,sha256=_fwAn1kiKmXe6m5y16Ipty3kvXIEFEmiUq74Ad1818U,1219
 sinatools/ner/nn/BertSeqTagger.py,sha256=dFcBBiMw2QCWsyy7aQDe_PS3aRuNn4DOxKIHgTblFvc,504
 sinatools/ner/nn/__init__.py,sha256=UgQD_XLNzQGBNSYc_Bw1aRJZjq4PJsnMT1iZwnJemqE,170
-sinatools/ner/trainers/BaseTrainer.py,sha256=oZgFJW-CawfCKT5gtaBHA7Q7XjNfiyqM62KnFsgVzPU,3919
+sinatools/ner/trainers/BaseTrainer.py,sha256=Ifz4SeTxJwVn1_uWZ3I9KbcSo2hLPN3ojsIYuoKE9wE,4050
 sinatools/ner/trainers/BertNestedTrainer.py,sha256=Pb4O2WeBmTvV3hHMT6DXjxrTzgtuh3OrKQZnogYy8RQ,8429
 sinatools/ner/trainers/BertTrainer.py,sha256=B_uVtUwfv_eFwMMPsKQvZgW_ZNLy6XEsX5ePR0s8d-k,6433
 sinatools/ner/trainers/__init__.py,sha256=UDok8pDDpYOpwRBBKVLKaOgSUlmqqb-zHZI1p0xPxzI,188
 sinatools/semantic_relatedness/__init__.py,sha256=S0xrmqtl72L02N56nbNMudPoebnYQgsaIyyX-587DsU,830
 sinatools/semantic_relatedness/compute_relatedness.py,sha256=JvI0cXgukKtuMpmAygMnlocCsPeAJ98LD1jZCP_6SyQ,1110
-sinatools/synonyms/__init__.py,sha256=BN1f99w4yqnuT9PrOBsbeOMepPJPi-Fh1hEMvxqfMdM,562
+sinatools/synonyms/__init__.py,sha256=yMuphNZrm5XLOR2T0weOHcUysJm-JKHUmVLoLQO8390,548
 sinatools/synonyms/synonyms_generator.py,sha256=FgAiuduSFyM6vJobWZKHg4KNWIQz8T6MGBPVIuVuw-8,6506
 sinatools/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sinatools/utils/charsets.py,sha256=rs82oZJqRqosZdTKXfFAJfJ5t4PxjMM_oAPsiWSWuwU,2817
@@ -105,18 +106,19 @@ sinatools/utils/implication.py,sha256=MsbI6S1LNY-fCxGMxFTuaV639r3QijkkdcfH48rvY7
 sinatools/utils/jaccard.py,sha256=S7OgvaMqkN5HFgTZkKhMCNAuAnQ0LhRyXPN79jAzmKM,10113
 sinatools/utils/parser.py,sha256=CPPtCrsbxUqsjhY5C9wTOgkAs6iw0k_WvMUxLEPM1IU,6168
 sinatools/utils/readfile.py,sha256=xE4LEaCqXJIk9v37QUSSmWb-aY3UnCFUNb7uVdx3cpM,133
+sinatools/utils/similarity.py,sha256=oEdpcn3hm8v3qvNvr1USQ7eQlK0zU-7a7W090rCIFyw,5688
 sinatools/utils/text_dublication_detector.py,sha256=6yAOUtdw4TKiJkUPDDi3oK7CEoIuBDbliJ4PU7kapfo,4249
 sinatools/utils/text_transliteration.py,sha256=NQoXrxI-h0UXnvVtDA3skNJduxIy0IW26r46N4tDxGk,8766
 sinatools/utils/tokenizer.py,sha256=QHyrVqJA_On4rKxexiWR2ovq4pI1-u6iZkdhRbK9tew,6676
 sinatools/utils/tokenizers_words.py,sha256=efNfOil9qDNVJ9yynk_8sqf65PsL-xtsHG7y2SZCkjQ,656
-sinatools/wsd/__init__.py,sha256=yV-SQSCzSrjbNkciMbDCqzGZ_EESchL7rlJk56uibVI,309
-sinatools/wsd/disambiguator.py,sha256=8HrVAGpEQyrzwiuEreLX9X82WSL-U2Aeca0ttrtIw2Y,19998
+sinatools/wsd/__init__.py,sha256=mwmCUurOV42rsNRpIUP3luG0oEzeTfEx3oeDl93Oif8,306
+sinatools/wsd/disambiguator.py,sha256=43Iq7NTZsiYWGFg-NUDrQuJKO1NT9QOnfBPB10IOJNs,19828
 sinatools/wsd/settings.py,sha256=6XflVTFKD8SVySX9Wj7zYQtV26WDTcQ2-uW8-gDNHKE,747
 sinatools/wsd/wsd.py,sha256=gHIBUFXegoY1z3rRnIlK6TduhYq2BTa_dHakOjOlT4k,4434
-SinaTools-0.1.25.dist-info/AUTHORS.rst,sha256=aTWeWlIdfLi56iLJfIUAwIrmqDcgxXKLji75_Fjzjyg,174
-SinaTools-0.1.25.dist-info/LICENSE,sha256=uwsKYG4TayHXNANWdpfMN2lVW4dimxQjA_7vuCVhD70,1088
-SinaTools-0.1.25.dist-info/METADATA,sha256=0tDl5B000Z0gvhNCSzdlOzfyC0YQrP2uSZSm4OEP_EQ,953
-SinaTools-0.1.25.dist-info/WHEEL,sha256=6T3TYZE4YFi2HTS1BeZHNXAi8N52OZT4O-dJ6-ome_4,116
-SinaTools-0.1.25.dist-info/entry_points.txt,sha256=ZwZLolnWog2fjdDrfaHNHob8SE_YtMbD6ayzsOzItxs,1234
-SinaTools-0.1.25.dist-info/top_level.txt,sha256=8tNdPTeJKw3TQCaua8IJIx6N6WpgZZmVekf1OdBNJpE,10
-SinaTools-0.1.25.dist-info/RECORD,,
+SinaTools-0.1.27.dist-info/AUTHORS.rst,sha256=aTWeWlIdfLi56iLJfIUAwIrmqDcgxXKLji75_Fjzjyg,174
+SinaTools-0.1.27.dist-info/LICENSE,sha256=uwsKYG4TayHXNANWdpfMN2lVW4dimxQjA_7vuCVhD70,1088
+SinaTools-0.1.27.dist-info/METADATA,sha256=jcBDhXCwjYOsgNkcEjQLG7l_kdWSHkEUpSsoMt7uI1c,3264
+SinaTools-0.1.27.dist-info/WHEEL,sha256=6T3TYZE4YFi2HTS1BeZHNXAi8N52OZT4O-dJ6-ome_4,116
+SinaTools-0.1.27.dist-info/entry_points.txt,sha256=ZwZLolnWog2fjdDrfaHNHob8SE_YtMbD6ayzsOzItxs,1234
+SinaTools-0.1.27.dist-info/top_level.txt,sha256=8tNdPTeJKw3TQCaua8IJIx6N6WpgZZmVekf1OdBNJpE,10
+SinaTools-0.1.27.dist-info/RECORD,,

sinatools/CLI/DataDownload/download_files.py

@@ -40,7 +40,7 @@ from sinatools.DataDownload.downloader import urls
 
 def main():
     parser = argparse.ArgumentParser(description="Download files from specified URLs.")
-    parser.add_argument('-f', '--files', nargs="*", choices=urls.keys(),
+    parser.add_argument('-f', '--files', nargs="*",
                         help="Names of the files to download. Available files are: "
                              f"{', '.join(urls.keys())}. If no file is specified, all files will be downloaded.")
     
@@ -50,8 +50,23 @@ def main():
 
     if args.files:
         for file in args.files:
-            url = urls[file]
-            download_file(url)
+            print("file: ", file)
+            if file == "wsd":
+                download_file(urls["morph"])
+                download_file(urls["ner"])
+                download_file(urls["wsd_model"])
+                download_file(urls["wsd_tokenizer"])
+                download_file(urls["one_gram"])
+                download_file(urls["five_grams"])
+                download_file(urls["four_grams"])
+                download_file(urls["three_grams"])
+                download_file(urls["two_grams"])
+            elif file == "synonyms":
+                download_file(urls["graph_l2"])
+                download_file(urls["graph_l3"])
+            else:
+               url = urls[file]
+               download_file(url)
     else:
         download_files()
 

sinatools/CLI/ner/corpus_entity_extractor.py

@@ -20,7 +20,7 @@ def jsons_to_list_of_lists(json_list):
     return [[d['token'], d['tags']] for d in json_list]
 
 def combine_tags(sentence):
-    output = jsons_to_list_of_lists(extract(sentence))
+    output = jsons_to_list_of_lists(extract(sentence, "nested"))
     return [word[1] for word in output]
 
 

sinatools/CLI/ner/entity_extractor.py

@@ -46,7 +46,7 @@ def jsons_to_list_of_lists(json_list):
     return [[d['token'], d['tags']] for d in json_list]
 
 def combine_tags(sentence):
-    output = jsons_to_list_of_lists(extract(sentence))
+    output = jsons_to_list_of_lists(extract(sentence, "nested"))
     return [word[1] for word in output]
 
 

sinatools/CLI/utils/jaccard.py

@@ -46,7 +46,7 @@ Examples:
 """
 
 import argparse
-from sinatools.utils.jaccard import jaccard
+from sinatools.utils.similarity import get_jaccard
 from sinatools.utils.readfile import read_file
 
 
@@ -76,7 +76,7 @@ def main():
         print("Either --file1 and --file2 arguments or both --set1 and --set2 arguments must be provided.")
         return
 
-    similarity = jaccard(args.delimiter, set1, set2, args.selection, args.ignoreAllDiacriticsButNotShadda, args.ignoreShaddaDiacritic)
+    similarity = get_jaccard(args.delimiter, set1, set2, args.selection, args.ignoreAllDiacriticsButNotShadda, args.ignoreShaddaDiacritic)
     
     print("Jaccard Result:", similarity)
 

sinatools/DataDownload/downloader.py

@@ -15,8 +15,8 @@ urls = {
     'four_grams':'https://sina.birzeit.edu/four_grams.pickle',
     'three_grams':'https://sina.birzeit.edu/three_grams.pickle',
     'two_grams':'https://sina.birzeit.edu/two_grams.pickle',
-    'synonyms_level2':'https://sina.birzeit.edu/synonyms_level2.pkl',
-    'synonyms_level3':'https://sina.birzeit.edu/synonyms_level3.pkl'
+    'synonyms_level2':'https://sina.birzeit.edu/graph_l2.pkl',
+    'synonyms_level3':'https://sina.birzeit.edu/graph_l3.pkl'
 }
 
 def get_appdatadir():

sinatools/VERSION

@@ -1 +1 @@
-0.1.25
+0.1.27

sinatools/ner/__init__.py

@@ -7,6 +7,8 @@ import torch
 import pickle
 import json
 from argparse import Namespace
+from transformers import pipeline
+#from transformers import AutoModelForSequenceClassification
 
 tagger = None
 tag_vocab = None
@@ -35,4 +37,7 @@ if torch.cuda.is_available():
 
 train_config.trainer_config["kwargs"]["model"] = model
 tagger = load_object(train_config.trainer_config["fn"], train_config.trainer_config["kwargs"])
-tagger.load(os.path.join(model_path,"checkpoints"))
+tagger.load(os.path.join(model_path,"checkpoints"))
+
+pipe = pipeline("sentiment-analysis", model= os.path.join(path, "best_model"), return_all_scores =True, max_length=128, truncation=True)
+#pipe = AutoModelForSequenceClassification.from_pretrained(os.path.join(path, "best_model"))

sinatools/ner/entity_extractor.py

@@ -3,43 +3,31 @@ from collections import namedtuple
 from sinatools.ner.data_format import get_dataloaders, text2segments
 from . import tagger, tag_vocab, train_config
 
-def extract(text, batch_size=32):
-    """
-    This method processes an input text and returns named entites for each token within the text, based on the specified batch size. As follows:
 
-    Args:
-        text (:obj:`str`): The Arabic text to be tagged.
-        batch_size (int, optional): Batch size for inference. Default is 32.
-
-    Returns:
-        list (:obj:`list`): A list of JSON objects, where each JSON could be contains:
-        token: The token from the original text.
-        NER tag: The label pairs for each segment.
-
-    **Example:**
-
-     .. highlight:: python
-     .. code-block:: python
+def convert_nested_to_flat(nested_tags):
+    flat_tags = []
+    
+    for entry in nested_tags:
+        word = entry['token']
+        tags = entry['tags'].split()
+        
+        # Initialize with the first tag in the sequence
+        flat_tag = tags[0]
+        
+        for tag in tags[1:]:
+            # Check if the tag is an "I-" tag, indicating continuation of an entity
+            if tag.startswith('I-'):
+                flat_tag = tag
+                break
+        
+        flat_tags.append({
+            'token': word,
+            'tags': flat_tag
+        })
+    
+    return flat_tags
 
-        from sinatools.ner.entity_extractor import extract
-        extract('ذهب محمد إلى جامعة بيرزيت')
-        [{
-            "word":"ذهب",
-            "tags":"O"
-          },{
-            "word":"محمد",
-            "tags":"B-PERS"
-          },{
-            "word":"إلى",
-            "tags":"O"
-          },{
-            "word":"جامعة",
-            "tags":"B-ORG"
-          },{
-            "word":"بيرزيت",
-            "tags":"B-GPE I-ORG"
-        }]
-    """    
+def extract(text, ner_method="nested"):
     
     dataset, token_vocab = text2segments(text)
 
@@ -50,7 +38,7 @@ def extract(text, batch_size=32):
         (dataset,),
         vocab,
         train_config.data_config,
-        batch_size=batch_size,
+        batch_size=32,
         shuffle=(False,),
     )[0]
 
@@ -69,4 +57,7 @@ def extract(text, batch_size=32):
             else:
                segments_list["tags"] = ' '.join(list_of_tags)
             segments_lists.append(segments_list)  
+    
+    if ner_method == "flat":
+      segments_lists = convert_nested_to_flat(segments_lists)          
     return segments_lists

sinatools/ner/relation_extractor.py

@@ -0,0 +1,201 @@
+import torch
+import json
+from urllib.request import Request, urlopen
+from sinatools.ner.entity_extractor import extract
+from . import pipe
+
+
+# ============================ Extract entities and their types ========================
+def jsons_to_list_of_lists(json_list):
+    return [[d['token'], d['tags']] for d in json_list]
+
+def entities_and_types(sentence):
+    output_list = jsons_to_list_of_lists(extract(sentence))
+    json_short = distill_entities(output_list)
+
+    entities = {}
+    for entity in json_short:
+        name = entity[0]
+        entity_type = entity[1]
+        entities[name] = entity_type
+
+    return entities
+
+def distill_entities(entities):
+    # This is list that we put the output what we need
+    list_output = list()
+
+    # This line go to sort function and save the output to temp_entities
+    temp_entities = sortTags(entities)
+
+    # This list help us to make the output,
+    temp_list = list()
+
+    # initlize the temp_list
+    temp_list.append(["", "", 0, 0])
+    word_position = 0
+
+    # For each entity, convert ibo to distllir list.
+    for entity in temp_entities:
+        # This is counter tag of this entity
+        counter_tag = 0
+        # For each tag
+        for tag in str(entity[1]).split():
+            # If the counter tag greater than or equal to lenght of templist, if yes then we will append the empty value in templist
+            if counter_tag >= len(temp_list):
+                temp_list.append(["", "", 0, 0])
+
+            # If tag equal O and word postion of this tag is not equal zero then it will add all
+            # not empty eliment of temp list in output list
+            if "O" == tag and word_position != 0:
+                for j in range(0, len(temp_list)):
+                    if temp_list[j][1] != "":
+                        list_output.append([temp_list[j][0].strip(), temp_list[j][1], temp_list[j][2], temp_list[j][3]])
+                        temp_list[j][0] = ""
+                        temp_list[j][1] = ""
+                        temp_list[j][2] = word_position
+                        temp_list[j][3] = word_position
+            # if this tag not equal O, and split by '-' the tag and check the lenght equals two and if the first eliment
+            # of the split its B
+            elif "O" != tag and len(tag.split("-")) == 2 and tag.split("-")[0] == "B":
+                # if the temp_list of counter is not empty then it will append in output list and hten it will
+                # initilize by new string and tag in templist of counter
+                if temp_list[counter_tag][1] != "":
+                    list_output.append([temp_list[counter_tag][0].strip(), temp_list[counter_tag][1], temp_list[counter_tag][2], temp_list[counter_tag][3]])
+                temp_list[counter_tag][0] = str(entity[0]) + " "
+                temp_list[counter_tag][1] = str(tag).split("-")[1]
+                temp_list[counter_tag][2] = word_position
+                temp_list[counter_tag][3] = word_position
+
+            # if this tag not equal O, and split by '-' the tag and check the lenght equals two and if the first eliment
+            # of the split its O
+            elif "O" != tag and len(tag.split("-")) == 2 and tag.split("-")[0] == "I" and word_position != 0:
+                # For each of temp_list, check if in this counter tag of templist is same tag with this.tag
+                # then will complete if not it will save in output list and cheak another
+                for j in range(counter_tag,len(temp_list)):
+                    if temp_list[j][1] == tag[2:] and temp_list[j][3] != word_position:
+                        temp_list[j][0] += str(entity[0]) + " "
+                        temp_list[j][3] += 1
+                        break
+                    else:
+                        if temp_list[j][1] != "":
+                            list_output.append([temp_list[j][0].strip(), temp_list[j][1], temp_list[j][2], temp_list[j][3]])
+                            temp_list[j][0] = ""
+                            temp_list[j][1] = ""
+                            temp_list[j][2] = word_position
+                            temp_list[j][3] = word_position
+            counter_tag += 1
+        word_position += 1
+    # For each temp_list, at the end of the previous loop, there will be some
+    # values in this list, we should save it to the output list
+    for j in range(0, len(temp_list)):
+        if temp_list[j][1] != "":
+            list_output.append([temp_list[j][0].strip(), temp_list[j][1], temp_list[j][2], temp_list[j][3]])
+    return sorted(list_output, key=lambda x: (x[2]))
+
+def sortTags(entities):
+    temp_entities = entities
+    temp_counter = 0
+    # For each entity, this loop will sort each tag of entitiy, first it will check if the
+    # previous tags has same count of this tag, second will sort the tags and check if this tags is correct
+    for entity in temp_entities:
+        tags = entity[1].split()
+        for tag in tags:
+            # if the counter is not 0 then, will complete
+            if temp_counter != 0:
+                # Check if this tag is equal I-, if yes then it will count how many tag in this tags and
+                # count how many tag in previous tags
+                if "I-" == tag[0:2]:
+                    counter_of_this_tag = 0
+                    counter_of_previous_tag = 0
+                    for word in tags:
+                        if tag.split("-")[1] in word:
+                            counter_of_this_tag+=1
+                    for word in temp_entities[temp_counter-1][1].split():
+                        if tag.split("-")[1] in word:
+                            counter_of_previous_tag+=1
+                    # if the counter of previous tag is bigger than counter of this tag, then we
+                    # need to add I-tag in this tags
+                    if counter_of_previous_tag > counter_of_this_tag:
+                        tags.append("I-"+tag.split("-")[1])
+        # Sort the tags
+        tags.sort()
+        # Need to revers the tags because it should begins with I
+        tags.reverse()
+        # If the counter is not 0 then we can complete
+        if temp_counter != 0:
+            this_tags = tags
+            previous_tags = temp_entities[temp_counter - 1][1].split()
+            sorted_tags = list()
+
+            # Check if the this tag is not O and previous tags is not O, then will complete,
+            # if not then it will ignor this tag
+            if "O" not in this_tags and "O" not in previous_tags:
+                index = 0
+                #For each previous tags, need sort this tag by previous tags if its I, B we can ignor
+                for i in previous_tags:
+                    j = 0
+                    while this_tags and j < len(this_tags):
+                        if this_tags[j][0:2] == "I-" and this_tags[j][2:] == i[2:]:
+                            sorted_tags.insert(index, this_tags.pop(j))
+                            break
+                        elif this_tags[j][0:2] == "B-":
+                            break
+                        j += 1
+                    index += 1
+            sorted_tags += this_tags
+            tags = sorted_tags
+        str_tag = " "
+        str_tag = str_tag.join(tags)
+        str_tag = str_tag.strip()
+        temp_entities[temp_counter][1] = str_tag
+        temp_counter += 1
+    return temp_entities
+
+# ============= Prepare Templates and Catergorize Extracted Entities ================
+temp03={'location':'مكان حدوث','agent':'أحد المتأثرين في','happened at':'تاريخ حدوث'}
+categories = {
+    'agent': ['PERS', 'NORP', 'OCC', 'ORG'],
+    'location': ['LOC', 'FAC', 'GPE'],
+    'happened at': ['DATE', 'TIME']
+    }
+
+def get_entity_category(entity_type, categories):
+    for category, types in categories.items():
+        if entity_type in types:
+            return category
+    return None
+
+
+# ============ Extract entities, their types and categorize them ===============
+def relation_extraction(sentence):
+    #test_sentence="صورة إعتقال طفل فلسطيني خلال انتفاضة الأقصى ."
+    entities=entities_and_types(sentence)
+
+    event_indices = [i for i, (_, entity_type) in enumerate(entities.items()) if entity_type == 'EVENT']
+    arg_event_indices = [i for i, (_, entity_type) in enumerate(entities.items()) if entity_type != 'EVENT']
+
+    output_list=[]
+
+    for i in event_indices:
+        event_entity=list(entities.keys())[i]
+        for j in arg_event_indices:
+            arg_name= list(entities.keys())[j]
+            arg_type=entities[arg_name]
+            category = get_entity_category(arg_type, categories)
+            
+            if category in temp03:
+                relation_sentence=f"[CLS] {sentence} [SEP] {event_entity} {temp03[category]} {arg_name}"
+                predicted_relation=pipe(relation_sentence)
+                score = predicted_relation[0][0]['score']  
+                if score > 0.50:
+                    #print(f"Event:{event_entity} Relation:{category} Argument:{arg_name}\n")
+                    #output_list.append([{event_entity} ,{category}, {arg_name}])
+                    output_list.append(f"Event:{event_entity}, Relation:{category}, Argument:{arg_name}")
+
+                else:
+                    #print(f"Event:{event_entity} Relation:No relation Argument:{arg_name}\n")
+                    #output_list.append([{event_entity} ,'No relation', {arg_name}])
+                    output_list.append(f"Event:{event_entity}, Relation:No relation, Argument:{arg_name}")
+    
+    return output_list

sinatools/ner/trainers/BaseTrainer.py

@@ -1,117 +1,117 @@
-import os
-import torch
-import logging
-import natsort
-import glob
-
-logger = logging.getLogger(__name__)
-
-
-class BaseTrainer:
-    def __init__(
-        self,
-        model=None,
-        max_epochs=50,
-        optimizer=None,
-        scheduler=None,
-        loss=None,
-        train_dataloader=None,
-        val_dataloader=None,
-        test_dataloader=None,
-        log_interval=10,
-        summary_writer=None,
-        output_path=None,
-        clip=5,
-        patience=5
-    ):
-        self.model = model
-        self.max_epochs = max_epochs
-        self.train_dataloader = train_dataloader
-        self.val_dataloader = val_dataloader
-        self.test_dataloader = test_dataloader
-        self.optimizer = optimizer
-        self.scheduler = scheduler
-        self.loss = loss
-        self.log_interval = log_interval
-        self.summary_writer = summary_writer
-        self.output_path = output_path
-        self.current_timestep = 0
-        self.current_epoch = 0
-        self.clip = clip
-        self.patience = patience
-
-    def tag(self, dataloader, is_train=True):
-        """
-        Given a dataloader containing segments, predict the tags
-        :param dataloader: torch.utils.data.DataLoader
-        :param is_train: boolean - True for training model, False for evaluation
-        :return: Iterator
-                    subwords (B x T x NUM_LABELS)- torch.Tensor - BERT subword ID
-                    gold_tags (B x T x NUM_LABELS) - torch.Tensor - ground truth tags IDs
-                    tokens - List[arabiner.data.dataset.Token] - list of tokens
-                    valid_len (B x 1) - int - valiud length of each sequence
-                    logits (B x T x NUM_LABELS) - logits for each token and each tag
-        """
-        for subwords, gold_tags, tokens, valid_len in dataloader:
-            self.model.train(is_train)
-
-            if torch.cuda.is_available():
-                subwords = subwords.cuda()
-                gold_tags = gold_tags.cuda()
-
-            if is_train:
-                self.optimizer.zero_grad()
-                logits = self.model(subwords)
-            else:
-                with torch.no_grad():
-                    logits = self.model(subwords)
-
-            yield subwords, gold_tags, tokens, valid_len, logits
-
-    def segments_to_file(self, segments, filename):
-        """
-        Write segments to file
-        :param segments: [List[arabiner.data.dataset.Token]] - list of list of tokens
-        :param filename: str - output filename
-        :return: None
-        """
-        with open(filename, "w") as fh:
-            results = "\n\n".join(["\n".join([t.__str__() for t in segment]) for segment in segments])
-            fh.write("Token\tGold Tag\tPredicted Tag\n")
-            fh.write(results)
-            logging.info("Predictions written to %s", filename)
-
-    def save(self):
-        """
-        Save model checkpoint
-        :return:
-        """
-        filename = os.path.join(
-            self.output_path,
-            "checkpoints",
-            "checkpoint_{}.pt".format(self.current_epoch),
-        )
-
-        checkpoint = {
-            "model": self.model.state_dict(),
-            "optimizer": self.optimizer.state_dict(),
-            "epoch": self.current_epoch
-        }
-
-        logger.info("Saving checkpoint to %s", filename)
-        torch.save(checkpoint, filename)
-
-    def load(self, checkpoint_path):
-        """
-        Load model checkpoint
-        :param checkpoint_path: str - path/to/checkpoints
-        :return: None
-        """
-        checkpoint_path = natsort.natsorted(glob.glob(f"{checkpoint_path}/checkpoint_*.pt"))
-        checkpoint_path = checkpoint_path[-1]
-
-        logger.info("Loading checkpoint %s", checkpoint_path)
-
-        device = None if torch.cuda.is_available() else torch.device('cpu')
-        checkpoint = torch.load(checkpoint_path, map_location=device)
-        self.model.load_state_dict(checkpoint["model"])
+import os
+import torch
+import logging
+import natsort
+import glob
+
+logger = logging.getLogger(__name__)
+
+
+class BaseTrainer:
+    def __init__(
+        self,
+        model=None,
+        max_epochs=50,
+        optimizer=None,
+        scheduler=None,
+        loss=None,
+        train_dataloader=None,
+        val_dataloader=None,
+        test_dataloader=None,
+        log_interval=10,
+        summary_writer=None,
+        output_path=None,
+        clip=5,
+        patience=5
+    ):
+        self.model = model
+        self.max_epochs = max_epochs
+        self.train_dataloader = train_dataloader
+        self.val_dataloader = val_dataloader
+        self.test_dataloader = test_dataloader
+        self.optimizer = optimizer
+        self.scheduler = scheduler
+        self.loss = loss
+        self.log_interval = log_interval
+        self.summary_writer = summary_writer
+        self.output_path = output_path
+        self.current_timestep = 0
+        self.current_epoch = 0
+        self.clip = clip
+        self.patience = patience
+
+    def tag(self, dataloader, is_train=True):
+        """
+        Given a dataloader containing segments, predict the tags
+        :param dataloader: torch.utils.data.DataLoader
+        :param is_train: boolean - True for training model, False for evaluation
+        :return: Iterator
+                    subwords (B x T x NUM_LABELS)- torch.Tensor - BERT subword ID
+                    gold_tags (B x T x NUM_LABELS) - torch.Tensor - ground truth tags IDs
+                    tokens - List[arabiner.data.dataset.Token] - list of tokens
+                    valid_len (B x 1) - int - valiud length of each sequence
+                    logits (B x T x NUM_LABELS) - logits for each token and each tag
+        """
+        for subwords, gold_tags, tokens, valid_len in dataloader:
+            self.model.train(is_train)
+
+            if torch.cuda.is_available():
+                subwords = subwords.cuda()
+                gold_tags = gold_tags.cuda()
+
+            if is_train:
+                self.optimizer.zero_grad()
+                logits = self.model(subwords)
+            else:
+                with torch.no_grad():
+                    logits = self.model(subwords)
+
+            yield subwords, gold_tags, tokens, valid_len, logits
+
+    def segments_to_file(self, segments, filename):
+        """
+        Write segments to file
+        :param segments: [List[arabiner.data.dataset.Token]] - list of list of tokens
+        :param filename: str - output filename
+        :return: None
+        """
+        with open(filename, "w") as fh:
+            results = "\n\n".join(["\n".join([t.__str__() for t in segment]) for segment in segments])
+            fh.write("Token\tGold Tag\tPredicted Tag\n")
+            fh.write(results)
+            logging.info("Predictions written to %s", filename)
+
+    def save(self):
+        """
+        Save model checkpoint
+        :return:
+        """
+        filename = os.path.join(
+            self.output_path,
+            "checkpoints",
+            "checkpoint_{}.pt".format(self.current_epoch),
+        )
+
+        checkpoint = {
+            "model": self.model.state_dict(),
+            "optimizer": self.optimizer.state_dict(),
+            "epoch": self.current_epoch
+        }
+
+        logger.info("Saving checkpoint to %s", filename)
+        torch.save(checkpoint, filename)
+
+    def load(self, checkpoint_path):
+        """
+        Load model checkpoint
+        :param checkpoint_path: str - path/to/checkpoints
+        :return: None
+        """
+        checkpoint_path = natsort.natsorted(glob.glob(f"{checkpoint_path}/checkpoint_*.pt"))
+        checkpoint_path = checkpoint_path[-1]
+
+        logger.info("Loading checkpoint %s", checkpoint_path)
+
+        device = None if torch.cuda.is_available() else torch.device('cpu')
+        checkpoint = torch.load(checkpoint_path, map_location=device)
+        self.model.load_state_dict(checkpoint["model"], strict=False)

sinatools/synonyms/__init__.py

@@ -3,7 +3,7 @@ from sinatools.DataDownload import downloader
 import os 
 
 synonyms_level2_dict = {} 
-level2_dict = 'synonyms_level2.pkl'
+level2_dict = 'graph_l2.pkl'
 path = downloader.get_appdatadir()
 file_path = os.path.join(path, level2_dict)
 with open(file_path, 'rb') as f:
@@ -11,7 +11,7 @@ with open(file_path, 'rb') as f:
 
 
 synonyms_level3_dict = {}    
-level3_dict = 'synonyms_level3.pkl'
+level3_dict = 'graph_l3.pkl'
 path = downloader.get_appdatadir()
 file_path = os.path.join(path, level3_dict)
 with open(file_path, 'rb') as f:

sinatools/utils/similarity.py

@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+
+from sinatools.utils.parser import arStrip
+from sinatools.utils.implication import Implication
+import argparse
+
+def normalize_word(word: str, ignore_all_diacritics_but_not_shadda: bool=True, ignore_shadda_diacritic: bool=True) -> str:
+    if ignore_all_diacritics_but_not_shadda:
+        word = arStrip(word, True, True, False, False, False, False)
+        
+    if ignore_shadda_diacritic:
+        word = arStrip(word, False, False, True, False, False, False)
+    
+    return word
+
+    
+def get_preferred_word(word1, word2):
+    implication = Implication(word1, word2)
+    
+    direction = implication.get_direction()
+    
+    if direction in (0, 2):
+        return word1
+       
+    elif direction == 1:
+        return word2
+       
+    elif direction == 3:
+        if not word1.endswith("َ") and not word1.endswith("ُ"):
+            return word2
+        return word1
+     
+    
+def get_non_preferred_word(word1, word2):
+
+    implication = Implication(word1, word2)
+    if implication.get_distance() < 15:
+        direction = implication.get_direction()
+        if direction == 0 or direction == 1:
+            return word1
+        elif direction == 2:
+            return word2
+        elif direction == 3:
+            if not word1.endswith("َ") and not word1.endswith("ُ"):
+                return word1
+            return word2
+    return "#"
+
+def get_intersection(list1, list2, ignore_all_diacritics_but_not_shadda=False, ignore_shadda_diacritic=False):
+
+    list1 = [str(i) for i in list1 if i not in (None, ' ', '')]
+    list1 = [str(i.strip()) for i in list1]
+
+    list2 = [str(i) for i in list2 if i not in (None, ' ', '')]
+    list2 = [str(i.strip()) for i in list2]
+
+    interection_list = []
+
+    for list1_word in list1:
+        for list2_word in list2:
+            word1 = normalize_word(list1_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+            word2 = normalize_word(list2_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+
+            implication = Implication(word1, word2)
+            if implication.get_direction() >= 0 and implication.get_distance() < 15:
+                interection_list.append(get_preferred_word(word1, word2))
+
+    i = 0
+    while i < len(interection_list):
+        j = i + 1
+        while j < len(interection_list):
+            non_preferred_word = get_non_preferred_word(interection_list[i], interection_list[j])
+            if non_preferred_word != "#":
+                interection_list.remove(non_preferred_word)
+            j += 1
+        i += 1
+
+    return interection_list
+             
+
+
+def get_union(list1, list2, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic):
+
+    list1 = [str(i) for i in list1 if i not in (None, ' ', '')]
+
+    list2 = [str(i) for i in list2 if i not in (None, ' ', '')]
+
+    union_list = []
+
+    for list1_word in list1:
+        word1 = normalize_word(list1_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+        union_list.append(word1)
+
+    for list2_word in list2:
+        word2 = normalize_word(list2_word, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+        union_list.append(word2)
+
+    i = 0
+    while i < len(union_list):
+        j = i + 1
+        while j < len(union_list):
+            non_preferred_word = get_non_preferred_word(union_list[i], union_list[j])
+            if (non_preferred_word != "#"):
+                union_list.remove(non_preferred_word)
+            j = j + 1
+        i = i + 1
+
+    return union_list
+      
+
+
+def get_jaccard_similarity(list1: list, list2: list, ignore_all_diacritics_but_not_shadda: bool, ignore_shadda_diacritic: bool) -> float:
+    
+    intersection_list = get_intersection(list1, list2, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+    
+    union_list = get_union(list1, list2, ignore_all_diacritics_but_not_shadda, ignore_shadda_diacritic)
+    
+    return float(len(intersection_list)) / float(len(union_list))
+
+def get_jaccard(delimiter, str1, str2, selection, ignoreAllDiacriticsButNotShadda=True, ignoreShaddaDiacritic=True):
+
+    try:
+        list1 = str1.split(delimiter)
+        list2 = str2.split(delimiter)
+
+        if selection == "intersection":
+            intersection = get_intersection(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            return intersection
+        elif selection == "union":
+            union = get_union(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            return union
+        elif selection == "jaccardSimilarity":      
+            similarity = get_jaccard_similarity(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            return similarity
+        elif selection == "jaccardAll":    
+            intersection = get_intersection(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            union = get_union(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            similarity = get_jaccard_similarity(list1, list2, ignoreAllDiacriticsButNotShadda, ignoreShaddaDiacritic)
+            output_list = ["intersection:", intersection, "union:", union, "similarity:", similarity]
+            return output_list
+        else:
+            return 'Invalid selection option'
+
+    except AttributeError as ae:
+        print(f"Attribute error occurred: {str(ae)}")
+        return 'Invalid input type'
+    except Exception as e:
+        print(f"Error occurred: {str(e)}")
+        return 'An error has occurred'

sinatools/wsd/__init__.py

@@ -4,7 +4,7 @@ from sinatools.DataDownload import downloader
 import os 
 
 glosses_dic = {}
-filename = 'glosses_dic.pickle'
+filename = 'one_gram.pickle'
 path =downloader.get_appdatadir()
 file_path = os.path.join(path, filename)
 with open(file_path, 'rb') as f:

sinatools/wsd/disambiguator.py

@@ -217,7 +217,7 @@ def jsons_to_list_of_lists(json_list):
 def find_named_entities(string):
    found_entities = []
    
-   ner_entites = extract(string)
+   ner_entites = extract(string, "nested")
    list_of_entites = jsons_to_list_of_lists(ner_entites)
    entites = distill_entities(list_of_entites)
    
@@ -288,17 +288,17 @@ def disambiguate_glosses_using_SALMA(glosses, Diac_lemma, Undiac_lemma, word, se
       concept_id, gloss = GlossPredictor(Diac_lemma, Undiac_lemma,word,sentence,glosses_dictionary)
 
       my_json = {}    
-      my_json['Concept_id'] = concept_id
+      my_json['concept_id'] = concept_id
     #   my_json['Gloss'] = gloss
       my_json['word'] = word
-      my_json['Undiac_lemma'] = Undiac_lemma
-      my_json['Diac_lemma'] = Diac_lemma
+      #my_json['Undiac_lemma'] = Undiac_lemma
+      my_json['lemma'] = Diac_lemma
       return my_json
    else:
       my_json = {}    
       my_json['word'] = word
-      my_json['Undiac_lemma'] = Undiac_lemma
-      my_json['Diac_lemma'] = Diac_lemma
+      #my_json['Undiac_lemma'] = Undiac_lemma
+      my_json['lemma'] = Diac_lemma
       return my_json
 
 
@@ -405,26 +405,26 @@ def disambiguate_glosses_main(word, sentence):
    if concept_count == 0:
       my_json = {}    
       my_json['word'] = word['word']
-      my_json['Diac_lemma'] = word['Diac_lemma']
-      my_json['Undiac_lemma'] = word['Undiac_lemma']
+      my_json['lemma'] = word['Diac_lemma']
+      #my_json['Undiac_lemma'] = word['Undiac_lemma']
       return my_json
    elif concept_count == 1:
       my_json = {}    
       my_json['word'] = word['word']
       glosses = word['glosses'][0]
     #   my_json['Gloss'] = glosses['gloss']
-      my_json['Concept_id'] = glosses['concept_id']
-      my_json['Diac_lemma'] = word['Diac_lemma']
-      my_json['Undiac_lemma'] = word['Undiac_lemma']
+      my_json['concept_id'] = glosses['concept_id']
+      my_json['lemma'] = word['Diac_lemma']
+      #my_json['Undiac_lemma'] = word['Undiac_lemma']
       return my_json
    elif concept_count == '*':
       my_json = {}    
       my_json['word'] = word['word']
       glosses = word['glosses'][0]
       my_json['Gloss'] = glosses['gloss']
-      my_json['Concept_id'] = glosses['concept_id']
-      my_json['Diac_lemma'] = word['Diac_lemma']
-      my_json['Undiac_lemma'] = word['Undiac_lemma']
+      my_json['concept_id'] = glosses['concept_id']
+      my_json['lemma'] = word['Diac_lemma']
+      #my_json['Undiac_lemma'] = word['Undiac_lemma']
       return my_json
    else:   
       input_word = word['word']
@@ -477,21 +477,18 @@ def disambiguate(sentence):
         #output
          [
              {
-                 "Concept_id": "303019218",
+                 "concept_id": "303019218",
                  "word": "ذهبت",
-                 "Undiac_lemma": "ذهب",
-                 "Diac_lemma": "ذَهَبَ۪ 1"
+                 "lemma": "ذَهَبَ۪ 1"
              },
              {
                  "word": "إلى",
-                 "Diac_lemma": إِلَى 1,
-                 "Undiac_lemma": "الى"
+                 "lemma": "إِلَى 1"
              },
              {
                  "word": "جامعة بيرزيت",
-                 "Concept_id": "334000099",
-                 "Diac_lemma": جامِعَة بيرزَيت,
-                 "Undiac_lemma": "جامعة بيرزيت"
+                 "concept_id": "334000099",
+                 "lemma": "جامِعَة بيرزَيت"
              }
          ]
     """      

SinaTools-0.1.25.dist-info/METADATA

@@ -1,34 +0,0 @@
-Metadata-Version: 2.1
-Name: SinaTools
-Version: 0.1.25
-Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
-Home-page: https://github.com/SinaLab/sinatools
-License: MIT license
-Keywords: sinatools
-Platform: UNKNOWN
-Description-Content-Type: text/markdown
-Requires-Dist: six
-Requires-Dist: farasapy
-Requires-Dist: tqdm
-Requires-Dist: requests
-Requires-Dist: regex
-Requires-Dist: pathlib
-Requires-Dist: torch (==1.13.0)
-Requires-Dist: transformers (==4.24.0)
-Requires-Dist: torchtext (==0.14.0)
-Requires-Dist: torchvision (==0.14.0)
-Requires-Dist: seqeval (==1.2.2)
-Requires-Dist: natsort (==7.1.1)
-
-SinaTools
----------
-
-Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
-
-Python APIs, command lines, colabs, and online demos.
-
-* Free software: MIT license
-* Documentation: https://sina.birzeit.edu/sinatools/
-
-
-