SinaTools 0.1.11__tar.gz → 0.1.13__tar.gz

{SinaTools-0.1.11 → SinaTools-0.1.13}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.11
+Version: 0.1.13
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license
-Description: sinatools
+Description: SinaTools
         ---------
         
         Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.

{SinaTools-0.1.11 → SinaTools-0.1.13}/README.rst

@@ -1,4 +1,4 @@
-sinatools
+SinaTools
 ---------
 
 Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.

{SinaTools-0.1.11 → SinaTools-0.1.13}/SinaTools.egg-info/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.11
+Version: 0.1.13
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license
-Description: sinatools
+Description: SinaTools
         ---------
         
         Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.

{SinaTools-0.1.11 → SinaTools-0.1.13}/SinaTools.egg-info/SOURCES.txt

@@ -82,7 +82,6 @@ sinatools/CLI/morphology/ALMA_multi_word.py
 sinatools/CLI/morphology/morph_analyzer.py
 sinatools/CLI/ner/corpus_entity_extractor.py
 sinatools/CLI/ner/entity_extractor.py
-sinatools/CLI/salma/salma_tools.py
 sinatools/CLI/utils/__init__.py
 sinatools/CLI/utils/arStrip.py
 sinatools/CLI/utils/corpus_tokenizer.py
@@ -91,6 +90,7 @@ sinatools/CLI/utils/jaccard.py
 sinatools/CLI/utils/remove_latin.py
 sinatools/CLI/utils/remove_punctuation.py
 sinatools/CLI/utils/sentence_tokenizer.py
+sinatools/CLI/utils/text_dublication_detector.py
 sinatools/CLI/utils/text_transliteration.py
 sinatools/DataDownload/__init__.py
 sinatools/DataDownload/downloader.py
@@ -150,18 +150,38 @@ sinatools/morphology/ALMA_multi_word.py
 sinatools/morphology/__init__.py
 sinatools/morphology/morph_analyzer.py
 sinatools/ner/__init__.py
+sinatools/ner/data_format.py
+sinatools/ner/datasets.py
 sinatools/ner/entity_extractor.py
-sinatools/salma/__init__.py
-sinatools/salma/settings.py
-sinatools/salma/views.py
-sinatools/salma/wsd.py
+sinatools/ner/helpers.py
+sinatools/ner/metrics.py
+sinatools/ner/transforms.py
+sinatools/ner/data/__init__.py
+sinatools/ner/data/datasets.py
+sinatools/ner/data/transforms.py
+sinatools/ner/nn/BaseModel.py
+sinatools/ner/nn/BertNestedTagger.py
+sinatools/ner/nn/BertSeqTagger.py
+sinatools/ner/nn/__init__.py
+sinatools/ner/trainers/BaseTrainer.py
+sinatools/ner/trainers/BertNestedTrainer.py
+sinatools/ner/trainers/BertTrainer.py
+sinatools/ner/trainers/__init__.py
+sinatools/semantic_relatedness/__init__.py
+sinatools/semantic_relatedness/compute_relatedness.py
+sinatools/synonyms/__init__.py
+sinatools/synonyms/synonyms_generator.py
 sinatools/utils/__init__.py
 sinatools/utils/charsets.py
 sinatools/utils/implication.py
 sinatools/utils/jaccard.py
 sinatools/utils/parser.py
 sinatools/utils/readfile.py
+sinatools/utils/text_dublication_detector.py
 sinatools/utils/text_transliteration.py
 sinatools/utils/tokenizer.py
 sinatools/utils/tokenizers_words.py
-sinatools/utils/utils.py
+sinatools/wsd/__init__.py
+sinatools/wsd/disambiguator.py
+sinatools/wsd/settings.py
+sinatools/wsd/wsd.py

{SinaTools-0.1.11 → SinaTools-0.1.13}/SinaTools.egg-info/entry_points.txt

@@ -2,17 +2,21 @@
 alma_multi_word = sinatools.CLI.morphology.ALMA_multi_word:main
 appdatadir = sinatools.CLI.DataDownload.get_appdatadir:main
 arStrip = sinatools.CLI.utils.arStrip:main
-arabi_ner = sinatools.CLI.ner.entity_extractor:main
-arabi_ner2 = sinatools.CLI.ner.corpus_entity_extractor:main
+corpus_entity_extractor = sinatools.CLI.ner.corpus_entity_extractor:main
 corpus_tokenizer = sinatools.CLI.utils.corpus_tokenizer:main
 download_files = sinatools.CLI.DataDownload.download_files:main
+entity_extractor = sinatools.CLI.ner.entity_extractor:main
+evaluate_synonyms = sinatools.CLI.synonyms.evaluate_synonyms:main
+extend_synonyms = sinatools.CLI.synonyms.extend_synonyms:main
 implication = sinatools.CLI.utils.implication:main
 install_env = sinatools.install_env:main
 jaccard_similarity = sinatools.CLI.utils.jaccard:main
 morphology_analyzer = sinatools.CLI.morphology.morph_analyzer:main
 remove_latin = sinatools.CLI.utils.remove_latin:main
 remove_punctuation = sinatools.CLI.utils.remove_punctuation:main
-salma = sinatools.CLI.salma.salma_tools:main
+semantic_relatedness = sinatools.CLI.semantic_relatedness.compute_relatedness:main
 sentence_tokenizer = sinatools.CLI.utils.sentence_tokenizer:main
+text_dublication_detector = sinatools.CLI.utils.text_dublication_detector:main
 transliterate = sinatools.CLI.utils.text_transliteration:main
+wsd = sinatools.CLI.wsd.disambiguator:main
 

{SinaTools-0.1.11 → SinaTools-0.1.13}/SinaTools.egg-info/requires.txt

@@ -10,4 +10,3 @@ torchtext==0.14.0
 torchvision==0.14.0
 seqeval==1.2.2
 natsort==7.1.1
-pandas==1.2.4

{SinaTools-0.1.11 → SinaTools-0.1.13}/setup.py

@@ -23,8 +23,7 @@ requirements = [
     'torchtext==0.14.0',
     'torchvision==0.14.0',
     'seqeval==1.2.2',
-    'natsort==7.1.1',
-    'pandas==1.2.4'
+    'natsort==7.1.1'
 ]
 
 
@@ -56,22 +55,30 @@ setup(
                 'sinatools.CLI.morphology.morph_analyzer:main'),
             ('alma_multi_word='
                 'sinatools.CLI.morphology.ALMA_multi_word:main'),
-            ('arabi_ner='
+            ('entity_extractor='
                 'sinatools.CLI.ner.entity_extractor:main'),
             ('remove_punctuation='
                 'sinatools.CLI.utils.remove_punctuation:main'),
             ('remove_latin='
                 'sinatools.CLI.utils.remove_latin:main'),
-            ('salma='
-                'sinatools.CLI.salma.salma_tools:main'),
+            ('wsd='
+                'sinatools.CLI.wsd.disambiguator:main'),
             ('corpus_tokenizer='
                 'sinatools.CLI.utils.corpus_tokenizer:main'),
             ('appdatadir='
                 'sinatools.CLI.DataDownload.get_appdatadir:main'),
             ('download_files='
                 'sinatools.CLI.DataDownload.download_files:main'),
-            ('arabi_ner2='
-                'sinatools.CLI.ner.corpus_entity_extractor:main')
+            ('corpus_entity_extractor='
+                'sinatools.CLI.ner.corpus_entity_extractor:main'),
+            ('text_dublication_detector='
+                'sinatools.CLI.utils.text_dublication_detector:main'),     
+            ('evaluate_synonyms='
+                'sinatools.CLI.synonyms.evaluate_synonyms:main'),  
+            ('extend_synonyms='
+                'sinatools.CLI.synonyms.extend_synonyms:main'),                    
+            ('semantic_relatedness='
+                'sinatools.CLI.semantic_relatedness.compute_relatedness:main'),                                 
         ],
     },
     data_files=[('sinatools', ['sinatools/environment.yml'])],

{SinaTools-0.1.11 → SinaTools-0.1.13}/sinatools/CLI/DataDownload/download_files.py

@@ -29,16 +29,6 @@ Examples:
     download_files -f morph ner 
     This command will download only the `morph` and `ner` files to the default directory.
 
-Note:
------
-
-.. code-block:: none
-
-    - The script automatically handles the extraction of zip and tar.gz files after downloading.
-    - Ensure you have the necessary permissions to write to the specified directory.
-    - The default download directory is based on the operating system and can be obtained using the `get_appdatadir` function.
-
-
 """
 
 import argparse

{SinaTools-0.1.11 → SinaTools-0.1.13}/sinatools/CLI/ner/corpus_entity_extractor.py

@@ -4,20 +4,23 @@ from sinatools.utils.tokenizer import sentence_tokenizer
 from sinatools.utils.tokenizers_words import simple_word_tokenize
 import pandas as pd
 import argparse
-from sinatools.ner.entity_extractor import ner
+from sinatools.ner.entity_extractor import extract
 
 """
-CSV NER Tagging Tool
+This tool processes a csv file and returns named entites for each token within the text, based on the specified batch size. As follows:
 
 Usage:
 ------
 Run the script with the following command:
 
-arabi_ner2  input.csv --text-columns "TextColumn1,TextColumn2" --additional-columns "Column3,Column4" --output-csv output.csv
+corpus_entity_extractor  input.csv --text-columns "TextColumn1,TextColumn2" --additional-columns "Column3,Column4" --output-csv output.csv
 """
 
-def infer(sentence):
-    output = ner(sentence)
+def jsons_to_list_of_lists(json_list):
+    return [[d['token'], d['tags']] for d in json_list]
+
+def combine_tags(sentence):
+    output = jsons_to_list_of_lists(extract(sentence))
     return [word[1] for word in output]
 
 
@@ -40,7 +43,7 @@ def corpus_tokenizer(input_csv, output_csv, text_column, additional_columns, row
                 words = simple_word_tokenize(sentence)
                 global_sentence_id += 1
 
-                tags = infer(sentence) 
+                tags = combine_tags(sentence) 
                 for word_position, word in enumerate(words, start=1):
                     row_id += 1
                     doc_sentence_filename = input_csv.split(".csv")[0]

SinaTools-0.1.13/sinatools/CLI/ner/entity_extractor.py

@@ -0,0 +1,89 @@
+"""
+About:
+------
+This tool processes an input text and returns named entites for each token within the text, based on the specified batch size. As follows:
+
+Usage:
+------
+Below is the usage information that can be generated by running entity_extractor --help.
+
+.. code-block:: none
+
+    entity_extractor --text=INPUT_TEXT
+    entity_extractor --dir=INPUT_FILE --output_csv=OUTPUT_FILE_NAME
+
+Options:
+--------
+
+.. code-block:: none
+
+  --text INPUT_TEXT
+        The text that needs to be analyzed for Named Entity Recognition.
+  --file INPUT_FILE
+        File containing the text to be analyzed for Named Entity Recognition.
+  --output_csv OUTPUT_FILE_NAME
+        A file containing the tokenized text and its Named Entity tags.
+
+
+Examples:
+---------
+
+.. code-block:: none
+
+    entity_extractor --text "Your text here"
+    entity_extractor --dir "/path/to/your/directory" --output_csv "output.csv"
+
+"""
+
+import argparse
+import json
+import pandas as pd
+from sinatools.ner.entity_extractor import extract
+from sinatools.utils.tokenizer import corpus_tokenizer
+from sinatools.utils.tokenizers_words import simple_word_tokenize
+
+
+def combine_tags(sentence):
+    output = extract(sentence)
+    return [word[1] for word in output]
+
+
+def main():
+    parser = argparse.ArgumentParser(description='NER Analysis using ArabiNER')
+
+    parser.add_argument('--text', type=str, help='Text to be analyzed for Named Entity Recognition')
+    parser.add_argument('--dir', type=str, help='dir containing the text files to be analyzed for Named Entity Recognition')
+    parser.add_argument('--output_csv', type=str, help='Output CSV file to write the results')
+
+    args = parser.parse_args()
+
+    if args.text is not None:
+        results = extract(args.text)
+        # Print the results in JSON format
+        print(json.dumps(results, ensure_ascii=False, indent=4))
+    elif args.dir is not None:
+        corpus_tokenizer(args.dir, args.output_csv)
+        df = pd.read_csv(args.output_csv)
+        df['NER tags'] = None
+        i = 0
+
+        result = df.drop_duplicates(subset=['Global Sentence ID', 'Sentence'])
+        unique_sentences = result['Sentence'].to_numpy()
+
+        for sentence in unique_sentences: 
+            ner_tags = combine_tags(sentence) 
+            if len(simple_word_tokenize(sentence)) > 300:
+                print(" Length of this sentence is more than 300 word:  ", sentence)
+                return
+            
+            df.loc[i:i+len(ner_tags)-1, 'NER tags'] = ner_tags 
+            i = i + len(ner_tags)
+        
+        df.to_csv(args.output_csv, index=False) 
+    else:    
+        print("Error: Either --text or --file argument must be provided.")
+        return
+
+
+if __name__ == '__main__':
+    main()

{SinaTools-0.1.11 → SinaTools-0.1.13}/sinatools/CLI/utils/arStrip.py

@@ -26,7 +26,7 @@ Below is the usage information that can be generated by running arStrip --help.
       --diacs BOOL [default=True]
             Indicates whether to strip diacritics.
 
-      --smallDiacs BOOL [default=True]
+      --small_diacs BOOL [default=True]
             Indicates whether to strip small diacritics.
 
       --shaddah BOOL [default=True]
@@ -38,15 +38,15 @@ Below is the usage information that can be generated by running arStrip --help.
       --alif BOOL [default=True]
             Indicates whether to strip alif.
 
-      --specialChars BOOL [default=True]
+      --special_chars BOOL [default=True]
             Indicates whether to strip special characters.
 
 Examples:
 ---------
 .. code-block:: none
 
-    arStrip --text "مُختَبَر سينا لحوسبة اللغة!" --diacs=True --smallDiacs=False --shaddah=True --digit=False --alif=False --specialChars=False    
-    arStrip --file "path/to/your/file.txt" --diacs=True --smallDiacs=False --shaddah=True --digit=False --alif=False --specialChars=False
+    arStrip --text "مُختَبَر سينا لحوسبة اللغة!" --diacs=True --small_diacs=False --shaddah=True --digit=False --alif=False --special_chars=False    
+    arStrip --file "path/to/your/file.txt" --diacs=True --small_diacs=False --shaddah=True --digit=False --alif=False --special_chars=False
 
 """
 
@@ -60,11 +60,11 @@ def main():
     parser.add_argument('--text', type=str, help='Text to be stripped')
     parser.add_argument('--file', type=str, help='File containing text to be stripped')
     parser.add_argument('--diacs', type=bool, default=True, help='Whether to strip diacritics')
-    parser.add_argument('--smallDiacs', type=bool, default=True, help='Whether to strip small diacritics')
+    parser.add_argument('--small_diacs', type=bool, default=True, help='Whether to strip small diacritics')
     parser.add_argument('--shaddah', type=bool, default=True, help='Whether to strip shaddah')
     parser.add_argument('--digit', type=bool, default=True, help='Whether to strip digits')
     parser.add_argument('--alif', type=bool, default=True, help='Whether to strip alif')
-    parser.add_argument('--specialChars', type=bool, default=True, help='Whether to strip special characters')
+    parser.add_argument('--special_chars', type=bool, default=True, help='Whether to strip special characters')
 
     args = parser.parse_args()
 
@@ -76,8 +76,8 @@ def main():
         print("Either --text or --file argument must be provided.")
         return
 
-    stripped_text = arStrip(text_content, diacs=args.diacs, smallDiacs=args.smallDiacs, 
-                            shaddah=args.shaddah, digit=args.digit, alif=args.alif, specialChars=args.specialChars)
+    stripped_text = arStrip(text_content, diacs=args.diacs, small_diacs=args.small_diacs, 
+                            shaddah=args.shaddah, digit=args.digit, alif=args.alif, special_chars=args.special_chars)
     
     print(stripped_text)
 

{SinaTools-0.1.11 → SinaTools-0.1.13}/sinatools/CLI/utils/implication.py

@@ -37,14 +37,6 @@ Examples:
       
       implication --file1 "path/to/your/file1.txt" --file2 "path/to/your/file2.txt"
 
-Note:
------
-
-.. code-block:: none
-
-    - The results are based on the underlying logic and data sets present in the `Implication` class of SinaTools.
-    - The tool compares the implication between two words, and the relationship might vary based on linguistic nuances.
-
 """
 import argparse
 from sinatools.utils.implication import Implication

{SinaTools-0.1.11 → SinaTools-0.1.13}/sinatools/CLI/utils/jaccard.py

@@ -5,14 +5,14 @@ The jaccard tool computes the Jaccard similarity between two sets of strings. Th
 
 Usage:
 ------
-Below is the usage information that can be generated by running jaccard --help.
+Below is the usage information that can be generated by running jaccard_similarity --help.
 
 .. code-block:: none
 
     Usage:
-        jaccard --list1="WORD1, WORD2"  --list2="WORD1,WORD2" --delimiter="DELIMITER"  --selection="SELECTION"  [OPTIONS]
+        jaccard_similarity --list1="WORD1, WORD2"  --list2="WORD1,WORD2" --delimiter="DELIMITER"  --selection="SELECTION"  [OPTIONS]
         
-        jaccard --file1=File1 --file2=File2 --delimiter="DELIMITER"  --selection="SELECTION"  [OPTIONS]
+        jaccard_similarity --file1=File1 --file2=File2 --delimiter="DELIMITER"  --selection="SELECTION"  [OPTIONS]
 
 .. code-block:: none
 
@@ -39,18 +39,9 @@ Examples:
 
 .. code-block:: none
 
-      jaccard --list1 "word1,word2"  --list2 "word1, word2" --delimiter ","  --selection "jaccardAll" --ignoreAllDiacriticsButNotShadda --ignoreShaddaDiacritic  
+      jaccard_similarity --list1 "word1,word2"  --list2 "word1, word2" --delimiter ","  --selection "jaccardAll" --ignoreAllDiacriticsButNotShadda --ignoreShaddaDiacritic  
       
-      jaccard --file1 "path/to/your/file1.txt"  --file2 "path/to/your/file2.txt" --delimiter ","  --selection "jaccardAll" --ignoreAllDiacriticsButNotShadda --ignoreShaddaDiacritic  
-
-Note:
------
-
-.. code-block:: none
-
-    - The Jaccard similarity ranges from 0 to 1. A value of 1 indicates that the sets are identical, while a value of 0 indicates no similarity between the sets.
-    - Diacritics refer to the Arabic Diacritics (like fatha, damma, kasra, etc.) and shadda.
-    - The two normalization options can be used individually or together. However, the combination will result in both rules being applied, and thus, 
+      jaccard_similarity --file1 "path/to/your/file1.txt"  --file2 "path/to/your/file2.txt" --delimiter ","  --selection "jaccardAll" --ignoreAllDiacriticsButNotShadda --ignoreShaddaDiacritic  
 
 """
 

{SinaTools-0.1.11 → SinaTools-0.1.13}/sinatools/CLI/utils/remove_latin.py

@@ -14,8 +14,8 @@ Below is the usage information that can be generated by running remove_latin --h
 Examples:
 ---------
 .. code-block:: none
-    latin_remove --text "123test"    
-    latin_remove --file "path/to/your/file.txt"
+    remove_latin --text "123test"    
+    remove_latin --file "path/to/your/file.txt"
 """
 
 import argparse

SinaTools-0.1.13/sinatools/CLI/utils/text_dublication_detector.py

@@ -0,0 +1,25 @@
+import argparse
+from sinatools.utils.text_dublication_detector import removal
+
+def main():
+    parser = argparse.ArgumentParser(description='Processes a CSV file of sentences to identify and remove duplicate sentences based on a specified threshold and cosine similarity. It saves the filtered results and the identified duplicates to separate files.')
+    
+    parser.add_argument('--csv_file', type=str, help='The path to the input CSV file that will be processed.')
+    parser.add_argument('--column_name', type=str, help='The name of the column from which duplicates will be removed.')
+    parser.add_argument('--final_file_name', type=str, help='The name of the output file that will contain the deduplicated results.')
+    parser.add_argument('--deleted_file_name', type=str, help='The name of the output file that will contain the records that were identified as duplicates and removed.')
+    parser.add_argument('--similarity_threshold', type=float, default=0.8, help='The similarity threshold for determining duplicates. Records with a similarity score above this value will be considered duplicates (default is 0.8).')
+
+    args = parser.parse_args()
+
+    if args.csv_file is None and args.column_name is None:
+        print("Either --csv_file or --column_name argument must be provided.")
+        return
+
+    removal(args.csv_file, args.column_name, args.final_file_name, args.deleted_file_name, args.similarity_threshold)
+    
+
+if __name__ == '__main__':
+    main()
+
+# text_dublication_detector --csv_file "text.csv" --column_name "A" --final_file_name "Final.csv" --deleted_file_name "deleted.csv" --similarity_threshold 0.8

SinaTools-0.1.13/sinatools/VERSION

@@ -0,0 +1 @@
+0.1.13

SinaTools-0.1.13/sinatools/morphology/ALMA_multi_word.py

@@ -0,0 +1,31 @@
+from sinatools.utils.parser import arStrip
+from . import five_grams_dict, four_grams_dict  , three_grams_dict , two_grams_dict
+
+def ALMA_multi_word(multi_word, n):
+    undiac_multi_word = arStrip(multi_word, True, True, True, False, True, False)  # diacs , smallDiacs , shaddah ,  digit , alif , specialChars
+    result_word = []
+    if n == 2:
+        if undiac_multi_word in two_grams_dict.keys():
+            result_word = two_grams_dict[undiac_multi_word]                    
+    elif n == 3:
+        if undiac_multi_word in three_grams_dict.keys():
+            result_word = three_grams_dict[undiac_multi_word]                    
+    elif n == 4:
+        if undiac_multi_word in four_grams_dict.keys():
+            result_word = four_grams_dict[undiac_multi_word]                    
+    else:    
+     if undiac_multi_word in five_grams_dict.keys():
+         result_word = five_grams_dict[undiac_multi_word]            
+    
+    my_json = {}
+    output_list = []
+    my_json['multi_word_lemma'] = multi_word
+    my_json['undiac_multi_word_lemma'] = multi_word
+    ids = []
+    if result_word != []:
+        my_json['POS'] = result_word[0][1] #POS
+        for result in result_word: 
+           ids.append(result[3])
+        my_json['ids'] = ids
+        output_list.append(my_json)    
+    return output_list  

SinaTools-0.1.13/sinatools/morphology/__init__.py

@@ -0,0 +1,43 @@
+import pickle
+from sinatools.DataDownload import downloader
+import os 
+
+dictionary = {}
+five_grams_dict = {} 
+four_grams_dict = {}
+three_grams_dict = {}
+two_grams_dict = {} 
+
+filename = 'lemmas_dic.pickle'
+path = downloader.get_appdatadir()
+file_path = os.path.join(path, filename)
+with open(file_path, 'rb') as f:
+    dictionary = pickle.load(f)
+
+filename_five = 'five_grams.pickle'
+path =downloader.get_appdatadir()
+file_path = os.path.join(path, filename_five)
+with open(file_path, 'rb') as f:
+   five_grams_dict = pickle.load(f, encoding='utf-8')
+
+
+filename_four = 'four_grams.pickle'
+path =downloader.get_appdatadir()
+file_path = os.path.join(path, filename_four)
+with open(file_path, 'rb') as f:
+  four_grams_dict = pickle.load(f, encoding='utf-8')
+  
+     
+filename_three = 'three_grams.pickle'
+path =downloader.get_appdatadir()
+file_path = os.path.join(path, filename_three)
+with open(file_path, 'rb') as f:
+  three_grams_dict = pickle.load(f, encoding='utf-8')
+  
+
+filename_two = 'two_grams.pickle'
+path =downloader.get_appdatadir()
+file_path = os.path.join(path, filename_two)
+with open(file_path, 'rb') as f:
+  two_grams_dict = pickle.load(f, encoding='utf-8')
+  

SinaTools-0.1.13/sinatools/ner/__init__.py

@@ -0,0 +1,38 @@
+from sinatools.DataDownload import downloader
+import os
+from sinatools.ner.helpers import load_object
+import pickle
+import os
+import torch
+import pickle
+import json
+from argparse import Namespace
+
+tagger = None
+tag_vocab = None
+train_config = None
+
+filename = 'Wj27012000.tar'
+path =downloader.get_appdatadir()
+model_path = os.path.join(path, filename)
+
+_path = os.path.join(model_path, "tag_vocab.pkl")
+
+with open(_path, "rb") as fh:
+    tag_vocab = pickle.load(fh)
+
+train_config = Namespace()
+args_path = os.path.join(model_path, "args.json")
+
+with open(args_path, "r") as fh:
+    train_config.__dict__ = json.load(fh)
+
+model = load_object(train_config.network_config["fn"], train_config.network_config["kwargs"])
+model = torch.nn.DataParallel(model)
+
+if torch.cuda.is_available():
+    model = model.cuda()
+
+train_config.trainer_config["kwargs"]["model"] = model
+tagger = load_object(train_config.trainer_config["fn"], train_config.trainer_config["kwargs"])
+tagger.load(os.path.join(model_path,"checkpoints"))

SinaTools-0.1.13/sinatools/ner/data/__init__.py

@@ -0,0 +1 @@
+from sinatools.ner.data.datasets import NestedTagsDataset