SinaTools 0.1.24__tar.gz → 0.1.26__tar.gz

{SinaTools-0.1.24 → SinaTools-0.1.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.24
+Version: 0.1.26
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.24 → SinaTools-0.1.26}/SinaTools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SinaTools
-Version: 0.1.24
+Version: 0.1.26
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license

{SinaTools-0.1.24 → SinaTools-0.1.26}/sinatools/CLI/DataDownload/download_files.py

@@ -40,7 +40,7 @@ from sinatools.DataDownload.downloader import urls
 
 def main():
     parser = argparse.ArgumentParser(description="Download files from specified URLs.")
-    parser.add_argument('-f', '--files', nargs="*", choices=urls.keys(),
+    parser.add_argument('-f', '--files', nargs="*",
                         help="Names of the files to download. Available files are: "
                              f"{', '.join(urls.keys())}. If no file is specified, all files will be downloaded.")
     
@@ -50,8 +50,23 @@ def main():
 
     if args.files:
         for file in args.files:
-            url = urls[file]
-            download_file(url)
+            print("file: ", file)
+            if file == "wsd":
+                download_file(urls["morph"])
+                download_file(urls["ner"])
+                download_file(urls["wsd_model"])
+                download_file(urls["wsd_tokenizer"])
+                download_file(urls["glosses_dic"])
+                download_file(urls["five_grams"])
+                download_file(urls["four_grams"])
+                download_file(urls["three_grams"])
+                download_file(urls["two_grams"])
+            elif file == "synonyms":
+                download_file(urls["synonyms_level2"])
+                download_file(urls["synonyms_level3"])
+            else:
+               url = urls[file]
+               download_file(url)
     else:
         download_files()
 

{SinaTools-0.1.24 → SinaTools-0.1.26}/sinatools/CLI/ner/corpus_entity_extractor.py

@@ -20,7 +20,7 @@ def jsons_to_list_of_lists(json_list):
     return [[d['token'], d['tags']] for d in json_list]
 
 def combine_tags(sentence):
-    output = jsons_to_list_of_lists(extract(sentence))
+    output = jsons_to_list_of_lists(extract(sentence, "nested"))
     return [word[1] for word in output]
 
 

{SinaTools-0.1.24 → SinaTools-0.1.26}/sinatools/CLI/ner/entity_extractor.py

@@ -46,7 +46,7 @@ def jsons_to_list_of_lists(json_list):
     return [[d['token'], d['tags']] for d in json_list]
 
 def combine_tags(sentence):
-    output = jsons_to_list_of_lists(extract(sentence))
+    output = jsons_to_list_of_lists(extract(sentence, "nested"))
     return [word[1] for word in output]
 
 

{SinaTools-0.1.24 → SinaTools-0.1.26}/sinatools/DataDownload/downloader.py

@@ -95,37 +95,41 @@ def download_file(url, dest_path=get_appdatadir()):
     print(filename)
     headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
 
-    # try:
-    with requests.get(url, headers=headers, stream=True) as r:
-        r.raise_for_status()
-        with open(file_path, 'wb') as f:
-            total_size = int(r.headers.get('content-length', 0))
-            block_size = 8192
-            progress_bar = tqdm(total=total_size, unit='iB', unit_scale=True)
-            for chunk in r.iter_content(chunk_size=block_size):
-                if chunk:
-                    f.write(chunk)
-                    progress_bar.update(len(chunk))
-            progress_bar.close()
-    # Check the file type and extract accordingly
-    file_extension = os.path.splitext(file_path)[1]
-    extracted_folder_name = os.path.splitext(file_path)[0]
-    
-    if file_extension == '.zip':
-        extract_zip(file_path, extracted_folder_name)
-    elif file_extension == '.gz':
-        extract_tar(file_path, extracted_folder_name)
-    elif file_extension =='.pickle':
-        print(f'Done: {file_extension}')
-    else:
-        print(f'Unsupported file type for extraction: {file_extension}')
-    return file_path
-
-    # except requests.exceptions.HTTPError as e:
-        # if e.response.status_code == 403:
-            # print(f'Error 403: Forbidden. The requested file URL {url} could not be downloaded due to insufficient permissions. Please check the URL and try again.')
-        # else:
-            # print('An error occurred while downloading the file:', e)
+    try:
+        with requests.get(url, headers=headers, stream=True) as r:
+            r.raise_for_status()
+            with open(file_path, 'wb') as f:
+                total_size = int(r.headers.get('content-length', 0))
+                block_size = 8192
+                progress_bar = tqdm(total=total_size, unit='iB', unit_scale=True)
+                for chunk in r.iter_content(chunk_size=block_size):
+                    if chunk:
+                        f.write(chunk)
+                        progress_bar.update(len(chunk))
+                progress_bar.close()
+
+        # Check the file type and extract accordingly
+        file_extension = os.path.splitext(file_path)[1]
+        extracted_folder_name = os.path.splitext(file_path)[0]
+        
+        if file_extension == '.zip':
+            extract_zip(file_path, extracted_folder_name)
+        elif file_extension == '.gz':
+
+            extract_tar(file_path, extracted_folder_name)
+        elif file_extension =='.pickle':
+            print(f'Done: {file_extension}')
+
+        else:
+            print(f'Unsupported file type for extraction: {file_extension}')
+
+        return file_path
+
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code == 403:
+            print(f'Error 403: Forbidden. The requested file URL {url} could not be downloaded due to insufficient permissions. Please check the URL and try again.')
+        else:
+            print('An error occurred while downloading the file:', e)
 
 def extract_zip(file_path, extracted_folder_name):
     """

SinaTools-0.1.26/sinatools/VERSION

@@ -0,0 +1 @@
+0.1.26

SinaTools-0.1.26/sinatools/ner/entity_extractor.py

@@ -0,0 +1,63 @@
+import os
+from collections import namedtuple
+from sinatools.ner.data_format import get_dataloaders, text2segments
+from . import tagger, tag_vocab, train_config
+
+
+def convert_nested_to_flat(nested_tags):
+    flat_tags = []
+    
+    for entry in nested_tags:
+        word = entry['token']
+        tags = entry['tags'].split()
+        
+        # Initialize with the first tag in the sequence
+        flat_tag = tags[0]
+        
+        for tag in tags[1:]:
+            # Check if the tag is an "I-" tag, indicating continuation of an entity
+            if tag.startswith('I-'):
+                flat_tag = tag
+                break
+        
+        flat_tags.append({
+            'token': word,
+            'tags': flat_tag
+        })
+    
+    return flat_tags
+
+def extract(text, ner_method):
+    
+    dataset, token_vocab = text2segments(text)
+
+    vocabs = namedtuple("Vocab", ["tags", "tokens"])
+    vocab = vocabs(tokens=token_vocab, tags=tag_vocab)
+
+    dataloader = get_dataloaders(
+        (dataset,),
+        vocab,
+        train_config.data_config,
+        batch_size=32,
+        shuffle=(False,),
+    )[0]
+
+
+    segments = tagger.infer(dataloader)
+    segments_lists = []
+    
+    for segment in segments:
+        for token in segment:
+            segments_list = {}
+            segments_list["token"] = token.text
+            list_of_tags = [t['tag'] for t in token.pred_tag]
+            list_of_tags = [i for i in list_of_tags if i not in('O',' ','')]
+            if list_of_tags == []:
+               segments_list["tags"] = ' '.join(['O'])
+            else:
+               segments_list["tags"] = ' '.join(list_of_tags)
+            segments_lists.append(segments_list)  
+    
+    if ner_method == "flat":
+      segments_lists = convert_nested_to_flat(segments_lists)          
+    return segments_lists

{SinaTools-0.1.24 → SinaTools-0.1.26}/sinatools/ner/trainers/BaseTrainer.py

@@ -1,117 +1,117 @@
-import os
-import torch
-import logging
-import natsort
-import glob
-
-logger = logging.getLogger(__name__)
-
-
-class BaseTrainer:
-    def __init__(
-        self,
-        model=None,
-        max_epochs=50,
-        optimizer=None,
-        scheduler=None,
-        loss=None,
-        train_dataloader=None,
-        val_dataloader=None,
-        test_dataloader=None,
-        log_interval=10,
-        summary_writer=None,
-        output_path=None,
-        clip=5,
-        patience=5
-    ):
-        self.model = model
-        self.max_epochs = max_epochs
-        self.train_dataloader = train_dataloader
-        self.val_dataloader = val_dataloader
-        self.test_dataloader = test_dataloader
-        self.optimizer = optimizer
-        self.scheduler = scheduler
-        self.loss = loss
-        self.log_interval = log_interval
-        self.summary_writer = summary_writer
-        self.output_path = output_path
-        self.current_timestep = 0
-        self.current_epoch = 0
-        self.clip = clip
-        self.patience = patience
-
-    def tag(self, dataloader, is_train=True):
-        """
-        Given a dataloader containing segments, predict the tags
-        :param dataloader: torch.utils.data.DataLoader
-        :param is_train: boolean - True for training model, False for evaluation
-        :return: Iterator
-                    subwords (B x T x NUM_LABELS)- torch.Tensor - BERT subword ID
-                    gold_tags (B x T x NUM_LABELS) - torch.Tensor - ground truth tags IDs
-                    tokens - List[arabiner.data.dataset.Token] - list of tokens
-                    valid_len (B x 1) - int - valiud length of each sequence
-                    logits (B x T x NUM_LABELS) - logits for each token and each tag
-        """
-        for subwords, gold_tags, tokens, valid_len in dataloader:
-            self.model.train(is_train)
-
-            if torch.cuda.is_available():
-                subwords = subwords.cuda()
-                gold_tags = gold_tags.cuda()
-
-            if is_train:
-                self.optimizer.zero_grad()
-                logits = self.model(subwords)
-            else:
-                with torch.no_grad():
-                    logits = self.model(subwords)
-
-            yield subwords, gold_tags, tokens, valid_len, logits
-
-    def segments_to_file(self, segments, filename):
-        """
-        Write segments to file
-        :param segments: [List[arabiner.data.dataset.Token]] - list of list of tokens
-        :param filename: str - output filename
-        :return: None
-        """
-        with open(filename, "w") as fh:
-            results = "\n\n".join(["\n".join([t.__str__() for t in segment]) for segment in segments])
-            fh.write("Token\tGold Tag\tPredicted Tag\n")
-            fh.write(results)
-            logging.info("Predictions written to %s", filename)
-
-    def save(self):
-        """
-        Save model checkpoint
-        :return:
-        """
-        filename = os.path.join(
-            self.output_path,
-            "checkpoints",
-            "checkpoint_{}.pt".format(self.current_epoch),
-        )
-
-        checkpoint = {
-            "model": self.model.state_dict(),
-            "optimizer": self.optimizer.state_dict(),
-            "epoch": self.current_epoch
-        }
-
-        logger.info("Saving checkpoint to %s", filename)
-        torch.save(checkpoint, filename)
-
-    def load(self, checkpoint_path):
-        """
-        Load model checkpoint
-        :param checkpoint_path: str - path/to/checkpoints
-        :return: None
-        """
-        checkpoint_path = natsort.natsorted(glob.glob(f"{checkpoint_path}/checkpoint_*.pt"))
-        checkpoint_path = checkpoint_path[-1]
-
-        logger.info("Loading checkpoint %s", checkpoint_path)
-
-        device = None if torch.cuda.is_available() else torch.device('cpu')
-        checkpoint = torch.load(checkpoint_path, map_location=device)
-        self.model.load_state_dict(checkpoint["model"])
+import os
+import torch
+import logging
+import natsort
+import glob
+
+logger = logging.getLogger(__name__)
+
+
+class BaseTrainer:
+    def __init__(
+        self,
+        model=None,
+        max_epochs=50,
+        optimizer=None,
+        scheduler=None,
+        loss=None,
+        train_dataloader=None,
+        val_dataloader=None,
+        test_dataloader=None,
+        log_interval=10,
+        summary_writer=None,
+        output_path=None,
+        clip=5,
+        patience=5
+    ):
+        self.model = model
+        self.max_epochs = max_epochs
+        self.train_dataloader = train_dataloader
+        self.val_dataloader = val_dataloader
+        self.test_dataloader = test_dataloader
+        self.optimizer = optimizer
+        self.scheduler = scheduler
+        self.loss = loss
+        self.log_interval = log_interval
+        self.summary_writer = summary_writer
+        self.output_path = output_path
+        self.current_timestep = 0
+        self.current_epoch = 0
+        self.clip = clip
+        self.patience = patience
+
+    def tag(self, dataloader, is_train=True):
+        """
+        Given a dataloader containing segments, predict the tags
+        :param dataloader: torch.utils.data.DataLoader
+        :param is_train: boolean - True for training model, False for evaluation
+        :return: Iterator
+                    subwords (B x T x NUM_LABELS)- torch.Tensor - BERT subword ID
+                    gold_tags (B x T x NUM_LABELS) - torch.Tensor - ground truth tags IDs
+                    tokens - List[arabiner.data.dataset.Token] - list of tokens
+                    valid_len (B x 1) - int - valiud length of each sequence
+                    logits (B x T x NUM_LABELS) - logits for each token and each tag
+        """
+        for subwords, gold_tags, tokens, valid_len in dataloader:
+            self.model.train(is_train)
+
+            if torch.cuda.is_available():
+                subwords = subwords.cuda()
+                gold_tags = gold_tags.cuda()
+
+            if is_train:
+                self.optimizer.zero_grad()
+                logits = self.model(subwords)
+            else:
+                with torch.no_grad():
+                    logits = self.model(subwords)
+
+            yield subwords, gold_tags, tokens, valid_len, logits
+
+    def segments_to_file(self, segments, filename):
+        """
+        Write segments to file
+        :param segments: [List[arabiner.data.dataset.Token]] - list of list of tokens
+        :param filename: str - output filename
+        :return: None
+        """
+        with open(filename, "w") as fh:
+            results = "\n\n".join(["\n".join([t.__str__() for t in segment]) for segment in segments])
+            fh.write("Token\tGold Tag\tPredicted Tag\n")
+            fh.write(results)
+            logging.info("Predictions written to %s", filename)
+
+    def save(self):
+        """
+        Save model checkpoint
+        :return:
+        """
+        filename = os.path.join(
+            self.output_path,
+            "checkpoints",
+            "checkpoint_{}.pt".format(self.current_epoch),
+        )
+
+        checkpoint = {
+            "model": self.model.state_dict(),
+            "optimizer": self.optimizer.state_dict(),
+            "epoch": self.current_epoch
+        }
+
+        logger.info("Saving checkpoint to %s", filename)
+        torch.save(checkpoint, filename)
+
+    def load(self, checkpoint_path):
+        """
+        Load model checkpoint
+        :param checkpoint_path: str - path/to/checkpoints
+        :return: None
+        """
+        checkpoint_path = natsort.natsorted(glob.glob(f"{checkpoint_path}/checkpoint_*.pt"))
+        checkpoint_path = checkpoint_path[-1]
+
+        logger.info("Loading checkpoint %s", checkpoint_path)
+
+        device = None if torch.cuda.is_available() else torch.device('cpu')
+        checkpoint = torch.load(checkpoint_path, map_location=device)
+        self.model.load_state_dict(checkpoint["model"], strict=False)

{SinaTools-0.1.24 → SinaTools-0.1.26}/sinatools/wsd/disambiguator.py

@@ -217,7 +217,7 @@ def jsons_to_list_of_lists(json_list):
 def find_named_entities(string):
    found_entities = []
    
-   ner_entites = extract(string)
+   ner_entites = extract(string, "nested")
    list_of_entites = jsons_to_list_of_lists(ner_entites)
    entites = distill_entities(list_of_entites)
    
@@ -288,17 +288,17 @@ def disambiguate_glosses_using_SALMA(glosses, Diac_lemma, Undiac_lemma, word, se
       concept_id, gloss = GlossPredictor(Diac_lemma, Undiac_lemma,word,sentence,glosses_dictionary)
 
       my_json = {}    
-      my_json['Concept_id'] = concept_id
+      my_json['concept_id'] = concept_id
     #   my_json['Gloss'] = gloss
       my_json['word'] = word
-      my_json['Undiac_lemma'] = Undiac_lemma
-      my_json['Diac_lemma'] = Diac_lemma
+      #my_json['Undiac_lemma'] = Undiac_lemma
+      my_json['lemma'] = Diac_lemma
       return my_json
    else:
       my_json = {}    
       my_json['word'] = word
-      my_json['Undiac_lemma'] = Undiac_lemma
-      my_json['Diac_lemma'] = Diac_lemma
+      #my_json['Undiac_lemma'] = Undiac_lemma
+      my_json['lemma'] = Diac_lemma
       return my_json
 
 
@@ -405,26 +405,26 @@ def disambiguate_glosses_main(word, sentence):
    if concept_count == 0:
       my_json = {}    
       my_json['word'] = word['word']
-      my_json['Diac_lemma'] = word['Diac_lemma']
-      my_json['Undiac_lemma'] = word['Undiac_lemma']
+      my_json['lemma'] = word['Diac_lemma']
+      #my_json['Undiac_lemma'] = word['Undiac_lemma']
       return my_json
    elif concept_count == 1:
       my_json = {}    
       my_json['word'] = word['word']
       glosses = word['glosses'][0]
     #   my_json['Gloss'] = glosses['gloss']
-      my_json['Concept_id'] = glosses['concept_id']
-      my_json['Diac_lemma'] = word['Diac_lemma']
-      my_json['Undiac_lemma'] = word['Undiac_lemma']
+      my_json['concept_id'] = glosses['concept_id']
+      my_json['lemma'] = word['Diac_lemma']
+      #my_json['Undiac_lemma'] = word['Undiac_lemma']
       return my_json
    elif concept_count == '*':
       my_json = {}    
       my_json['word'] = word['word']
       glosses = word['glosses'][0]
       my_json['Gloss'] = glosses['gloss']
-      my_json['Concept_id'] = glosses['concept_id']
-      my_json['Diac_lemma'] = word['Diac_lemma']
-      my_json['Undiac_lemma'] = word['Undiac_lemma']
+      my_json['concept_id'] = glosses['concept_id']
+      my_json['lemma'] = word['Diac_lemma']
+      #my_json['Undiac_lemma'] = word['Undiac_lemma']
       return my_json
    else:   
       input_word = word['word']
@@ -477,21 +477,18 @@ def disambiguate(sentence):
         #output
          [
              {
-                 "Concept_id": "303019218",
+                 "concept_id": "303019218",
                  "word": "ذهبت",
-                 "Undiac_lemma": "ذهب",
-                 "Diac_lemma": "ذَهَبَ۪ 1"
+                 "lemma": "ذَهَبَ۪ 1"
              },
              {
                  "word": "إلى",
-                 "Diac_lemma": إِلَى 1,
-                 "Undiac_lemma": "الى"
+                 "lemma": "إِلَى 1"
              },
              {
                  "word": "جامعة بيرزيت",
-                 "Concept_id": "334000099",
-                 "Diac_lemma": جامِعَة بيرزَيت,
-                 "Undiac_lemma": "جامعة بيرزيت"
+                 "concept_id": "334000099",
+                 "lemma": "جامِعَة بيرزَيت"
              }
          ]
     """      

SinaTools-0.1.24/sinatools/VERSION

@@ -1 +0,0 @@
-0.1.24

SinaTools-0.1.24/sinatools/ner/entity_extractor.py

@@ -1,72 +0,0 @@
-import os
-from collections import namedtuple
-from sinatools.ner.data_format import get_dataloaders, text2segments
-from . import tagger, tag_vocab, train_config
-
-def extract(text, batch_size=32):
-    """
-    This method processes an input text and returns named entites for each token within the text, based on the specified batch size. As follows:
-
-    Args:
-        text (:obj:`str`): The Arabic text to be tagged.
-        batch_size (int, optional): Batch size for inference. Default is 32.
-
-    Returns:
-        list (:obj:`list`): A list of JSON objects, where each JSON could be contains:
-        token: The token from the original text.
-        NER tag: The label pairs for each segment.
-
-    **Example:**
-
-     .. highlight:: python
-     .. code-block:: python
-
-        from sinatools.ner.entity_extractor import extract
-        extract('ذهب محمد إلى جامعة بيرزيت')
-        [{
-            "word":"ذهب",
-            "tags":"O"
-          },{
-            "word":"محمد",
-            "tags":"B-PERS"
-          },{
-            "word":"إلى",
-            "tags":"O"
-          },{
-            "word":"جامعة",
-            "tags":"B-ORG"
-          },{
-            "word":"بيرزيت",
-            "tags":"B-GPE I-ORG"
-        }]
-    """    
-    
-    dataset, token_vocab = text2segments(text)
-
-    vocabs = namedtuple("Vocab", ["tags", "tokens"])
-    vocab = vocabs(tokens=token_vocab, tags=tag_vocab)
-
-    dataloader = get_dataloaders(
-        (dataset,),
-        vocab,
-        train_config.data_config,
-        batch_size=batch_size,
-        shuffle=(False,),
-    )[0]
-
-
-    segments = tagger.infer(dataloader)
-    segments_lists = []
-    
-    for segment in segments:
-        for token in segment:
-            segments_list = {}
-            segments_list["token"] = token.text
-            list_of_tags = [t['tag'] for t in token.pred_tag]
-            list_of_tags = [i for i in list_of_tags if i not in('O',' ','')]
-            if list_of_tags == []:
-               segments_list["tags"] = ' '.join(['O'])
-            else:
-               segments_list["tags"] = ' '.join(list_of_tags)
-            segments_lists.append(segments_list)  
-    return segments_lists