SinaTools 0.1.35__py2.py3-none-any.whl → 0.1.37__py2.py3-none-any.whl

sinatools/ner/data.py

@@ -1,124 +0,0 @@
-from torch.utils.data import DataLoader
-from torchtext.vocab import vocab
-from collections import Counter, namedtuple
-import logging
-import re
-import itertools
-from sinatools.ner.helpers import load_object
-from sinatools.ner.datasets import Token
-from sinatools.utils.tokenizers_words import simple_word_tokenize
-
-logger = logging.getLogger(__name__)
-
-
-def conll_to_segments(filename):
-    """
-    Convert CoNLL files to segments. This return list of segments and each segment is
-    a list of tuples (token, tag)
-    :param filename: Path
-    :return: list[[tuple]] - [[(token, tag), (token, tag), ...], [(token, tag), ...]]
-    """
-    segments, segment = list(), list()
-
-    with open(filename, "r") as fh:
-        for token in fh.read().splitlines():
-            if not token.strip():
-                segments.append(segment)
-                segment = list()
-            else:
-                parts = token.split()
-                token = Token(text=parts[0], gold_tag=parts[1:])
-                segment.append(token)
-
-        segments.append(segment)
-
-    return segments
-
-
-def parse_conll_files(data_paths):
-    """
-    Parse CoNLL formatted files and return list of segments for each file and index
-    the vocabs and tags across all data_paths
-    :param data_paths: tuple(Path) - tuple of filenames
-    :return: tuple( [[(token, tag), ...], [(token, tag), ...]], -> segments for data_paths[i]
-                    [[(token, tag), ...], [(token, tag), ...]], -> segments for data_paths[i+1],
-                    ...
-                  )
-             List of segments for each dataset and each segment has list of (tokens, tags)
-    """
-    vocabs = namedtuple("Vocab", ["tags", "tokens"])
-    datasets, tags, tokens = list(), list(), list()
-
-    for data_path in data_paths:
-        dataset = conll_to_segments(data_path)
-        datasets.append(dataset)
-        tokens += [token.text for segment in dataset for token in segment]
-        tags += [token.gold_tag for segment in dataset for token in segment]
-
-    # Flatten list of tags
-    tags = list(itertools.chain(*tags))
-
-    # Generate vocabs for tags and tokens
-    tag_vocabs = tag_vocab_by_type(tags)
-    tag_vocabs.insert(0, vocab(Counter(tags)))
-    vocabs = vocabs(tokens=vocab(Counter(tokens), specials=["UNK"]), tags=tag_vocabs)
-    return tuple(datasets), vocabs
-
-
-def tag_vocab_by_type(tags):
-    vocabs = list()
-    c = Counter(tags)
-    tag_names = c.keys()
-    tag_types = sorted(list(set([tag.split("-", 1)[1] for tag in tag_names if "-" in tag])))
-
-    for tag_type in tag_types:
-        r = re.compile(".*-" + tag_type)
-        t = list(filter(r.match, tags)) + ["O"]
-        vocabs.append(vocab(Counter(t), specials=["<pad>"]))
-
-    return vocabs
-
-
-def text2segments(text):
-    """
-    Convert text to a datasets and index the tokens
-    """
-    #dataset = [[Token(text=token, gold_tag=["O"]) for token in text.split()]]
-    list_of_tokens = simple_word_tokenize(text)
-    dataset = [[Token(text=token, gold_tag=["O"]) for token in list_of_tokens]]
-    tokens = [token.text for segment in dataset for token in segment]
-
-    # Generate vocabs for the tokens
-    segment_vocab = vocab(Counter(tokens), specials=["UNK"])
-    return dataset, segment_vocab
-
-
-def get_dataloaders(
-    datasets, vocab, data_config, batch_size=32, num_workers=0, shuffle=(True, False, False)
-):
-    """
-    From the datasets generate the dataloaders
-    :param datasets: list - list of the datasets, list of list of segments and tokens
-    :param batch_size: int
-    :param num_workers: int
-    :param shuffle: boolean - to shuffle the data or not
-    :return: List[torch.utils.data.DataLoader]
-    """
-    dataloaders = list()
-
-    for i, examples in enumerate(datasets):
-        data_config["kwargs"].update({"examples": examples, "vocab": vocab})
-        dataset = load_object("sinatools."+data_config["fn"], data_config["kwargs"])
-
-        dataloader = DataLoader(
-            dataset=dataset,
-            shuffle=shuffle[i],
-            batch_size=batch_size,
-            num_workers=num_workers,
-            collate_fn=dataset.collate_fn,
-        )
-
-        logger.info("%s batches found", len(dataloader))
-        dataloaders.append(dataloader)
-
-    return dataloaders

sinatools/ner/relation_extractor.py

@@ -1,201 +0,0 @@
-import torch
-import json
-from urllib.request import Request, urlopen
-from sinatools.ner.entity_extractor import extract
-from . import pipe
-
-
-# ============================ Extract entities and their types ========================
-def jsons_to_list_of_lists(json_list):
-    return [[d['token'], d['tags']] for d in json_list]
-
-def entities_and_types(sentence):
-    output_list = jsons_to_list_of_lists(extract(sentence))
-    json_short = distill_entities(output_list)
-
-    entities = {}
-    for entity in json_short:
-        name = entity[0]
-        entity_type = entity[1]
-        entities[name] = entity_type
-
-    return entities
-
-def distill_entities(entities):
-    # This is list that we put the output what we need
-    list_output = list()
-
-    # This line go to sort function and save the output to temp_entities
-    temp_entities = sortTags(entities)
-
-    # This list help us to make the output,
-    temp_list = list()
-
-    # initlize the temp_list
-    temp_list.append(["", "", 0, 0])
-    word_position = 0
-
-    # For each entity, convert ibo to distllir list.
-    for entity in temp_entities:
-        # This is counter tag of this entity
-        counter_tag = 0
-        # For each tag
-        for tag in str(entity[1]).split():
-            # If the counter tag greater than or equal to lenght of templist, if yes then we will append the empty value in templist
-            if counter_tag >= len(temp_list):
-                temp_list.append(["", "", 0, 0])
-
-            # If tag equal O and word postion of this tag is not equal zero then it will add all
-            # not empty eliment of temp list in output list
-            if "O" == tag and word_position != 0:
-                for j in range(0, len(temp_list)):
-                    if temp_list[j][1] != "":
-                        list_output.append([temp_list[j][0].strip(), temp_list[j][1], temp_list[j][2], temp_list[j][3]])
-                        temp_list[j][0] = ""
-                        temp_list[j][1] = ""
-                        temp_list[j][2] = word_position
-                        temp_list[j][3] = word_position
-            # if this tag not equal O, and split by '-' the tag and check the lenght equals two and if the first eliment
-            # of the split its B
-            elif "O" != tag and len(tag.split("-")) == 2 and tag.split("-")[0] == "B":
-                # if the temp_list of counter is not empty then it will append in output list and hten it will
-                # initilize by new string and tag in templist of counter
-                if temp_list[counter_tag][1] != "":
-                    list_output.append([temp_list[counter_tag][0].strip(), temp_list[counter_tag][1], temp_list[counter_tag][2], temp_list[counter_tag][3]])
-                temp_list[counter_tag][0] = str(entity[0]) + " "
-                temp_list[counter_tag][1] = str(tag).split("-")[1]
-                temp_list[counter_tag][2] = word_position
-                temp_list[counter_tag][3] = word_position
-
-            # if this tag not equal O, and split by '-' the tag and check the lenght equals two and if the first eliment
-            # of the split its O
-            elif "O" != tag and len(tag.split("-")) == 2 and tag.split("-")[0] == "I" and word_position != 0:
-                # For each of temp_list, check if in this counter tag of templist is same tag with this.tag
-                # then will complete if not it will save in output list and cheak another
-                for j in range(counter_tag,len(temp_list)):
-                    if temp_list[j][1] == tag[2:] and temp_list[j][3] != word_position:
-                        temp_list[j][0] += str(entity[0]) + " "
-                        temp_list[j][3] += 1
-                        break
-                    else:
-                        if temp_list[j][1] != "":
-                            list_output.append([temp_list[j][0].strip(), temp_list[j][1], temp_list[j][2], temp_list[j][3]])
-                            temp_list[j][0] = ""
-                            temp_list[j][1] = ""
-                            temp_list[j][2] = word_position
-                            temp_list[j][3] = word_position
-            counter_tag += 1
-        word_position += 1
-    # For each temp_list, at the end of the previous loop, there will be some
-    # values in this list, we should save it to the output list
-    for j in range(0, len(temp_list)):
-        if temp_list[j][1] != "":
-            list_output.append([temp_list[j][0].strip(), temp_list[j][1], temp_list[j][2], temp_list[j][3]])
-    return sorted(list_output, key=lambda x: (x[2]))
-
-def sortTags(entities):
-    temp_entities = entities
-    temp_counter = 0
-    # For each entity, this loop will sort each tag of entitiy, first it will check if the
-    # previous tags has same count of this tag, second will sort the tags and check if this tags is correct
-    for entity in temp_entities:
-        tags = entity[1].split()
-        for tag in tags:
-            # if the counter is not 0 then, will complete
-            if temp_counter != 0:
-                # Check if this tag is equal I-, if yes then it will count how many tag in this tags and
-                # count how many tag in previous tags
-                if "I-" == tag[0:2]:
-                    counter_of_this_tag = 0
-                    counter_of_previous_tag = 0
-                    for word in tags:
-                        if tag.split("-")[1] in word:
-                            counter_of_this_tag+=1
-                    for word in temp_entities[temp_counter-1][1].split():
-                        if tag.split("-")[1] in word:
-                            counter_of_previous_tag+=1
-                    # if the counter of previous tag is bigger than counter of this tag, then we
-                    # need to add I-tag in this tags
-                    if counter_of_previous_tag > counter_of_this_tag:
-                        tags.append("I-"+tag.split("-")[1])
-        # Sort the tags
-        tags.sort()
-        # Need to revers the tags because it should begins with I
-        tags.reverse()
-        # If the counter is not 0 then we can complete
-        if temp_counter != 0:
-            this_tags = tags
-            previous_tags = temp_entities[temp_counter - 1][1].split()
-            sorted_tags = list()
-
-            # Check if the this tag is not O and previous tags is not O, then will complete,
-            # if not then it will ignor this tag
-            if "O" not in this_tags and "O" not in previous_tags:
-                index = 0
-                #For each previous tags, need sort this tag by previous tags if its I, B we can ignor
-                for i in previous_tags:
-                    j = 0
-                    while this_tags and j < len(this_tags):
-                        if this_tags[j][0:2] == "I-" and this_tags[j][2:] == i[2:]:
-                            sorted_tags.insert(index, this_tags.pop(j))
-                            break
-                        elif this_tags[j][0:2] == "B-":
-                            break
-                        j += 1
-                    index += 1
-            sorted_tags += this_tags
-            tags = sorted_tags
-        str_tag = " "
-        str_tag = str_tag.join(tags)
-        str_tag = str_tag.strip()
-        temp_entities[temp_counter][1] = str_tag
-        temp_counter += 1
-    return temp_entities
-
-# ============= Prepare Templates and Catergorize Extracted Entities ================
-temp03={'location':'مكان حدوث','agent':'أحد المتأثرين في','happened at':'تاريخ حدوث'}
-categories = {
-    'agent': ['PERS', 'NORP', 'OCC', 'ORG'],
-    'location': ['LOC', 'FAC', 'GPE'],
-    'happened at': ['DATE', 'TIME']
-    }
-
-def get_entity_category(entity_type, categories):
-    for category, types in categories.items():
-        if entity_type in types:
-            return category
-    return None
-
-
-# ============ Extract entities, their types and categorize them ===============
-def relation_extraction(sentence):
-    #test_sentence="صورة إعتقال طفل فلسطيني خلال انتفاضة الأقصى ."
-    entities=entities_and_types(sentence)
-
-    event_indices = [i for i, (_, entity_type) in enumerate(entities.items()) if entity_type == 'EVENT']
-    arg_event_indices = [i for i, (_, entity_type) in enumerate(entities.items()) if entity_type != 'EVENT']
-
-    output_list=[]
-
-    for i in event_indices:
-        event_entity=list(entities.keys())[i]
-        for j in arg_event_indices:
-            arg_name= list(entities.keys())[j]
-            arg_type=entities[arg_name]
-            category = get_entity_category(arg_type, categories)
-            
-            if category in temp03:
-                relation_sentence=f"[CLS] {sentence} [SEP] {event_entity} {temp03[category]} {arg_name}"
-                predicted_relation=pipe(relation_sentence)
-                score = predicted_relation[0][0]['score']  
-                if score > 0.50:
-                    #print(f"Event:{event_entity} Relation:{category} Argument:{arg_name}\n")
-                    #output_list.append([{event_entity} ,{category}, {arg_name}])
-                    output_list.append(f"Event:{event_entity}, Relation:{category}, Argument:{arg_name}")
-
-                else:
-                    #print(f"Event:{event_entity} Relation:No relation Argument:{arg_name}\n")
-                    #output_list.append([{event_entity} ,'No relation', {arg_name}])
-                    output_list.append(f"Event:{event_entity}, Relation:No relation, Argument:{arg_name}")
-    
-    return output_list