ddi-fw 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

ddi_fw/datasets/embedding_generator.py

@@ -0,0 +1,66 @@
+# !pip install -U sentence-transformers
+
+# from transformers import BertTokenizer,BertForPreTraining,BertModel
+# from sentence_transformers import SentenceTransformer, util
+import pandas as pd
+import numpy as np
+import nltk
+from nltk import sent_tokenize
+from tqdm import tqdm
+
+
+nltk.download('punkt')
+
+import os
+def check_file_exists(path):
+  return os.path.isdir(path)
+
+def get_model_name_or_local_path(model_local_path, model_name):
+  if check_file_exists(model_local_path):
+    return model_local_path
+  return model_name
+
+import re
+def process_text(text):
+  text = re.sub("\[L\d*\]", "",text)
+  text = text.replace("[","")
+  text = text.replace("]","")
+  return text
+
+
+
+from collections import defaultdict
+from functools import partial
+
+# NOT modelden input size'ı anlama,
+def create_embeddings(model, data, column, drop_column=True):
+  # model._modules['1'].get_sentence_embedding_dimension()
+  # shape = (1,model._modules['0'].get_word_embedding_dimension())
+  shape = model._modules['0'].get_word_embedding_dimension()
+  column_embeddings_dict = defaultdict(lambda: np.zeros(shape))
+  for index, row in tqdm(data.iterrows()):
+    # if index == 10:
+    #   break
+    text = data[column][index]
+    # else'de zero
+    if text == None or type(text) != str:
+      embeddings = None
+    else:
+      sentences = sent_tokenize(text)
+      embeddings = model.encode(sentences)
+
+  #TODO benzer olan ilacın embedding değerini vererek dene
+    if embeddings is None or len(embeddings) == 0: #embedding check none type
+      sum_of_embeddings = np.zeros(shape)
+    else:
+      sum_of_embeddings = np.sum(embeddings, axis = 0)
+    # column_embeddings_dict[row['id']] = sum_of_embeddings.reshape(1, -1) # 2d
+    column_embeddings_dict[row['id']] = sum_of_embeddings
+    # data.iloc[index][column+'_embedding']=sum_of_embeddings
+  
+  data[column+'_embedding'] = pd.Series(column_embeddings_dict.values())
+  if(drop_column):
+    data.drop([column], axis = 1, inplace = True)
+  # data[column+'_embedding'] = [column_embeddings_dict[row['name']] for index, row in data.iterrows()]
+  return column_embeddings_dict
+

ddi_fw/datasets/embedding_generator_new.py

@@ -0,0 +1,105 @@
+# !pip install -U sentence-transformers
+
+# from transformers import BertTokenizer,BertForPreTraining,BertModel
+# from sentence_transformers import SentenceTransformer, util
+import pandas as pd
+import numpy as np
+from nltk import sent_tokenize
+import torch
+from tqdm import tqdm
+
+
+from collections import defaultdict
+from functools import partial
+from abc import ABC, abstractmethod
+from transformers import AutoModel, AutoTokenizer
+from sentence_transformers import SentenceTransformer, util
+
+
+class EmbeddingGenerator(ABC):
+
+    def __init__(self):
+        self.shape = None
+
+    @abstractmethod
+    def generate(self, text):
+        pass
+
+# https://github.com/huggingface/transformers/issues/1791
+class PretrainedEmbeddingGenerator(EmbeddingGenerator):
+    def __init__(self, model_name, split_text=True):
+        self.model_name = model_name
+        self.model = AutoModel.from_pretrained(model_name)
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        self.shape = self.model.get_input_embeddings().weight.shape
+        self.split_text = split_text
+
+    def generate(self, text):
+        if self.split_text:
+            sentences = sent_tokenize(text)
+            output_embeddings = None
+            for sentence in sentences:
+                input_ids  = self.tokenizer.encode(sentence, return_tensors='pt', padding=True)
+                if output_embeddings == None:
+                    output_embeddings = self.model(input_ids).last_hidden_state.mean(dim=1)
+                else:
+                    output_embeddings += self.model(input_ids).last_hidden_state.mean(dim=1)
+            if output_embeddings == None:
+                output_embeddings = torch.empty((1,self.model.get_input_embeddings().weight.shape[1]))
+        else:
+            encoded_input = self.tokenizer(text, return_tensors='pt')
+            input_ids = self.tokenizer.encode(text, add_special_tokens=True, max_length=self.tokenizer.model_max_length, return_tensors='pt')
+            # input_ids  = encoded_input.input_ids[:self.tokenizer.model_max_length]
+            output_embeddings = self.model(input_ids)
+            # output_embeddings = self.model(**encoded_input)
+            # sentence embedding
+            output_embeddings = output_embeddings.last_hidden_state.mean(dim=1)
+        return torch.flatten(output_embeddings).detach().numpy()
+
+
+class LLMEmbeddingGenerator(EmbeddingGenerator):
+    pass
+
+
+class SBertEmbeddingGenerator(PretrainedEmbeddingGenerator):
+    def __init__(self, model_name, split_text=True):
+        self.model = SentenceTransformer(model_name)
+        self.shape = self.model._modules['0'].get_word_embedding_dimension()
+        self.split_text = split_text
+
+    def generate(self, text):
+        if text == None or type(text) != str:
+            embeddings = None
+        else:
+            if self.split_text:
+                sentences = sent_tokenize(text)
+                embeddings = self.model.encode(sentences)
+            else:
+                embeddings = self.model.encode(text)
+        return embeddings
+
+
+# NOT modelden input size'ı anlama,
+def create_embeddings_new(generator: EmbeddingGenerator, data, column, drop_column=True):
+    column_embeddings_dict = defaultdict(lambda: np.zeros(generator.shape))
+    for index, row in tqdm(data.iterrows()):
+        # if index == 10:
+        #   break
+        text = data[column][index]
+        embeddings = generator.generate(text)
+
+    # TODO benzer olan ilacın embedding değerini vererek dene
+        # embedding check none type
+        if embeddings is None or len(embeddings) == 0:
+            sum_of_embeddings = np.zeros(generator.shape)
+        else:
+            sum_of_embeddings = np.sum(embeddings, axis=0)
+        # column_embeddings_dict[row['id']] = sum_of_embeddings.reshape(1, -1) # 2d
+        column_embeddings_dict[row['id']] = sum_of_embeddings
+        # data.iloc[index][column+'_embedding']=sum_of_embeddings
+
+    data[column+'_embedding'] = pd.Series(column_embeddings_dict.values())
+    if (drop_column):
+        data.drop([column], axis=1, inplace=True)
+    # data[column+'_embedding'] = [column_embeddings_dict[row['name']] for index, row in data.iterrows()]
+    return column_embeddings_dict

ddi_fw/datasets/feature_vector_generation.py

@@ -0,0 +1,100 @@
+import numpy as np
+from scipy.spatial.distance import pdist, squareform
+
+# todo pd.unique kullan
+def find_distinct_elements(frame):
+    # y = set(pd.unique(frame))
+    y = set()
+    for x in frame:
+        if x is not None:
+            for k in x:
+                #     if type(k) == list:
+                #         for i in k:
+                #             y.add(i)
+                #     else:
+                y.add(k)
+    return y
+
+
+def find_distinct_elements_count(frame):
+    y = set()
+    for x in frame:
+        if x is not None:
+            y.update(x)
+    return len(y)
+
+
+class SimilarityMatrixGenerator:
+    def __init__(self):
+        pass
+
+    def create_jaccard_similarity_matrices_ex(self, array):
+        jaccard_sim = 1 - pdist(array, metric='jaccard')
+        jaccard_sim_matrix = squareform(jaccard_sim)
+        return jaccard_sim_matrix
+
+    # https://github.com/YifanDengWHU/DDIMDL/blob/master/DDIMDL.py , def Jaccard(matrix):
+    def create_jaccard_similarity_matrices(self, matrix):
+        matrix = np.mat(matrix)
+        numerator = matrix * matrix.T
+        denominator = np.ones(np.shape(matrix)) * matrix.T + \
+            matrix * np.ones(np.shape(matrix.T)) - matrix * matrix.T
+        matrix = numerator / denominator
+        np.nan_to_num(matrix, nan=0.0)
+        return matrix
+
+
+class VectorGenerator:
+    def __init__(self, df):
+        self.df = df
+
+    def generate_feature_vector(self, column):
+        bit_vectors = []
+        map = dict()
+        idx = 0
+        count = find_distinct_elements_count(self.df[column])
+        print(f"find_distinct_elements_count bitti, boyut: {count}")
+        for ind in self.df.index:
+            e = self.df[column][ind]
+            # vector = np.zeros(len(sorted_features))
+            vector = np.zeros(count)
+            if e is not None:
+                for item in e:
+                    if item in map:
+                        vector[map[item]] = 1
+                    else:
+                        vector[idx]=1
+                        map[item] = idx
+                        idx += 1 
+ 
+            bit_vectors.append(vector)
+        print("array oluşturuldu")
+        return np.array(bit_vectors)
+    
+    # def generate_feature_vector(self, column):
+    #     bit_vectors = []
+    #     distinct_feature = find_distinct_elements(self.df[column])
+    #     sorted_features = sorted(distinct_feature)
+    #     for ind in self.df.index:
+    #         e = self.df[column][ind]
+    #         vector = np.zeros(len(sorted_features))
+    #         if e is not None:
+    #             indexes = [i for i, x in enumerate(sorted_features) if x in e]
+    #             np.put(vector, indexes, np.ones(len(indexes)))
+    #         bit_vectors.append(vector)
+    #     return bit_vectors
+
+# bit_vectors ndarray olacak
+    def generate_feature_vectors(self, columns):
+        vectors = dict()
+        for column in columns:
+            bit_vectors = self.generate_feature_vector(column)
+            vectors[column] = bit_vectors
+        return vectors
+
+
+# generate feature vector
+# np.hstack
+
+# https://www.datasciencelearner.com/how-to-create-an-array-of-bits-in-python/
+#

ddi_fw/datasets/idf_helper.py

@@ -0,0 +1,71 @@
+from collections import defaultdict
+import numpy as np
+import pandas as pd
+
+def find_distinct_elements(frame):
+    y = set()
+    for x in frame:
+        if x is not None:
+            for k in x:
+                y.add(k)
+    return y
+
+class IDF:
+    def __init__(self, dataframe, columns):
+        self.dataframe = dataframe
+        self.columns = columns
+
+    def calculate(self):
+        idf_scores = defaultdict(dict)
+        total_document_number = self.dataframe.shape[0]
+        for column in self.columns:
+            score = dict()
+            idf_scores[column] = score
+            for e in self.dataframe[column]:
+                if e is not None:
+                    for item in e:
+                        if item in score:
+                            score[item] = score[item] +1
+                        else:
+                            score[item] = 1.0
+            for key,value in score.items():
+                score[key]= np.log(1.0 * total_document_number /  value)
+        self.idf_scores = idf_scores
+
+    def calculate_old(self):
+        self.idf_scores = defaultdict(dict)
+        for column in self.columns:
+            data = self.dataframe[column]
+            self.distinct_items = find_distinct_elements(data)
+            #sorted_distinct_items = sorted(self.distinct_items)
+            total_document_number = data.shape[0]
+            for item in self.distinct_items:
+                document_freq = data.map(set([item]).issubset).sum()
+                idf = np.log(total_document_number/document_freq)
+                self.idf_scores[column][item] = idf
+
+    def to_dataframe(self):
+        return pd.DataFrame.from_dict(self.idf_scores)
+
+
+# class IDF:
+#     def __init__(self, data, threshold = 0):
+#         self.data = data
+#         self.threshold = threshold
+#         self.distinct_items = find_distinct_elements(data)
+
+#     def calculate(self):
+#         self.idf_scores = {}
+#         sorted_distinct_items = sorted(self.distinct_items)
+#         total_document_number = self.data.shape[0]
+#         for item in sorted_distinct_items:
+#             document_freq = self.data.map(set([item]).issubset).sum()
+#             idf = np.log(total_document_number/document_freq)
+#             self.idf_scores[item] = idf
+
+#     def find_items_over_threshold(self):
+#         return [k for k,v in self.idf_scores.items() if v > self.threshold]
+    
+#     def filter_dict_by_threshold(self):
+#         return {k:v for k,v in self.idf_scores.items() if v > self.threshold}
+ 

ddi_fw/drugbank/__init__.py

@@ -0,0 +1,2 @@
+from .drugbank_parser import DrugBankParser
+from .drugbank_processor import DrugBankProcessor

ddi_fw/drugbank/drugbank_parser.py

@@ -0,0 +1,154 @@
+# # https://caseolap.github.io/docs/drug/drugbank/
+# #https://gist.github.com/rosherbal/56461421c69a8a7da775336c95fa62e0
+
+import os
+import zipfile
+import xml.etree.ElementTree as ET
+from xml.etree.ElementTree import XMLParser, XMLPullParser
+import pandas as pd
+import xmlschema
+import json as json
+import sys
+import unicodedata
+import re
+
+from ddi_fw.utils import ZipHelper
+
+
+def slugify(value, allow_unicode=False):
+    """
+    Taken from https://github.com/django/django/blob/master/django/utils/text.py
+    Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated
+    dashes to single dashes. Remove characters that aren't alphanumerics,
+    underscores, or hyphens. Convert to lowercase. Also strip leading and
+    trailing whitespace, dashes, and underscores.
+    """
+    value = str(value)
+    if allow_unicode:
+        value = unicodedata.normalize('NFKC', value)
+    else:
+        value = unicodedata.normalize('NFKD', value).encode(
+            'ascii', 'ignore').decode('ascii')
+    value = re.sub(r'[^\w\s-]', '', value.lower())
+    return re.sub(r'[-\s]+', '-', value).strip('-_')
+
+
+def replace_key(key: str):
+    if key.startswith('@'):
+        key = key[1:]
+    if key == '$':
+        key = "value"
+    elif '{http://www.drugbank.ca}' in key:
+        key = key.replace('{http://www.drugbank.ca}', '')
+    return key
+
+
+def modify_keys(d):
+    for k, v in d.copy().items():
+        if isinstance(v, dict):
+            d.pop(k)
+            d[replace_key(k)] = v
+            modify_keys(v)
+        elif isinstance(v, list):
+            d.pop(k)
+            d[replace_key(k)] = v
+            for i in v:
+                if isinstance(i, list) or isinstance(i, dict):
+                    modify_keys(i)
+                # print(i)
+
+        else:
+            if k == "keyToChange":
+                v = int(v)
+            d.pop(k)
+            d[replace_key(k)] = v
+    return d
+
+
+class DrugBankParser:
+    def __init__(self, xsd_file='drugbank.xsd', zip_file='drugbank.zip', input_path='./drugbank'):
+
+        # sys.path.insert(0,'/content/drive/My Drive/drugbank')
+        # HERE = '/content/drive/My Drive/drugbank'
+        HERE = input_path
+        DRUGBANK_XSD = HERE + '/' + xsd_file
+        DRUGBANK_ZIP = HERE + '/' + zip_file
+        xsd = xmlschema.XMLSchema(DRUGBANK_XSD)
+        self.drug_type_schema = xsd.complex_types[1]
+        self.zf = zipfile.ZipFile(DRUGBANK_ZIP, 'r')
+
+    def parse(self, save_path='./drugbank/drugs', override = False):
+        if not override:
+            print('No parsing process has been executed!!!')
+            return
+
+        elements = []
+        k = 0
+
+        for name in self.zf.namelist():
+            f = self.zf.open(name)
+            # tree = ET.parse(f)
+            # root = tree.getroot()
+            previous_element = None
+            for event, element in ET.iterparse(f, events=('end',)):  # "end"
+                # if k == 10:
+                #     break
+                if len(elements) == 0:
+                    elements.append(element)
+                elif len(elements) == 1:
+                    elements.append(element)
+                elif len(elements) == 2:
+                    elements[0] = elements[1]
+                    elements[1] = element
+                if len(elements) == 2:
+                    previous_element = elements[len(elements)-2]
+                drug = None
+                # previous_element = element.find("..")
+                #
+                if previous_element is not None and previous_element.tag == '{http://www.drugbank.ca}transporters' and event == 'end' and element.tag == "{http://www.drugbank.ca}drug":
+                    drug = element
+                    elements = []
+
+                    # for child in element:
+                    #     print(child.text)
+
+                if drug is None:
+                    continue
+
+                name = drug.find("{http://www.drugbank.ca}name")
+
+                d_name = None
+                if name is not None:
+                    d_name = name.text
+                    line = name.text
+
+                if d_name is None:
+                    continue
+
+                k = k + 1
+
+                # print(d_name)
+
+                # if lax is used we have to send d[0] as a parameter
+                d = self.drug_type_schema.decode(drug, validation='strict')
+                # pretty_dict = {replace_key(k): v for k, v in d[0].items()}
+                pretty_dict = modify_keys(d)
+                # for key, value in pretty_dict.items():
+                #     print(key, '->', value)
+                # file_name = slugify(d_name)
+
+                from pathlib import Path
+
+                Path(save_path).mkdir(parents=True, exist_ok=True)
+
+                primary_id = [
+                    id['value'] for id in pretty_dict["drugbank-id"] if id['primary'] == True][0]
+                with open(f'{save_path}/{primary_id}.json', 'w', encoding='utf-8') as f:
+                    json.dump(pretty_dict, f, ensure_ascii=False, indent=4)
+
+        print("Done")
+
+    def zip_files(self, chunk_size=1000, input_path='./drugbank/drugs', output_path='./drugbank/zips'):
+        zip_helper = ZipHelper()
+        zip_helper.zip(zip_prefix='drugs', input_path=input_path,
+                       output_path=output_path, chunk_size=chunk_size)