ddi-fw 0.0.110__py3-none-any.whl → 0.0.112__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -274,7 +274,7 @@ class BaseDataset(ABC):
         filtered_df = self.drugs_df
         combined_df = filtered_df.copy()
 
-        if self.ner_df:
+        if self.ner_df is not None and not self.ner_df.empty:
             filtered_ner_df = self.ner_df[self.ner_df['drugbank_id'].isin(
                 drug_ids)]
             filtered_ner_df = self.ner_df.copy()

ddi_fw/datasets/ddi_mdl/base.py

@@ -1,7 +1,9 @@
 import pathlib
 
+import numpy as np
 import pandas as pd
 
+from ddi_fw.datasets.feature_vector_generation import SimilarityMatrixGenerator
 from ddi_fw.langchain.embeddings import PoolingStrategy
 from .. import BaseDataset
 from ..db_utils import create_connection
@@ -22,6 +24,18 @@ list_of_chemical_property_columns = ['enzyme',
 list_of_ner_columns = ['tui', 'cui', 'entities']
 
 
+def indices_to_binary_vector(indices, vector_length=881):
+    # vector_length = len(indices)
+    # Initialize a zero vector of the given length
+    binary_vector = [0] * vector_length
+    
+    # Set the positions specified by indices to 1
+    for index in indices:
+        if 0 <= index < vector_length:
+            binary_vector[index] = 1
+    
+    return binary_vector
+
 class DDIMDLDataset(BaseDataset):
     def __init__(self, embedding_size,
                  embedding_dict,
@@ -70,6 +84,34 @@ class DDIMDLDataset(BaseDataset):
         print("db bitti")
         self.index_path = kwargs.get('index_path')
 
+        jaccard_sim_dict = {}
+        sim_matrix_gen = SimilarityMatrixGenerator()
+        jaccard_sim_dict["smile_2"] = sim_matrix_gen.create_jaccard_similarity_matrices(
+                        self.drugs_df["smile_2"].to_list())
+        
+        similarity_matrices = {}
+        drugbank_ids = self.drugs_df['id'].to_list()
+        new_columns = {}
+        for idx in range(len(drugbank_ids)):
+            new_columns[idx] = drugbank_ids[idx]
+        for idx in range(len(drugbank_ids)):
+            new_columns[idx] = drugbank_ids[idx]
+        new_df = pd.DataFrame.from_dict(jaccard_sim_dict["smile_2"])
+        new_df = new_df.rename(index=new_columns, columns=new_columns)
+        similarity_matrices["smile_2"] = new_df
+        
+        def lambda_fnc(row, value):
+            if row['id1'] in value and row['id2'] in value:
+                return np.float16(np.hstack(
+                    (value[row['id1']], value[row['id2']])))
+        for key, value in similarity_matrices.items():
+
+            print(f'sim matrix: {key}')
+            self.ddis_df[key] = self.ddis_df.apply(
+                lambda_fnc, args=(value,), axis=1)
+            print(self.ddis_df[key].head())
+        print("init finished")
+
     def __select_all_drugs_as_dataframe__(self, conn):
         headers = ['index', 'id', 'name',
                    'target', 'enzyme', 'pathway', 'smile']
@@ -82,6 +124,11 @@ class DDIMDLDataset(BaseDataset):
         df['target'] = df['target'].apply(lambda x: x.split('|'))
         df['pathway'] = df['pathway'].apply(lambda x: x.split('|'))
         df['smile'] = df['smile'].apply(lambda x: x.split('|'))
+        df['smile_2'] = df['smile'].apply(lambda x: indices_to_binary_vector(indices = list(map(int, x.split('|'))), vector_length = 881))
+       
+       
+        
+            
         return df
 
     def __select_all_events__(self, conn):

ddi_fw/datasets/feature_vector_generation.py

@@ -1,4 +1,5 @@
 import numpy as np
+import pandas as pd
 from scipy.spatial.distance import pdist, squareform
 
 # todo pd.unique kullan
@@ -48,12 +49,40 @@ class VectorGenerator:
     def __init__(self, df):
         self.df = df
 
+    # https://github.com/YifanDengWHU/DDIMDL/blob/master/DDIMDL.py#L86
+    # def generate_feature_vector(self, column):
+    #     # Initialize list to store all distinct features across all rows
+    #     all_features = []
+        
+    #     # Loop through the column to extract features, split by '|', and collect all distinct ones
+    #     drug_list = np.array(self.df[column]).tolist()
+    #     for i in drug_list:
+    #         for each_feature in i.split('|'):
+    #             if each_feature not in all_features:
+    #                 all_features.append(each_feature)
+        
+    #     # Initialize a matrix to hold feature vectors (rows for each element, columns for each distinct feature)
+    #     feature_matrix = np.zeros((len(drug_list), len(all_features)), dtype=float)
+        
+    #     # Create a DataFrame to store the feature matrix with the column names as the distinct features
+    #     df_feature = pd.DataFrame(feature_matrix, columns=all_features)
+        
+    #     # Fill the feature matrix (set value to 1 if feature is present for the specific item in the column)
+    #     for i in range(len(drug_list)):
+    #         for each_feature in drug_list[i].split('|'):
+    #             if each_feature in all_features:
+    #                 df_feature[each_feature].iloc[i] = 1
+        
+    #     # Convert DataFrame to numpy array and return
+    #     print("Feature vectors generated")
+    #     return df_feature.to_numpy()
+
     def generate_feature_vector(self, column):
         bit_vectors = []
         map = dict()
         idx = 0
         count = find_distinct_elements_count(self.df[column])
-        print(f"find_distinct_elements_count bitti, boyut: {count}")
+        print(f"{column} has {count} different items")
         for ind in self.df.index:
             e = self.df[column][ind]
             # vector = np.zeros(len(sorted_features))

ddi_fw/langchain/storage.py

@@ -3,6 +3,7 @@ from langchain.vectorstores import Chroma
 # from langchain_community.vectorstores import Chroma
 from langchain_community.vectorstores.utils import filter_complex_metadata
 from langchain_core.embeddings import Embeddings
+import time
 
 
 from langchain.docstore.document import Document
@@ -102,7 +103,7 @@ class DataFrameToVectorDB:
                  persist_directory,
                  embeddings: Embeddings,
                  text_splitter: TextSplitter,
-                 batch_size=1000):
+                 batch_size=1024):
         self.collection_name = collection_name
         self.persist_directory = persist_directory
         self.embeddings = embeddings
@@ -118,10 +119,10 @@ class DataFrameToVectorDB:
 
     def __split_list(self, input_list, batch_size):
         # for i in range(0, len(input_list), batch_size):
-        for s, e in split_dataframe_indices(input_list):
+        for s, e in split_dataframe_indices(input_list, batch_size):
             yield input_list[s:e+1]
 
-    def store_documents(self, df, columns, page_content_columns):
+    def store_documents(self, df, columns, page_content_columns, partial_df_size=None):
         """
         Core function that processes the documents and adds them to the vector database.
         """
@@ -131,39 +132,49 @@ class DataFrameToVectorDB:
             col_df = df[copy_columns].copy()
             col_df.dropna(subset=[page_content_column], inplace=True)
             col_df['type'] = page_content_column  # Set the type column
-            # if partial_df_size:
-            #     documents = []
-
-            #     partial_dfs = split_dataframe(col_df, min_size=partial_df_size)
-            #     for partial_df in partial_dfs:
-            #         print(f"{page_content_column}, size:{len(partial_df)}")
-            #         print(partial_df.count())
-            #         loader = DataFrameLoader(
-            #             data_frame=partial_df, page_content_column=page_content_column)
-            #         loaded_docs = loader.load()
-            #         documents.extend(self.__split_docs(loaded_docs))
-            #         print(f"Documents size: {len(documents)}")
-            #         split_docs_chunked = self.__split_list(
-            #             documents, self.batch_size)
-
-            #         for split_docs_chunk in split_docs_chunked:
-            #             print(f"Split docs size: {len(split_docs_chunk)}")
-            #             self.vectordb.add_documents(split_docs_chunk)
-            #             self.vectordb.persist()
-            # else:
-            documents = []
-
-            loader = DataFrameLoader(
-                data_frame=col_df, page_content_column=page_content_column)
-            loaded_docs = loader.load()
-            documents.extend(self.__split_docs(loaded_docs))
-
-            split_docs_chunked = self.__split_list(
-                documents, self.batch_size)
-
-            for split_docs_chunk in split_docs_chunked:
-                self.vectordb.add_documents(split_docs_chunk)
-                self.vectordb.persist()
+            if partial_df_size:
+                total = 0
+                partial_dfs = split_dataframe(col_df, min_size=partial_df_size)
+                for partial_df in partial_dfs:
+                    import torch
+
+                    documents = []
+                    loader = DataFrameLoader(
+                        data_frame=partial_df, page_content_column=page_content_column)
+                    loaded_docs = loader.load()
+                    # print(loaded_docs)
+                    # documents.extend(self.__split_docs(loaded_docs))
+                    total += len(partial_df)
+
+                    self.vectordb.add_documents(loaded_docs)
+                    self.vectordb.persist()
+                    print(f"{page_content_column}: {total}/{len(col_df)}")
+                    torch.cuda.empty_cache()
+                    # time.sleep(30)  # The GPU will not be used during this period
+
+                    # split_docs_chunked = self.__split_list(
+                    #     loaded_docs, self.batch_size)
+                    # print(f"Number of chunks: {len(split_docs_chunked)}")
+                    # for split_docs_chunk in split_docs_chunked:
+                    #     print(f"Split docs size: {len(split_docs_chunk)}")
+                    #     self.vectordb.add_documents(split_docs_chunk)
+                    #     self.vectordb.persist()
+            else:
+                documents = []
+                print(col_df.shape)
+                loader = DataFrameLoader(
+                    data_frame=col_df, page_content_column=page_content_column)
+                loaded_docs = loader.load()
+                documents.extend(self.__split_docs(loaded_docs))
+                print(f"Documents size: {len(loaded_docs)}")
+                split_docs_chunked = self.__split_list(
+                    documents, self.batch_size)
+                for split_docs_chunk in split_docs_chunked:
+                    import torch
+                    torch.cuda.empty_cache()
+                    self.vectordb.add_documents(split_docs_chunk)
+                    self.vectordb.persist()
+                    print(f"{page_content_column}, size:{len(split_docs_chunk)}")
 
 
 def generate_embeddings(df, config_file, new_model_names, collections=None, persist_directory="embeddings"):
@@ -239,6 +250,6 @@ def generate_embeddings(df, config_file, new_model_names, collections=None, pers
                                                persist_directory=persist_directory,
                                                embeddings=model,
                                                text_splitter=text_splitter,
-                                               batch_size=1024)
+                                               batch_size=batch_size)
             to_vector_db.store_documents(
-                df, columns, page_content_columns)
+                df, columns, page_content_columns, partial_df_size=batch_size)

{ddi_fw-0.0.110.dist-info → ddi_fw-0.0.112.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.110
+Version: 0.0.112
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.110.dist-info → ddi_fw-0.0.112.dist-info}/RECORD

@@ -1,11 +1,11 @@
 ddi_fw/datasets/__init__.py,sha256=HSwQrqnzrEjIG4gif41pwJ_cST3t2XHGDxqFyuEBRwo,351
-ddi_fw/datasets/core.py,sha256=9RaUPhAYCn4RDeTZpHATtJaqNWsO17bduYyVqxAZWs0,17001
+ddi_fw/datasets/core.py,sha256=0bEJSxqO22x0XBoCKOYmxXNa2j7_CqFqKiBZ4KFd9Mk,17039
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/embedding_generator.py,sha256=Jqrlv88RCu0Lg812KsA12X0cSaZuxbckJ4LNRKNy_qw,2173
-ddi_fw/datasets/feature_vector_generation.py,sha256=EImavcALxkIB0YG_smOzagMNzuWMbK9SaWSKwARx_qU,3254
+ddi_fw/datasets/feature_vector_generation.py,sha256=gvjpEzkgVV8dp4V8NMMv59u0v-1tNAJ7v83R-keWGoA,4748
 ddi_fw/datasets/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
-ddi_fw/datasets/ddi_mdl/base.py,sha256=45cUmDRyyD8CC07oj5Dka2DWfgWU4Qi7-Am0vCvRKbo,4237
+ddi_fw/datasets/ddi_mdl/base.py,sha256=QtDAaXpqDiPxQ-xOVA0Xd-wcm1sRY5FpO_4zNAKoksM,6146
 ddi_fw/datasets/ddi_mdl/readme.md,sha256=WC6lpmsEKvIISnZqENY7TWtzCQr98HPpE3oRsBl8pIw,625
 ddi_fw/datasets/ddi_mdl/data/event.db,sha256=cmlSsf9MYjRzqR-mw3cUDnTnfT6FkpOG2yCl2mMwwew,30580736
 ddi_fw/datasets/ddi_mdl/indexes/test_indexes.txt,sha256=XVlDqYATckrQwNSXqMSKVBqyoN_Hg8SK6CL-XMdLADY,102176
@@ -58,7 +58,7 @@ ddi_fw/drugbank/event_extractor.py,sha256=6odoZohhK7OdLF-LF0l-5BFq0_NMG_5jrFJbHr
 ddi_fw/langchain/__init__.py,sha256=zS0CQrakWEP19biSRewFJGcBT8WBZq4899HrEKiMqUY,269
 ddi_fw/langchain/embeddings.py,sha256=lU64a5AZ62jP8U3hTSwK0kXt7gThbwPACLfJMZ1baPA,7538
 ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
-ddi_fw/langchain/storage.py,sha256=gIQfpRG1t8SY1r3jWZGp-MOblQ_-8EgqZ55A1ZQ8kBg,10047
+ddi_fw/langchain/storage.py,sha256=Vz1aICIyZzKwOnOwxkhWAMYWJ9X6kOfqEkIeQJyQIHY,10762
 ddi_fw/ml/__init__.py,sha256=tIxiW0g6q1VsmDYVXR_ovvHQR3SCir8g2bKxx_CrS7s,221
 ddi_fw/ml/evaluation_helper.py,sha256=o4-w5Xa3t4olLW4ymx_8L-Buhe5wfQEmT2bh4Zz544c,13066
 ddi_fw/ml/ml_helper.py,sha256=fySjIAFzkeEOvaLJhDwtCOgRhgYQ7H106eqaP16GhDY,4489
@@ -91,7 +91,7 @@ ddi_fw/utils/package_helper.py,sha256=erl8_onmhK-41zQoaED2qyDUV9GQxmT9sdoyRp9_q5
 ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
 ddi_fw/utils/utils.py,sha256=szwnxMTDRrZoeNRyDuf3aCbtzriwtaRk4mHSH3asLdA,4301
 ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
-ddi_fw-0.0.110.dist-info/METADATA,sha256=bozy0pU7E9nOOqwWvqXof28dNdfr9UQqoyb1OVKyfHw,1967
-ddi_fw-0.0.110.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-ddi_fw-0.0.110.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.110.dist-info/RECORD,,
+ddi_fw-0.0.112.dist-info/METADATA,sha256=tF5FdHznWj9YznDQbTGQtsspXYlHOxmEJLxeEcI7oLg,1967
+ddi_fw-0.0.112.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+ddi_fw-0.0.112.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.112.dist-info/RECORD,,