ddi-fw 0.0.149__py3-none-any.whl → 0.0.150__py3-none-any.whl

ddi_fw/datasets/ddi_mdl/base.py

@@ -1,149 +1,213 @@
+import glob
 import pathlib
-
+from typing import List, Optional, Tuple
+from ddi_fw.datasets.core import BaseDataset, TextDatasetMixin, generate_sim_matrices_new, generate_vectors
+from ddi_fw.datasets.dataset_splitter import DatasetSplitter
+from ddi_fw.datasets.db_utils import create_connection
+from ddi_fw.datasets.idf_helper import IDF
+from ddi_fw.utils.utils import create_folder_if_not_exists
 import numpy as np
 import pandas as pd
-
-from ddi_fw.datasets.feature_vector_generation import SimilarityMatrixGenerator
+from pydantic import BaseModel, Field, model_validator, root_validator
+from ddi_fw.datasets.feature_vector_generation import SimilarityMatrixGenerator,VectorGenerator
 from ddi_fw.langchain.embeddings import PoolingStrategy
-from .. import BaseDataset
-from ..db_utils import create_connection
+from abc import ABC, abstractmethod
+from sklearn.preprocessing import LabelBinarizer
+
+from sklearn.model_selection import KFold, StratifiedKFold, train_test_split
+
+# Constants for embedding, chemical properties, and NER columns
+LIST_OF_EMBEDDING_COLUMNS = [
+    'all_text', 'description', 'synthesis_reference', 'indication',
+    'pharmacodynamics', 'mechanism_of_action', 'toxicity', 'metabolism',
+    'absorption', 'half_life', 'protein_binding', 'route_of_elimination',
+    'volume_of_distribution', 'clearance'
+]
+
+LIST_OF_CHEMICAL_PROPERTY_COLUMNS = ['enzyme', 'target', 'pathway', 'smile']
+LIST_OF_NER_COLUMNS = ['tui', 'cui', 'entities']
 
 HERE = pathlib.Path(__file__).resolve().parent
-list_of_embedding_columns = ['all_text', 'description',
-                     'synthesis_reference', 'indication',
-                     'pharmacodynamics', 'mechanism_of_action',
-                     'toxicity', 'metabolism',
-                     'absorption', 'half_life',
-                     'protein_binding', 'route_of_elimination',
-                     'volume_of_distribution', 'clearance']
-
-list_of_chemical_property_columns = ['enzyme',
-                             'target',
-                             'pathway',
-                             'smile']
-list_of_ner_columns = ['tui', 'cui', 'entities']
-
-
-def indices_to_binary_vector(indices, vector_length=881):
-    # vector_length = len(indices)
-    # Initialize a zero vector of the given length
-    binary_vector = [0] * vector_length
-    
-    # Set the positions specified by indices to 1
-    for index in indices:
-        if 0 <= index < vector_length:
-            binary_vector[index] = 1
-    
-    return binary_vector
-
-class DDIMDLDataset(BaseDataset):
-    def __init__(self, embedding_size,
-                 embedding_dict,
-                 embeddings_pooling_strategy: PoolingStrategy,
-                 ner_df,
-                 chemical_property_columns=['enzyme',
-                                            'target',
-                                            'pathway',
-                                            'smile'],
-                 embedding_columns=[],
-                 ner_columns=[],
-                 **kwargs):
-        columns = kwargs['columns']
-        if columns:
-            chemical_property_columns = []
-            embedding_columns=[]
-            ner_columns=[]
-            for column in columns:
-                if column in list_of_chemical_property_columns:
-                    chemical_property_columns.append(column)
-                elif column in list_of_embedding_columns:
-                    embedding_columns.append(column)
-                elif column in list_of_ner_columns:
-                    ner_columns.append(column)
-                # elif column == 'smile_2':
-                #     continue
-                else:
-                    raise Exception(f"{column} is not related this dataset")
-
-
-        super().__init__(embedding_size=embedding_size,
-                         embedding_dict=embedding_dict,
-                         embeddings_pooling_strategy=embeddings_pooling_strategy,
-                         ner_df=ner_df,
-                         chemical_property_columns=chemical_property_columns,
-                         embedding_columns=embedding_columns,
-                         ner_columns=ner_columns,
-                         **kwargs)
-
-        # kwargs = {'index_path': str(HERE.joinpath('indexes'))}
-        kwargs['index_path'] = str(HERE.joinpath('indexes'))
-
-        db = HERE.joinpath('data/event.db')
-        conn = create_connection(db)
-        print("db prep")
-        self.drugs_df = self.__select_all_drugs_as_dataframe__(conn)
-        self.ddis_df = self.__select_all_events__(conn)
-        print("db bitti")
-        self.index_path = kwargs.get('index_path')
-
-        # jaccard_sim_dict = {}
-        # sim_matrix_gen = SimilarityMatrixGenerator()
-        # jaccard_sim_dict["smile_2"] = sim_matrix_gen.create_jaccard_similarity_matrices(
-        #                 self.drugs_df["smile_2"].to_list())
-        
-        # similarity_matrices = {}
-        # drugbank_ids = self.drugs_df['id'].to_list()
-        # new_columns = {}
-        # for idx in range(len(drugbank_ids)):
-        #     new_columns[idx] = drugbank_ids[idx]
-        # new_df = pd.DataFrame.from_dict(jaccard_sim_dict["smile_2"])
-        # new_df = new_df.rename(index=new_columns, columns=new_columns)
-        # similarity_matrices["smile_2"] = new_df
-        
+
+class DDIMDLDataset(TextDatasetMixin):
+    index_path: str = Field(default_factory=lambda: str(
+        pathlib.Path(__file__).resolve().parent.joinpath('indexes')))
+    # drugs_df: pd.DataFrame = Field(default_factory=pd.DataFrame)
+    # ddis_df: pd.DataFrame = Field(default_factory=pd.DataFrame)
+    drugs_df: Optional[pd.DataFrame] = None 
+    ddis_df: Optional[pd.DataFrame] = None 
+
+    chemical_property_columns: list[str] = Field(
+        default_factory=lambda: LIST_OF_CHEMICAL_PROPERTY_COLUMNS)
+    embedding_columns: list[str] = Field(default_factory=list)
+    ner_columns: list[str] = Field(default_factory=list)
+    ner_df: pd.DataFrame | None = None
+    tui_threshold: float | None = None
+    cui_threshold: float | None = None
+    entities_threshold: float | None = None
+
+
+    # @model_validator
+    def validate_columns(self, values):
+        if not set(values['chemical_property_columns']).issubset(LIST_OF_CHEMICAL_PROPERTY_COLUMNS):
+            raise ValueError("Invalid chemical property columns")
+        if not set(values['ner_columns']).issubset(LIST_OF_NER_COLUMNS):
+            raise ValueError("Invalid NER columns")
+        return values
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.class_column = 'event_category'
+        _db_path = HERE.joinpath('data/event.db')
         
-        # def lambda_fnc(row, value):
-        #     if row['id1'] in value and row['id2'] in value:
-        #         return np.float16(np.hstack(
-        #             (value[row['id1']], value[row['id2']])))
-        # for key, value in similarity_matrices.items():
-
-        #     print(f'sim matrix: {key}')
-        #     self.ddis_df[key] = self.ddis_df.apply(
-        #         lambda_fnc, args=(value,), axis=1)
-        #     print(self.ddis_df[key].head())
-        # print("init finished")
-
-    def __select_all_drugs_as_dataframe__(self, conn):
+        self.__similarity_related_columns__ = []
+        self.__similarity_related_columns__.extend(self.chemical_property_columns)
+        self.__similarity_related_columns__.extend(self.ner_columns)
+        # TODO with resource
+        self._conn = create_connection(_db_path.absolute().as_posix())
+        self.load_drugs_and_events()
+
+    def load_drugs_and_events(self):
+        self.drugs_df = self.__select_all_drugs_as_dataframe__()
+        self.ddis_df = self.__select_all_events__()
+
+    def __select_all_drugs_as_dataframe__(self):
         headers = ['index', 'id', 'name',
                    'target', 'enzyme', 'pathway', 'smile']
-        cur = conn.cursor()
+        if self._conn is None:
+            raise Exception("There is no connection")
+        cur = self._conn.cursor()
         cur.execute(
-            '''select "index", id, name, target, enzyme, pathway, smile from drug''')
+            '''SELECT "index", id, name, target, enzyme, pathway, smile FROM drug'''
+        )
         rows = cur.fetchall()
         df = pd.DataFrame(columns=headers, data=rows)
-        df['enzyme'] = df['enzyme'].apply(lambda x: x.split('|'))
-        df['target'] = df['target'].apply(lambda x: x.split('|'))
-        df['pathway'] = df['pathway'].apply(lambda x: x.split('|'))
-        # df['smile_2'] = df['smile'].apply(lambda x: indices_to_binary_vector(indices = list(map(int, x.split('|'))), vector_length = 881))
-        df['smile'] = df['smile'].apply(lambda x: x.split('|'))
-        
-            
+
+        # Convert string fields to lists
+        for col in ['enzyme', 'target', 'pathway', 'smile']:
+            df[col] = df[col].apply(lambda x: x.split('|'))
+
         return df
 
-    def __select_all_events__(self, conn):
-        """
-        Query all rows in the event table
-        :param conn: the Connection object
-        :return:
-        """
-        cur = conn.cursor()
+    def __select_all_events__(self):
+        if self._conn is None:
+            raise Exception("There is no connection")
+        cur = self._conn.cursor()
         cur.execute('''
-                select ex."index", d1.id, d1.name, d2.id, d2.name,  mechanism || ' ' ||action from extraction ex
-                join drug d1 on  d1.name = ex.drugA
-                join drug d2 on  d2.name = ex.drugB
+            SELECT ex."index", d1.id, d1.name, d2.id, d2.name, mechanism || ' ' || action
+            FROM extraction ex
+            JOIN drug d1 ON d1.name = ex.drugA
+            JOIN drug d2 ON d2.name = ex.drugB
         ''')
-
         rows = cur.fetchall()
-
         headers = ["index", "id1", "name1", "id2", "name2", "event_category"]
         return pd.DataFrame(columns=headers, data=rows)
+
+    def prep(self):
+        if self.drugs_df is None or self.ddis_df is None:
+            raise Exception("There is no data")
+        
+        drug_ids = self.drugs_df['id'].to_list()
+        
+        filtered_df = self.drugs_df
+        combined_df = filtered_df.copy()
+
+        if self.ner_df is not None and not self.ner_df.empty:
+            filtered_ner_df = self.ner_df[self.ner_df['drugbank_id'].isin(
+                drug_ids)]
+            filtered_ner_df = self.ner_df.copy()
+
+            # TODO: eğer kullanılan veri setinde tui, cui veya entity bilgileri yoksa o veri setine bu sütunları eklemek için aşağısı gerekli
+
+            # idf_calc = IDF(filtered_ner_df, [f for f in filtered_ner_df.keys()])
+            idf_calc = IDF(filtered_ner_df, self.ner_columns)
+            idf_calc.calculate()
+            idf_scores_df = idf_calc.to_dataframe()
+
+            # for key in filtered_ner_df.keys():
+            for key in self.ner_columns:
+                threshold = 0
+                if key.startswith('tui'):
+                    threshold = self.tui_threshold
+                if key.startswith('cui'):
+                    threshold = self.cui_threshold
+                if key.startswith('entities'):
+                    threshold = self.entities_threshold
+                combined_df[key] = filtered_ner_df[key]
+                valid_codes = idf_scores_df[idf_scores_df[key] > threshold].index
+
+                # print(f'{key}: valid code size = {len(valid_codes)}')
+                combined_df[key] = combined_df[key].apply(lambda items:
+                                                        [item for item in items if item in valid_codes])
+
+        moved_columns = ['id']
+        moved_columns.extend(self.__similarity_related_columns__)
+        chemical_properties_df = combined_df[moved_columns]
+
+        chemical_properties_df = chemical_properties_df.fillna("").apply(list)
+
+        # generate vectors dictionary içinde ndarray dönecek
+        generated_vectors = generate_vectors(chemical_properties_df, self.__similarity_related_columns__)
+
+        similarity_matrices = generate_sim_matrices_new(
+            chemical_properties_df,generated_vectors,  self.__similarity_related_columns__, key_column= "id")
+
+        event_categories = self.ddis_df['event_category']
+        labels = event_categories.tolist()
+        lb = LabelBinarizer()
+        lb.fit(labels)
+        classes = lb.transform(labels)
+      
+        def similarity_lambda_fnc(row, value):
+            if row['id1'] in value:
+                return value[row['id1']]
+
+        def lambda_fnc(row: pd.Series, value)-> Optional[np.float16]:
+            if row['id1'] in value and row['id2'] in value:
+                return np.float16(np.hstack(
+                    (value[row['id1']], value[row['id2']])))
+            return None
+                # return np.hstack(
+                #     (value[row['id1']], value[row['id2']]), dtype=np.float16)
+
+        def x_fnc(row, embeddings_after_pooling):
+            if row['id1'] in embeddings_after_pooling:
+                v1 = embeddings_after_pooling[row['id1']]
+            else:
+                v1 = np.zeros(self.embedding_size)
+            if row['id2'] in embeddings_after_pooling:
+                v2 = embeddings_after_pooling[row['id2']]
+            else:
+                v2 = np.zeros(self.embedding_size)
+            return np.float16(np.hstack(
+                (v1, v2)))
+
+        for key, value in similarity_matrices.items():
+
+            print(f'sim matrix: {key}')
+            self.ddis_df[key] = self.ddis_df.apply(
+                lambda_fnc, args=(value,), axis=1)
+            self.columns.append(key)
+            print(self.ddis_df[key].head())
+
+        for embedding_column in self.embedding_columns:
+            print(f"concat {embedding_column} embeddings")
+            embeddings_after_pooling = {k: self.embeddings_pooling_strategy.apply(
+                v) for k, v in self.embedding_dict[embedding_column].items()}
+            # column_embeddings_dict = embedding_values[embedding_column]
+            self.ddis_df[embedding_column+'_embedding'] = self.ddis_df.apply(
+                x_fnc, args=(embeddings_after_pooling,), axis=1)
+            self.columns.append(embedding_column+'_embedding')
+
+        dataframe = self.ddis_df.copy()
+        if not isinstance(classes, (list, pd.Series, np.ndarray)):
+         raise TypeError("classes must be an iterable (list, Series, or ndarray)")
+
+        if len(classes) != len(dataframe):
+            raise ValueError("Length of classes must match the number of rows in the DataFrame")
+
+        dataframe['class'] = list(classes)
+        self.set_dataframe(dataframe)
+        
+
+    

ddi_fw/datasets/ddi_mdl/debug.log

@@ -0,0 +1 @@
+[0217/121135.683:ERROR:registration_protocol_win.cc(108)] CreateFile: Sistem belirtilen dosyayı bulamıyor. (0x2)

ddi_fw/datasets/embedding_generator.py

@@ -58,7 +58,8 @@ def create_embeddings(model, data, column, drop_column=True):
     column_embeddings_dict[row['id']] = sum_of_embeddings
     # data.iloc[index][column+'_embedding']=sum_of_embeddings
   
-  data[column+'_embedding'] = pd.Series(column_embeddings_dict.values())
+  # data[column+'_embedding'] = pd.Series(column_embeddings_dict.values())
+  data[column+'_embedding'] = pd.Series(list(column_embeddings_dict.values()))
   if(drop_column):
     data.drop([column], axis = 1, inplace = True)
   # data[column+'_embedding'] = [column_embeddings_dict[row['name']] for index, row in data.iterrows()]

ddi_fw/langchain/embeddings.py

@@ -82,6 +82,7 @@ class PretrainedEmbeddings(Embeddings):
                 text, return_tensors='pt', padding=True)
             output_embeddings.append(self.model(
                 input_ids).last_hidden_state.mean(dim=1))
+        return output_embeddings
 
     def embed_query(self, text: str) -> List[float]:
         return self.embed_documents([text])[0]