ddi-fw 0.0.149__py3-none-any.whl → 0.0.151__py3-none-any.whl

ddi_fw/datasets/__init__.py

@@ -5,7 +5,7 @@ from .mdf_sa_ddi.base import MDFSADDIDataset
 from .embedding_generator import create_embeddings
 from .idf_helper import IDF
 from .feature_vector_generation import SimilarityMatrixGenerator, VectorGenerator
-
+from .dataset_splitter import DatasetSplitter
 __all__  = ['BaseDataset','DDIMDLDataset','MDFSADDIDataset']
 
 

ddi_fw/datasets/core.py

@@ -1,234 +1,129 @@
 import glob
-from sklearn.model_selection import KFold, StratifiedKFold, train_test_split
-from sklearn.preprocessing import LabelBinarizer
-from abc import ABC, abstractmethod
+from typing import List, Optional, Type
 import numpy as np
 import pandas as pd
-import pathlib
+from pydantic import BaseModel, Field, computed_field
+from ddi_fw.datasets.dataset_splitter import DatasetSplitter
+from ddi_fw.datasets.feature_vector_generation import SimilarityMatrixGenerator, VectorGenerator
 from ddi_fw.langchain.embeddings import PoolingStrategy
-from ddi_fw.datasets.idf_helper import IDF
-
-from ddi_fw.utils.zip_helper import ZipHelper
-from .feature_vector_generation import SimilarityMatrixGenerator, VectorGenerator
-# from ddi_fw.ner.ner import CTakesNER
-from ddi_fw.utils import create_folder_if_not_exists
-from stopwatch import Stopwatch, profile
-
-HERE = pathlib.Path(__file__).resolve().parent
+from ddi_fw.utils.utils import create_folder_if_not_exists
 
 
 def stack(df_column):
     return np.stack(df_column.values)
 
 
-class BaseDataset(ABC):
-    def __init__(self,
-                 embedding_size,
-                 embedding_dict,
-                 embeddings_pooling_strategy: PoolingStrategy,
-                 ner_df,
-                 chemical_property_columns,
-                 embedding_columns,
-                 ner_columns,
-                 **kwargs):
-        self.embedding_size = embedding_size
-        self.embedding_dict = embedding_dict
-        self.embeddings_pooling_strategy = embeddings_pooling_strategy
-        self.ner_df = ner_df
-        self.__similarity_related_columns__ = []
-        self.__similarity_related_columns__.extend(chemical_property_columns)
-        self.__similarity_related_columns__.extend(ner_columns)
-
-        self.chemical_property_columns = chemical_property_columns
-        self.embedding_columns = embedding_columns
-        self.ner_columns = ner_columns
-        self.threshold_method = kwargs.get('threshold_method', 'idf')
-        self.tui_threshold = kwargs.get('tui_threshold', 0)
-        self.cui_threshold = kwargs.get('cui_threshold', 0)
-        self.entities_threshold = kwargs.get('entities_threshold', 0)
-
-        self.stopwatch = Stopwatch()
-
-        # self.store_similarity_matrices = kwargs.get('store_similarity_matrices', True)
-        # self.similarity_matrices_path = kwargs.get('similarity_matrices_path', True)
-
-# önce load veya split çalıştırılmalı
+def generate_vectors(df, columns):
+    vectorGenerator = VectorGenerator(df)
+    generated_vectors = vectorGenerator.generate_feature_vectors(
+        columns)
+    return generated_vectors
+
+
+def generate_sim_matrices_new(df, generated_vectors, columns, key_column="id"):
+    jaccard_sim_dict = {}
+    sim_matrix_gen = SimilarityMatrixGenerator()
+
+    for column in columns:
+        # key = '2D_'+column
+        key = column
+        jaccard_sim_dict[column] = sim_matrix_gen.create_jaccard_similarity_matrices(
+            generated_vectors[key])
+
+    similarity_matrices = {}
+    keys = df[key_column].to_list()
+    new_columns = {}
+    for idx in range(len(keys)):
+        new_columns[idx] = keys[idx]
+    for column in columns:
+        new_df = pd.DataFrame.from_dict(jaccard_sim_dict[column])
+        new_df = new_df.rename(index=new_columns, columns=new_columns)
+        similarity_matrices[column] = new_df
+    return similarity_matrices
+
+
+class BaseDataset(BaseModel):
+    dataset_name: str
+    index_path: str
+    dataset_splitter_type: Type[DatasetSplitter]
+    class_column: str = 'class'
+    dataframe: Optional[pd.DataFrame] = None
+    X_train:	Optional[pd.DataFrame] = None
+    X_test:	Optional[pd.DataFrame] = None
+    y_train:	Optional[pd.Series] = None
+    y_test:	Optional[pd.Series] = None
+    train_indexes:	Optional[pd.Index] = None
+    test_indexes:	Optional[pd.Index] = None
+    train_idx_arr:	List|None = None
+    val_idx_arr:	List|None = None
+    # train_idx_arr:	Optional[List[np.ndarray]] = None
+    # val_idx_arr:	Optional[List[np.ndarray]] = None
+    columns: List[str] = []
+
+    # feature_process: FeatureProcessor
+    # similarity_matrix_service: SimilarityMatrixService
+
+    class Config:
+        arbitrary_types_allowed = True
+
     def produce_inputs(self):
         items = []
+        if self.X_train is None or self.X_test is None:
+            raise Exception("There is no data to produce inputs")
         y_train_label, y_test_label = stack(self.y_train), stack(self.y_test)
-        # self.__similarity_related_columns__.append("smile_2") #TODO
-        for column in self.__similarity_related_columns__:
+
+        for column in self.columns:
             train_data, test_data = stack(
                 self.X_train[column]), stack(self.X_test[column])
             items.append([f'{column}', np.nan_to_num(train_data),
                           y_train_label, np.nan_to_num(test_data), y_test_label])
-        for column in self.embedding_columns:
-            train_data, test_data = stack(
-                self.X_train[column+'_embedding']), stack(self.X_test[column+'_embedding'])
-            items.append([f'{column}_embedding', train_data,
-                          y_train_label, test_data, y_test_label])
+
+            # items.append([f'{column}_embedding', train_data,
+            #             y_train_label, test_data, y_test_label])
         return items
+    
+    @computed_field
+    @property
+    def dataset_splitter(self) -> DatasetSplitter:
+        return self.dataset_splitter_type()
+
+    def set_dataframe(self, dataframe: pd.DataFrame):
+        self.dataframe = dataframe
+
+    # @abstractmethod
+    def prep(self):
+        pass
+
+    def load(self):
+        if self.index_path is None:
+            raise Exception(
+                "There is no index path, please call split function")
+
+        try:
+            train_idx_all, test_idx_all, train_idx_arr, val_idx_arr = self.__get_indexes__(
+                self.index_path)
+        except FileNotFoundError as e:
+            raise FileNotFoundError(f"Index files not found: {e.filename}")
 
-# remove this function
-    def generate_sim_matrices(self, chemical_properties_df, two_d_dict):
-
-        jaccard_sim_dict = {}
-        sim_matrix_gen = SimilarityMatrixGenerator()
-
-        for column in self.__similarity_related_columns__:
-            key = '2D_'+column
-            jaccard_sim_dict[column] = sim_matrix_gen.create_jaccard_similarity_matrices(
-                two_d_dict[key])
-
-        drugbank_ids = chemical_properties_df['id'].to_list()
-
-        similarity_matrices = {}
-
-        for column in self.__similarity_related_columns__:
-            sim_matrix = jaccard_sim_dict[column]
-            jaccard_sim_feature = {}
-            for i in range(len(drugbank_ids)):
-                jaccard_sim_feature[drugbank_ids[i]] = sim_matrix[i]
-            similarity_matrices[column] = jaccard_sim_feature
-
-        return similarity_matrices
-
-    def generate_sim_matrices_new(self, chemical_properties_df):
-        self.stopwatch.reset()
-        self.stopwatch.start()
-        jaccard_sim_dict = {}
-        sim_matrix_gen = SimilarityMatrixGenerator()
-
-        for column in self.__similarity_related_columns__:
-            # key = '2D_'+column
-            key = column
-            jaccard_sim_dict[column] = sim_matrix_gen.create_jaccard_similarity_matrices(
-                self.generated_vectors[key])
-        self.stopwatch.stop()
-        print(f'similarity_matrix_generation_part_1: {self.stopwatch.elapsed}')
-
-        self.stopwatch.reset()
-        self.stopwatch.start()
-        similarity_matrices = {}
-        drugbank_ids = chemical_properties_df['id'].to_list()
-        new_columns = {}
-        for idx in range(len(drugbank_ids)):
-            new_columns[idx] = drugbank_ids[idx]
-        for column in self.__similarity_related_columns__:
-            new_df = pd.DataFrame.from_dict(jaccard_sim_dict[column])
-            new_df = new_df.rename(index=new_columns, columns=new_columns)
-            similarity_matrices[column] = new_df
-        self.stopwatch.stop()
-        print(f'similarity_matrix_generation_part_2: {self.stopwatch.elapsed}')
-        return similarity_matrices
-
-    # matris formuna çevirmek için
-    def transform_2d(self, chemical_properties_df):
-        two_d_dict = {}
-        for column in self.__similarity_related_columns__:
-            key = '2D_'+column
-            new_column = column + '_vectors'
-            two_d_dict[key] = np.stack(
-                chemical_properties_df[new_column].to_numpy())
-
-        return two_d_dict
-
-    # todo dictionary içinde ndarray dönsün
-    def generate_vectors(self, chemical_properties_df):
-        self.stopwatch.reset()
-        self.stopwatch.start()
-        vectorGenerator = VectorGenerator(chemical_properties_df)
-
-        new_columns = [
-            c+'_vectors' for c in self.__similarity_related_columns__]
-        self.generated_vectors = vectorGenerator.generate_feature_vectors(
-            self.__similarity_related_columns__)
-
-        # for column, new_column in zip(self.__similarity_related_columns__, new_columns):
-        #     chemical_properties_df.loc[:,
-        #                                new_column] = generated_vectors[column]
-        # self.generated_vectors = generated_vectors
-        self.stopwatch.stop()
-        print(f'vector_generation: {self.stopwatch.elapsed}')
-
-
-# remove this function
-
-
-    def sim(self, chemical_properties_df):
-        self.stopwatch.reset()
-        self.stopwatch.start()
-        from scipy.spatial.distance import pdist
-        sim_matrix_gen = SimilarityMatrixGenerator()
-
-        drugbank_ids = chemical_properties_df['id'].to_list()
-        similarity_matrices = {}
-        for column in self.__similarity_related_columns__:
-            df = pd.DataFrame(np.stack(
-                chemical_properties_df[f'{column}_vectors'].values), index=drugbank_ids)
-        #   similarity_matrices[column] = 1 - pdist(df.to_numpy(), metric='jaccard')
-            similarity_matrices[column] = sim_matrix_gen.create_jaccard_similarity_matrices(
-                df.to_numpy())
-        self.stopwatch.stop()
-        print(f'sim: {self.stopwatch.elapsed}')
-        return similarity_matrices
-
-# import pandas as pd
-# a = [[0,0,1],[0,0,1],[0,0,0]]
-# s = pd.Series(a)
-# # print(np.vstack(s.to_numpy()))
-# l = np.argmax(np.vstack(s.to_numpy()),axis = 1)
-# l
-    def split_dataset(self,
-                      fold_size=5,
-                      shuffle=True,
-                      test_size=0.2,
-                      save_indexes=False):
-        save_path = self.index_path
         self.prep()
-        X = self.dataframe.drop('class', axis=1)
-        y = self.dataframe['class']
-        X_train, X_test, y_train, y_test = train_test_split(
-            X, y, shuffle=shuffle, test_size=test_size, stratify=np.argmax(np.vstack(y.to_numpy()), axis=1))
-        # k_fold = KFold(n_splits=fold_size, shuffle=shuffle, random_state=1)
-        # folds = k_fold.split(X_train)
-
-        k_fold = StratifiedKFold(
-            n_splits=fold_size, shuffle=shuffle, random_state=1)
-        folds = k_fold.split(X_train, np.argmax(
-            np.vstack(y_train.to_numpy()), axis=1))
-        train_idx_arr = []
-        val_idx_arr = []
-        for i, (train_index, val_index) in enumerate(folds):
-            train_idx_arr.append(train_index)
-            val_idx_arr.append(val_index)
 
-        if save_indexes:
-            # train_pairs = [row['id1'].join(',').row['id2'] for index, row in X_train.iterrows()]
-            self.__save_indexes__(
-                save_path, 'train_indexes.txt', X_train['index'].values)
-            self.__save_indexes__(
-                save_path, 'test_indexes.txt',  X_test['index'].values)
-            # self.__save_indexes__(
-            #     save_path, 'train_indexes.txt', X_train.index.values)
-            # self.__save_indexes__(
-            #     save_path, 'test_indexes.txt',  X_test.index.values)
+        if self.dataframe is None:
+            raise Exception("There is no dataframe")
 
-            for i, (train_idx, val_idx) in enumerate(zip(train_idx_arr, val_idx_arr)):
-                self.__save_indexes__(
-                    save_path, f'train_fold_{i}.txt', train_idx)
-                self.__save_indexes__(
-                    save_path, f'validation_fold_{i}.txt', val_idx)
+        train = self.dataframe[self.dataframe.index.isin(train_idx_all)]
+        test = self.dataframe[self.dataframe.index.isin(test_idx_all)]
 
-        self.X_train = X_train
-        self.X_test = X_test
-        self.y_train = y_train
-        self.y_test = y_test
-        self.train_indexes = X_train.index
-        self.test_indexes = X_test.index
+        self.X_train = train.drop(self.class_column, axis=1)
+        self.y_train = train[self.class_column]
+        self.X_test = test.drop(self.class_column, axis=1)
+        self.y_test = test[self.class_column]
+
+        self.train_indexes = self.X_train.index
+        self.test_indexes = self.X_test.index
         self.train_idx_arr = train_idx_arr
         self.val_idx_arr = val_idx_arr
-        return X_train, X_test, y_train, y_test, X_train.index, X_test.index, train_idx_arr, val_idx_arr
+
+        return self.X_train, self.X_test, self.y_train, self.y_test, self.X_train.index, self.X_test.index, train_idx_arr, val_idx_arr
 
     def __get_indexes__(self, path):
         train_index_path = path+'/train_indexes.txt'
@@ -259,147 +154,58 @@ class BaseDataset(ABC):
         with open(file_path, 'w') as f:
             f.write('\n'.join(str_indexes))
 
-    # @abstractmethod
-    # def prep(self):
-    #     pass
+    def split_dataset(self, save_indexes: bool = False):
+        # TODO class type should be parametric
 
-    # @abstractmethod
-    # def load(self):
-    #     pass
+        save_path = self.index_path
+        self.prep()
 
-# her bir metin tipi için embedding oluşturursan burayı düzenle
-    def prep(self):
-        drug_names = self.drugs_df['name'].to_list()
-        drug_ids = self.drugs_df['id'].to_list()
-
-        filtered_df = self.drugs_df
-        combined_df = filtered_df.copy()
-
-        if self.ner_df is not None and not self.ner_df.empty:
-            filtered_ner_df = self.ner_df[self.ner_df['drugbank_id'].isin(
-                drug_ids)]
-            filtered_ner_df = self.ner_df.copy()
-
-            # TODO: eğer kullanılan veri setinde tui, cui veya entity bilgileri yoksa o veri setine bu sütunları eklemek için aşağısı gerekli
-
-            # idf_calc = IDF(filtered_ner_df, [f for f in filtered_ner_df.keys()])
-            idf_calc = IDF(filtered_ner_df, self.ner_columns)
-            idf_calc.calculate()
-            idf_scores_df = idf_calc.to_dataframe()
-
-            # for key in filtered_ner_df.keys():
-            for key in self.ner_columns:
-                threshold = 0
-                if key.startswith('tui'):
-                    threshold = self.tui_threshold
-                if key.startswith('cui'):
-                    threshold = self.cui_threshold
-                if key.startswith('entities'):
-                    threshold = self.entities_threshold
-                combined_df[key] = filtered_ner_df[key]
-                valid_codes = idf_scores_df[idf_scores_df[key] > threshold].index
-
-                # print(f'{key}: valid code size = {len(valid_codes)}')
-                combined_df[key] = combined_df[key].apply(lambda items:
-                                                        [item for item in items if item in valid_codes])
-
-        moved_columns = ['id']
-        moved_columns.extend(self.__similarity_related_columns__)
-        chemical_properties_df = combined_df[moved_columns]
-
-        chemical_properties_df = chemical_properties_df.fillna("").apply(list)
-
-        # generate vectors dictionary içinde ndarray dönecek
-        self.generate_vectors(chemical_properties_df)
-
-        # two_d_dict = self.transform_2d(chemical_properties_df)
-
-        similarity_matrices = self.generate_sim_matrices_new(
-            chemical_properties_df)
-
-        # similarity_matrices = self.sim(chemical_properties_df)
-
-        event_categories = self.ddis_df['event_category']
-        labels = event_categories.tolist()
-        lb = LabelBinarizer()
-        lb.fit(labels)
-        classes = lb.transform(labels)
-
-        # def similarity_lambda_fnc(row, value):
-        #     if row['id1'] in value and row['id2'] in value:
-        #         return value[row['id1']][row['id2']]
-
-        def similarity_lambda_fnc(row, value):
-            if row['id1'] in value:
-                return value[row['id1']]
-
-        def lambda_fnc(row, value):
-            if row['id1'] in value and row['id2'] in value:
-                return np.float16(np.hstack(
-                    (value[row['id1']], value[row['id2']])))
-                # return np.hstack(
-                #     (value[row['id1']], value[row['id2']]), dtype=np.float16)
-
-        def x_fnc(row, embeddings_after_pooling):
-            if row['id1'] in embeddings_after_pooling:
-                v1 = embeddings_after_pooling[row['id1']]
-            else:
-                v1 = np.zeros(self.embedding_size)
-            if row['id2'] in embeddings_after_pooling:
-                v2 = embeddings_after_pooling[row['id2']]
-            else:
-                v2 = np.zeros(self.embedding_size)
-            return np.float16(np.hstack(
-                (v1, v2)))
-
-        for key, value in similarity_matrices.items():
-
-            print(f'sim matrix: {key}')
-            self.ddis_df[key] = self.ddis_df.apply(
-                lambda_fnc, args=(value,), axis=1)
-            print(self.ddis_df[key].head())
-
-        for embedding_column in self.embedding_columns:
-            print(f"concat {embedding_column} embeddings")
-            embeddings_after_pooling = {k: self.embeddings_pooling_strategy.apply(
-                v) for k, v in self.embedding_dict[embedding_column].items()}
-            # column_embeddings_dict = embedding_values[embedding_column]
-            self.ddis_df[embedding_column+'_embedding'] = self.ddis_df.apply(
-                x_fnc, args=(embeddings_after_pooling,), axis=1)
-
-        self.dataframe = self.ddis_df.copy()
-        self.dataframe['class'] = list(classes)
-        print(self.dataframe.shape)
+        if self.dataframe is None:
+            raise Exception("There is no data")
 
-    def load(self):
-        if self.index_path == None:
-            raise Exception(
-                "There is no index path, please call split function")
+        X = self.dataframe.drop(self.class_column, axis=1)
+        y = self.dataframe[self.class_column]
 
-        # prep - split - load
-        train_idx_all, test_idx_all, train_idx_arr, val_idx_arr = self.__get_indexes__(
-            self.index_path)
+        X_train, X_test, y_train, y_test, X_train.index, X_test.index, train_idx_arr, val_idx_arr = self.dataset_splitter.split(
+            X=X, y=y)
+        self.X_train = X_train
+        self.X_test = X_test
+        self.y_train = y_train
+        self.y_test = y_test
+        self.train_indexes = X_train.index
+        self.test_indexes = X_test.index
+        self.train_idx_arr = train_idx_arr
+        self.val_idx_arr = val_idx_arr
 
-        self.prep()
-        train = self.dataframe[self.dataframe['index'].isin(train_idx_all)]
-        test = self.dataframe[self.dataframe['index'].isin(test_idx_all)]
+        if save_indexes:
+            # train_pairs = [row['id1'].join(',').row['id2'] for index, row in X_train.iterrows()]
+            self.__save_indexes__(
+                save_path, 'train_indexes.txt', self.train_indexes.values)
+            self.__save_indexes__(
+                save_path, 'test_indexes.txt',  self.test_indexes.values)
+
+            for i, (train_idx, val_idx) in enumerate(zip(train_idx_arr, val_idx_arr)):
+                self.__save_indexes__(
+                    save_path, f'train_fold_{i}.txt', train_idx)
+                self.__save_indexes__(
+                    save_path, f'validation_fold_{i}.txt', val_idx)
 
-        self.X_train = train.drop('class', axis=1)
-        self.y_train = train['class']
-        self.X_test = test.drop('class', axis=1)
-        self.y_test = test['class']
+        # return X_train, X_test, y_train, y_test, folds
 
-        self.train_indexes = self.X_train.index
-        self.test_indexes = self.X_test.index
-        self.train_idx_arr = train_idx_arr
-        self.val_idx_arr = val_idx_arr
 
-        return self.X_train, self.X_test, self.y_train, self.y_test, self.X_train.index, self.X_test.index, train_idx_arr, val_idx_arr
+class TextDatasetMixin(BaseDataset):
+    embedding_size: int
+    embedding_dict: dict
+    embeddings_pooling_strategy: PoolingStrategy | None = None
+
+    def process_text(self):
+        pass
+
+
+# class ImageDatasetMixin(BaseModel):
+#     image_size: tuple[int, int] = Field(default=(224, 224))
+#     augmentations: list[str] = Field(default_factory=list)
 
-        def export_as_csv(self, output_file_path, not_change: list):
-            copy = self.dataframe.copy()
-            for col in copy.columns:
-                if col not in not_change:
-                    copy[col] = [
-                        '[' + ','.join(f"{value:.3f}" for value in row) + ']' for row in copy[col]]
-            copy.to_csv(output_file_path, index=False)
+#     def process_image_data(self):
+#         print(
+#             f"Processing image data with size {self.image_size} and augmentations {self.augmentations}...")

ddi_fw/datasets/dataset_splitter.py

@@ -0,0 +1,39 @@
+from typing import List, Tuple
+import numpy as np
+import pandas as pd
+from pydantic import BaseModel, Field
+from sklearn.model_selection import StratifiedKFold, train_test_split
+
+
+class DatasetSplitter(BaseModel):
+    fold_size: int = Field(default=5, ge=2)
+    test_size: float = Field(default=0.2, ge=0.0, le=1.0)
+    shuffle: bool = True
+    random_state: int = Field(default=42)
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def split(self, X: pd.DataFrame, y: pd.Series)-> Tuple[
+    pd.DataFrame, pd.DataFrame, pd.Series, pd.Series, pd.Index, pd.Index, List[np.ndarray], List[np.ndarray]]:
+        print(
+            f"Splitting dataset into {self.fold_size} folds with shuffle={self.shuffle}...")
+        #TODO check it
+        if len(y.shape) == 1:
+            y = pd.Series(np.expand_dims(y.to_numpy(), axis=1).flatten())
+        stacked = np.vstack(tuple(y.to_numpy()))
+        stratify = np.argmax(stacked, axis=1)
+        X_train, X_test, y_train, y_test = train_test_split(
+            X, y, shuffle=self.shuffle, test_size=self.test_size, stratify=stratify)
+
+        k_fold = StratifiedKFold(
+            n_splits=self.fold_size, shuffle=self.shuffle, random_state=self.random_state)
+        folds = k_fold.split(X_train, np.argmax(
+            np.vstack(y_train.to_numpy()), axis=1))
+        train_idx_arr = []
+        val_idx_arr = []
+        for i, (train_index, val_index) in enumerate(folds):
+            train_idx_arr.append(train_index)
+            val_idx_arr.append(val_index)
+
+        return X_train, X_test, y_train, y_test, X_train.index, X_test.index, train_idx_arr, val_idx_arr