ddi-fw 0.0.197__py3-none-any.whl → 0.0.198__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -1,3 +1,4 @@
+import abc
 from collections import defaultdict
 import glob
 import logging
@@ -57,7 +58,7 @@ def generate_sim_matrices_new(df, generated_vectors, columns, key_column="id"):
     return similarity_matrices
 
 
-class BaseDataset(BaseModel):
+class BaseDataset(BaseModel, abc.ABC):
     dataset_name: str
     index_path: Optional[str] = None
     dataset_splitter_type: Type[DatasetSplitter]
@@ -125,19 +126,26 @@ class BaseDataset(BaseModel):
     def set_dataframe(self, dataframe: pd.DataFrame):
         self.dataframe = dataframe
 
-    # @abstractmethod
+    @abc.abstractmethod
     def prep(self):
-        pass
+        """Prepare the dataset. This method should be overridden in subclasses."""
+        
 
+    def handle_mixins(self):
+        """Handle mixin-specific logic."""
+        if isinstance(self, TextDatasetMixin):
+            self.process_text()
+        # if isinstance(self, ImageDatasetMixin):
+        #     self.process_image_data()
+        # Add other mixin-specific logic here
+        
     def load(self):
         """
         Load the dataset. If X_train, y_train, X_test, and y_test are already provided,
         skip deriving them. Otherwise, derive them from the dataframe and indices.
         """
-        self.prep()
-
-        if isinstance(self, TextDatasetMixin):
-            self.process_text()
+        self.prep()  # Prepare the dataset
+        self.handle_mixins()  # Centralized mixin handling
 
         if self.X_train is not None or self.y_train is not None or self.X_test is not None or self.y_test is not None:
             # Data is already provided, no need to calculate
@@ -158,9 +166,11 @@ class BaseDataset(BaseModel):
                 self.index_path)
         except FileNotFoundError as e:
             raise FileNotFoundError(f"Index files not found: {e.filename}")
-
-        train = self.dataframe[self.dataframe.index.isin(train_idx_all)]
-        test = self.dataframe[self.dataframe.index.isin(test_idx_all)]
+        
+        # train = self.dataframe[self.dataframe.index.isin(train_idx_all)]
+        # test = self.dataframe[self.dataframe.index.isin(test_idx_all)]
+        train = self.dataframe.loc[self.dataframe.index.isin(train_idx_all), self.columns]
+        test = self.dataframe.loc[self.dataframe.index.isin(test_idx_all), self.columns]
         X_train = train.drop(self.class_column, axis=1)
         X_train = train.drop(self.class_column, axis=1)
         y_train = train[self.class_column]
@@ -259,13 +269,18 @@ class BaseDataset(BaseModel):
 
 
 class TextDatasetMixin(BaseModel):
-    embedding_size: Optional[int] = None
     embedding_dict: Dict[str, Any] | None = Field(
         default_factory=dict, description="Dictionary for embeddings")
     pooling_strategy: PoolingStrategy | None = None
     column_embedding_configs: Optional[Dict] = None
     vector_db_persist_directory: Optional[str] = None
     vector_db_collection_name: Optional[str] = None
+    _embedding_size: int
+
+    @computed_field
+    @property
+    def embedding_size(self) -> int:
+        return self._embedding_size
 
     class Config:
         arbitrary_types_allowed = True
@@ -317,44 +332,43 @@ class TextDatasetMixin(BaseModel):
         else:
             raise ValueError(
                 "Persistent directory for the vector DB is not specified.")
+            
+    def __initialize_embedding_dict(self):
+        embedding_dict = defaultdict(lambda: defaultdict(list))
+        if self.column_embedding_configs:
+            for item in self.column_embedding_configs:
+                col = item["column"]
+                col_db_dir = item["vector_db_persist_directory"]
+                col_db_collection = item["vector_db_collection_name"]
+                self.__create_or_update_embeddings__(embedding_dict, col_db_dir, col_db_collection, col)
+        elif self.vector_db_persist_directory:
+            self.__create_or_update_embeddings__(embedding_dict, self.vector_db_persist_directory, self.vector_db_collection_name)
+        else:
+            logging.warning("There is no configuration of Embeddings")
+            raise ValueError(
+                "There is no configuration of Embeddings. Please provide a vector database directory and collection name.")
+        return embedding_dict
 
-    def process_text(self):
-        # key, value = next(iter(embedding_dict.items()))
-        #     embedding_size = value[next(iter(value))][0].shape[0]
-        #     pooling_strategy = self.embedding_pooling_strategy_type(
-        #     ) if self.embedding_pooling_strategy_type else None
+    def __calculate_embedding_size(self):
+        if self.embedding_dict is None:
+            raise ValueError("Embedding dictionary is not initialized, embedding size cannot be calculated.")
+            
+        key, value = next(iter(self.embedding_dict.items()))
+        self._embedding_size = value[next(iter(value))][0].shape[0]
 
+    def process_text(self):
+        logging.info("Processing text data...")
+ 
         # 'enzyme','target','pathway','smile','all_text','indication', 'description','mechanism_of_action','pharmacodynamics', 'tui', 'cui', 'entities'
         # kwargs = {"columns": self.columns}
         # if self.ner_threshold:
         #     for k, v in self.ner_threshold.items():
         #         kwargs[k] = v
-        if self.embedding_dict == None:
-            embedding_dict = defaultdict(lambda: defaultdict(list))
-            # TODO find more effective solution
-
-            if self.column_embedding_configs:
-                for item in self.column_embedding_configs:
-                    col = item["column"]
-                    col_db_dir = item["vector_db_persist_directory"]
-                    col_db_collection = item["vector_db_collection_name"]
-                    self.__create_or_update_embeddings__(
-                        embedding_dict, col_db_dir, col_db_collection, col)
-
-            elif self.vector_db_persist_directory:
-                self.__create_or_update_embeddings__(
-                    embedding_dict, self.vector_db_persist_directory, self.vector_db_collection_name)
+        if self.embedding_dict is None:
+            self.embedding_dict = self.__initialize_embedding_dict()
 
-            else:
-                print(
-                    f"There is no configuration of Embeddings")
-            self.embedding_dict = embedding_dict
-
-        # else:
-        #     embedding_dict = self.embedding_dict
-            # TODO make generic
-            # embedding_size = list(embedding_dict['all_text'].values())[
-            #     0][0].shape
+        self.__calculate_embedding_size()
+         
 
 
 # class ImageDatasetMixin(BaseModel):

ddi_fw/datasets/ddi_mdl/base.py

@@ -91,7 +91,7 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
         self.__similarity_related_columns__.extend(self.ner_columns)
         # TODO with resource
         self._conn = create_connection(_db_path.absolute().as_posix())
-        self.load_drugs_and_events()
+        # self.load_drugs_and_events()
         logger.info(f'{self.dataset_name} is initialized')
 
     def load_drugs_and_events(self):
@@ -131,6 +131,7 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
         return pd.DataFrame(columns=headers, data=rows)
 
     def prep(self):
+        self.load_drugs_and_events()
         if self.drugs_df is None or self.ddis_df is None:
             raise Exception("There is no data")
 
@@ -220,14 +221,15 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
             self.columns.append(key)
             print(self.ddis_df[key].head())
 
-        for embedding_column in self.embedding_columns:
-            print(f"concat {embedding_column} embeddings")
-            embeddings_after_pooling = {k: self.embeddings_pooling_strategy.apply(
-                v) for k, v in self.embedding_dict[embedding_column].items()}
-            # column_embeddings_dict = embedding_values[embedding_column]
-            self.ddis_df[embedding_column+'_embedding'] = self.ddis_df.apply(
-                x_fnc, args=(embeddings_after_pooling,), axis=1)
-            self.columns.append(embedding_column+'_embedding')
+        if self.embedding_dict is not None:
+            for embedding_column in self.embedding_columns:
+                print(f"concat {embedding_column} embeddings")
+                embeddings_after_pooling = {k: self.pooling_strategy.apply(
+                    v) for k, v in self.embedding_dict[embedding_column].items()}
+                # column_embeddings_dict = embedding_values[embedding_column]
+                self.ddis_df[embedding_column+'_embedding'] = self.ddis_df.apply(
+                    x_fnc, args=(embeddings_after_pooling,), axis=1)
+                self.columns.append(embedding_column+'_embedding')
 
         dataframe = self.ddis_df.copy()
         if not isinstance(classes, (list, pd.Series, np.ndarray)):

{ddi_fw-0.0.197.dist-info → ddi_fw-0.0.198.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.197
+Version: 0.0.198
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.197.dist-info → ddi_fw-0.0.198.dist-info}/RECORD

@@ -1,9 +1,9 @@
 ddi_fw/datasets/__init__.py,sha256=_I3iDHARwzmg7_EL5XKtB_TgG1yAkLSOVTujLL9Wz9Q,280
-ddi_fw/datasets/core.py,sha256=eLS4TtQN1_1kI0huMt7eTOCz5hY3da9PHhEeiLjWtQg,15605
+ddi_fw/datasets/core.py,sha256=yfnJwyF9oV2RUErFSAKSyxQQeL1tmLiq7SfADhn1Cgk,16379
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
-ddi_fw/datasets/ddi_mdl/base.py,sha256=rS8lSGE-SLeoE3GuElJ-TNaRHIGhaZBeOM2UH3JUS4M,10218
+ddi_fw/datasets/ddi_mdl/base.py,sha256=8WFc0iLT5PF6IOUStqKVIKR74D8WBuwXm_uMiV4OFsk,10324
 ddi_fw/datasets/ddi_mdl/debug.log,sha256=eWz05j8RFqZuHFDTCF7Rck5w4rvtTanFN21iZsgxO7Y,115
 ddi_fw/datasets/ddi_mdl/readme.md,sha256=WC6lpmsEKvIISnZqENY7TWtzCQr98HPpE3oRsBl8pIw,625
 ddi_fw/datasets/ddi_mdl/data/event.db,sha256=cmlSsf9MYjRzqR-mw3cUDnTnfT6FkpOG2yCl2mMwwew,30580736
@@ -99,7 +99,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=EBf-XAiwQwr68az91erEYNegfeqssBR29kVgrliIyac,4765
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.197.dist-info/METADATA,sha256=pVnij5JFvkPUgjVqvHmFLdI2OKSFRYxt7-vLVXhpldU,2542
-ddi_fw-0.0.197.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddi_fw-0.0.197.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.197.dist-info/RECORD,,
+ddi_fw-0.0.198.dist-info/METADATA,sha256=z3otymNU3l4737h3tkMaP0UMhZdLBtzS4ELP4wIcVt8,2542
+ddi_fw-0.0.198.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddi_fw-0.0.198.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.198.dist-info/RECORD,,