ddi-fw 0.0.197__tar.gz → 0.0.199__tar.gz

{ddi_fw-0.0.197 → ddi_fw-0.0.199}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.197
+Version: 0.0.199
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.197 → ddi_fw-0.0.199}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.197"
+version = "0.0.199"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.197 → ddi_fw-0.0.199}/src/ddi_fw/datasets/core.py

@@ -1,3 +1,4 @@
+import abc
 from collections import defaultdict
 import glob
 import logging
@@ -57,7 +58,7 @@ def generate_sim_matrices_new(df, generated_vectors, columns, key_column="id"):
     return similarity_matrices
 
 
-class BaseDataset(BaseModel):
+class BaseDataset(BaseModel, abc.ABC):
     dataset_name: str
     index_path: Optional[str] = None
     dataset_splitter_type: Type[DatasetSplitter]
@@ -125,19 +126,26 @@ class BaseDataset(BaseModel):
     def set_dataframe(self, dataframe: pd.DataFrame):
         self.dataframe = dataframe
 
-    # @abstractmethod
+    @abc.abstractmethod
     def prep(self):
-        pass
+        """Prepare the dataset. This method should be overridden in subclasses."""
+        
 
+    def handle_mixins(self):
+        """Handle mixin-specific logic."""
+        if isinstance(self, TextDatasetMixin):
+            self.process_text()
+        # if isinstance(self, ImageDatasetMixin):
+        #     self.process_image_data()
+        # Add other mixin-specific logic here
+        
     def load(self):
         """
         Load the dataset. If X_train, y_train, X_test, and y_test are already provided,
         skip deriving them. Otherwise, derive them from the dataframe and indices.
         """
-        self.prep()
-
-        if isinstance(self, TextDatasetMixin):
-            self.process_text()
+        self.handle_mixins()  # Centralized mixin handling
+        self.prep()  # Prepare the dataset
 
         if self.X_train is not None or self.y_train is not None or self.X_test is not None or self.y_test is not None:
             # Data is already provided, no need to calculate
@@ -158,9 +166,12 @@ class BaseDataset(BaseModel):
                 self.index_path)
         except FileNotFoundError as e:
             raise FileNotFoundError(f"Index files not found: {e.filename}")
-
-        train = self.dataframe[self.dataframe.index.isin(train_idx_all)]
-        test = self.dataframe[self.dataframe.index.isin(test_idx_all)]
+        
+        # train = self.dataframe[self.dataframe.index.isin(train_idx_all)]
+        # test = self.dataframe[self.dataframe.index.isin(test_idx_all)]
+        columns = self.columns + [self.class_column] 
+        train = self.dataframe.loc[self.dataframe.index.isin(train_idx_all), columns]
+        test = self.dataframe.loc[self.dataframe.index.isin(test_idx_all), columns]
         X_train = train.drop(self.class_column, axis=1)
         X_train = train.drop(self.class_column, axis=1)
         y_train = train[self.class_column]
@@ -259,13 +270,18 @@ class BaseDataset(BaseModel):
 
 
 class TextDatasetMixin(BaseModel):
-    embedding_size: Optional[int] = None
     embedding_dict: Dict[str, Any] | None = Field(
         default_factory=dict, description="Dictionary for embeddings")
     pooling_strategy: PoolingStrategy | None = None
     column_embedding_configs: Optional[Dict] = None
     vector_db_persist_directory: Optional[str] = None
     vector_db_collection_name: Optional[str] = None
+    _embedding_size: int
+
+    @computed_field
+    @property
+    def embedding_size(self) -> int:
+        return self._embedding_size
 
     class Config:
         arbitrary_types_allowed = True
@@ -317,44 +333,43 @@ class TextDatasetMixin(BaseModel):
         else:
             raise ValueError(
                 "Persistent directory for the vector DB is not specified.")
+            
+    def __initialize_embedding_dict(self):
+        embedding_dict = defaultdict(lambda: defaultdict(list))
+        if self.column_embedding_configs:
+            for item in self.column_embedding_configs:
+                col = item["column"]
+                col_db_dir = item["vector_db_persist_directory"]
+                col_db_collection = item["vector_db_collection_name"]
+                self.__create_or_update_embeddings__(embedding_dict, col_db_dir, col_db_collection, col)
+        elif self.vector_db_persist_directory:
+            self.__create_or_update_embeddings__(embedding_dict, self.vector_db_persist_directory, self.vector_db_collection_name)
+        else:
+            logging.warning("There is no configuration of Embeddings")
+            raise ValueError(
+                "There is no configuration of Embeddings. Please provide a vector database directory and collection name.")
+        return embedding_dict
 
-    def process_text(self):
-        # key, value = next(iter(embedding_dict.items()))
-        #     embedding_size = value[next(iter(value))][0].shape[0]
-        #     pooling_strategy = self.embedding_pooling_strategy_type(
-        #     ) if self.embedding_pooling_strategy_type else None
+    def __calculate_embedding_size(self):
+        if self.embedding_dict is None:
+            raise ValueError("Embedding dictionary is not initialized, embedding size cannot be calculated.")
+            
+        key, value = next(iter(self.embedding_dict.items()))
+        self._embedding_size = value[next(iter(value))][0].shape[0]
 
+    def process_text(self):
+        logging.info("Processing text data...")
+ 
         # 'enzyme','target','pathway','smile','all_text','indication', 'description','mechanism_of_action','pharmacodynamics', 'tui', 'cui', 'entities'
         # kwargs = {"columns": self.columns}
         # if self.ner_threshold:
         #     for k, v in self.ner_threshold.items():
         #         kwargs[k] = v
-        if self.embedding_dict == None:
-            embedding_dict = defaultdict(lambda: defaultdict(list))
-            # TODO find more effective solution
-
-            if self.column_embedding_configs:
-                for item in self.column_embedding_configs:
-                    col = item["column"]
-                    col_db_dir = item["vector_db_persist_directory"]
-                    col_db_collection = item["vector_db_collection_name"]
-                    self.__create_or_update_embeddings__(
-                        embedding_dict, col_db_dir, col_db_collection, col)
-
-            elif self.vector_db_persist_directory:
-                self.__create_or_update_embeddings__(
-                    embedding_dict, self.vector_db_persist_directory, self.vector_db_collection_name)
+        if self.embedding_dict is None:
+            self.embedding_dict = self.__initialize_embedding_dict()
 
-            else:
-                print(
-                    f"There is no configuration of Embeddings")
-            self.embedding_dict = embedding_dict
-
-        # else:
-        #     embedding_dict = self.embedding_dict
-            # TODO make generic
-            # embedding_size = list(embedding_dict['all_text'].values())[
-            #     0][0].shape
+        self.__calculate_embedding_size()
+         
 
 
 # class ImageDatasetMixin(BaseModel):

{ddi_fw-0.0.197 → ddi_fw-0.0.199}/src/ddi_fw/datasets/ddi_mdl/base.py

@@ -81,6 +81,7 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
             self.chemical_property_columns = chemical_property_columns
             self.embedding_columns = embedding_columns 
             self.ner_columns = ner_columns 
+            self.columns = [] # these variable is modified in prep method
         
         self.class_column = 'event_category'
         _db_path = HERE.joinpath('data/event.db')
@@ -91,7 +92,7 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
         self.__similarity_related_columns__.extend(self.ner_columns)
         # TODO with resource
         self._conn = create_connection(_db_path.absolute().as_posix())
-        self.load_drugs_and_events()
+        # self.load_drugs_and_events()
         logger.info(f'{self.dataset_name} is initialized')
 
     def load_drugs_and_events(self):
@@ -131,6 +132,7 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
         return pd.DataFrame(columns=headers, data=rows)
 
     def prep(self):
+        self.load_drugs_and_events()
         if self.drugs_df is None or self.ddis_df is None:
             raise Exception("There is no data")
 
@@ -220,14 +222,15 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
             self.columns.append(key)
             print(self.ddis_df[key].head())
 
-        for embedding_column in self.embedding_columns:
-            print(f"concat {embedding_column} embeddings")
-            embeddings_after_pooling = {k: self.embeddings_pooling_strategy.apply(
-                v) for k, v in self.embedding_dict[embedding_column].items()}
-            # column_embeddings_dict = embedding_values[embedding_column]
-            self.ddis_df[embedding_column+'_embedding'] = self.ddis_df.apply(
-                x_fnc, args=(embeddings_after_pooling,), axis=1)
-            self.columns.append(embedding_column+'_embedding')
+        if self.embedding_dict is not None:
+            for embedding_column in self.embedding_columns:
+                print(f"concat {embedding_column} embeddings")
+                embeddings_after_pooling = {k: self.pooling_strategy.apply(
+                    v) for k, v in self.embedding_dict[embedding_column].items()}
+                # column_embeddings_dict = embedding_values[embedding_column]
+                self.ddis_df[embedding_column+'_embedding'] = self.ddis_df.apply(
+                    x_fnc, args=(embeddings_after_pooling,), axis=1)
+                self.columns.append(embedding_column+'_embedding')
 
         dataframe = self.ddis_df.copy()
         if not isinstance(classes, (list, pd.Series, np.ndarray)):

{ddi_fw-0.0.197 → ddi_fw-0.0.199}/src/ddi_fw/ml/ml_helper.py

@@ -32,9 +32,10 @@ import ddi_fw.utils as utils
 
 class MultiModalRunner:
     # todo model related parameters to config
-    def __init__(self, library, multi_modal, use_mlflow=False):
+    def __init__(self, library, multi_modal, default_model, use_mlflow=False):
         self.library = library
         self.multi_modal = multi_modal
+        self.default_model = default_model
         self.use_mlflow = use_mlflow
         self.result = Result()
 
@@ -60,14 +61,13 @@ class MultiModalRunner:
     # TODO check single_results, 1d,2d ...
     def __predict(self, single_results):
         item_dict = {t[0]: t for t in self.items}
-        print("multi_modal")
-        print(self.multi_modal)
-        print(item_dict.keys())
+        if self.default_model is None and not self.multi_modal:
+            raise Exception("Default model and multi modal cannot be None at the same time")
 
         if self.multi_modal:
             for m in self.multi_modal:
                 name = m.get('name')
-                input_type = m.get('input_type')
+                # input_type = m.get('input_type')
                 input = m.get('input')
                 inputs = m.get('inputs')
                 model_type = get_import(m.get("model_type"))
@@ -100,6 +100,15 @@ class MultiModalRunner:
                 else:
                     raise Exception("check configurations")
             else: # TODO default model maybe?
+                print("Default model will be used")
+                name = self.default_model.get('name')
+                # input_type = m.get('input_type')
+                input = self.default_model.get('input')
+                inputs = self.default_model.get('inputs')
+                model_type = get_import(self.default_model.get("model_type"))
+                kwargs = self.default_model.get('params')
+                single_modal = T(self.date, name, model_type,
+                                use_mlflow=self.use_mlflow,  **kwargs)
                 item = self.items[0]
                 single_modal.set_data(
                         self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])

{ddi_fw-0.0.197 → ddi_fw-0.0.199}/src/ddi_fw/pipeline/multi_pipeline.py

@@ -57,6 +57,7 @@ class MultiPipeline():
         tracking_uri = config.get("tracking_uri")
         artifact_location = config.get("artifact_location")
         #new
+        default_model = config.get("default_model"),
         multi_modal = config.get("multi_modal")
         columns = config.get("columns")
         ner_data_file = config.get("ner_data_file")
@@ -101,6 +102,7 @@ class MultiPipeline():
                 ner_data_file=ner_data_file,
                 ner_threshold=ner_threshold,
                 combinations=combinations,
+                default_model=default_model,
                 multi_modal= multi_modal)
         elif type== "ner_search":
             pipeline = NerParameterSearch(

{ddi_fw-0.0.197 → ddi_fw-0.0.199}/src/ddi_fw/pipeline/pipeline.py

@@ -35,6 +35,7 @@ class Pipeline(BaseModel):
     ner_threshold: Optional[dict] = None
     combinations: Optional[List[str]] = None
     model: Optional[Any] = None
+    default_model:  Optional[Any] = None
     multi_modal:  Optional[Any] = None
     use_mlflow: bool = False
     _dataset: BaseDataset = []
@@ -193,7 +194,7 @@ class Pipeline(BaseModel):
 
         y_test_label = self.items[0][4]
         multi_modal_runner = MultiModalRunner(
-            library=self.library, multi_modal=self.multi_modal, use_mlflow=self.use_mlflow)
+            library=self.library, multi_modal=self.multi_modal, default_model= self.default_model , use_mlflow=self.use_mlflow)
         # multi_modal_runner = MultiModalRunner(
         #     library=self.library, model_func=model_func, batch_size=batch_size,  epochs=epochs)
         # multi_modal = TFMultiModal(

{ddi_fw-0.0.197 → ddi_fw-0.0.199}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.197
+Version: 0.0.199
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>