ddi-fw 0.0.217__py3-none-any.whl → 0.0.218__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -73,6 +73,7 @@ class BaseDataset(BaseModel, abc.ABC):
     train_idx_arr: Optional[List[np.ndarray]] = None
     val_idx_arr: Optional[List[np.ndarray]] = None
     columns: List[str] = []
+    additional_config: Optional[Dict[str, Any]] = None
 
     class Config:
         arbitrary_types_allowed = True

ddi_fw/datasets/ddi_mdl/base.py

@@ -9,6 +9,8 @@ from abc import ABC, abstractmethod
 from sklearn.preprocessing import LabelBinarizer
 import logging
 
+from ddi_fw.ner.ner import CTakesNER
+
 
 try:
     from ddi_fw.vectorization import IDF
@@ -63,6 +65,18 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
 
         super().__init__(**kwargs)
         
+        # self.additional_config = kwargs.get('dataset_additional_config', {})
+        if self.additional_config:
+            ner = self.additional_config.get('ner', {})
+            self.ner_data_file = ner.get('data_file', None)
+            self.ner_threshold = ner.get('thresholds', None)
+            # if self.ner_threshold:
+            #     for k, v in self.ner_threshold.items():
+            #         kwargs[k] = v
+        
+            self.ner_df = CTakesNER(df=None).load(
+                filename=self.ner_data_file) if self.ner_data_file else None
+        
         columns = kwargs['columns']
         if columns:
             chemical_property_columns = []
@@ -155,13 +169,14 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
 
             # for key in filtered_ner_df.keys():
             for key in self.ner_columns:
-                threshold = 0
-                if key.startswith('tui'):
-                    threshold = self.tui_threshold
-                if key.startswith('cui'):
-                    threshold = self.cui_threshold
-                if key.startswith('entities'):
-                    threshold = self.entities_threshold
+                threshold = self.ner_threshold.get(key, 0)
+                # threshold = 0
+                # if key.startswith('tui'):
+                #     threshold = self.tui_threshold
+                # if key.startswith('cui'):
+                #     threshold = self.cui_threshold
+                # if key.startswith('entities'):
+                #     threshold = self.entities_threshold
                 combined_df[key] = filtered_ner_df[key]
                 valid_codes = idf_scores_df[idf_scores_df[key]
                                             > threshold].index

ddi_fw/datasets/mdf_sa_ddi/base.py

@@ -1,68 +1,157 @@
 import os
 import pathlib
-import sqlite3
-from sqlite3 import Error
+from typing import List, Optional, Tuple
+from ddi_fw.datasets.core import BaseDataset, TextDatasetMixin, generate_sim_matrices_new, generate_vectors
+from ddi_fw.datasets.db_utils import create_connection
+import numpy as np
 import pandas as pd
+from pydantic import BaseModel, Field, model_validator, root_validator
+from abc import ABC, abstractmethod
+from sklearn.preprocessing import LabelBinarizer
+import logging
 
-from ddi_fw.utils import ZipHelper
+from ddi_fw.ner.ner import CTakesNER
+from ddi_fw.utils.zip_helper import ZipHelper
 
-from .. import BaseDataset
-from ddi_fw.langchain.embeddings import PoolingStrategy
-from ..db_utils import create_connection
+
+try:
+    from ddi_fw.vectorization import IDF
+except ImportError:
+    raise ImportError(
+        "Failed to import vectorization module. Ensure that the module exists and is correctly installed. ")
+
+logger = logging.getLogger(__name__)
+
+# Constants for embedding, chemical properties, and NER columns
+LIST_OF_EMBEDDING_COLUMNS = [
+    'all_text', 'description', 'synthesis_reference', 'indication',
+    'pharmacodynamics', 'mechanism_of_action', 'toxicity', 'metabolism',
+    'absorption', 'half_life', 'protein_binding', 'route_of_elimination',
+    'volume_of_distribution', 'clearance'
+]
+
+LIST_OF_CHEMICAL_PROPERTY_COLUMNS = ['enzyme', 'target', 'smile']
+LIST_OF_NER_COLUMNS = ['tui', 'cui', 'entities']
 
 HERE = pathlib.Path(__file__).resolve().parent
-list_of_embedding_columns = ['all_text', 'description',
-                     'synthesis_reference', 'indication',
-                     'pharmacodynamics', 'mechanism_of_action',
-                     'toxicity', 'metabolism',
-                     'absorption', 'half_life',
-                     'protein_binding', 'route_of_elimination',
-                     'volume_of_distribution', 'clearance']
-
-list_of_chemical_property_columns = ['enzyme',
-                             'target',
-                             'smile']
-
-list_of_ner_columns = ['tui', 'cui', 'entities']
-class MDFSADDIDataset(BaseDataset):
-    def __init__(self, embedding_size,
-                 embedding_dict,
-                 embeddings_pooling_strategy: PoolingStrategy,
-                 ner_df,
-                 chemical_property_columns=['enzyme',
-                                                  'target',
-                                                  'smile'],
-                 embedding_columns=[],
-                 ner_columns=[],
-                 **kwargs):
 
+class MDFSADDIDataset(BaseDataset,TextDatasetMixin):
+    # def __init__(self, embedding_size,
+    #              embedding_dict,
+    #              embeddings_pooling_strategy: PoolingStrategy,
+    #              ner_df,
+    #              chemical_property_columns=['enzyme',
+    #                                               'target',
+    #                                               'smile'],
+    #              embedding_columns=[],
+    #              ner_columns=[],
+    #              **kwargs):
+
+    #     columns = kwargs['columns']
+    #     if columns:
+    #         chemical_property_columns = []
+    #         embedding_columns=[]
+    #         ner_columns=[]
+    #         for column in columns:
+    #             if column in list_of_chemical_property_columns:
+    #                 chemical_property_columns.append(column)
+    #             elif column in list_of_embedding_columns:
+    #                 embedding_columns.append(column)
+    #             elif column in list_of_ner_columns:
+    #                 ner_columns.append(column)
+    #             # elif column == 'smile_2':
+    #             #     continue
+    #             else:
+    #                 raise Exception(f"{column} is not related this dataset")
+
+
+    #     super().__init__(embedding_size=embedding_size,
+    #                      embedding_dict=embedding_dict,
+    #                      embeddings_pooling_strategy=embeddings_pooling_strategy,
+    #                      ner_df=ner_df,
+    #                      chemical_property_columns=chemical_property_columns,
+    #                      embedding_columns=embedding_columns,
+    #                      ner_columns=ner_columns,
+    #                      **kwargs)
+
+    #     db_zip_path = HERE.joinpath('mdf-sa-ddi.zip')
+    #     db_path = HERE.joinpath('mdf-sa-ddi.db')
+    #     if not os.path.exists(db_zip_path):
+    #         self.__to_db__(db_path)
+    #     else:
+    #         ZipHelper().extract(
+    #             input_path=str(HERE), output_path=str(HERE))
+    #         conn = create_connection(db_path)
+    #         self.drugs_df = select_all_drugs_as_dataframe(conn)
+    #         self.ddis_df = select_all_events_as_dataframe(conn)
+    #     # kwargs = {'index_path': str(HERE.joinpath('indexes'))}
+    #     kwargs['index_path'] = str(HERE.joinpath('indexes'))
+
+    #     self.index_path = kwargs.get('index_path')
+    
+    dataset_name: str = "MDFSADDIDataset"
+    index_path: str = Field(default_factory=lambda: str(
+        pathlib.Path(__file__).resolve().parent.joinpath('indexes')))
+    # drugs_df: pd.DataFrame = Field(default_factory=pd.DataFrame)
+    # ddis_df: pd.DataFrame = Field(default_factory=pd.DataFrame)
+    drugs_df: Optional[pd.DataFrame] = None
+    ddis_df: Optional[pd.DataFrame] = None
+
+    chemical_property_columns: list[str] = Field(
+        default_factory=lambda: LIST_OF_CHEMICAL_PROPERTY_COLUMNS)
+    embedding_columns: list[str] = Field(default_factory=list)
+    ner_columns: list[str] = Field(default_factory=list)
+    ner_df: pd.DataFrame | None = None
+    tui_threshold: float | None = None
+    cui_threshold: float | None = None
+    entities_threshold: float | None = None
+
+    # @model_validator
+
+    def validate_columns(self, values):
+        if not set(values['chemical_property_columns']).issubset(LIST_OF_CHEMICAL_PROPERTY_COLUMNS):
+            raise ValueError("Invalid chemical property columns")
+        if not set(values['ner_columns']).issubset(LIST_OF_NER_COLUMNS):
+            raise ValueError("Invalid NER columns")
+        return values
+
+    def __init__(self, **kwargs):
+
+        super().__init__(**kwargs)
+        
+        # self.additional_config = kwargs.get('dataset_additional_config', {})
+        if self.additional_config:
+            ner = self.additional_config.get('ner', {})
+            self.ner_data_file = ner.get('data_file', None)
+            self.ner_threshold = ner.get('thresholds', None)
+            # if self.ner_threshold:
+            #     for k, v in self.ner_threshold.items():
+            #         kwargs[k] = v
+        
+            self.ner_df = CTakesNER(df=None).load(
+                filename=self.ner_data_file) if self.ner_data_file else None
+        
         columns = kwargs['columns']
         if columns:
             chemical_property_columns = []
-            embedding_columns=[]
-            ner_columns=[]
+            embedding_columns = []
+            ner_columns = []
             for column in columns:
-                if column in list_of_chemical_property_columns:
+                if column in LIST_OF_CHEMICAL_PROPERTY_COLUMNS:
                     chemical_property_columns.append(column)
-                elif column in list_of_embedding_columns:
+                elif column in LIST_OF_EMBEDDING_COLUMNS:
                     embedding_columns.append(column)
-                elif column in list_of_ner_columns:
+                elif column in LIST_OF_NER_COLUMNS:
                     ner_columns.append(column)
-                # elif column == 'smile_2':
-                #     continue
                 else:
                     raise Exception(f"{column} is not related this dataset")
-
-
-        super().__init__(embedding_size=embedding_size,
-                         embedding_dict=embedding_dict,
-                         embeddings_pooling_strategy=embeddings_pooling_strategy,
-                         ner_df=ner_df,
-                         chemical_property_columns=chemical_property_columns,
-                         embedding_columns=embedding_columns,
-                         ner_columns=ner_columns,
-                         **kwargs)
-
+        
+            self.chemical_property_columns = chemical_property_columns
+            self.embedding_columns = embedding_columns 
+            self.ner_columns = ner_columns 
+            self.columns = [] # these variable is modified in prep method
+        
+        
         db_zip_path = HERE.joinpath('mdf-sa-ddi.zip')
         db_path = HERE.joinpath('mdf-sa-ddi.db')
         if not os.path.exists(db_zip_path):
@@ -70,13 +159,19 @@ class MDFSADDIDataset(BaseDataset):
         else:
             ZipHelper().extract(
                 input_path=str(HERE), output_path=str(HERE))
-            conn = create_connection(db_path)
+            conn = create_connection(db_path.absolute().as_posix())
             self.drugs_df = select_all_drugs_as_dataframe(conn)
             self.ddis_df = select_all_events_as_dataframe(conn)
         # kwargs = {'index_path': str(HERE.joinpath('indexes'))}
-        kwargs['index_path'] = str(HERE.joinpath('indexes'))
+        
+        
+        self.class_column = 'event_category'
 
-        self.index_path = kwargs.get('index_path')
+        self.__similarity_related_columns__ = []
+        self.__similarity_related_columns__.extend(
+            self.chemical_property_columns)
+        self.__similarity_related_columns__.extend(self.ner_columns)
+        logger.info(f'{self.dataset_name} is initialized')
 
     def __to_db__(self, db_path):
         conn = create_connection(db_path)
@@ -118,10 +213,11 @@ class MDFSADDIDataset(BaseDataset):
             lambda_fnc1)  # , axis=1
         self.ddis_df['id2'] = self.ddis_df['name2'].apply(
             lambda_fnc1)  # , axis=1
-        self.drugs_df.to_sql('drug', conn, if_exists='replace', index=False)
-        self.ddis_df.to_sql('event', conn, if_exists='replace', index=False)
-        ZipHelper().zip_single_file(
-            file_path=db_path, output_path=HERE, name='mdf-sa-ddi')
+        if conn:
+            self.drugs_df.to_sql('drug', conn, if_exists='replace', index=False)
+            self.ddis_df.to_sql('event', conn, if_exists='replace', index=False)
+            ZipHelper().zip_single_file(
+                file_path=db_path, output_path=HERE, zip_name='mdf-sa-ddi')
 
 
 def select_all_drugs(conn):

ddi_fw/ml/__init__.py

@@ -2,4 +2,5 @@ from .ml_helper import MultiModalRunner
 from .model_wrapper import ModelWrapper,Result
 from .tensorflow_wrapper import TFModelWrapper
 from .pytorch_wrapper import PTModelWrapper
-from .evaluation_helper import evaluate
+from .evaluation_helper import evaluate
+from .tracking_service import TrackingService

ddi_fw/ml/ml_helper.py

@@ -1,23 +1,9 @@
-from typing import Callable, Dict, List, Tuple
-from matplotlib import pyplot as plt
 from ddi_fw.ml.model_wrapper import Result
 from ddi_fw.ml.pytorch_wrapper import PTModelWrapper
 from ddi_fw.ml.tensorflow_wrapper import TFModelWrapper
 from ddi_fw.utils.package_helper import get_import
-import tensorflow as tf
-from tensorflow.python import keras
-from tensorflow.python.keras import Model, Sequential
-from tensorflow.python.keras.layers import Dense, Dropout, Input, Activation
-from tensorflow.python.keras.callbacks import EarlyStopping
-from sklearn.model_selection import train_test_split, KFold, StratifiedKFold
 import numpy as np
-
-import mlflow
-from mlflow.utils.autologging_utils import batch_metrics_logger
-import time
-
-from mlflow.models import infer_signature
-from ddi_fw.ml.evaluation_helper import Metrics, evaluate
+from ddi_fw.ml.evaluation_helper import  evaluate
 
 # import tf2onnx
 # import onnx
@@ -32,16 +18,16 @@ import ddi_fw.utils as utils
 
 class MultiModalRunner:
     # todo model related parameters to config
-    def __init__(self, library, multi_modal, default_model, use_mlflow=False):
+    def __init__(self, library, multi_modal, default_model, tracking_service):
         self.library = library
         self.multi_modal = multi_modal
         self.default_model = default_model
-        self.use_mlflow = use_mlflow
+        self.tracking_service = tracking_service
         self.result = Result()
 
-    def _mlflow_(self, func: Callable):
-        if self.use_mlflow:
-            func()
+    # def _mlflow_(self, func: Callable):
+    #     if self.use_mlflow:
+    #         func()
 
     def set_data(self, items, train_idx_arr, val_idx_arr, y_test_label):
         self.items = items
@@ -74,7 +60,7 @@ class MultiModalRunner:
                 kwargs = m.get('params')
                 T = self.__create_model(self.library)
                 single_modal = T(self.date, name, model_type,
-                                use_mlflow=self.use_mlflow,  **kwargs)
+                                tracking_service=self.tracking_service,  **kwargs)
                 
                 if input is not None and inputs is not None:
                     raise Exception("input and inputs should not be used together")
@@ -110,7 +96,7 @@ class MultiModalRunner:
                 name = item[0]
                 T = self.__create_model(self.library)
                 single_modal = T(self.date, name, model_type,
-                                use_mlflow=self.use_mlflow,  **kwargs)
+                                tracking_service=self.tracking_service,  **kwargs)
                 single_modal.set_data(
                         self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
 
@@ -130,9 +116,12 @@ class MultiModalRunner:
             combinations = []
             for i in range(2, len(l) + 1):
                 combinations.extend(list(itertools.combinations(l, i)))  # all
-        if self.use_mlflow:
-            with mlflow.start_run(run_name=self.prefix, description="***") as run:
-                self.__predict(single_results)
+                
+        def _f():
+            self.__predict(single_results)
+            
+        if self.tracking_service:
+            self.tracking_service.run(run_name=self.prefix, description="***", func = _f , nested_run=False)
         else:
             self.__predict(single_results)
         if combinations:
@@ -143,10 +132,17 @@ class MultiModalRunner:
     def evaluate_combinations(self, single_results, combinations):
         for combination in combinations:
             combination_descriptor = '-'.join(combination)
-            if self.use_mlflow:
-                with mlflow.start_run(run_name=combination_descriptor, description="***", nested=True) as combination_run:
+            if self.tracking_service:
+                def evaluate_combination(artifact_uri=None):
                     self.__evaluate_combinations(
-                        single_results, combination, combination_descriptor, combination_run.info.artifact_uri)
+                        single_results, combination, combination_descriptor, artifact_uri
+                )
+                 
+                self.tracking_service.run(run_name=combination_descriptor, description="***", nested_run=True, func=evaluate_combination)
+                
+                # with mlflow.start_run(run_name=combination_descriptor, description="***", nested=True) as combination_run:
+                #     self.__evaluate_combinations(
+                #         single_results, combination, combination_descriptor, combination_run.info.artifact_uri)
             else:
                 self.__evaluate_combinations(
                     single_results, combination, combination_descriptor, None)
@@ -159,8 +155,8 @@ class MultiModalRunner:
         prediction = utils.to_one_hot_encode(prediction)
         logs, metrics = evaluate(
             actual=self.y_test_label, pred=prediction, info=combination_descriptor)
-        if self.use_mlflow:
-            mlflow.log_metrics(logs)
+        if self.tracking_service:
+            self.tracking_service.log_metrics(logs)
         metrics.format_float()
         # TODO path bulunamadı hatası aldık
         if artifact_uri:

ddi_fw/ml/model_wrapper.py

@@ -29,7 +29,6 @@ class ModelWrapper:
         self.train_label = train_label
         self.test_data = test_data
         self.test_label = test_label
-# https://github.com/mlflow/mlflow/blob/master/examples/tensorflow/train.py
 
     def predict(self)-> Any:
         pass