ddi-fw 0.0.214__py3-none-any.whl → 0.0.215__py3-none-any.whl

ddi_fw/datasets/db_utils.py

@@ -20,160 +20,160 @@ def create_connection(db_file=r"./event.db"):
     return conn
 
 
-def select_all_drugs(conn):
-    cur = conn.cursor()
-    cur.execute(
-        '''select "index", id, name, target, enzyme, pathway, smile from drug''')
-    rows = cur.fetchall()
-    return rows
-
-
-def select_all_drugs_as_dataframe(conn):
-    headers = ['index','id', 'name', 'target', 'enzyme', 'pathway', 'smile']
-    rows = select_all_drugs(conn)
-    df = pd.DataFrame(columns=headers, data=rows)
-    df['enzyme'] = df['enzyme'].apply(lambda x: x.split('|'))
-    df['target'] = df['target'].apply(lambda x: x.split('|'))
-    df['pathway'] = df['pathway'].apply(lambda x: x.split('|'))
-    df['smile'] = df['smile'].apply(lambda x: x.split('|'))
-    return df
-
-
-def select_all_events(conn):
-    """
-    Query all rows in the event table
-    :param conn: the Connection object
-    :return:
-    """
-    cur = conn.cursor()
-    cur.execute("select * from event")
-
-    rows = cur.fetchall()
-    return rows
-
-
-def select_all_events_as_dataframe(conn):
-    headers = ["index", "id1", "name1", "id2", "name2", "event_category"]
-    rows = select_all_events(conn)
-    return pd.DataFrame(columns=headers, data=rows)
-
-
-def select_events_with_category(conn):
-    sql = '''select id1, name1, id2, name2, mechanism || ' ' ||action from event ev
-            join extraction ex
-            on ev.name1 = ex.drugA and ev.name2 = ex.drugB
-            union 
-            select id1, name1, id2, name2, mechanism || ' ' ||action from event ev
-            join extraction ex
-            on ev.name1 = ex.drugB and ev.name2 = ex.drugA
-            '''
-    cur = conn.cursor()
-    cur.execute(sql)
-
-    rows = cur.fetchall()
-
-    headers = ['id1', 'name1', 'id2', 'name2', 'event_category']
-    return pd.DataFrame(columns=headers, data=rows)
-
-
-def select_all_interactions_tuple_as_dataframe(conn):
-    cur = conn.cursor()
-    cur.execute("select id1, id2 from event")
-    rows = cur.fetchall()
-    headers = ['id1', 'id2']
-
-    return pd.DataFrame(columns=headers, data=rows)
-
-
-def select_ddi_pairs(conn):
-    cur = conn.cursor()
-    cur.execute('''
-        select d1.[index] as Drug1Index, d2.[index] as Drug2Index, 1 from event e 
-        join drug d1 on e.id1 = d1.id 
-        join drug d2 on e.id2 = d2.id 
-    ''')
-    rows = cur.fetchall()
-    return rows
-
-
-def select_ddi_pairs_as_dataframe(conn):
-    headers = ["Drug1Index", "Drug2Index", "Interaction"]
-    rows = select_ddi_pairs(conn)
-    return pd.DataFrame(columns=headers, data=rows)
-
-
-def get_interactions(conn):
-    cur = conn.cursor()
-    cur.execute('''
-        select
-            drug_1_id,
-            drug_1,
-            drug_2_id,
-            drug_2,
-            mechanism_action, 
-            interaction, 
-            masked_interaction
-        from _Interactions
-                ''')
-
-    rows = cur.fetchall()
-
-    headers = ['id1', 'name1', 'id2', 'name2',
-               'event_category', 'interaction', 'masked_interaction']
-    df = pd.DataFrame(columns=headers, data=rows)
-    return df
-
-
-def get_extended_version(conn):
-    cur = conn.cursor()
-    cur.execute('''
-        select 
-        _Drugs."index",
-        drugbank_id,
-        _Drugs.name,
-        description,
-        synthesis_reference,
-        indication,
-        pharmacodynamics,
-        mechanism_of_action,
-        toxicity,
-        metabolism,
-        absorption,
-        half_life,
-        protein_binding,
-        route_of_elimination,
-        volume_of_distribution,
-        clearance,
-        smiles,
-        smiles_morgan_fingerprint,
-        enzymes_polypeptides,
-        targets_polypeptides
+# def select_all_drugs(conn):
+#     cur = conn.cursor()
+#     cur.execute(
+#         '''select "index", id, name, target, enzyme, pathway, smile from drug''')
+#     rows = cur.fetchall()
+#     return rows
+
+
+# def select_all_drugs_as_dataframe(conn):
+#     headers = ['index','id', 'name', 'target', 'enzyme', 'pathway', 'smile']
+#     rows = select_all_drugs(conn)
+#     df = pd.DataFrame(columns=headers, data=rows)
+#     df['enzyme'] = df['enzyme'].apply(lambda x: x.split('|'))
+#     df['target'] = df['target'].apply(lambda x: x.split('|'))
+#     df['pathway'] = df['pathway'].apply(lambda x: x.split('|'))
+#     df['smile'] = df['smile'].apply(lambda x: x.split('|'))
+#     return df
+
+
+# def select_all_events(conn):
+#     """
+#     Query all rows in the event table
+#     :param conn: the Connection object
+#     :return:
+#     """
+#     cur = conn.cursor()
+#     cur.execute("select * from event")
+
+#     rows = cur.fetchall()
+#     return rows
+
+
+# def select_all_events_as_dataframe(conn):
+#     headers = ["index", "id1", "name1", "id2", "name2", "event_category"]
+#     rows = select_all_events(conn)
+#     return pd.DataFrame(columns=headers, data=rows)
+
+
+# def select_events_with_category(conn):
+#     sql = '''select id1, name1, id2, name2, mechanism || ' ' ||action from event ev
+#             join extraction ex
+#             on ev.name1 = ex.drugA and ev.name2 = ex.drugB
+#             union 
+#             select id1, name1, id2, name2, mechanism || ' ' ||action from event ev
+#             join extraction ex
+#             on ev.name1 = ex.drugB and ev.name2 = ex.drugA
+#             '''
+#     cur = conn.cursor()
+#     cur.execute(sql)
+
+#     rows = cur.fetchall()
+
+#     headers = ['id1', 'name1', 'id2', 'name2', 'event_category']
+#     return pd.DataFrame(columns=headers, data=rows)
+
+
+# def select_all_interactions_tuple_as_dataframe(conn):
+#     cur = conn.cursor()
+#     cur.execute("select id1, id2 from event")
+#     rows = cur.fetchall()
+#     headers = ['id1', 'id2']
+
+#     return pd.DataFrame(columns=headers, data=rows)
+
+
+# def select_ddi_pairs(conn):
+#     cur = conn.cursor()
+#     cur.execute('''
+#         select d1.[index] as Drug1Index, d2.[index] as Drug2Index, 1 from event e 
+#         join drug d1 on e.id1 = d1.id 
+#         join drug d2 on e.id2 = d2.id 
+#     ''')
+#     rows = cur.fetchall()
+#     return rows
+
+
+# def select_ddi_pairs_as_dataframe(conn):
+#     headers = ["Drug1Index", "Drug2Index", "Interaction"]
+#     rows = select_ddi_pairs(conn)
+#     return pd.DataFrame(columns=headers, data=rows)
+
+
+# def get_interactions(conn):
+#     cur = conn.cursor()
+#     cur.execute('''
+#         select
+#             drug_1_id,
+#             drug_1,
+#             drug_2_id,
+#             drug_2,
+#             mechanism_action, 
+#             interaction, 
+#             masked_interaction
+#         from _Interactions
+#                 ''')
+
+#     rows = cur.fetchall()
+
+#     headers = ['id1', 'name1', 'id2', 'name2',
+#                'event_category', 'interaction', 'masked_interaction']
+#     df = pd.DataFrame(columns=headers, data=rows)
+#     return df
+
+
+# def get_extended_version(conn):
+#     cur = conn.cursor()
+#     cur.execute('''
+#         select 
+#         _Drugs."index",
+#         drugbank_id,
+#         _Drugs.name,
+#         description,
+#         synthesis_reference,
+#         indication,
+#         pharmacodynamics,
+#         mechanism_of_action,
+#         toxicity,
+#         metabolism,
+#         absorption,
+#         half_life,
+#         protein_binding,
+#         route_of_elimination,
+#         volume_of_distribution,
+#         clearance,
+#         smiles,
+#         smiles_morgan_fingerprint,
+#         enzymes_polypeptides,
+#         targets_polypeptides
                 
-        from drug 
-        join _Drugs on drug.id = _Drugs.drugbank_id  
-        where 
-                targets_polypeptides is not null and 
-                enzymes_polypeptides is not null and 
-                smiles_morgan_fingerprint is not null
-''')
-    # pathway is absent
-
-    rows = cur.fetchall()
-    headers = ['index', 'id', 'name', 'description', 'synthesis_reference', 'indication', 'pharmacodynamics', 'mechanism_of_action', 'toxicity', 'metabolism', 'absorption', 'half_life',
-               'protein_binding', 'route_of_elimination', 'volume_of_distribution', 'clearance', 'smiles_notation', 'smile', 'enzyme', 'target']
-    df = pd.DataFrame(columns=headers, data=rows)
-    df['smile'] = df['smile'].apply(lambda x:
-                                    np.fromstring(
-                                        x.replace(
-                                            '\n', '')
-                                        .replace('[', '')
-                                        .replace(']', '')
-                                        .replace('  ', ' '), sep=','))
-    df['enzyme'] = df['enzyme'].apply(
-        lambda x: x.split('|'))
-    df['target'] = df['target'].apply(
-        lambda x: x.split('|'))
-    return df
+#         from drug 
+#         join _Drugs on drug.id = _Drugs.drugbank_id  
+#         where 
+#                 targets_polypeptides is not null and 
+#                 enzymes_polypeptides is not null and 
+#                 smiles_morgan_fingerprint is not null
+# ''')
+#     # pathway is absent
+
+#     rows = cur.fetchall()
+#     headers = ['index', 'id', 'name', 'description', 'synthesis_reference', 'indication', 'pharmacodynamics', 'mechanism_of_action', 'toxicity', 'metabolism', 'absorption', 'half_life',
+#                'protein_binding', 'route_of_elimination', 'volume_of_distribution', 'clearance', 'smiles_notation', 'smile', 'enzyme', 'target']
+#     df = pd.DataFrame(columns=headers, data=rows)
+#     df['smile'] = df['smile'].apply(lambda x:
+#                                     np.fromstring(
+#                                         x.replace(
+#                                             '\n', '')
+#                                         .replace('[', '')
+#                                         .replace(']', '')
+#                                         .replace('  ', ' '), sep=','))
+#     df['enzyme'] = df['enzyme'].apply(
+#         lambda x: x.split('|'))
+#     df['target'] = df['target'].apply(
+#         lambda x: x.split('|'))
+#     return df
 
 
 # SELECT
@@ -190,15 +190,15 @@ def get_extended_version(conn):
 # where LENGTH(masked_interaction) = LENGTH(REPLACE(masked_interaction, 'DRUG', ''))
 # or LENGTH(masked_interaction) = LENGTH(REPLACE(masked_interaction, 'DRUG', '')) + 4
 
-if __name__ == "__main__":
-    conn = create_connection(r"./event-extended.db")
-    extended_version_df = get_extended_version(conn)
+# if __name__ == "__main__":
+#     conn = create_connection(r"./event-extended.db")
+#     extended_version_df = get_extended_version(conn)
 
-    df = select_all_events_as_dataframe(conn)
-    print(df.head())
+#     df = select_all_events_as_dataframe(conn)
+#     print(df.head())
 
-    events_with_category_df = select_events_with_category(conn)
-    print(events_with_category_df.head())
+#     events_with_category_df = select_events_with_category(conn)
+#     print(events_with_category_df.head())
 
-    u = events_with_category_df['event_category'].unique()
-    print(len(u))
+#     u = events_with_category_df['event_category'].unique()
+#     print(len(u))

ddi_fw/datasets/ddi_mdl/base.py

@@ -96,10 +96,10 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
         logger.info(f'{self.dataset_name} is initialized')
 
     def load_drugs_and_events(self):
-        self.drugs_df = self.__select_all_drugs_as_dataframe__()
-        self.ddis_df = self.__select_all_events__()
+        self.drugs_df = self.__select_all_drugs_as_dataframe()
+        self.ddis_df = self.__select_all_events()
 
-    def __select_all_drugs_as_dataframe__(self):
+    def __select_all_drugs_as_dataframe(self):
         headers = ['index', 'id', 'name',
                    'target', 'enzyme', 'pathway', 'smile']
         if self._conn is None:
@@ -117,7 +117,7 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
 
         return df
 
-    def __select_all_events__(self):
+    def __select_all_events(self):
         if self._conn is None:
             raise Exception("There is no connection")
         cur = self._conn.cursor()
@@ -221,16 +221,16 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
                 lambda_fnc, args=(value,), axis=1)
             self.columns.append(key)
             print(self.ddis_df[key].head())
-
-        if self.embedding_dict is not None:
-            for embedding_column in self.embedding_columns:
-                print(f"concat {embedding_column} embeddings")
-                embeddings_after_pooling = {k: self.pooling_strategy.apply(
-                    v) for k, v in self.embedding_dict[embedding_column].items()}
-                # column_embeddings_dict = embedding_values[embedding_column]
-                self.ddis_df[embedding_column+'_embedding'] = self.ddis_df.apply(
-                    x_fnc, args=(embeddings_after_pooling,), axis=1)
-                self.columns.append(embedding_column+'_embedding')
+        if isinstance(self, TextDatasetMixin):
+            if self.embedding_dict is not None:
+                for embedding_column in self.embedding_columns:
+                    print(f"concat {embedding_column} embeddings")
+                    embeddings_after_pooling = {k: self.pooling_strategy.apply(
+                        v) for k, v in self.embedding_dict[embedding_column].items()}
+                    # column_embeddings_dict = embedding_values[embedding_column]
+                    self.ddis_df[embedding_column+'_embedding'] = self.ddis_df.apply(
+                        x_fnc, args=(embeddings_after_pooling,), axis=1)
+                    self.columns.append(embedding_column+'_embedding')
 
         dataframe = self.ddis_df.copy()
         if not isinstance(classes, (list, pd.Series, np.ndarray)):

ddi_fw/datasets/ddi_mdl_text/base.py

@@ -90,8 +90,8 @@ class DDIMDLDatasetV2(BaseDataset):
         db = HERE.joinpath('data/event.db')
         conn = create_connection(db)
         print("db prep")
-        self.drugs_df = self.__select_all_drugs_as_dataframe__(conn)
-        self.ddis_df = self.__select_all_events__(conn)
+        self.drugs_df = self.__select_all_drugs_as_dataframe(conn)
+        self.ddis_df = self.__select_all_events(conn)
         print("db bitti")
         self.index_path = kwargs.get('index_path')
 
@@ -121,7 +121,7 @@ class DDIMDLDatasetV2(BaseDataset):
         #     print(self.ddis_df[key].head())
         # print("init finished")
 
-    def __select_all_drugs_as_dataframe__(self, conn):
+    def __select_all_drugs_as_dataframe(self, conn):
         headers = ['index', 'id', 'name',
                    'target', 'enzyme', 'pathway', 'smile']
         cur = conn.cursor()
@@ -137,7 +137,7 @@ class DDIMDLDatasetV2(BaseDataset):
 
         return df
 
-    def __select_all_events__(self, conn):
+    def __select_all_events(self, conn):
         """
         Query all rows in the event table
         :param conn: the Connection object

ddi_fw/datasets/mdf_sa_ddi/base.py

@@ -9,7 +9,6 @@ from ddi_fw.utils import ZipHelper
 from .. import BaseDataset
 from ddi_fw.langchain.embeddings import PoolingStrategy
 from ..db_utils import create_connection
-# from ..db_utils import create_connection, select_all_drugs_as_dataframe, select_events_with_category
 
 HERE = pathlib.Path(__file__).resolve().parent
 list_of_embedding_columns = ['all_text', 'description',

ddi_fw/pipeline/multi_pipeline.py

@@ -120,6 +120,8 @@ class MultiPipeline():
         columns = config.get("columns")
         ner_data_file = config.get("ner_data_file")
         ner_threshold = config.get("ner_threshold")
+        ner_min_threshold_dict = config.get("ner_min_threshold_dict")
+        ner_max_threshold_dict = config.get("ner_max_threshold_dict")
         column_embedding_configs = config.get("column_embedding_configs")
         vector_db_persist_directory = config.get("vector_db_persist_directory")
         vector_db_collection_name = config.get("vector_db_collection_name")
@@ -170,10 +172,14 @@ class MultiPipeline():
                 experiment_tags=experiment_tags,
                 tracking_uri=tracking_uri,
                 dataset_type=dataset_type,
+                dataset_splitter_type=dataset_splitter_type,
                 umls_code_types = None,
                 text_types = None,
-                columns=['tui', 'cui', 'entities'],
+                min_threshold_dict=ner_min_threshold_dict,
+                max_threshold_dict=ner_max_threshold_dict,
+                columns=columns,
                 ner_data_file=ner_data_file,
+                default_model=default_model,
                 multi_modal= multi_modal
             )
 

ddi_fw/pipeline/multi_pipeline_v2.py

@@ -0,0 +1,231 @@
+import json
+from typing import Optional
+from ddi_fw.pipeline.pipeline import Pipeline
+from ddi_fw.pipeline.ner_pipeline import NerParameterSearch
+import importlib
+
+
+def load_config(file_path):
+    with open(file_path, 'r') as file:
+        config = json.load(file)
+    return config
+
+
+def get_import(full_path_of_import):
+    """Dynamically imports an object from a module given its full path.
+
+    Args:
+        full_path_of_import (str): The full path of the import (e.g., 'module.submodule.ClassName').
+
+    Returns:
+        object: The imported object.
+
+    Raises:
+        ImportError: If the module cannot be imported.
+        AttributeError: If the attribute does not exist in the module.
+    """
+    if not full_path_of_import:
+        raise ValueError("The import path cannot be empty.")
+
+    parts = full_path_of_import.split('.')
+    import_name = parts[-1]
+    module_name = ".".join(parts[:-1]) if len(parts) > 1 else ""
+
+    try:
+        module = importlib.import_module(module_name)
+        return getattr(module, import_name)
+    except ModuleNotFoundError as e:
+        raise ImportError(f"Module '{module_name}' could not be found.") from e
+    except AttributeError as e:
+        raise AttributeError(
+            f"'{module_name}' has no attribute '{import_name}'") from e
+
+
+class MultiPipeline():
+    # def __init__(self, experiments_config_file, experiments_config):
+    #     if experiments_config_file is None and experiments_config is None:
+    #         raise ValueError("Either experiments_config_file or experiments_config must be provided.")
+    #     if experiments_config_file is not None and experiments_config is not None:  
+    #         raise ValueError("Only one of experiments_config_file or experiments_config should be provided.")
+    #     if experiments_config_file is not None:
+    #         self.experiments_config = load_config(experiments_config_file)
+    #     else:
+    #         self.experiments_config = experiments_config
+    #     self.items = []
+    #     self.pipeline_resuts = dict()
+    
+    def __init__(self, experiments_config_file: Optional[str] = None, experiments_config: Optional[dict] = None):
+        """
+        Initialize the MultiPipeline.
+
+        Args:
+            experiments_config_file (str, optional): Path to the experiments configuration file.
+            experiments_config (dict, optional): Dictionary containing the experiments configuration.
+
+        Raises:
+            ValueError: If neither or both of the parameters are provided.
+        """
+        self.experiments_config = self._validate_and_load_config(experiments_config_file, experiments_config)
+        self.items = []
+        # self.pipeline_results = {}
+        self.pipeline_resuts = dict()
+
+    def _validate_and_load_config(self, experiments_config_file: Optional[str], experiments_config: Optional[dict]) -> dict:
+        """
+        Validate and load the experiments configuration.
+
+        Args:
+            experiments_config_file (str, optional): Path to the experiments configuration file.
+            experiments_config (dict, optional): Dictionary containing the experiments configuration.
+
+        Returns:
+            dict: The loaded experiments configuration.
+
+        Raises:
+            ValueError: If neither or both of the parameters are provided.
+        """
+        if experiments_config_file is None and experiments_config is None:
+            raise ValueError("Either 'experiments_config_file' or 'experiments_config' must be provided.")
+        if experiments_config_file is not None and experiments_config is not None:
+            raise ValueError("Only one of 'experiments_config_file' or 'experiments_config' should be provided.")
+
+        if experiments_config_file is not None:
+            try:
+               config = load_config(experiments_config_file)
+            except FileNotFoundError:
+                raise FileNotFoundError(f"Configuration file '{experiments_config_file}' not found.")
+        else:
+            config = experiments_config
+        if config is None:
+            raise ValueError("Configuration cannot be None.")
+        if not isinstance(config, dict):
+            raise ValueError("Configuration must be a dictionary.")
+        # if "experiments" not in config: 
+        #     raise ValueError("Configuration must contain 'experiments' key.")
+        return config
+
+    def __create_pipeline(self, config):
+        type = config.get("type")
+        library = config.get("library")
+        experiment_name = config.get("experiment_name")
+        experiment_description = config.get("experiment_description")
+        experiment_tags = config.get("experiment_tags")
+        
+        # Tracking configuration
+        tracking_config = config.get("tracking_config", {})
+        tracking_library = tracking_config.get("library")
+        use_tracking = tracking_config.get("use_tracking", False)
+        tracking_params = tracking_config.get("params", {}).get(tracking_library, {})
+        
+        # tracking_uri = config.get("tracking_uri")
+        # artifact_location = config.get("artifact_location")
+      
+         # Dataset configuration
+        dataset_config = config.get("dataset", {})
+        dataset_type = get_import(dataset_config.get("dataset_type"))
+        dataset_splitter_type = get_import(dataset_config.get("dataset_splitter_type"))
+        columns = dataset_config.get("columns", [])
+        additional_config = dataset_config.get("additional_config", {})
+        
+         # Vector database configuration
+        vector_database = config.get("vector_databases", {})
+        vector_db_persist_directory = None
+        vector_db_collection_name = None
+        embedding_pooling_strategy = None
+        if vector_database:
+            vector_db_persist_directory = vector_database.get("vector_db_persist_directory")
+            vector_db_collection_name = vector_database.get("vector_db_collection_name")
+            embedding_pooling_strategy = get_import(vector_database.get("embedding_pooling_strategy"))
+            column_embedding_configs = vector_database.get("column_embedding_configs")
+            
+        # Combination strategy
+        combination_strategy_config = config.get("combination_strategy", {})
+        combination_type = get_import(combination_strategy_config.get("type")) if combination_strategy_config else None
+        kwargs_combination_params = combination_strategy_config.get("params", {})
+        combinations = combination_type(**kwargs_combination_params).generate() if combination_type else []
+
+        # Default model configuration
+        default_model_config = config.get("default_model", {})
+        default_model_type = get_import(default_model_config.get("model_type"))
+        default_model_params = default_model_config.get("params", {})
+        
+        multi_modal = config.get("multi_modal")
+        
+
+
+        #ner move it to related dataset
+        
+        # ner_data_file = config.get("ner_data_file")
+        # ner_threshold = config.get("ner_threshold")
+      
+
+        combination_type = None
+        kwargs_combination_params=None
+        if config.get("combination_strategy"):
+            combination_type = get_import(config.get("combination_strategy").get("type"))
+            kwargs_combination_params = config.get("combination_strategy").get("params")
+        combinations = []
+        if combination_type is not None:
+            combinations = combination_type(**kwargs_combination_params).generate()
+      
+
+        pipeline = None
+        if type == "general":
+            pipeline = Pipeline(
+                library=library,
+                use_mlflow=use_mlflow,
+                experiment_name=experiment_name,
+                experiment_description=experiment_description,
+                experiment_tags=experiment_tags,
+                artifact_location=artifact_location,
+                tracking_uri=tracking_uri,
+                dataset_type=dataset_type,
+                dataset_splitter_type=dataset_splitter_type,
+                columns=columns,
+                column_embedding_configs=column_embedding_configs,
+                vector_db_persist_directory=vector_db_persist_directory,
+                vector_db_collection_name=vector_db_collection_name,
+                embedding_pooling_strategy_type=embedding_pooling_strategy,
+                ner_data_file=ner_data_file,
+                ner_threshold=ner_threshold,
+                combinations=combinations,
+                default_model=default_model,
+                multi_modal= multi_modal)
+        elif type== "ner_search":
+            pipeline = NerParameterSearch(
+                library=library,
+                experiment_name=experiment_name,
+                experiment_description=experiment_description,
+                experiment_tags=experiment_tags,
+                tracking_uri=tracking_uri,
+                dataset_type=dataset_type,
+                umls_code_types = None,
+                text_types = None,
+                columns=['tui', 'cui', 'entities'],
+                ner_data_file=ner_data_file,
+                multi_modal= multi_modal
+            )
+
+
+        return {
+            "name": experiment_name,
+            "library": library,
+            "pipeline": pipeline}
+
+    def build(self):
+        for config in self.experiments_config['experiments']:
+            item = self.__create_pipeline(config)
+            self.items.append(item)
+        return self
+
+    def run(self):
+        for item in self.items:
+            print(f"{item['name']} is running")
+            pipeline = item['pipeline']
+            pipeline.build()
+            result = pipeline.run()
+            self.pipeline_resuts[item['name']] = result
+        return self
+
+    def results(self):
+        return self.pipeline_resuts

ddi_fw/pipeline/ner_pipeline.py

@@ -1,139 +1,134 @@
 from collections import defaultdict
+from typing import Any, Dict, List, Optional, Type
+from itertools import product
 import numpy as np
+import mlflow
+from pydantic import BaseModel, Field, model_validator, root_validator, validator
 from ddi_fw.datasets.core import BaseDataset
+from ddi_fw.datasets.dataset_splitter import DatasetSplitter
 from ddi_fw.vectorization.idf_helper import IDF
-from typing import Any, Dict, List, Optional
-from itertools import product
-
+from ddi_fw.ner.ner import CTakesNER
 from ddi_fw.ml.ml_helper import MultiModalRunner
 from ddi_fw.utils.enums import DrugBankTextDataTypes, UMLSCodeTypes
-import mlflow
-from ddi_fw.ner.ner import CTakesNER
 
 
-def stack(df_column):
-    return np.stack(df_column.values)
-
-
-class NerParameterSearch:
-    def __init__(self,
-                 library,
-                 multi_modal,
-                 experiment_name,
-                 experiment_description,
-                 experiment_tags,
-                 tracking_uri,
-                 dataset_type: BaseDataset,
-                 ner_data_file,
-                 columns: list,
-                 umls_code_types: List[UMLSCodeTypes]|None,
-                 text_types:List[DrugBankTextDataTypes]|None,
-                 min_threshold_dict: Dict[str, float] = defaultdict(float),
-                 max_threshold_dict: Dict[str, float] = defaultdict(float),
-                 increase_step=0.5):
-        self.library = library
-        self.multi_modal = multi_modal
-        self.experiment_name = experiment_name
-        self.experiment_description = experiment_description
-        self.experiment_tags = experiment_tags
-        self.tracking_uri = tracking_uri
-
-        self.dataset_type = dataset_type
-        self.ner_data_file = ner_data_file
-        self.columns = columns
-        self.umls_code_types = umls_code_types
-        self.text_types = text_types
-
-        self.min_threshold_dict = min_threshold_dict
-        self.max_threshold_dict = max_threshold_dict
-        self.increase_step = increase_step
+class NerParameterSearch(BaseModel):
+    library: str
+    default_model:  Optional[Any] = None
+    multi_modal:  Optional[Any] = None
+    experiment_name: str
+    experiment_description: Optional[str] = None
+    experiment_tags: Optional[Dict[str, Any]] = None
+    tracking_uri: str
+    dataset_type: Type[BaseDataset]
+    dataset_splitter_type: Type[DatasetSplitter] = DatasetSplitter
+    ner_data_file: Optional[str] = None
+    columns: List[str] = Field(default_factory=list)
+    umls_code_types: Optional[List[UMLSCodeTypes]] = None
+    text_types: Optional[List[DrugBankTextDataTypes]] = None
+    min_threshold_dict: Dict[str, float] = Field(default_factory=lambda: defaultdict(float))
+    max_threshold_dict: Dict[str, float] = Field(default_factory=lambda: defaultdict(float))
+    increase_step: float = 0.5
+
+    # Internal fields (not part of the input)
+    datasets: Dict[str, Any] = Field(default_factory=dict, exclude=True)
+    items: List[Any] = Field(default_factory=list, exclude=True)
+    ner_df: Optional[Any] = Field(default=None, exclude=True)
+    train_idx_arr: Optional[List[np.ndarray]] = Field(default=None, exclude=True)
+    val_idx_arr: Optional[List[np.ndarray]] = Field(default=None, exclude=True)
+    y_test_label: Optional[np.ndarray] = Field(default=None, exclude=True)
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    # @root_validator(pre=True)
+    @model_validator(mode="before")
+    def validate_columns_and_thresholds(cls, values):
+        """Validate and initialize columns and thresholds."""
+        umls_code_types = values.get("umls_code_types")
+        text_types = values.get("text_types")
+        columns = values.get("columns", [])
+
+        if umls_code_types and text_types:
+            _umls_codes = [t.value[0] for t in umls_code_types]
+            _text_types = [t.value[0] for t in text_types]
+            _columns = [f"{item[0]}_{item[1]}" for item in product(_umls_codes, _text_types)]
+            columns.extend(_columns)
+
+        values["columns"] = columns
+        return values
 
     def build(self):
+        """Build the datasets and items for the parameter search."""
         if not isinstance(self.dataset_type, type):
             raise TypeError("self.dataset_type must be a class, not an instance")
-        self.datasets = {}
-        self.items = []
-        # columns = ['tui', 'cui', 'entities']
-        if self.umls_code_types is not None and self.text_types is not None:
-            # add checking statements
-            _umls_codes = [t.value[0] for t in self.umls_code_types]
-            _text_types = [t.value[0] for t in self.text_types]
-            _columns = [f'{item[0]}_{item[1]}' for item in product(
-                _umls_codes, _text_types)]
-            self.columns.extend(_columns)
-        print(f'Columns: {self.columns}')
-        self.ner_df = CTakesNER(df = None).load(
-            filename=self.ner_data_file) if self.ner_data_file else None
 
+        # Load NER data
+        if self.ner_data_file:
+            self.ner_df = CTakesNER(df=None).load(filename=self.ner_data_file)
+
+        # Initialize thresholds if not provided
         if not self.min_threshold_dict or not self.max_threshold_dict:
-            idf2 = IDF(self.ner_df, self.columns)
-            idf2.calculate()
-            # df = pd.DataFrame.from_dict(idf2.idf_scores)
-            df = idf2.to_dataframe()
-            import math
-            self.min_threshold_dict = {key: math.floor(
-                df.describe()[key]['min']) for key in df.describe().keys()}
-            self.max_threshold_dict = {key: math.ceil(
-                df.describe()[key]['max']) for key in df.describe().keys()}
-
-        train_idx_arr, val_idx_arr = None, None
+            idf = IDF(self.ner_df, self.columns)
+            idf.calculate()
+            df = idf.to_dataframe()
+            self.min_threshold_dict = {key: np.floor(df.describe()[key]["min"]) for key in df.describe().keys()}
+            self.max_threshold_dict = {key: np.ceil(df.describe()[key]["max"]) for key in df.describe().keys()}
+
+        # Generate datasets and items
         for column in self.columns:
             min_threshold = self.min_threshold_dict[column]
             max_threshold = self.max_threshold_dict[column]
-            kwargs = {}
-            kwargs['threshold_method'] = 'idf'
-            kwargs['tui_threshold'] = 0
-            kwargs['cui_threshold'] = 0
-            kwargs['entities_threshold'] = 0
+            kwargs = {
+                "threshold_method": "idf",
+                "tui_threshold": 0,
+                "cui_threshold": 0,
+                "entities_threshold": 0,
+            }
 
             for threshold in np.arange(min_threshold, max_threshold, self.increase_step):
-                print(threshold)
-                if column.startswith('tui'):
-                    kwargs['tui_threshold'] = threshold
-                if column.startswith('cui'):
-                    kwargs['cui_threshold'] = threshold
-                if column.startswith('entities'):
-                    kwargs['entities_threshold'] = threshold
+                if column.startswith("tui"):
+                    kwargs["tui_threshold"] = threshold
+                if column.startswith("cui"):
+                    kwargs["cui_threshold"] = threshold
+                if column.startswith("entities"):
+                    kwargs["entities_threshold"] = threshold
+
                 dataset = self.dataset_type(
-                    # chemical_property_columns=[],
-                    # embedding_columns=[],
-                    # ner_columns=[column],
                     columns=[column],
                     ner_df=self.ner_df,
-                    embedding_size=None,
-                    embedding_dict=None,
-                    embeddings_pooling_strategy=None,
-                    **kwargs)
-
-                # train_idx_arr, val_idx_arr  bir kez hesaplanması yeterli aslında
+                    dataset_splitter_type=self.dataset_splitter_type,
+                    **kwargs,
+                )
                 dataset.load()
                 group_items = dataset.produce_inputs()
+
                 for item in group_items:
-                    # item[0] = f'threshold_{threshold}_{item[0]}'
-                    item[0] = f'threshold_{item[0]}_{threshold}'
-                    self.datasets[item[0]] = dataset.ddis_df
+                    item[0] = f"threshold_{item[0]}_{threshold}"
+                    self.datasets[item[0]] = dataset
 
                 self.items.extend(group_items)
+
         self.y_test_label = self.items[0][4]
         self.train_idx_arr = dataset.train_idx_arr
         self.val_idx_arr = dataset.val_idx_arr
 
-    def run(self, model_func, batch_size=128, epochs=100):
+    def run(self):
+        """Run the parameter search."""
         mlflow.set_tracking_uri(self.tracking_uri)
 
-        if mlflow.get_experiment_by_name(self.experiment_name) == None:
+        if mlflow.get_experiment_by_name(self.experiment_name) is None:
             mlflow.create_experiment(self.experiment_name)
+        if self.experiment_tags:
             mlflow.set_experiment_tags(self.experiment_tags)
         mlflow.set_experiment(self.experiment_name)
 
-        y_test_label = self.items[0][4]
         multi_modal_runner = MultiModalRunner(
-            library=self.library, multi_modal=self.multi_modal)
-        # multi_modal_runner = MultiModalRunner(
-        #     library=self.library, model_func=model_func, batch_size=batch_size,  epochs=epochs)
-        multi_modal_runner.set_data(
-            self.items, self.train_idx_arr, self.val_idx_arr, y_test_label)
+            library=self.library,
+            multi_modal=self.multi_modal,
+            default_model=self.default_model,
+            use_mlflow=True,
+        )
+        multi_modal_runner.set_data(self.items, self.train_idx_arr, self.val_idx_arr, self.y_test_label)
         result = multi_modal_runner.predict()
-        
-         
-        return result
+        return result

{ddi_fw-0.0.214.dist-info → ddi_fw-0.0.215.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.214
+Version: 0.0.215
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.214.dist-info → ddi_fw-0.0.215.dist-info}/RECORD

@@ -1,9 +1,9 @@
 ddi_fw/datasets/__init__.py,sha256=_I3iDHARwzmg7_EL5XKtB_TgG1yAkLSOVTujLL9Wz9Q,280
 ddi_fw/datasets/core.py,sha256=ciIOma5--rArgpCjtALpgBBZaKk2oI6rJeGCs05h4iU,16685
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
-ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
+ddi_fw/datasets/db_utils.py,sha256=xRj28U_uXTRPHcz3yIICczFUHXUPiAOZtAj5BM6kH44,6465
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
-ddi_fw/datasets/ddi_mdl/base.py,sha256=afk5ToGSnHQ-N7dnhiJYIiFTBM08vwEJgb6mwiacY1w,10409
+ddi_fw/datasets/ddi_mdl/base.py,sha256=Vvyzxd2BnFK9Bn2mn-3aS5ZczlPElQ0-TKMAqgkyJiI,10483
 ddi_fw/datasets/ddi_mdl/debug.log,sha256=eWz05j8RFqZuHFDTCF7Rck5w4rvtTanFN21iZsgxO7Y,115
 ddi_fw/datasets/ddi_mdl/readme.md,sha256=WC6lpmsEKvIISnZqENY7TWtzCQr98HPpE3oRsBl8pIw,625
 ddi_fw/datasets/ddi_mdl/data/event.db,sha256=cmlSsf9MYjRzqR-mw3cUDnTnfT6FkpOG2yCl2mMwwew,30580736
@@ -31,7 +31,7 @@ ddi_fw/datasets/ddi_mdl/indexes_old/validation_fold_1.txt,sha256=KuRIXhQ4uUrYK0d
 ddi_fw/datasets/ddi_mdl/indexes_old/validation_fold_2.txt,sha256=bkZuigOiSJleKx4hJ7L0aF0ltMgaI7wfrODWiJ1ppkY,69374
 ddi_fw/datasets/ddi_mdl/indexes_old/validation_fold_3.txt,sha256=HNBs4vHhyPepnZyeeqRWfrr25heB9CKIo9-5Xh02Pyk,69393
 ddi_fw/datasets/ddi_mdl/indexes_old/validation_fold_4.txt,sha256=-pphbOlEyKD2xedpzkhWvPExDm43FKZ2RSCiie-ZHHo,69356
-ddi_fw/datasets/ddi_mdl_text/base.py,sha256=JCQvqJPAaOks0VTNRf_birph70CrHoIow5Z3rfX_cdw,6721
+ddi_fw/datasets/ddi_mdl_text/base.py,sha256=UHhrb8ts3IOpl7MdxopBjiKK5xGfzMysc-OdGyDKpCM,6713
 ddi_fw/datasets/ddi_mdl_text/data/event.db,sha256=cmlSsf9MYjRzqR-mw3cUDnTnfT6FkpOG2yCl2mMwwew,30580736
 ddi_fw/datasets/ddi_mdl_text/indexes/test_indexes.txt,sha256=XVlDqYATckrQwNSXqMSKVBqyoN_Hg8SK6CL-XMdLADY,102176
 ddi_fw/datasets/ddi_mdl_text/indexes/train_fold_0.txt,sha256=YDQwMNEpEjkOJiUtX_BoymdezE34W3mj1Q0oAYG3mIs,325015
@@ -46,7 +46,7 @@ ddi_fw/datasets/ddi_mdl_text/indexes/validation_fold_2.txt,sha256=fFJbN0DbKH4mve
 ddi_fw/datasets/ddi_mdl_text/indexes/validation_fold_3.txt,sha256=NhiLF_5INQCpjOlE-RIxDKy7rYwksLdx60L6HCmDKoY,81247
 ddi_fw/datasets/ddi_mdl_text/indexes/validation_fold_4.txt,sha256=bPvMCJVy7jtcaYbR-5bmdB6s7gT8NSfK2wDC7iJ0O10,81308
 ddi_fw/datasets/mdf_sa_ddi/__init__.py,sha256=UEFBM92y2aJjlMJw4Jx405tOAwJ88r_nHAVgAszSjuo,68
-ddi_fw/datasets/mdf_sa_ddi/base.py,sha256=kYNmtg-s0V7mP-wjLMaAstNCG3vckMPQSE651RA_LAE,6502
+ddi_fw/datasets/mdf_sa_ddi/base.py,sha256=ILdvu7pBMazt-FxRWzIaqO2PmbkyooEOT3U9vSoV3PY,6398
 ddi_fw/datasets/mdf_sa_ddi/df_extraction_cleanxiaoyu50.csv,sha256=EOOLF_0vVVzShoofcGYlOzpztlM1m9jJdftepHicix4,25787699
 ddi_fw/datasets/mdf_sa_ddi/drug_information_del_noDDIxiaoyu50.csv,sha256=lpuMz5KxPsG6MKNuIIUmT5cZquWHQiIao8tXlmOHzq8,381321
 ddi_fw/datasets/mdf_sa_ddi/mdf-sa-ddi.zip,sha256=DfN8mczGvWba2y45cPqtWtXjUDXy49VOtRfpcb0tn8c,4382827
@@ -83,8 +83,9 @@ ddi_fw/ner/mmlrestclient.py,sha256=NZta7m2Qm6I_qtVguMZhqtAUjVBmmXn0-TMnsNp0jpg,6
 ddi_fw/ner/ner.py,sha256=FHyyX53Xwpdw8Hec261dyN88yD7Z9LmJua2mIrQLguI,17967
 ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,212
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
-ddi_fw/pipeline/multi_pipeline.py,sha256=SZFJ9QSPD_3mcG9NHZOtMqKyNvyWrodsdsLryMyDdUw,8686
-ddi_fw/pipeline/ner_pipeline.py,sha256=Bp6BA6nozfWFaMHH6jKlzesnCGO6qiMkzdGy_ed6nh0,5947
+ddi_fw/pipeline/multi_pipeline.py,sha256=AbErwu05-3YIPnCcXRsj-jxPJG8HG2H7cMZlGjzaYa8,9037
+ddi_fw/pipeline/multi_pipeline_v2.py,sha256=7IGtaGFhgJqW29a6nDheUrVtn_7_xvWFdD6GC--sehM,10003
+ddi_fw/pipeline/ner_pipeline.py,sha256=yp-Met2794EKcgr8_3gqt03l4v2efOdaZuAcIXTubvQ,5780
 ddi_fw/pipeline/pipeline.py,sha256=YhUBVLC29ZD2tmVd0e8X1FVBLhSKECZL2OP57oEW6HE,9171
 ddi_fw/utils/__init__.py,sha256=WNxkQXk-694roG50D355TGLXstfdWVb_tUyr-PM-8rg,537
 ddi_fw/utils/categorical_data_encoding_checker.py,sha256=T1X70Rh4atucAuqyUZmz-iFULllY9dY0NRyV9-jTjJ0,3438
@@ -99,7 +100,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=EBf-XAiwQwr68az91erEYNegfeqssBR29kVgrliIyac,4765
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.214.dist-info/METADATA,sha256=IEDJdH40Nw4B0aJXnUwuxeNRdXMX5rw1RBsX93Zbj1A,2631
-ddi_fw-0.0.214.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddi_fw-0.0.214.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.214.dist-info/RECORD,,
+ddi_fw-0.0.215.dist-info/METADATA,sha256=A_IcjN3qyvwbfFH-51g6NGR-8DUmwWEHWPu4sLfGuzc,2631
+ddi_fw-0.0.215.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddi_fw-0.0.215.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.215.dist-info/RECORD,,