ddi-fw 0.0.84__py3-none-any.whl → 0.0.87__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -24,11 +24,11 @@ def stack(df_column):
 class BaseDataset(ABC):
     def __init__(self,
                  embedding_size,
-                 embedding_dict, 
-                 embeddings_pooling_strategy: PoolingStrategy, 
-                 ner_df, 
-                 chemical_property_columns, 
-                 embedding_columns, 
+                 embedding_dict,
+                 embeddings_pooling_strategy: PoolingStrategy,
+                 ner_df,
+                 chemical_property_columns,
+                 embedding_columns,
                  ner_columns,
                  **kwargs):
         self.embedding_size = embedding_size
@@ -409,6 +409,7 @@ class BaseDataset(ABC):
                 x_fnc, args=(embeddings_after_pooling,), axis=1)
 
         self.dataframe = self.ddis_df.copy()
+        self.dataframe['class_as_txt'] = labels
         self.dataframe['class'] = list(classes)
         print(self.dataframe.shape)
 
@@ -436,3 +437,11 @@ class BaseDataset(ABC):
         self.val_idx_arr = val_idx_arr
 
         return self.X_train, self.X_test, self.y_train, self.y_test, self.X_train.index, self.X_test.index, train_idx_arr, val_idx_arr
+
+        def export_as_csv(self, output_file_path, not_change: list):
+            copy = self.dataframe.copy()
+            for col in copy.columns:
+                if col not in not_change:
+                    copy[col] = [
+                        '[' + ','.join(f"{value:.3f}" for value in row) + ']' for row in copy[col]]
+            copy.to_csv(output_file_path, index=False)

ddi_fw/datasets/ddi_mdl/base.py

@@ -35,7 +35,7 @@ class DDIMDLDataset(BaseDataset):
                  ner_columns=[],
                  **kwargs):
         columns = kwargs['columns']
-        if columns is not None:
+        if columns:
             chemical_property_columns = []
             embedding_columns=[]
             ner_columns=[]

ddi_fw/ml/ml_helper.py

@@ -74,7 +74,8 @@ class MultiModalRunner:
                     self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
                 logs, metrics, prediction = single_modal.predict()
                 # self.result.add_log(item[0], logs)
-                # self.result.add_metric(item[0], metrics)
+                #Check
+                self.result.add_metric(item[0], metrics)
                 single_results[item[0]] = prediction
                 # sum = sum + prediction
 

ddi_fw/pipeline/__init__.py

@@ -1,3 +1,4 @@
 from .pipeline import Pipeline
 from .multi_pipeline import MultiPipeline
-from .multi_modal_combination_strategy import CombinationStrategy,CustomCombinationStrategy
+from .multi_modal_combination_strategy import CombinationStrategy,CustomCombinationStrategy
+from .ner_pipeline import NerParameterSearch

ddi_fw/pipeline/multi_modal_combination_strategy.py

@@ -9,8 +9,8 @@ class CombinationStrategy():
 class CustomCombinationStrategy(CombinationStrategy):
     def __init__(self, **kwargs_combination_params):
         # kwargs fonksiyona da alınabilir
-        self.group1 = kwargs_combination_params.get("group_1", None)
-        self.group2 = kwargs_combination_params.get("group_2", None)
+        self.group_1 = kwargs_combination_params.get("group_1", None)
+        self.group_2 = kwargs_combination_params.get("group_2", None)
 
     def generate(self):
         # Handle edge cases

ddi_fw/pipeline/multi_pipeline.py

@@ -1,5 +1,6 @@
 import json
 from ddi_fw.pipeline import Pipeline
+from ddi_fw.pipeline import NerParameterSearch
 import importlib
 
 
@@ -46,41 +47,41 @@ class MultiPipeline():
         self.pipeline_resuts = dict()
 
     def __create_pipeline(self, config):
-        library = config["library"]
-        batch_size = config["batch_size"]
-        epochs = config["epochs"]
-
-        # dataset_module = config["dataset_module"]
-        # dataset_name = config["dataset_name"]
-
-        experiment_name = config["experiment_name"]
-        experiment_description = config["experiment_description"]
-        experiment_tags = config["experiment_tags"]
-        tracking_uri = config["tracking_uri"]
-        artifact_location = config["artifact_location"]
-        columns = config["columns"]
-        ner_data_file = config["ner_data_file"]
-        ner_threshold = config["ner_threshold"]
-        vector_db_persist_directory = config["vector_db_persist_directory"]
-        vector_db_collection_name = config["vector_db_collection_name"]
+        type = config.get("type")
+        library = config.get("library")
+        batch_size = config.get("batch_size")
+        epochs = config.get("epochs")
+
+        # dataset_module = config.get("dataset_module")
+        # dataset_name = config.get("dataset_name")
+
+        experiment_name = config.get("experiment_name")
+        experiment_description = config.get("experiment_description")
+        experiment_tags = config.get("experiment_tags")
+        tracking_uri = config.get("tracking_uri")
+        artifact_location = config.get("artifact_location")
+        columns = config.get("columns")
+        ner_data_file = config.get("ner_data_file")
+        ner_threshold = config.get("ner_threshold")
+        vector_db_persist_directory = config.get("vector_db_persist_directory")
+        vector_db_collection_name = config.get("vector_db_collection_name")
         embedding_pooling_strategy = get_import(
-            config["embedding_pooling_strategy_type"])
+            config.get("embedding_pooling_strategy_type"))
         # Dynamically import the model and dataset classes
-        model_type = get_import(config["model_type"])
-        dataset_type = get_import(config["dataset_type"])
-        combination_type = get_import(config["combination_strategy"]["type"])
-        kwargs_combination_params = config["combination_strategy"]["params"]
-
+        model_type = get_import(config.get("model_type"))
+        dataset_type = get_import(config.get("dataset_type"))
+        combination_type = get_import(config.get("combination_strategy").get("type"))
+        kwargs_combination_params = config.get("combination_strategy").get("params")
+        combinations = []
+        if combination_type is not None:
+            combinations = combination_type(**kwargs_combination_params).generate()
         # # Instantiate the classes
         # model_instance = model_class()
         # dataset_instance = dataset_class()
-        return {
-            "name": experiment_name,
-            "library": library,
-            "batch_size": batch_size,
-            "epochs": epochs,
-            "model_type": model_type,
-            "pipeline": Pipeline(
+
+        pipeline = None
+        if type == "general":
+            pipeline = Pipeline(
                 library=library,
                 experiment_name=experiment_name,
                 experiment_description=experiment_description,
@@ -94,7 +95,28 @@ class MultiPipeline():
                 embedding_pooling_strategy_type=embedding_pooling_strategy,
                 ner_data_file=ner_data_file,
                 ner_threshold=ner_threshold,
-                combinations=combination_type(**kwargs_combination_params).generate())}
+                combinations=combinations)
+        elif type== "ner_search":
+            pipeline = NerParameterSearch(
+                experiment_name=experiment_name,
+                experiment_description=experiment_description,
+                experiment_tags=experiment_tags,
+                tracking_uri=tracking_uri,
+                dataset_type=dataset_type,
+                umls_code_types = None,
+                text_types = None,
+                columns=['tui', 'cui', 'entities'],
+                ner_data_file=ner_data_file,
+            )
+
+
+        return {
+            "name": experiment_name,
+            "library": library,
+            "batch_size": batch_size,
+            "epochs": epochs,
+            "model_type": model_type,
+            "pipeline": pipeline}
 
     def build(self):
         for config in self.experiments_config['experiments']:

ddi_fw/pipeline/ner_pipeline.py

@@ -0,0 +1,128 @@
+from collections import defaultdict
+import numpy as np
+from ddi_fw.datasets.core import BaseDataset
+from ddi_fw.datasets.idf_helper import IDF
+from typing import Dict, List
+from itertools import product
+
+from ddi_fw.ml.ml_helper import MultiModalRunner
+from ddi_fw.utils.enums import DrugBankTextDataTypes, UMLSCodeTypes
+import mlflow
+from ddi_fw.ner.ner import CTakesNER
+
+
+def stack(df_column):
+    return np.stack(df_column.values)
+
+
+class NerParameterSearch:
+    def __init__(self,
+                 experiment_name,
+                 experiment_description,
+                 experiment_tags,
+                 tracking_uri,
+                 dataset_type: BaseDataset,
+                 ner_data_file,
+                 columns: list,
+                 umls_code_types: List[UMLSCodeTypes],
+                 text_types=List[DrugBankTextDataTypes],
+                 min_threshold_dict: Dict[str, float] = defaultdict(float),
+                 max_threshold_dict: Dict[str, float] = defaultdict(float),
+                 increase_step=0.5):
+        self.experiment_name = experiment_name
+        self.experiment_description = experiment_description
+        self.experiment_tags = experiment_tags
+        self.tracking_uri = tracking_uri
+
+        self.dataset_type = dataset_type
+        self.ner_data_file = ner_data_file
+        self.columns = columns
+        self.umls_code_types = umls_code_types
+        self.text_types = text_types
+
+        self.min_threshold_dict = min_threshold_dict
+        self.max_threshold_dict = max_threshold_dict
+        self.increase_step = increase_step
+
+    def build(self):
+        self.datasets = {}
+        self.items = []
+        # columns = ['tui', 'cui', 'entities']
+        if self.umls_code_types is not None and self.text_types is not None:
+            # add checking statements
+            _umls_codes = [t.value[0] for t in self.umls_code_types]
+            _text_types = [t.value[0] for t in self.text_types]
+            _columns = [f'{item[0]}_{item[1]}' for item in product(
+                _umls_codes, _text_types)]
+            self.columns.extend(_columns)
+        print(f'Columns: {self.columns}')
+        self.ner_df = CTakesNER().load(
+            filename=self.ner_data_file) if self.ner_data_file else None
+
+        if not self.min_threshold_dict or not self.max_threshold_dict:
+            idf2 = IDF(self.ner_df, self.columns)
+            idf2.calculate()
+            # df = pd.DataFrame.from_dict(idf2.idf_scores)
+            df = idf2.to_dataframe()
+            import math
+            self.min_threshold_dict = {key: math.floor(
+                df.describe()[key]['min']) for key in df.describe().keys()}
+            self.max_threshold_dict = {key: math.ceil(
+                df.describe()[key]['max']) for key in df.describe().keys()}
+
+        for column in self.columns:
+            min_threshold = self.min_threshold_dict[column]
+            max_threshold = self.max_threshold_dict[column]
+            kwargs = {}
+            kwargs['threshold_method'] = 'idf'
+            kwargs['tui_threshold'] = 0
+            kwargs['cui_threshold'] = 0
+            kwargs['entities_threshold'] = 0
+
+            for threshold in np.arange(min_threshold, max_threshold, self.increase_step):
+                print(threshold)
+                if column.startswith('tui'):
+                    kwargs['tui_threshold'] = threshold
+                if column.startswith('cui'):
+                    kwargs['cui_threshold'] = threshold
+                if column.startswith('entities'):
+                    kwargs['entities_threshold'] = threshold
+                dataset = self.dataset_type(
+                    # chemical_property_columns=[],
+                    # embedding_columns=[],
+                    # ner_columns=[column],
+                    columns=[column],
+                    ner_df=self.ner_df,
+                    embedding_size=None,
+                    embedding_dict=None,
+                    embeddings_pooling_strategy=None,
+                    **kwargs)
+
+                # train_idx_arr, val_idx_arr  bir kez hesaplanması yeterli aslında
+                X_train, X_test, y_train, y_test, X_train.index, X_test.index, train_idx_arr, val_idx_arr = dataset.load()
+                group_items = dataset.produce_inputs()
+                for item in group_items:
+                    # item[0] = f'threshold_{threshold}_{item[0]}'
+                    item[0] = f'threshold_{item[0]}_{threshold}'
+                self.datasets[item[0]] = dataset.ddis_df
+
+                self.items.extend(group_items)
+        self.y_test_label = self.items[0][4]
+        self.train_idx_arr = train_idx_arr
+        self.val_idx_arr = val_idx_arr
+
+    def run(self, model_func, batch_size=128, epochs=100):
+        mlflow.set_tracking_uri(self.tracking_uri)
+
+        if mlflow.get_experiment_by_name(self.experiment_name) == None:
+            mlflow.create_experiment(self.experiment_name)
+            mlflow.set_experiment_tags(self.experiment_tags)
+        mlflow.set_experiment(self.experiment_name)
+
+        y_test_label = self.items[0][4]
+        multi_modal_runner = MultiModalRunner(
+            library=self.library, model_func=model_func, batch_size=batch_size,  epochs=epochs)
+        multi_modal_runner.set_data(
+            self.items, self.train_idx_arr, self.val_idx_arr, y_test_label)
+        result = multi_modal_runner.predict()
+        return result

ddi_fw/pipeline/pipeline.py

@@ -12,7 +12,7 @@ from ddi_fw.ml import MultiModalRunner
 
 class Pipeline:
     def __init__(self,
-                 library='TF',
+                 library='tensorflow',
                  experiment_name=None,
                  experiment_description=None,
                  experiment_tags=None,
@@ -68,6 +68,7 @@ class Pipeline:
                 embedding_size = dictionary['embeddings'].shape[1]
         else:
             embedding_dict = self.embedding_dict
+            #TODO make generic
             embedding_size = list(embedding_dict['all_text'].values())[
                 0][0].shape
 

{ddi_fw-0.0.84.dist-info → ddi_fw-0.0.87.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.84
+Version: 0.0.87
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.84.dist-info → ddi_fw-0.0.87.dist-info}/RECORD

@@ -1,11 +1,11 @@
 ddi_fw/datasets/__init__.py,sha256=HSwQrqnzrEjIG4gif41pwJ_cST3t2XHGDxqFyuEBRwo,351
-ddi_fw/datasets/core.py,sha256=lGVP2P8CIeSEG5fH230XV8bLoycblJxBQKYbdMSBITM,19021
+ddi_fw/datasets/core.py,sha256=cL_H7-osGTNG5W8X8LLpIcSJ-GUXoI3LjNwvffmEGzA,19452
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/embedding_generator.py,sha256=Jqrlv88RCu0Lg812KsA12X0cSaZuxbckJ4LNRKNy_qw,2173
 ddi_fw/datasets/feature_vector_generation.py,sha256=EImavcALxkIB0YG_smOzagMNzuWMbK9SaWSKwARx_qU,3254
 ddi_fw/datasets/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
-ddi_fw/datasets/ddi_mdl/base.py,sha256=ZW8uJIvEizK2x_VkoyhNYcKh3ki3kQRsKxl8d2_hVYQ,4249
+ddi_fw/datasets/ddi_mdl/base.py,sha256=45cUmDRyyD8CC07oj5Dka2DWfgWU4Qi7-Am0vCvRKbo,4237
 ddi_fw/datasets/ddi_mdl/readme.md,sha256=WC6lpmsEKvIISnZqENY7TWtzCQr98HPpE3oRsBl8pIw,625
 ddi_fw/datasets/ddi_mdl/data/event.db,sha256=cmlSsf9MYjRzqR-mw3cUDnTnfT6FkpOG2yCl2mMwwew,30580736
 ddi_fw/datasets/ddi_mdl/indexes/test_indexes.txt,sha256=XVlDqYATckrQwNSXqMSKVBqyoN_Hg8SK6CL-XMdLADY,102176
@@ -67,17 +67,18 @@ ddi_fw/langchain/embeddings.py,sha256=8J_SfO9pyET2W-Ltzq0_r9EchFzBsYdUabiOMma42U
 ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
 ddi_fw/langchain/storage.py,sha256=uy5clVB07So2eFbRGdAKzHIPdfEk4se33cPktis7Aa4,2716
 ddi_fw/ml/__init__.py,sha256=0YubqmEpJKp3OfqlLKkD5N9L6WDWew3QEtnbdY3mqKg,180
-ddi_fw/ml/ml_helper.py,sha256=ibHVWMfCfC4jePRDsI8kC6e1e1x_ybOmigy6hkHOrAs,4470
+ddi_fw/ml/ml_helper.py,sha256=8ll5cMfcHUfwPhm8Gbmy7UQA91SRuf3MKoTaXTSzunY,4492
 ddi_fw/ml/model_wrapper.py,sha256=ZExnsLMjHKL3BaI4aKkbyWTp8vbswLeF2_T3cZ73YpQ,1144
 ddi_fw/ml/pytorch_wrapper.py,sha256=YdwzR5qAHFNajYB_elFqDhVKRLeajaRpopNzyQ6gIIA,3725
 ddi_fw/ml/tensorflow_wrapper.py,sha256=pSeiJDuaLf9MhZVlLuLJBA-LH-H-Dl2TyYbB39iGsto,5748
 ddi_fw/ner/__init__.py,sha256=JwhGXrepomxPSsGsg2b_xPRC72AjvxOIn2CW5Mvscn0,26
 ddi_fw/ner/mmlrestclient.py,sha256=NZta7m2Qm6I_qtVguMZhqtAUjVBmmXn0-TMnsNp0jpg,6859
 ddi_fw/ner/ner.py,sha256=BEs9AFljAxOQrC2BEP1raSzRoypcfELS5UTdl4bjTqw,15863
-ddi_fw/pipeline/__init__.py,sha256=1oLf4sGCwyLc_zPsZDRkEcpjtJJP09Y1EUv61PqmJrw,166
-ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=YkPixHVo9-4SPkY8VaWvBe1aaI5IiV4oZT4kBrm2WHQ,1635
-ddi_fw/pipeline/multi_pipeline.py,sha256=7PqeafCD--UUI7Xg2att9DdaB7b-ZRGPQY-E5F8qlgU,4529
-ddi_fw/pipeline/pipeline.py,sha256=k2LevGe7e5kRWraKvy-8i312zd6w5VfFHzFs3EQ9F1E,5519
+ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,212
+ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=qIst7vxHaOAhRv4lgozszwa3b1QE4aIrN74t41Xnvr4,1637
+ddi_fw/pipeline/multi_pipeline.py,sha256=ZSVBR5UJIgCh1sLaTDAE_F4u7cyWyOjYTqvdN8uTPnI,5425
+ddi_fw/pipeline/ner_pipeline.py,sha256=bf9amT8I-Ed9cdudWzPOpI6-X0oLgg8O37GJMSyX_bo,5527
+ddi_fw/pipeline/pipeline.py,sha256=NvRTHcccIZU-N17dgZRPkO0TCjpzj932cMIsmpaj7Qs,5559
 ddi_fw/test/basic_test.py,sha256=fEOGcZm1ObnsDvMiXNmdmz6YCeUrGc8V0DwlSwGhsq8,376
 ddi_fw/test/combination_test.py,sha256=TWNE8sf-DSh1Q9-yRaRBc774Sn1kSMGXLwQhd2_Qynk,324
 ddi_fw/test/compress_json_test.py,sha256=BGny56YqiG-pzhMoDzLKQBQI1E7o3jU0S7VYWtclAx4,1045
@@ -94,7 +95,7 @@ ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
 ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
 ddi_fw/utils/utils.py,sha256=szwnxMTDRrZoeNRyDuf3aCbtzriwtaRk4mHSH3asLdA,4301
 ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
-ddi_fw-0.0.84.dist-info/METADATA,sha256=8NTw5XSYOQa6b9yzlC7O2AanU1rIxtOi9j0WQ1J_Xvo,1966
-ddi_fw-0.0.84.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-ddi_fw-0.0.84.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.84.dist-info/RECORD,,
+ddi_fw-0.0.87.dist-info/METADATA,sha256=qAB0NEo8r0O-l27PecHa6fY1KOrSFSjbEO2bNF3-xZg,1966
+ddi_fw-0.0.87.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+ddi_fw-0.0.87.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.87.dist-info/RECORD,,

{ddi_fw-0.0.84.dist-info → ddi_fw-0.0.87.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (75.5.0)
 Root-Is-Purelib: true
 Tag: py3-none-any