ddi-fw 0.0.49__tar.gz → 0.0.51__tar.gz

{ddi_fw-0.0.49 → ddi_fw-0.0.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.49
+Version: 0.0.51
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.49 → ddi_fw-0.0.51}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.49"
+version = "0.0.51"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.49 → ddi_fw-0.0.51}/src/ddi_fw/datasets/ddi_mdl/base.py

@@ -7,16 +7,48 @@ from .. import BaseDataset
 from ..db_utils import create_connection
 
 HERE = pathlib.Path(__file__).resolve().parent
+list_of_embedding_columns = ['all_text', 'description',
+                     'synthesis_reference', 'indication',
+                     'pharmacodynamics', 'mechanism_of_action',
+                     'toxicity', 'metabolism',
+                     'absorption', 'half_life',
+                     'protein_binding', 'route_of_elimination',
+                     'volume_of_distribution', 'clearance']
+
+list_of_chemical_property_columns = ['enzyme',
+                             'target',
+                             'pathway',
+                             'smile']
+list_of_ner_columns = ['tui', 'cui', 'entities']
 
 
 class DDIMDLDataset(BaseDataset):
-    def __init__(self, embedding_size, embedding_dict, embeddings_pooling_strategy: PoolingStrategy, ner_df, chemical_property_columns=['enzyme',
-                                                                                                                                        'target',
-                                                                                                                                        'pathway',
-                                                                                                                                        'smile'],
+    def __init__(self, embedding_size,
+                 embedding_dict,
+                 embeddings_pooling_strategy: PoolingStrategy,
+                 ner_df,
+                 chemical_property_columns=['enzyme',
+                                            'target',
+                                            'pathway',
+                                            'smile'],
                  embedding_columns=[],
                  ner_columns=[],
                  **kwargs):
+        columns = kwargs['columns']
+        if columns is not None:
+            chemical_property_columns = []
+            embedding_columns=[]
+            ner_columns=[]
+            for column in columns:
+                if column in list_of_chemical_property_columns:
+                    chemical_property_columns.append(column)
+                elif column in list_of_embedding_columns:
+                    embedding_columns.append(column)
+                elif column in list_of_ner_columns:
+                    ner_columns.append(column)
+                else:
+                    raise Exception(f"{column} is not related this dataset")
+
 
         super().__init__(embedding_size=embedding_size,
                          embedding_dict=embedding_dict,

ddi_fw-0.0.51/src/ddi_fw/experiments/__init__.py

@@ -0,0 +1,2 @@
+from .tensorflow_helper import TFMultiModal, TFSingleModal,Result
+from .evaluation_helper import evaluate, Metrics

ddi_fw-0.0.51/src/ddi_fw/experiments/pipeline.py

@@ -0,0 +1,126 @@
+import sqlite3
+from sklearn.model_selection import train_test_split, KFold, StratifiedKFold
+from keras.models import Model, Sequential
+from keras.callbacks import EarlyStopping
+from keras.layers import Dense, Dropout, Input, Activation, BatchNormalization
+from tensorflow import keras
+from ddi_fw.experiments import TFSingleModal, TFMultiModal
+from ddi_fw.experiments import evaluate
+from sklearn.preprocessing import LabelBinarizer
+import numpy as np
+import pandas as pd
+from ddi_fw.utils import ZipHelper, Py7ZipHelper
+import os
+import chromadb
+from collections import defaultdict
+from langchain_community.vectorstores import Chroma
+from ddi_fw.ner.ner import CTakesNER
+from ddi_fw.datasets.embedding_generator_new import PoolingStrategy
+
+from ddi_fw.datasets import BaseDataset, DDIMDLDataset
+
+from ddi_fw.datasets import SumPoolingStrategy
+from keras import metrics
+from ddi_fw.experiments.evaluation_helper import evaluate
+
+import mlflow
+
+
+class Experiment:
+    def __init__(self, 
+                 experiment_name=None, 
+                 experiment_description=None, 
+                 experiment_tags=None, 
+                 tracking_uri=None, 
+                 dataset_type:BaseDataset=None, 
+                 columns=None, 
+                 vector_db_persist_directory=None, 
+                 vector_db_collection_name=None, 
+                 embedding_pooling_strategy_type:PoolingStrategy=None, 
+                 ner_data_file=None, 
+                 ner_threshold=None, 
+                 combinations=None, 
+                 model=None):
+        
+        self.experiment_name = experiment_name
+        self.experiment_description = experiment_description
+        self.experiment_tags = experiment_tags
+        self.tracking_uri = tracking_uri
+        self.dataset_type = dataset_type
+        self.columns = columns
+        self.vector_db_persist_directory = vector_db_persist_directory
+        self.vector_db_collection_name = vector_db_collection_name
+        self.embedding_pooling_strategy_type = embedding_pooling_strategy_type
+        self.ner_data_file = ner_data_file
+        self.ner_threshold = ner_threshold
+        self.combinations = combinations
+        self.model = model
+
+    def build(self):
+        # 'enzyme','target','pathway','smile','all_text','indication', 'description','mechanism_of_action','pharmacodynamics', 'tui', 'cui', 'entities'
+        kwargs = {"columns": self.columns}
+        for k, v in self.ner_threshold.items():
+            kwargs[k] = v
+
+        self.vector_db = chromadb.PersistentClient(
+            path=self.vector_db_persist_directory)
+        self.collection = self.vector_db.get_collection(
+            self.vector_db_collection_name)
+        dictionary = self.collection.get(include=['embeddings', 'metadatas'])
+
+        embedding_dict = defaultdict(lambda: defaultdict(list))
+
+        for metadata, embedding in zip(dictionary['metadatas'], dictionary['embeddings']):
+            embedding_dict[metadata["type"]][metadata["id"]].append(embedding)
+
+        embedding_size = dictionary['embeddings'].shape[1]
+
+        pooling_strategy = self.embedding_pooling_strategy_type()
+
+        self.ner_df = CTakesNER().load(filename=self.ner_data_file)  if self.ner_data_file else None
+
+        self.dataset = self.dataset_type(
+            embedding_dict=embedding_dict,
+            embedding_size=embedding_size,
+            embeddings_pooling_strategy=pooling_strategy,
+            ner_df=self.ner_df, kwargs=kwargs)
+
+        X_train, X_test, y_train, y_test, X_train.index, X_test.index, train_idx_arr, val_idx_arr = self.dataset.load()
+
+        self.dataframe = self.dataset.dataframe
+        # dataframe.dropna()
+        self.X_train = self.dataset.X_train
+        self.X_test = self.dataset.X_test
+        self.y_train = self.dataset.y_train
+        self.y_test = self.dataset.y_test
+        self.train_idx_arr = self.dataset.train_idx_arr
+        self.val_idx_arr = self.dataset.val_idx_arr
+        # Logic to set up the experiment
+        self.items = self.dataset.produce_inputs()
+
+        unique_classes = pd.unique(self.dataframe['event_category'])
+        event_num = len(unique_classes)
+        # droprate = 0.3
+        vector_size = self.dataset.drugs_df.shape[0]
+
+        print("Building the experiment with the following settings:")
+        print(
+            f"Name: {self.experiment_name}, Dataset: {self.dataset}, Model: {self.model}")
+        # Implement additional build logic as needed
+        return self
+
+    def run(self, model_func, batch_size=128, epochs=100):
+        mlflow.set_tracking_uri(self.tracking_uri)
+
+        if mlflow.get_experiment_by_name(self.experiment_name) == None:
+            mlflow.create_experiment(self.experiment_name)
+            mlflow.set_experiment_tags(self.experiment_tags)
+        mlflow.set_experiment(self.experiment_name)
+
+        y_test_label = self.items[0][4]
+        multi_modal = TFMultiModal(
+            model_func=model_func, batch_size=batch_size,  epochs=epochs)  # 100
+        multi_modal.set_data(
+            self.items, self.train_idx_arr, self.val_idx_arr, y_test_label)
+        result = multi_modal.predict(self.combinations)
+        return result

ddi_fw-0.0.51/src/ddi_fw/experiments/pipeline_builder_pattern.py

@@ -0,0 +1,152 @@
+import sqlite3
+from sklearn.model_selection import train_test_split, KFold, StratifiedKFold
+from keras.models import Model, Sequential
+from keras.callbacks import EarlyStopping
+from keras.layers import Dense, Dropout, Input, Activation, BatchNormalization
+from tensorflow import keras
+from ddi_fw.experiments import TFSingleModal, TFMultiModal
+from ddi_fw.experiments import evaluate
+from sklearn.preprocessing import LabelBinarizer
+import numpy as np
+import pandas as pd
+from ddi_fw.utils import ZipHelper, Py7ZipHelper
+import os
+import chromadb
+from collections import defaultdict
+from langchain_community.vectorstores import Chroma
+from ddi_fw.ner.ner import CTakesNER
+from ddi_fw.datasets.embedding_generator_new import PoolingStrategy
+
+from ddi_fw.datasets import BaseDataset, DDIMDLDataset
+
+from ddi_fw.datasets import SumPoolingStrategy
+from keras import metrics
+from ddi_fw.experiments.evaluation_helper import evaluate
+
+import mlflow
+
+
+class Experiment:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def create():
+        return Experiment()
+
+    def name(self, name):
+        self.experiment_name = name
+        return self
+
+    def description(self, description):
+        self.experiment_description = description
+        return self
+
+    def tags(self, tags):
+        self.experiment_tags = tags
+        return self
+
+    def tracking_uri(self, uri):
+        self.tracking_uri = uri
+        return self
+
+    def dataset(self, dataset_type: BaseDataset):
+        self.dataset_type = dataset_type
+        return self
+
+    def columns(self, cols):
+        self.columns = cols
+        return self
+
+    def vectordb_collection(self, persist_directory, collection_name):
+        self.vector_db_persist_directory = persist_directory
+        self.vector_db_collection_name = collection_name
+        return self
+
+    def embedding_pooling_strategy(self, strategy_type: PoolingStrategy):
+        self.embedding_pooling_strategy_type = strategy_type
+        return self
+
+    def ner_data_file(self, ner_data_file):
+        self.ner_data_file = ner_data_file
+        self.ner_df = CTakesNER().load(filename=ner_data_file)
+        return self
+
+    def ner_threshold(self, threshold):
+        self.ner_threshold = threshold
+        return self
+
+    def combinations(self, combs):
+        self.combinations = combs
+        return self
+
+    def model(self, model):
+        self.model = model
+        return self
+
+    def build(self):
+        # 'enzyme','target','pathway','smile','all_text','indication', 'description','mechanism_of_action','pharmacodynamics', 'tui', 'cui', 'entities'
+        kwargs = {"columns": self.columns}
+        for k, v in self.ner_threshold.items():
+            kwargs[k] = v
+
+        self.vector_db = chromadb.PersistentClient(
+            path=self.vector_db_persist_directory)
+        self.collection = self.vector_db.get_collection(
+            self.vector_db_collection_name)
+        dictionary = self.collection.get(include=['embeddings', 'metadatas'])
+
+        embedding_dict = defaultdict(lambda: defaultdict(list))
+
+        for metadata, embedding in zip(dictionary['metadatas'], dictionary['embeddings']):
+            embedding_dict[metadata["type"]][metadata["id"]].append(embedding)
+
+        embedding_size = dictionary['embeddings'].shape[1]
+
+        pooling_strategy = self.embedding_pooling_strategy_type()
+
+        self.dataset = self.dataset_type(
+            embedding_dict=embedding_dict,
+            embedding_size=embedding_size,
+            embeddings_pooling_strategy=pooling_strategy,
+            ner_df=self.ner_df, kwargs=kwargs)
+
+        X_train, X_test, y_train, y_test, X_train.index, X_test.index, train_idx_arr, val_idx_arr = self.dataset.load()
+
+        self.dataframe = self.dataset.dataframe
+        # dataframe.dropna()
+        self.X_train = self.dataset.X_train
+        self.X_test = self.dataset.X_test
+        self.y_train = self.dataset.y_train
+        self.y_test = self.dataset.y_test
+        self.train_idx_arr = self.dataset.train_idx_arr
+        self.val_idx_arr = self.dataset.val_idx_arr
+        # Logic to set up the experiment
+        self.items = self.dataset.produce_inputs()
+
+        unique_classes = pd.unique(self.dataframe['event_category'])
+        event_num = len(unique_classes)
+        # droprate = 0.3
+        vector_size = self.dataset.drugs_df.shape[0]
+
+        print("Building the experiment with the following settings:")
+        print(
+            f"Name: {self.experiment_name}, Dataset: {self.dataset}, Model: {self.model}")
+        # Implement additional build logic as needed
+        return self
+
+    def run(self, model_func, batch_size=128, epochs=100):
+        mlflow.set_tracking_uri(self.tracking_uri)
+
+        if mlflow.get_experiment_by_name(self.experiment_name) == None:
+            mlflow.create_experiment(self.experiment_name)
+            mlflow.set_experiment_tags(self.experiment_tags)
+        mlflow.set_experiment(self.experiment_name)
+
+        y_test_label = self.items[0][4]
+        multi_modal = TFMultiModal(
+            model_func=model_func, batch_size=batch_size,  epochs=epochs)  # 100
+        multi_modal.set_data(
+            self.items, self.train_idx_arr, self.val_idx_arr, y_test_label)
+        pred, self.single_results = multi_modal.predict(self.combinations)
+        return self

{ddi_fw-0.0.49 → ddi_fw-0.0.51}/src/ddi_fw/experiments/tensorflow_helper.py

@@ -30,12 +30,25 @@ np.random.seed(2)
 np.set_printoptions(precision=4)
 
 
+class Result:
+    def __init__(self) -> None:
+        self.log_dict = {}
+        self.metric_dict = {}
+
+    def add_log(self, key, logs):
+        self.log_dict[key] = logs
+
+    def add_metric(self, key, metrics):
+        self.metric_dict[key] = metrics
+
+
 class TFMultiModal:
     # todo model related parameters to config
     def __init__(self, model_func, batch_size=128, epochs=100):
         self.model_func = model_func
         self.batch_size = batch_size
         self.epochs = epochs
+        self.result = Result()
 
     def set_data(self, items, train_idx_arr, val_idx_arr, y_test_label):
         self.items = items
@@ -64,14 +77,16 @@ class TFMultiModal:
                     self.date, item[0], self.model_func, self.batch_size, self.epochs)
                 single_modal.set_data(
                     self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
-                r = single_modal.predict()
-                single_results[item[0]] = r
-                sum = sum + r
+                logs, metrics, prediction = single_modal.predict()
+                self.result.add_log(item[0], logs)
+                self.result.add_metric(item[0], metrics)
+                single_results[item[0]] = prediction
+                # sum = sum + prediction
 
             if combinations:
                 self.evaluate_combinations(single_results, combinations)
         # TODO: sum'a gerek yok
-        return sum, single_results
+        return self.result
 
     def evaluate_combinations(self, single_results, combinations):
         for combination in combinations:
@@ -90,6 +105,8 @@ class TFMultiModal:
                     f'combination_artifact_uri:{combination_run.info.artifact_uri}')
                 utils.compress_and_save_data(
                     metrics.__dict__, combination_run.info.artifact_uri, f'{self.date}_metrics.gzip')
+                self.result.add_log(combination_descriptor,logs)
+                self.result.add_metric(combination_descriptor,metrics)
 
 
 class TFSingleModal:
@@ -172,39 +189,8 @@ class TFSingleModal:
             #           '/model/model.onnx')
             utils.compress_and_save_data(
                 metrics.__dict__, run.info.artifact_uri, f'{self.date}_metrics.gzip')
-            # mlflow.log_dict(metrics.__dict__, "metrics.json")
-
-            # Plot Precision-Recall curves for each class and micro-average
-            # fig = plt.figure()
-            # plt.step(metrics.recall['micro_event'], metrics.precision['micro_event'],
-            #          color='b', alpha=0.2, where='post')
-            # plt.fill_between(
-            #     metrics.recall["micro_event"], metrics.precision["micro_event"], step='post', alpha=0.2, color='b')
-
-            # for i in range(pred.shape[1]):
-            #     plt.step(metrics.recall[i], metrics.precision[i], where='post',
-            #              label='Class {0} (AUC={1:0.2f})'.format(i, metrics.roc_aupr[i]))
-
-            # plt.xlabel('Recall')
-            # plt.ylabel('Precision')
-            # plt.ylim([0.0, 1.05])
-            # plt.xlim([0.0, 1.0])
-            # plt.title(
-            #     'Micro-average Precision-Recall curve: AUC={0:0.2f}'.format(metrics.roc_aupr["micro"]))
-            # plt.legend(loc='best')
-            # # plt.savefig(run.info.artifact_uri + '/auprc.png')
-            # mlflow.log_figure(fig, 'auprc.png')
-
-            # mlflow.log_model(
-            #         model,
-            #         artifact_path=run.info.artifact_uri + '/model',
-            #         signature=signature,
-            #     )
-            # mlflow.log_artifact(run.info.artifact_uri + '/model')
-
-            # mlflow.MlflowClient().log_artifact(run.info.run_id,
-            #                                    run.info.artifact_uri, None)
-        return pred
+
+        return logs, metrics, pred
 
 
 class CustomCallback(keras.callbacks.Callback):

ddi_fw-0.0.51/src/ddi_fw/utils/__init__.py

@@ -0,0 +1,4 @@
+from .utils import clear_directory,create_folder_if_not_exists, utc_time_as_string,utc_time_as_string_simple_format, compress_and_save_data
+from .zip_helper import ZipHelper
+from .py7zr_helper import Py7ZipHelper
+from .enums import UMLSCodeTypes, DrugBankTextDataTypes

{ddi_fw-0.0.49 → ddi_fw-0.0.51}/src/ddi_fw/utils/py7zr_helper.py

@@ -1,11 +1,13 @@
 from collections import defaultdict
 import math
-from ddi_fw.utils.utils import create_folder_if_not_exists
+from ddi_fw.utils.utils import clear_directory, create_folder_if_not_exists
 import py7zr
 import os
 from os.path import basename
 
-
+# https://py7zr.readthedocs.io/en/latest/user_guide.html
+# import multivolumefile
+#https://github.com/miurahr/py7zr/issues/497
 class Py7ZipHelper:
     def __init__(self):
         pass
@@ -24,9 +26,10 @@ class Py7ZipHelper:
                                                   os.path.join(folder_path, '..')))
                     # archive.write(root+"/"+file)
 
-    def create_archive_multiparts(self, zip_name, file_path, output_path, chunk_size):
+    def create_archive_multiparts(self, zip_name, file_path, output_path, chunk_size, delete_existing_files=True):
         parent_folder = os.path.dirname(file_path)
-
+        if delete_existing_files:
+            clear_directory(output_path)
         # parts_path = f"{parent_folder}/parts"
         create_folder_if_not_exists(output_path)
         # file_name, file_extension = os.path.splitext(file_path)

{ddi_fw-0.0.49 → ddi_fw-0.0.51}/src/ddi_fw/utils/utils.py

@@ -1,11 +1,9 @@
 import gzip
 import json
 import os
-
 from datetime import datetime, timezone
-
 from matplotlib import pyplot as plt
-
+import shutil
 
 def create_folder_if_not_exists(path):
     if not os.path.exists(path):
@@ -51,6 +49,22 @@ def decompress(gzip_file):
     return data
 
 
+def clear_directory(directory_path):
+    # Check if the directory exists
+    if os.path.exists(directory_path) and os.path.isdir(directory_path):
+        # Iterate through all files and directories in the directory
+        for item in os.listdir(directory_path):
+            item_path = os.path.join(directory_path, item)
+            # Check if it's a file or a directory and remove it
+            if os.path.isfile(item_path):
+                os.remove(item_path)  # Remove file
+            elif os.path.isdir(item_path):
+                shutil.rmtree(item_path)  # Remove directory
+        print(f"Cleared contents of directory: {directory_path}")
+    else:
+        print(f"The directory does not exist: {directory_path}")
+
+
 if __name__ == "__main__":
     # json_file = f'C:\\Users\\kivanc\\Downloads\\metrics.json'
     # file_data = open(json_file, "r", 1).read()

{ddi_fw-0.0.49 → ddi_fw-0.0.51}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.49
+Version: 0.0.51
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.49 → ddi_fw-0.0.51}/src/ddi_fw.egg-info/SOURCES.txt

@@ -66,6 +66,8 @@ src/ddi_fw/drugbank/event_extractor.py
 src/ddi_fw/experiments/__init__.py
 src/ddi_fw/experiments/custom_torch_model.py
 src/ddi_fw/experiments/evaluation_helper.py
+src/ddi_fw/experiments/pipeline.py
+src/ddi_fw/experiments/pipeline_builder_pattern.py
 src/ddi_fw/experiments/tensorflow_helper.py
 src/ddi_fw/experiments/test.py
 src/ddi_fw/ner/__init__.py

ddi_fw-0.0.49/src/ddi_fw/experiments/__init__.py

@@ -1,2 +0,0 @@
-from .tensorflow_helper import TFMultiModal, TFSingleModal
-from .evaluation_helper import evaluate, Metrics

ddi_fw-0.0.49/src/ddi_fw/utils/__init__.py

@@ -1,4 +0,0 @@
-from .utils import create_folder_if_not_exists, utc_time_as_string,utc_time_as_string_simple_format, compress_and_save_data
-from .zip_helper import ZipHelper
-from .py7zr_helper import Py7ZipHelper
-from .enums import UMLSCodeTypes, DrugBankTextDataTypes