ddi-fw 0.0.131__py3-none-any.whl → 0.0.133__py3-none-any.whl

ddi_fw/ml/ml_helper.py

@@ -3,6 +3,7 @@ from matplotlib import pyplot as plt
 from ddi_fw.ml.model_wrapper import Result
 from ddi_fw.ml.pytorch_wrapper import PTModelWrapper
 from ddi_fw.ml.tensorflow_wrapper import TFModelWrapper
+from ddi_fw.utils.package_helper import get_import
 import tensorflow as tf
 from tensorflow import keras
 from keras.models import Model, Sequential
@@ -30,11 +31,9 @@ import ddi_fw.utils as utils
 
 class MultiModalRunner:
     # todo model related parameters to config
-    def __init__(self, library ,model_func, batch_size=128, epochs=100):
+    def __init__(self, library, multi_modal):
         self.library = library
-        self.model_func = model_func
-        self.batch_size = batch_size
-        self.epochs = epochs
+        self.epochs = multi_modal
         self.result = Result()
 
     def set_data(self, items, train_idx_arr, val_idx_arr, y_test_label):
@@ -43,7 +42,7 @@ class MultiModalRunner:
         self.val_idx_arr = val_idx_arr
         self.y_test_label = y_test_label
 
-    def __create_multi_modal(self,library):
+    def __create_model(self,library):
         if library == 'tensorflow':
             return TFModelWrapper
         elif library == 'pytorch':
@@ -66,18 +65,49 @@ class MultiModalRunner:
 
         with mlflow.start_run(run_name=self.prefix, description="***") as run:
             self.level_0_run_id = run.info.run_id
-            for item in self.items:
-                print(item[0])
-                T = self.__create_multi_modal(self.library)
-                single_modal=T(self.date, item[0], self.model_func, self.batch_size, self.epochs)
-                single_modal.set_data(
-                    self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
+            item_dict = {t[0]: t for t in self.items}
+            for m in self.multi_modal:
+                name = m.get('name')
+                input_type = m.get('input_type')
+                input = m.get('input')
+                inputs = m.get('inputs')
+                model_type = get_import(m.get("model_type"))
+                kwargs = m.get('params')
+                T = self.__create_model(self.library)
+                single_modal=T(self.date, name, model_type, **kwargs)
+                if input_type == '1D':
+                    item = item_dict[input]
+                    single_modal.set_data(
+                        self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
+                elif input_type == '2D':
+                    filtered_dict = {k: item_dict[k] for k in inputs if k in item_dict}
+                    first_input = next(iter(item_dict.values()))
+                    train_data_list = [f[1] for f in filtered_dict.values()]
+                    test_data_list = [f[3] for f in filtered_dict.values()]
+                    train_data = np.stack(train_data_list, axis=1)
+                    test_data = np.stack(test_data_list, axis=1)
+                    train_label = first_input[2]
+                    test_label = first_input[2]
+                    single_modal.set_data(
+                        self.train_idx_arr, self.val_idx_arr, train_data, train_label, test_data, test_label)
                 logs, metrics, prediction = single_modal.predict()
-                # self.result.add_log(item[0], logs)
-                #Check
                 self.result.add_metric(item[0], metrics)
-                single_results[item[0]] = prediction
-                # sum = sum + prediction
+                single_results[name] = prediction
+
+
+            # for item in self.items:
+            #     print(item[0])
+            #     T = self.__create_model(self.library)
+            #     # parameters of model should be dictionary 
+            #     single_modal=T(self.date, item[0], self.model_func, self.batch_size, self.epochs)
+            #     single_modal.set_data(
+            #         self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
+            #     logs, metrics, prediction = single_modal.predict()
+            #     # self.result.add_log(item[0], logs)
+            #     #Check
+            #     self.result.add_metric(item[0], metrics)
+            #     single_results[item[0]] = prediction
+            #     # sum = sum + prediction
 
             if combinations:
                 self.evaluate_combinations(single_results, combinations)

ddi_fw/ml/model_wrapper.py

@@ -15,12 +15,12 @@ class Result:
 
 
 class ModelWrapper:
-    def __init__(self, date, descriptor, model_func, batch_size=128, epochs=100):
+    def __init__(self, date, descriptor, model_func ,**kwargs):
         self.date = date
         self.descriptor = descriptor
         self.model_func = model_func
-        self.batch_size = batch_size
-        self.epochs = epochs
+        self.kwargs = kwargs
+       
 
     def set_data(self, train_idx_arr, val_idx_arr, train_data, train_label, test_data, test_label):
         self.train_idx_arr = train_idx_arr

ddi_fw/ml/tensorflow_wrapper.py

@@ -1,7 +1,7 @@
 from ddi_fw.ml.model_wrapper import ModelWrapper
 import tensorflow as tf
 from tensorflow import keras
-from keras.callbacks import EarlyStopping,ModelCheckpoint
+from keras.callbacks import EarlyStopping, ModelCheckpoint
 from sklearn.model_selection import train_test_split, KFold, StratifiedKFold
 import numpy as np
 
@@ -15,91 +15,86 @@ from ddi_fw.ml.evaluation_helper import Metrics, evaluate
 # import onnx
 
 import ddi_fw.utils as utils
+import os
 
 
 class TFModelWrapper(ModelWrapper):
-    # https://github.com/mlflow/mlflow/blob/master/examples/tensorflow/train.py
-    def predict(self):
+
+    def __init__(self, date, descriptor, model_func, **kwargs):
+        super().__init__(date, descriptor, model_func, **kwargs)
+        self.batch_size = kwargs.get('batch_size',128)
+        self.epochs = kwargs.get('epochs',100)
+
+    def fit_model(self, X_train, y_train, X_valid, y_valid):
+        self.kwargs['input_shape'] = self.train_data.shape
+        model = self.model_func(**self.kwargs)
+        checkpoint = ModelCheckpoint(
+            filepath=f'{self.descriptor}_validation.weights.h5',
+            monitor='val_loss',
+            save_best_only=True,
+            save_weights_only=True,
+            verbose=1,
+            mode='min'
+        )
+        early_stopping = EarlyStopping(
+            monitor='val_loss', patience=10, mode='auto')
+        custom_callback = CustomCallback()
+
+        history = model.fit(
+            X_train, y_train,
+            batch_size=self.batch_size,
+            epochs=self.epochs,
+            validation_data=(X_valid, y_valid),
+            callbacks=[early_stopping, checkpoint, custom_callback]
+        )
+
+        if os.path.exists(f'{self.descriptor}_validation.weights.h5'):
+            os.remove(f'{self.descriptor}_validation.weights.h5')
+
+        return model, checkpoint
+
+    def fit(self):
         print(self.train_data.shape)
+        models = {}
+        models_val_acc = {}
+        for i, (train_idx, val_idx) in enumerate(zip(self.train_idx_arr, self.val_idx_arr)):
+            print(f"Validation {i}")
+            with mlflow.start_run(run_name=f'Validation {i}', description='CV models', nested=True) as cv_fit:
+                X_train_cv = self.train_data[train_idx]
+                y_train_cv = self.train_label[train_idx]
+                X_valid_cv = self.train_data[val_idx]
+                y_valid_cv = self.train_label[val_idx]
+                model, best_val_acc = self.fit_model(
+                    X_train_cv, y_train_cv, X_valid_cv, y_valid_cv)
+                models[f'validation_{i}'] = model
+                models_val_acc[f'{self.descriptor}_validation_{i}'] = best_val_acc
+
+        best_model_key = max(models_val_acc, key=models_val_acc.get)
+        best_model = models[best_model_key]
+        return best_model, best_model_key
 
-        # Failed to convert a NumPy array to a Tensor
-        with mlflow.start_run(run_name=self.descriptor, description="***", nested=True) as run:
-            models = dict()
-            histories = dict()
-            models_val_acc = dict()
-            # with batch_metrics_logger(run_id) as metrics_logger:
-            for i, (train_idx, val_idx) in enumerate(zip(self.train_idx_arr, self.val_idx_arr)):
-                print(f"Validation {i}")
-
-                with mlflow.start_run(run_name=f'Validation {i}', description='CV models', nested=True) as cv_fit:
-                    model = self.model_func(self.train_data.shape[1])
-                    models[f'validation_{i}'] = model
-                    X_train_cv = self.train_data[train_idx]
-                    y_train_cv = self.train_label[train_idx]
-                    X_valid_cv = self.train_data[val_idx]
-                    y_valid_cv = self.train_label[val_idx]
-
-                    checkpoint = ModelCheckpoint(
-                        filepath=f'{self.descriptor}_validation_{i}.weights.h5',
-                        monitor='val_loss',
-                        save_best_only=True,
-                        save_weights_only=True,
-                        verbose=1,
-                        mode='min'
-                    )
-
-                    early_stopping = EarlyStopping(
-                        monitor='val_loss', patience=10, verbose=0, mode='auto')
-                    custom_callback = CustomCallback()
-                    history = model.fit(X_train_cv, y_train_cv,
-                                        batch_size=self.batch_size,
-                                        epochs=self.epochs,
-                                        validation_data=(
-                                            X_valid_cv, y_valid_cv),
-                                        callbacks=[early_stopping, checkpoint, custom_callback])
-                    # histories[f'validation_{i}'] = history
-                    # models_val_acc[f'validation_{i}'] = history.history['val_accuracy'][-1]
-                    models_val_acc[f'{self.descriptor}_validation_{i}'] = checkpoint.best
-                    models[f'{self.descriptor}_validation_{i}'] = checkpoint.model
-                    import os
-                    if os.path.exists(f'{self.descriptor}_validation_{i}.weights.h5'):
-                        os.remove(f'{self.descriptor}_validation_{i}.weights.h5')
-                    # Saving each CV model
-
-            best_model_key = max(models_val_acc, key=models_val_acc.get)
-            best_model = models[best_model_key]
-            # mlflow.tensorflow.log_model(best_model, "model")
-            # best_model.evaluate(self.test_data, self.test_label,
-            #                     callbacks=[custom_callback])
-            pred = best_model.predict(self.test_data)
+    # https://github.com/mlflow/mlflow/blob/master/examples/tensorflow/train.py
+
+    def predict(self, best_model):
+        pred = best_model.predict(self.test_data)
+        return pred
+
+    def fit_and_evaluate(self):
 
+        with mlflow.start_run(run_name=self.descriptor, description="***", nested=True) as run:
+            print(run.info.artifact_uri)
+            best_model, best_model_key = self.fit()
+            pred = self.predict(best_model)
             logs, metrics = evaluate(
                 actual=self.test_label, pred=pred, info=self.descriptor)
             metrics.format_float()
             mlflow.log_metrics(logs)
             mlflow.log_param('best_cv', best_model_key)
-            # signature = infer_signature(
-            #     self.train_data,
-            #     # generate_signature_output(model,X_valid_cv)
-            #     # params=params,
-            # )
-            
-            # mlflow.keras.save_model(
-            #     best_model,
-            #     path=run.info.artifact_uri + '/model',
-            #     signature=signature,
-            # )
-            print(run.info.artifact_uri)
-            # todo tf2onnx not compatible with keras > 2.15
-            # onnx_model, _ = tf2onnx.convert.from_keras(
-            #     best_model, input_signature=None, opset=13)
-            # onnx.save(onnx_model, run.info.artifact_uri +
-            #           '/model/model.onnx')
+
             utils.compress_and_save_data(
                 metrics.__dict__, run.info.artifact_uri, f'{self.date}_metrics.gzip')
 
-        return logs, metrics, pred
-
+            return logs, metrics, pred
 
 class CustomCallback(keras.callbacks.Callback):
     def on_train_begin(self, logs=None):

ddi_fw/pipeline/multi_pipeline.py

@@ -49,8 +49,8 @@ class MultiPipeline():
     def __create_pipeline(self, config):
         type = config.get("type")
         library = config.get("library")
-        batch_size = config.get("batch_size")
-        epochs = config.get("epochs")
+        # batch_size = config.get("batch_size")
+        # epochs = config.get("epochs")
 
         # dataset_module = config.get("dataset_module")
         # dataset_name = config.get("dataset_name")
@@ -60,6 +60,8 @@ class MultiPipeline():
         experiment_tags = config.get("experiment_tags")
         tracking_uri = config.get("tracking_uri")
         artifact_location = config.get("artifact_location")
+        #new
+        multi_modal = config.get("multi_modal")
         columns = config.get("columns")
         ner_data_file = config.get("ner_data_file")
         ner_threshold = config.get("ner_threshold")
@@ -69,8 +71,9 @@ class MultiPipeline():
         embedding_pooling_strategy = get_import(
             config.get("embedding_pooling_strategy_type")) if config.get("embedding_pooling_strategy_type") else None
         # Dynamically import the model and dataset classes
-        model_type = get_import(config.get("model_type"))
+        # model_type = get_import(config.get("model_type"))
         dataset_type = get_import(config.get("dataset_type"))
+
         combination_type = None
         kwargs_combination_params=None
         if config.get("combination_strategy"):
@@ -100,7 +103,8 @@ class MultiPipeline():
                 embedding_pooling_strategy_type=embedding_pooling_strategy,
                 ner_data_file=ner_data_file,
                 ner_threshold=ner_threshold,
-                combinations=combinations)
+                combinations=combinations,
+                multi_modal= multi_modal)
         elif type== "ner_search":
             pipeline = NerParameterSearch(
                 library=library,
@@ -119,9 +123,9 @@ class MultiPipeline():
         return {
             "name": experiment_name,
             "library": library,
-            "batch_size": batch_size,
-            "epochs": epochs,
-            "model_type": model_type,
+            # "batch_size": batch_size,
+            # "epochs": epochs,
+            # "model_type": model_type,
             "pipeline": pipeline}
 
     def build(self):
@@ -134,12 +138,12 @@ class MultiPipeline():
         for item in self.items:
             print(f"{item['name']} is running")
             pipeline = item['pipeline']
-            model_type = item['model_type']
-            batch_size = item['batch_size']
-            epochs = item['epochs']
+            # model_type = item['model_type']
+            # batch_size = item['batch_size']
+            # epochs = item['epochs']
             # It can be moved to build function
             pipeline.build()
-            result = pipeline.run(model_type, epochs=epochs, batch_size=batch_size)
+            result = pipeline.run()
             self.pipeline_resuts[item['name']] = result
         return self
 

ddi_fw/pipeline/pipeline.py

@@ -28,7 +28,8 @@ class Pipeline:
                  ner_data_file=None,
                  ner_threshold=None,
                  combinations=None,
-                 model=None):
+                 model=None,
+                 multi_modal = None ):
         self.library = library
         self.experiment_name = experiment_name
         self.experiment_description = experiment_description
@@ -46,6 +47,7 @@ class Pipeline:
         self.ner_threshold = ner_threshold
         self.combinations = combinations
         self.model = model
+        self.multi_modal = multi_modal
 
     def __create_or_update_embeddings__(self, embedding_dict, vector_db_persist_directory, vector_db_collection_name, column=None):
         """
@@ -155,6 +157,7 @@ class Pipeline:
         self.train_idx_arr = self.dataset.train_idx_arr
         self.val_idx_arr = self.dataset.val_idx_arr
         # Logic to set up the experiment
+        # column name, train data, train label, test data, test label
         self.items = self.dataset.produce_inputs()
 
         unique_classes = pd.unique(self.dataframe['event_category'])
@@ -168,7 +171,7 @@ class Pipeline:
         # Implement additional build logic as needed
         return self
 
-    def run(self, model_func, batch_size=128, epochs=100):
+    def run(self):
         mlflow.set_tracking_uri(self.tracking_uri)
 
         if mlflow.get_experiment_by_name(self.experiment_name) == None:
@@ -178,8 +181,9 @@ class Pipeline:
         mlflow.set_experiment(self.experiment_name)
 
         y_test_label = self.items[0][4]
-        multi_modal_runner = MultiModalRunner(
-            library=self.library, model_func=model_func, batch_size=batch_size,  epochs=epochs)
+        multi_modal_runner = MultiModalRunner(library=self.library, multi_modal = self.multi_modal)
+        # multi_modal_runner = MultiModalRunner(
+        #     library=self.library, model_func=model_func, batch_size=batch_size,  epochs=epochs)
         # multi_modal = TFMultiModal(
         #     model_func=model_func, batch_size=batch_size,  epochs=epochs)  # 100
         multi_modal_runner.set_data(

{ddi_fw-0.0.131.dist-info → ddi_fw-0.0.133.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.131
+Version: 0.0.133
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.131.dist-info → ddi_fw-0.0.133.dist-info}/RECORD

@@ -75,18 +75,19 @@ ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSz
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=tIxiW0g6q1VsmDYVXR_ovvHQR3SCir8g2bKxx_CrS7s,221
 ddi_fw/ml/evaluation_helper.py,sha256=o4-w5Xa3t4olLW4ymx_8L-Buhe5wfQEmT2bh4Zz544c,13066
-ddi_fw/ml/ml_helper.py,sha256=fySjIAFzkeEOvaLJhDwtCOgRhgYQ7H106eqaP16GhDY,4489
-ddi_fw/ml/model_wrapper.py,sha256=W-bed6NOJxuXs7d3nG6iU8bkUX8LBPFQ0gMjZ7Qc0Sw,1135
+ddi_fw/ml/ml_helper.py,sha256=poZKeIRK4S9SJVgQzMtahHvoekwNxCARCExUeTCjARY,6162
+ddi_fw/ml/model_wrapper.py,sha256=kc01_TVJuriUvNI6ABnLngnJWvmG_Y7-XJ6XMusLJ8U,1088
 ddi_fw/ml/pytorch_wrapper.py,sha256=AkG-2sKDXr0IBhgmkbjG0i20OuwQv3mhdvqp6UvJDCA,3716
-ddi_fw/ml/tensorflow_wrapper.py,sha256=E46lC9qMkM5NvFTL-eOuMcMhEUso5UYfP66Du4BOhfQ,6423
+ddi_fw/ml/tensorflow_wrapper.py,sha256=jaWPZfsQMSG_Vzl1OTALAbgb2sdJklcgQD5kokt7b18,5409
 ddi_fw/ner/__init__.py,sha256=JwhGXrepomxPSsGsg2b_xPRC72AjvxOIn2CW5Mvscn0,26
 ddi_fw/ner/mmlrestclient.py,sha256=NZta7m2Qm6I_qtVguMZhqtAUjVBmmXn0-TMnsNp0jpg,6859
 ddi_fw/ner/ner.py,sha256=BEs9AFljAxOQrC2BEP1raSzRoypcfELS5UTdl4bjTqw,15863
 ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,212
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=qIst7vxHaOAhRv4lgozszwa3b1QE4aIrN74t41Xnvr4,1637
-ddi_fw/pipeline/multi_pipeline.py,sha256=t_Z7d7xRfDnhpQTlqCf7c0isZ5hZlyXavKhC7ePsnJY,5903
+ddi_fw/pipeline/multi_pipeline.py,sha256=G8ONZdfwjGZRI2PrzMOaET6w5AUcmgYzMtaV6j5Hbz0,5981
 ddi_fw/pipeline/ner_pipeline.py,sha256=wB7hz4YCOv7UAz6bGE6sSpPXXIdoOflOVK5UCc1fO-o,5586
-ddi_fw/pipeline/pipeline.py,sha256=WQk9vZOLao6YDpqPK7O7NDMqb5pjcPfUaG6_LZL2Qis,8769
+ddi_fw/pipeline/pipeline.py,sha256=-1zGbSJapmUSx9xltJLQajmUCeZdT-9Ow0cC6JZ92y0,8984
+ddi_fw/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ddi_fw/test/basic_test.py,sha256=fEOGcZm1ObnsDvMiXNmdmz6YCeUrGc8V0DwlSwGhsq8,376
 ddi_fw/test/combination_test.py,sha256=TWNE8sf-DSh1Q9-yRaRBc774Sn1kSMGXLwQhd2_Qynk,324
 ddi_fw/test/compress_json_test.py,sha256=BGny56YqiG-pzhMoDzLKQBQI1E7o3jU0S7VYWtclAx4,1045
@@ -105,7 +106,7 @@ ddi_fw/utils/package_helper.py,sha256=erl8_onmhK-41zQoaED2qyDUV9GQxmT9sdoyRp9_q5
 ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
 ddi_fw/utils/utils.py,sha256=szwnxMTDRrZoeNRyDuf3aCbtzriwtaRk4mHSH3asLdA,4301
 ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
-ddi_fw-0.0.131.dist-info/METADATA,sha256=Q9yK_3SVMUOviMB1_9_5p9vOwCm-yLAfgysKkCw86wQ,1965
-ddi_fw-0.0.131.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ddi_fw-0.0.131.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.131.dist-info/RECORD,,
+ddi_fw-0.0.133.dist-info/METADATA,sha256=LpoElNyGbr_R9jW90vl4RPpMbi9oRwi0PjYKTuLvwZM,1965
+ddi_fw-0.0.133.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ddi_fw-0.0.133.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.133.dist-info/RECORD,,