ddi-fw 0.0.198__py3-none-any.whl → 0.0.200__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -144,8 +144,8 @@ class BaseDataset(BaseModel, abc.ABC):
         Load the dataset. If X_train, y_train, X_test, and y_test are already provided,
         skip deriving them. Otherwise, derive them from the dataframe and indices.
         """
-        self.prep()  # Prepare the dataset
         self.handle_mixins()  # Centralized mixin handling
+        self.prep()  # Prepare the dataset
 
         if self.X_train is not None or self.y_train is not None or self.X_test is not None or self.y_test is not None:
             # Data is already provided, no need to calculate
@@ -169,8 +169,9 @@ class BaseDataset(BaseModel, abc.ABC):
         
         # train = self.dataframe[self.dataframe.index.isin(train_idx_all)]
         # test = self.dataframe[self.dataframe.index.isin(test_idx_all)]
-        train = self.dataframe.loc[self.dataframe.index.isin(train_idx_all), self.columns]
-        test = self.dataframe.loc[self.dataframe.index.isin(test_idx_all), self.columns]
+        columns = self.columns + [self.class_column] 
+        train = self.dataframe.loc[self.dataframe.index.isin(train_idx_all), columns]
+        test = self.dataframe.loc[self.dataframe.index.isin(test_idx_all), columns]
         X_train = train.drop(self.class_column, axis=1)
         X_train = train.drop(self.class_column, axis=1)
         y_train = train[self.class_column]

ddi_fw/datasets/ddi_mdl/base.py

@@ -81,6 +81,7 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
             self.chemical_property_columns = chemical_property_columns
             self.embedding_columns = embedding_columns 
             self.ner_columns = ner_columns 
+            self.columns = [] # these variable is modified in prep method
         
         self.class_column = 'event_category'
         _db_path = HERE.joinpath('data/event.db')

ddi_fw/ml/ml_helper.py

@@ -32,9 +32,10 @@ import ddi_fw.utils as utils
 
 class MultiModalRunner:
     # todo model related parameters to config
-    def __init__(self, library, multi_modal, use_mlflow=False):
+    def __init__(self, library, multi_modal, default_model, use_mlflow=False):
         self.library = library
         self.multi_modal = multi_modal
+        self.default_model = default_model
         self.use_mlflow = use_mlflow
         self.result = Result()
 
@@ -60,14 +61,13 @@ class MultiModalRunner:
     # TODO check single_results, 1d,2d ...
     def __predict(self, single_results):
         item_dict = {t[0]: t for t in self.items}
-        print("multi_modal")
-        print(self.multi_modal)
-        print(item_dict.keys())
+        if self.default_model is None and not self.multi_modal:
+            raise Exception("Default model and multi modal cannot be None at the same time")
 
         if self.multi_modal:
             for m in self.multi_modal:
                 name = m.get('name')
-                input_type = m.get('input_type')
+                # input_type = m.get('input_type')
                 input = m.get('input')
                 inputs = m.get('inputs')
                 model_type = get_import(m.get("model_type"))
@@ -99,14 +99,25 @@ class MultiModalRunner:
                         self.train_idx_arr, self.val_idx_arr, train_data, train_label, test_data, test_label)
                 else:
                     raise Exception("check configurations")
-            else: # TODO default model maybe?
-                item = self.items[0]
+                logs, metrics, prediction = single_modal.fit_and_evaluate()
+                self.result.add_metric(name, metrics)
+                single_results[name] = prediction
+        else: # TODO default model maybe?
+            print("Default model will be used")
+            name = self.default_model.get('name')
+            # input_type = m.get('input_type')
+            model_type = get_import(self.default_model.get("model_type"))
+            kwargs = self.default_model.get('params')
+            for item in self.items:
+                T = self.__create_model(self.library)
+                single_modal = T(self.date, name, model_type,
+                                use_mlflow=self.use_mlflow,  **kwargs)
                 single_modal.set_data(
                         self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
 
-            logs, metrics, prediction = single_modal.fit_and_evaluate()
-            self.result.add_metric(name, metrics)
-            single_results[name] = prediction
+                logs, metrics, prediction = single_modal.fit_and_evaluate()
+                self.result.add_metric(name, metrics)
+                single_results[name] = prediction
 
     def predict(self, combinations: list = [], generate_combinations=False):
         self.prefix = utils.utc_time_as_string()

ddi_fw/pipeline/multi_pipeline.py

@@ -57,6 +57,7 @@ class MultiPipeline():
         tracking_uri = config.get("tracking_uri")
         artifact_location = config.get("artifact_location")
         #new
+        default_model = config.get("default_model"),
         multi_modal = config.get("multi_modal")
         columns = config.get("columns")
         ner_data_file = config.get("ner_data_file")
@@ -101,6 +102,7 @@ class MultiPipeline():
                 ner_data_file=ner_data_file,
                 ner_threshold=ner_threshold,
                 combinations=combinations,
+                default_model=default_model,
                 multi_modal= multi_modal)
         elif type== "ner_search":
             pipeline = NerParameterSearch(

ddi_fw/pipeline/pipeline.py

@@ -35,6 +35,7 @@ class Pipeline(BaseModel):
     ner_threshold: Optional[dict] = None
     combinations: Optional[List[str]] = None
     model: Optional[Any] = None
+    default_model:  Optional[Any] = None
     multi_modal:  Optional[Any] = None
     use_mlflow: bool = False
     _dataset: BaseDataset = []
@@ -193,7 +194,7 @@ class Pipeline(BaseModel):
 
         y_test_label = self.items[0][4]
         multi_modal_runner = MultiModalRunner(
-            library=self.library, multi_modal=self.multi_modal, use_mlflow=self.use_mlflow)
+            library=self.library, multi_modal=self.multi_modal, default_model= self.default_model , use_mlflow=self.use_mlflow)
         # multi_modal_runner = MultiModalRunner(
         #     library=self.library, model_func=model_func, batch_size=batch_size,  epochs=epochs)
         # multi_modal = TFMultiModal(

{ddi_fw-0.0.198.dist-info → ddi_fw-0.0.200.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.198
+Version: 0.0.200
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.198.dist-info → ddi_fw-0.0.200.dist-info}/RECORD

@@ -1,9 +1,9 @@
 ddi_fw/datasets/__init__.py,sha256=_I3iDHARwzmg7_EL5XKtB_TgG1yAkLSOVTujLL9Wz9Q,280
-ddi_fw/datasets/core.py,sha256=yfnJwyF9oV2RUErFSAKSyxQQeL1tmLiq7SfADhn1Cgk,16379
+ddi_fw/datasets/core.py,sha256=vxatq85GDrvp_cdHQsCHHE-PBneGYRSMb9VyRHlBce8,16424
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
-ddi_fw/datasets/ddi_mdl/base.py,sha256=8WFc0iLT5PF6IOUStqKVIKR74D8WBuwXm_uMiV4OFsk,10324
+ddi_fw/datasets/ddi_mdl/base.py,sha256=lmHKlpaxyH3HZDcC_kOuvDjuzb_rtOvdD7CkwA00w7k,10399
 ddi_fw/datasets/ddi_mdl/debug.log,sha256=eWz05j8RFqZuHFDTCF7Rck5w4rvtTanFN21iZsgxO7Y,115
 ddi_fw/datasets/ddi_mdl/readme.md,sha256=WC6lpmsEKvIISnZqENY7TWtzCQr98HPpE3oRsBl8pIw,625
 ddi_fw/datasets/ddi_mdl/data/event.db,sha256=cmlSsf9MYjRzqR-mw3cUDnTnfT6FkpOG2yCl2mMwwew,30580736
@@ -74,7 +74,7 @@ ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSz
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=tIxiW0g6q1VsmDYVXR_ovvHQR3SCir8g2bKxx_CrS7s,221
 ddi_fw/ml/evaluation_helper.py,sha256=2-7CLSgGTqLEk4HkgCVIOt-GxfLAn6SBozJghAtHb5M,11581
-ddi_fw/ml/ml_helper.py,sha256=-pgnLhuPBp60IZxAvS11oDYggrgQY3GRHkAwQVWM9XI,6943
+ddi_fw/ml/ml_helper.py,sha256=IFLpMa5oxwnhRXHNNYCITyaXPp7Rrid9bpsjp_Ss5SQ,7725
 ddi_fw/ml/model_wrapper.py,sha256=kabPXuo7S8tGkp9a00V04n4rXDmv7dD8wYGMjotISRc,1050
 ddi_fw/ml/pytorch_wrapper.py,sha256=pe6UsjP2XeTgLxDnIUiodoyhJTGCxV27wD4Cjxysu2Q,8553
 ddi_fw/ml/tensorflow_wrapper.py,sha256=lNJvg3odqMKmILecOMdcOCAOrwzWZDzxB0DWGcYWsPg,12952
@@ -83,9 +83,9 @@ ddi_fw/ner/mmlrestclient.py,sha256=NZta7m2Qm6I_qtVguMZhqtAUjVBmmXn0-TMnsNp0jpg,6
 ddi_fw/ner/ner.py,sha256=FHyyX53Xwpdw8Hec261dyN88yD7Z9LmJua2mIrQLguI,17967
 ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,212
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
-ddi_fw/pipeline/multi_pipeline.py,sha256=fYyvwIOscUahjXd3QO5RSFrp1LliGR7RzOZyAXrXXz4,5637
+ddi_fw/pipeline/multi_pipeline.py,sha256=L0Apy2Z909GnR8KMX_q7-kluZmIG3CT0yfMuL22poaQ,5737
 ddi_fw/pipeline/ner_pipeline.py,sha256=Bp6BA6nozfWFaMHH6jKlzesnCGO6qiMkzdGy_ed6nh0,5947
-ddi_fw/pipeline/pipeline.py,sha256=CUHuy1nNgGD-eUcLnWFXcmSoTGssmg4ZFRAY1Cufey0,9047
+ddi_fw/pipeline/pipeline.py,sha256=s6fYtgAyWcap4oC6bAKZATju8BnasRtLopnM2Blt2Fo,9125
 ddi_fw/utils/__init__.py,sha256=HC32XkYQTYH_9vt0eX6tqQngEFG-R70hGrYkT-BcHCk,519
 ddi_fw/utils/categorical_data_encoding_checker.py,sha256=gzb_vUDBrCMUhBxY1fBYTe8hmK72p0_uw3DTga8cqP8,1580
 ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
@@ -99,7 +99,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=EBf-XAiwQwr68az91erEYNegfeqssBR29kVgrliIyac,4765
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.198.dist-info/METADATA,sha256=z3otymNU3l4737h3tkMaP0UMhZdLBtzS4ELP4wIcVt8,2542
-ddi_fw-0.0.198.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddi_fw-0.0.198.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.198.dist-info/RECORD,,
+ddi_fw-0.0.200.dist-info/METADATA,sha256=IkLebplT9KXZhVHSinJkay9r_ZmHs7n-jVeLpKW_kC4,2542
+ddi_fw-0.0.200.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddi_fw-0.0.200.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.200.dist-info/RECORD,,