ddi-fw 0.0.189__py3-none-any.whl → 0.0.191__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -73,6 +73,7 @@ class BaseDataset(BaseModel):
     class Config:
         arbitrary_types_allowed = True
 
+    # TODO columns yoksa tüm feature'lar alınıyor, bu pipeline'da nasıl yapılacak?
     def produce_inputs(self):
         items = []
         if self.X_train is None or self.X_test is None:
@@ -127,15 +128,15 @@ class BaseDataset(BaseModel):
         Load the dataset. If X_train, y_train, X_test, and y_test are already provided,
         skip deriving them. Otherwise, derive them from the dataframe and indices.
         """
-        if self.X_train  and self.y_train and self.X_test and self.y_test :
+        self.prep()
+
+        if self.X_train is not None or self.y_train is not None or self.X_test is not None or self.y_test is not None:
             # Data is already provided, no need to calculate
             logging.info(
                 "X_train, y_train, X_test, and y_test are already provided. Skipping calculation.")
             return
             # return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
 
-        self.prep()
-
         if self.index_path is None:
             raise Exception(
                 "There is no index path. Please call split_dataset or provide indices.")
@@ -156,7 +157,7 @@ class BaseDataset(BaseModel):
         y_train = train[self.class_column]
         X_test = test.drop(self.class_column, axis=1)
         y_test = test[self.class_column]
-        
+
         self.X_train = np.array(X_train)
         # self.y_train = np.array(y_train)
         self.y_train = np.array(y_train.tolist())
@@ -170,7 +171,6 @@ class BaseDataset(BaseModel):
         self.val_idx_arr = val_idx_arr
 
         # Dataframe to numpy array conversion
-       
 
         # return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
 
@@ -226,14 +226,13 @@ class BaseDataset(BaseModel):
             X=X, y=y)
         self.X_train = np.array(X_train)
         self.X_test = np.array(X_test)
-        self.y_train =  np.array(y_train.tolist())
+        self.y_train = np.array(y_train.tolist())
         self.y_test = np.array(y_test.tolist())
         self.train_indexes = X_train.index
         self.test_indexes = X_test.index
         self.train_idx_arr = train_idx_arr
         self.val_idx_arr = val_idx_arr
 
-       
         if save_indexes:
             # train_pairs = [row['id1'].join(',').row['id2'] for index, row in X_train.iterrows()]
             self.__save_indexes__(

ddi_fw/ml/ml_helper.py

@@ -57,42 +57,53 @@ class MultiModalRunner:
             raise ValueError(
                 "Unsupported library type. Choose 'tensorflow' or 'pytorch'.")
 
+    # TODO check single_results, 1d,2d ...
     def __predict(self, single_results):
         item_dict = {t[0]: t for t in self.items}
         print("multi_modal")
         print(self.multi_modal)
         print(item_dict.keys())
 
-        for m in self.multi_modal:
-            name = m.get('name')
-            input_type = m.get('input_type')
-            input = m.get('input')
-            inputs = m.get('inputs')
-            model_type = get_import(m.get("model_type"))
-            kwargs = m.get('params')
-            T = self.__create_model(self.library)
-            single_modal = T(self.date, name, model_type,
-                             use_mlflow=self.use_mlflow,  **kwargs)
-            if input_type == '1D':
-                item = item_dict[input]
-                single_modal.set_data(
-                    self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
-            elif input_type == '2D':
-                # check keys
-                filtered_dict = {k: item_dict[k]
-                                 for k in inputs if k in item_dict}
-                print(filtered_dict.keys())
-                first_input = next(iter(filtered_dict.values()))
-                train_data_list = [f[1] for f in filtered_dict.values()]
-                test_data_list = [f[3] for f in filtered_dict.values()]
-                train_data = np.stack(train_data_list, axis=1)
-                test_data = np.stack(test_data_list, axis=1)
-                train_label = first_input[2]
-                test_label = first_input[4]
-                single_modal.set_data(
-                    self.train_idx_arr, self.val_idx_arr, train_data, train_label, test_data, test_label)
+        if self.multi_modal:
+            for m in self.multi_modal:
+                name = m.get('name')
+                input_type = m.get('input_type')
+                input = m.get('input')
+                inputs = m.get('inputs')
+                model_type = get_import(m.get("model_type"))
+                kwargs = m.get('params')
+                T = self.__create_model(self.library)
+                single_modal = T(self.date, name, model_type,
+                                use_mlflow=self.use_mlflow,  **kwargs)
+                
+                if input is not None and inputs is not None:
+                    raise Exception("input and inputs should not be used together")
+                
+                if input_type == '1D':
+                    item = item_dict[input]
+                    single_modal.set_data(
+                        self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
+                elif input_type == '2D':
+                    # check keys
+                    filtered_dict = {k: item_dict[k]
+                                    for k in inputs if k in item_dict}
+                    print(filtered_dict.keys())
+                    first_input = next(iter(filtered_dict.values()))
+                    train_data_list = [f[1] for f in filtered_dict.values()]
+                    test_data_list = [f[3] for f in filtered_dict.values()]
+                    train_data = np.stack(train_data_list, axis=1)
+                    test_data = np.stack(test_data_list, axis=1)
+                    train_label = first_input[2]
+                    test_label = first_input[4]
+                    single_modal.set_data(
+                        self.train_idx_arr, self.val_idx_arr, train_data, train_label, test_data, test_label)
+                else:
+                    raise Exception("check configurations")
             else:
-                raise Exception("check configurations")
+                item = self.items[0]
+                single_modal.set_data(
+                        self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
+
             logs, metrics, prediction = single_modal.fit_and_evaluate()
             self.result.add_metric(name, metrics)
             single_results[name] = prediction

ddi_fw/pipeline/multi_pipeline.py

@@ -50,6 +50,7 @@ class MultiPipeline():
         type = config.get("type")
         library = config.get("library")
 
+        use_mlflow = config.get("use_mlflow")
         experiment_name = config.get("experiment_name")
         experiment_description = config.get("experiment_description")
         experiment_tags = config.get("experiment_tags")
@@ -84,6 +85,7 @@ class MultiPipeline():
         if type == "general":
             pipeline = Pipeline(
                 library=library,
+                use_mlflow=use_mlflow,
                 experiment_name=experiment_name,
                 experiment_description=experiment_description,
                 experiment_tags=experiment_tags,

ddi_fw/pipeline/pipeline.py

@@ -37,10 +37,15 @@ class Pipeline(BaseModel):
     model: Optional[Any] = None
     multi_modal:  Optional[Any] = None
     use_mlflow: bool = False
+    _dataset: BaseDataset = []
     _items: List = []
     _train_idx_arr: List | None = []
     _val_idx_arr: List | None = []
 
+    @property
+    def dataset(self) -> BaseDataset:
+        return self._dataset
+    
     @property
     def items(self) -> List:
         return self._items
@@ -168,7 +173,10 @@ class Pipeline(BaseModel):
             dataset = self.dataset_type(**kwargs)
 
         # X_train, X_test, y_train, y_test, train_indexes, test_indexes, train_idx_arr, val_idx_arr = dataset.load()
+        
         dataset.load()
+        self._dataset = dataset
+        
         dataframe = dataset.dataframe
 
         # Check if any of the arrays are None or empty
@@ -180,7 +188,7 @@ class Pipeline(BaseModel):
         # Check if the dataframe is None or empty
         is_dataframe_valid = dataframe is not None and not dataframe.empty
 
-        if not (is_data_valid and is_dataframe_valid):
+        if not (is_data_valid or is_dataframe_valid):
             raise ValueError("The dataset is not loaded")
 
         # column name, train data, train label, test data, test label

ddi_fw/utils/__init__.py

@@ -4,4 +4,5 @@ from .py7zr_helper import Py7ZipHelper
 from .enums import UMLSCodeTypes, DrugBankTextDataTypes
 from .package_helper import get_import
 from .kaggle import create_kaggle_dataset
-from .categorical_data_encoding_checker import is_one_hot_encoded, is_binary_encoded, is_binary_vector,is_label_encoded
+from .categorical_data_encoding_checker import is_one_hot_encoded, is_binary_encoded, is_binary_vector,is_label_encoded
+from .numpy_utils import adjust_array_dims

ddi_fw/utils/numpy_utils.py

@@ -0,0 +1,27 @@
+import numpy as np
+
+def adjust_array_dims(arr, final_ndim=2):
+    # Add axes if array has fewer dimensions than final_ndim
+    while arr.ndim < final_ndim:
+        arr = arr[:, np.newaxis]  # Add a new axis
+        
+    # Drop axes if array has more dimensions than final_ndim
+    while arr.ndim > final_ndim:
+        arr = np.squeeze(arr, axis=-1)  # Remove the last axis
+    
+    return arr
+
+
+
+# # Example usage
+# arr_1d = np.array([1, 2, 3, 4, 5])
+
+# # Convert to a 3D array (iteratively adds axes)
+# arr_3d = adjust_array_dims(arr_1d, final_ndim=3)
+# print(arr_3d)
+# print("Shape of arr_3d:", arr_3d.shape)
+
+# # Convert to a 2D array (iteratively drops axes)
+# arr_2d = adjust_array_dims(arr_3d, final_ndim=2)
+# print(arr_2d)
+# print("Shape of arr_2d:", arr_2d.shape)

{ddi_fw-0.0.189.dist-info → ddi_fw-0.0.191.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.189
+Version: 0.0.191
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.189.dist-info → ddi_fw-0.0.191.dist-info}/RECORD

@@ -1,5 +1,5 @@
 ddi_fw/datasets/__init__.py,sha256=_I3iDHARwzmg7_EL5XKtB_TgG1yAkLSOVTujLL9Wz9Q,280
-ddi_fw/datasets/core.py,sha256=WWWd5SGHVUpJn-IJF1p1PScSWpb7VfQdcMTroufkgUk,10734
+ddi_fw/datasets/core.py,sha256=IaEk4T7f590rAAAG7Nc45mofeutX85mNxhikFs7mzpE,10839
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
@@ -74,7 +74,7 @@ ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSz
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=tIxiW0g6q1VsmDYVXR_ovvHQR3SCir8g2bKxx_CrS7s,221
 ddi_fw/ml/evaluation_helper.py,sha256=2-7CLSgGTqLEk4HkgCVIOt-GxfLAn6SBozJghAtHb5M,11581
-ddi_fw/ml/ml_helper.py,sha256=xbIg0fAJeJuB7rlgUMzCFhQ4WLBXS35x5N5gCcs6-so,6367
+ddi_fw/ml/ml_helper.py,sha256=ENjdpu6stRxGxqhFweZLIglCnLREtMh6ypwanf0qMGc,6940
 ddi_fw/ml/model_wrapper.py,sha256=kabPXuo7S8tGkp9a00V04n4rXDmv7dD8wYGMjotISRc,1050
 ddi_fw/ml/pytorch_wrapper.py,sha256=pe6UsjP2XeTgLxDnIUiodoyhJTGCxV27wD4Cjxysu2Q,8553
 ddi_fw/ml/tensorflow_wrapper.py,sha256=lNJvg3odqMKmILecOMdcOCAOrwzWZDzxB0DWGcYWsPg,12952
@@ -83,14 +83,15 @@ ddi_fw/ner/mmlrestclient.py,sha256=NZta7m2Qm6I_qtVguMZhqtAUjVBmmXn0-TMnsNp0jpg,6
 ddi_fw/ner/ner.py,sha256=FHyyX53Xwpdw8Hec261dyN88yD7Z9LmJua2mIrQLguI,17967
 ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,212
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
-ddi_fw/pipeline/multi_pipeline.py,sha256=NfcH4Ze5U-JRiH3lrxEDWj-VPxYQYtp7tq6bLCImBzs,5550
+ddi_fw/pipeline/multi_pipeline.py,sha256=fYyvwIOscUahjXd3QO5RSFrp1LliGR7RzOZyAXrXXz4,5637
 ddi_fw/pipeline/ner_pipeline.py,sha256=Bp6BA6nozfWFaMHH6jKlzesnCGO6qiMkzdGy_ed6nh0,5947
-ddi_fw/pipeline/pipeline.py,sha256=dCXZuXOlW74ZO0e_OhS9OX0dqI9abj7CQz_lkKrDIWY,9787
-ddi_fw/utils/__init__.py,sha256=bqIC0YjbD0YSHtO0nWUkRs4w5nu7qBV0yU72sRzwCj8,475
+ddi_fw/pipeline/pipeline.py,sha256=GMMauyp0GvdaQLyQ5dPBffDDxFK28hdDtPUzdFX9-Yk,9961
+ddi_fw/utils/__init__.py,sha256=HC32XkYQTYH_9vt0eX6tqQngEFG-R70hGrYkT-BcHCk,519
 ddi_fw/utils/categorical_data_encoding_checker.py,sha256=gzb_vUDBrCMUhBxY1fBYTe8hmK72p0_uw3DTga8cqP8,1580
 ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
 ddi_fw/utils/json_helper.py,sha256=BVU6wmJgdXPxyqLPu3Ck_9Es5RrP1PDanKvE-OSj1D4,571
 ddi_fw/utils/kaggle.py,sha256=wKRJ18KpQ6P-CubpZklEgsDtyFpR9RUL1_HyyF6ttEE,2425
+ddi_fw/utils/numpy_utils.py,sha256=gd1WNq5NpWD2MBEMTtFuS5I0h8B6FAUNcq6BVOlxdhY,797
 ddi_fw/utils/package_helper.py,sha256=erl8_onmhK-41zQoaED2qyDUV9GQxmT9sdoyRp9_q5I,1056
 ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
 ddi_fw/utils/utils.py,sha256=szwnxMTDRrZoeNRyDuf3aCbtzriwtaRk4mHSH3asLdA,4301
@@ -98,7 +99,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=Z1A_DOBqDFPqLN4YB-3oYlOQWJK-X6Oes6UFjpzR47Q,4760
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.189.dist-info/METADATA,sha256=y3hik68p2UsqKr7ur8R2ix8TyPfGsveDbKWboSaRkzA,2542
-ddi_fw-0.0.189.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddi_fw-0.0.189.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.189.dist-info/RECORD,,
+ddi_fw-0.0.191.dist-info/METADATA,sha256=xrlNEz8W_iIfKVZidKR8kNa_WtrDr4EnVQwbtex_sbQ,2542
+ddi_fw-0.0.191.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddi_fw-0.0.191.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.191.dist-info/RECORD,,