ddi-fw 0.0.190__py3-none-any.whl → 0.0.192__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -73,6 +73,7 @@ class BaseDataset(BaseModel):
     class Config:
         arbitrary_types_allowed = True
 
+    # TODO columns yoksa tüm feature'lar alınıyor, bu pipeline'da nasıl yapılacak?
     def produce_inputs(self):
         items = []
         if self.X_train is None or self.X_test is None:
@@ -80,7 +81,10 @@ class BaseDataset(BaseModel):
         y_train_label, y_test_label = np.array(
             self.y_train), np.array(self.y_test)
 
-        if self.columns is None or len(self.columns) == 0:
+        if self.columns is None or len(self.columns) == 0 or len(self.columns) == 1:
+            # If no columns or only one column are provided, do not change the data
+            # and use the entire dataset as a single input.
+            column = self.columns[0] if self.columns else 'default'
             items.append([f'default', np.nan_to_num(self.X_train),
                           y_train_label, np.nan_to_num(self.X_test), y_test_label])
         else:
@@ -127,15 +131,15 @@ class BaseDataset(BaseModel):
         Load the dataset. If X_train, y_train, X_test, and y_test are already provided,
         skip deriving them. Otherwise, derive them from the dataframe and indices.
         """
-        if self.X_train  and self.y_train and self.X_test and self.y_test :
+        self.prep()
+
+        if self.X_train is not None or self.y_train is not None or self.X_test is not None or self.y_test is not None:
             # Data is already provided, no need to calculate
             logging.info(
                 "X_train, y_train, X_test, and y_test are already provided. Skipping calculation.")
             return
             # return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
 
-        self.prep()
-
         if self.index_path is None:
             raise Exception(
                 "There is no index path. Please call split_dataset or provide indices.")
@@ -156,7 +160,7 @@ class BaseDataset(BaseModel):
         y_train = train[self.class_column]
         X_test = test.drop(self.class_column, axis=1)
         y_test = test[self.class_column]
-        
+
         self.X_train = np.array(X_train)
         # self.y_train = np.array(y_train)
         self.y_train = np.array(y_train.tolist())
@@ -170,7 +174,6 @@ class BaseDataset(BaseModel):
         self.val_idx_arr = val_idx_arr
 
         # Dataframe to numpy array conversion
-       
 
         # return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
 
@@ -226,14 +229,13 @@ class BaseDataset(BaseModel):
             X=X, y=y)
         self.X_train = np.array(X_train)
         self.X_test = np.array(X_test)
-        self.y_train =  np.array(y_train.tolist())
+        self.y_train = np.array(y_train.tolist())
         self.y_test = np.array(y_test.tolist())
         self.train_indexes = X_train.index
         self.test_indexes = X_test.index
         self.train_idx_arr = train_idx_arr
         self.val_idx_arr = val_idx_arr
 
-       
         if save_indexes:
             # train_pairs = [row['id1'].join(',').row['id2'] for index, row in X_train.iterrows()]
             self.__save_indexes__(

ddi_fw/ml/ml_helper.py

@@ -57,42 +57,53 @@ class MultiModalRunner:
             raise ValueError(
                 "Unsupported library type. Choose 'tensorflow' or 'pytorch'.")
 
+    # TODO check single_results, 1d,2d ...
     def __predict(self, single_results):
         item_dict = {t[0]: t for t in self.items}
         print("multi_modal")
         print(self.multi_modal)
         print(item_dict.keys())
 
-        for m in self.multi_modal:
-            name = m.get('name')
-            input_type = m.get('input_type')
-            input = m.get('input')
-            inputs = m.get('inputs')
-            model_type = get_import(m.get("model_type"))
-            kwargs = m.get('params')
-            T = self.__create_model(self.library)
-            single_modal = T(self.date, name, model_type,
-                             use_mlflow=self.use_mlflow,  **kwargs)
-            if input_type == '1D':
-                item = item_dict[input]
+        if self.multi_modal:
+            for m in self.multi_modal:
+                name = m.get('name')
+                input_type = m.get('input_type')
+                input = m.get('input')
+                inputs = m.get('inputs')
+                model_type = get_import(m.get("model_type"))
+                kwargs = m.get('params')
+                T = self.__create_model(self.library)
+                single_modal = T(self.date, name, model_type,
+                                use_mlflow=self.use_mlflow,  **kwargs)
+                
+                if input is not None and inputs is not None:
+                    raise Exception("input and inputs should not be used together")
+                
+                if input:
+                    item = item_dict[input]
+                    single_modal.set_data(
+                        self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
+                elif inputs:
+                    # check keys
+                    filtered_dict = {k: item_dict[k]
+                                    for k in inputs if k in item_dict}
+                    print(filtered_dict.keys())
+                    first_input = next(iter(filtered_dict.values()))
+                    train_data_list = [f[1] for f in filtered_dict.values()]
+                    test_data_list = [f[3] for f in filtered_dict.values()]
+                    train_data = np.stack(train_data_list, axis=1)
+                    test_data = np.stack(test_data_list, axis=1)
+                    train_label = first_input[2]
+                    test_label = first_input[4]
+                    single_modal.set_data(
+                        self.train_idx_arr, self.val_idx_arr, train_data, train_label, test_data, test_label)
+                else:
+                    raise Exception("check configurations")
+            else: # TODO default model maybe?
+                item = self.items[0]
                 single_modal.set_data(
-                    self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
-            elif input_type == '2D':
-                # check keys
-                filtered_dict = {k: item_dict[k]
-                                 for k in inputs if k in item_dict}
-                print(filtered_dict.keys())
-                first_input = next(iter(filtered_dict.values()))
-                train_data_list = [f[1] for f in filtered_dict.values()]
-                test_data_list = [f[3] for f in filtered_dict.values()]
-                train_data = np.stack(train_data_list, axis=1)
-                test_data = np.stack(test_data_list, axis=1)
-                train_label = first_input[2]
-                test_label = first_input[4]
-                single_modal.set_data(
-                    self.train_idx_arr, self.val_idx_arr, train_data, train_label, test_data, test_label)
-            else:
-                raise Exception("check configurations")
+                        self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
+
             logs, metrics, prediction = single_modal.fit_and_evaluate()
             self.result.add_metric(name, metrics)
             single_results[name] = prediction

ddi_fw/pipeline/multi_pipeline.py

@@ -50,6 +50,7 @@ class MultiPipeline():
         type = config.get("type")
         library = config.get("library")
 
+        use_mlflow = config.get("use_mlflow")
         experiment_name = config.get("experiment_name")
         experiment_description = config.get("experiment_description")
         experiment_tags = config.get("experiment_tags")
@@ -84,6 +85,7 @@ class MultiPipeline():
         if type == "general":
             pipeline = Pipeline(
                 library=library,
+                use_mlflow=use_mlflow,
                 experiment_name=experiment_name,
                 experiment_description=experiment_description,
                 experiment_tags=experiment_tags,

ddi_fw/pipeline/pipeline.py

@@ -37,10 +37,15 @@ class Pipeline(BaseModel):
     model: Optional[Any] = None
     multi_modal:  Optional[Any] = None
     use_mlflow: bool = False
+    _dataset: BaseDataset = []
     _items: List = []
     _train_idx_arr: List | None = []
     _val_idx_arr: List | None = []
 
+    @property
+    def dataset(self) -> BaseDataset:
+        return self._dataset
+    
     @property
     def items(self) -> List:
         return self._items
@@ -168,7 +173,10 @@ class Pipeline(BaseModel):
             dataset = self.dataset_type(**kwargs)
 
         # X_train, X_test, y_train, y_test, train_indexes, test_indexes, train_idx_arr, val_idx_arr = dataset.load()
+        
         dataset.load()
+        self._dataset = dataset
+        
         dataframe = dataset.dataframe
 
         # Check if any of the arrays are None or empty
@@ -180,7 +188,7 @@ class Pipeline(BaseModel):
         # Check if the dataframe is None or empty
         is_dataframe_valid = dataframe is not None and not dataframe.empty
 
-        if not (is_data_valid and is_dataframe_valid):
+        if not (is_data_valid or is_dataframe_valid):
             raise ValueError("The dataset is not loaded")
 
         # column name, train data, train label, test data, test label

{ddi_fw-0.0.190.dist-info → ddi_fw-0.0.192.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.190
+Version: 0.0.192
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.190.dist-info → ddi_fw-0.0.192.dist-info}/RECORD

@@ -1,5 +1,5 @@
 ddi_fw/datasets/__init__.py,sha256=_I3iDHARwzmg7_EL5XKtB_TgG1yAkLSOVTujLL9Wz9Q,280
-ddi_fw/datasets/core.py,sha256=WWWd5SGHVUpJn-IJF1p1PScSWpb7VfQdcMTroufkgUk,10734
+ddi_fw/datasets/core.py,sha256=4705a94kKBueyWFXRJ3cnivAGKjrR89uBBKpxtMozOM,11080
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
@@ -74,7 +74,7 @@ ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSz
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=tIxiW0g6q1VsmDYVXR_ovvHQR3SCir8g2bKxx_CrS7s,221
 ddi_fw/ml/evaluation_helper.py,sha256=2-7CLSgGTqLEk4HkgCVIOt-GxfLAn6SBozJghAtHb5M,11581
-ddi_fw/ml/ml_helper.py,sha256=xbIg0fAJeJuB7rlgUMzCFhQ4WLBXS35x5N5gCcs6-so,6367
+ddi_fw/ml/ml_helper.py,sha256=-pgnLhuPBp60IZxAvS11oDYggrgQY3GRHkAwQVWM9XI,6943
 ddi_fw/ml/model_wrapper.py,sha256=kabPXuo7S8tGkp9a00V04n4rXDmv7dD8wYGMjotISRc,1050
 ddi_fw/ml/pytorch_wrapper.py,sha256=pe6UsjP2XeTgLxDnIUiodoyhJTGCxV27wD4Cjxysu2Q,8553
 ddi_fw/ml/tensorflow_wrapper.py,sha256=lNJvg3odqMKmILecOMdcOCAOrwzWZDzxB0DWGcYWsPg,12952
@@ -83,9 +83,9 @@ ddi_fw/ner/mmlrestclient.py,sha256=NZta7m2Qm6I_qtVguMZhqtAUjVBmmXn0-TMnsNp0jpg,6
 ddi_fw/ner/ner.py,sha256=FHyyX53Xwpdw8Hec261dyN88yD7Z9LmJua2mIrQLguI,17967
 ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,212
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
-ddi_fw/pipeline/multi_pipeline.py,sha256=NfcH4Ze5U-JRiH3lrxEDWj-VPxYQYtp7tq6bLCImBzs,5550
+ddi_fw/pipeline/multi_pipeline.py,sha256=fYyvwIOscUahjXd3QO5RSFrp1LliGR7RzOZyAXrXXz4,5637
 ddi_fw/pipeline/ner_pipeline.py,sha256=Bp6BA6nozfWFaMHH6jKlzesnCGO6qiMkzdGy_ed6nh0,5947
-ddi_fw/pipeline/pipeline.py,sha256=dCXZuXOlW74ZO0e_OhS9OX0dqI9abj7CQz_lkKrDIWY,9787
+ddi_fw/pipeline/pipeline.py,sha256=GMMauyp0GvdaQLyQ5dPBffDDxFK28hdDtPUzdFX9-Yk,9961
 ddi_fw/utils/__init__.py,sha256=HC32XkYQTYH_9vt0eX6tqQngEFG-R70hGrYkT-BcHCk,519
 ddi_fw/utils/categorical_data_encoding_checker.py,sha256=gzb_vUDBrCMUhBxY1fBYTe8hmK72p0_uw3DTga8cqP8,1580
 ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
@@ -99,7 +99,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=Z1A_DOBqDFPqLN4YB-3oYlOQWJK-X6Oes6UFjpzR47Q,4760
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.190.dist-info/METADATA,sha256=kTsCriMdtxc5BZWLUlThWAMPoYSWjPRkLFZntodwCwM,2542
-ddi_fw-0.0.190.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddi_fw-0.0.190.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.190.dist-info/RECORD,,
+ddi_fw-0.0.192.dist-info/METADATA,sha256=MhOs-typXfDNCIRm8_9NheEkyNYmnmQbArQGstSOAas,2542
+ddi_fw-0.0.192.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddi_fw-0.0.192.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.192.dist-info/RECORD,,