ddi-fw 0.0.185__py3-none-any.whl → 0.0.187__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -60,10 +60,10 @@ class BaseDataset(BaseModel):
     dataset_splitter_type: Type[DatasetSplitter]
     class_column: str = 'class'
     dataframe: Optional[pd.DataFrame] = None
-    X_train: Optional[pd.DataFrame | np.ndarray] = None
-    X_test: Optional[pd.DataFrame | np.ndarray] = None
-    y_train: Optional[pd.Series | np.ndarray] = None
-    y_test: Optional[pd.Series | np.ndarray] = None
+    X_train: Optional[np.ndarray] = None
+    X_test: Optional[np.ndarray] = None
+    y_train: Optional[np.ndarray] = None
+    y_test: Optional[np.ndarray] = None
     train_indexes: Optional[pd.Index] = None
     test_indexes: Optional[pd.Index] = None
     train_idx_arr: Optional[List[np.ndarray]] = None
@@ -81,7 +81,7 @@ class BaseDataset(BaseModel):
             self.y_train), np.array(self.y_test)
 
         if self.columns is None or len(self.columns) == 0:
-            items.append([f'defaukt', np.nan_to_num(self.X_train),
+            items.append([f'default', np.nan_to_num(self.X_train),
                           y_train_label, np.nan_to_num(self.X_test), y_test_label])
         else:
             for index, column in enumerate(self.columns):
@@ -127,11 +127,12 @@ class BaseDataset(BaseModel):
         Load the dataset. If X_train, y_train, X_test, and y_test are already provided,
         skip deriving them. Otherwise, derive them from the dataframe and indices.
         """
-        if self.X_train is not None and self.y_train is not None and self.X_test is not None and self.y_test is not None:
+        if self.X_train  and self.y_train and self.X_test and self.y_test :
             # Data is already provided, no need to calculate
             logging.info(
                 "X_train, y_train, X_test, and y_test are already provided. Skipping calculation.")
-            return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
+            return
+            # return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
 
         self.prep()
 
@@ -150,24 +151,26 @@ class BaseDataset(BaseModel):
 
         train = self.dataframe[self.dataframe.index.isin(train_idx_all)]
         test = self.dataframe[self.dataframe.index.isin(test_idx_all)]
+        X_train = train.drop(self.class_column, axis=1)
+        X_train = train.drop(self.class_column, axis=1)
+        y_train = train[self.class_column]
+        X_test = test.drop(self.class_column, axis=1)
+        y_test = test[self.class_column]
+        
+        self.X_train = np.array(X_train)
+        self.y_train = np.array(y_train)
+        self.X_test = np.array(X_test)
+        self.y_test = np.array(y_test)
 
-        self.X_train = train.drop(self.class_column, axis=1)
-        self.y_train = train[self.class_column]
-        self.X_test = test.drop(self.class_column, axis=1)
-        self.y_test = test[self.class_column]
-
-        self.train_indexes = self.X_train.index
-        self.test_indexes = self.X_test.index
+        self.train_indexes = X_train.index
+        self.test_indexes = X_test.index
         self.train_idx_arr = train_idx_arr
         self.val_idx_arr = val_idx_arr
 
         # Dataframe to numpy array conversion
-        self.X_train = np.array(self.X_train)
-        self.y_train = np.array(self.y_train)
-        self.X_test = np.array(self.X_test)
-        self.y_test = np.array(self.y_test)
+       
 
-        return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
+        # return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
 
     def __get_indexes__(self, path):
         train_index_path = path+'/train_indexes.txt'
@@ -219,21 +222,16 @@ class BaseDataset(BaseModel):
 
         X_train, X_test, y_train, y_test, X_train.index, X_test.index, train_idx_arr, val_idx_arr = self.dataset_splitter.split(
             X=X, y=y)
-        self.X_train = X_train
-        self.X_test = X_test
-        self.y_train = y_train
-        self.y_test = y_test
+        self.X_train = np.array(self.X_train)
+        self.X_test = np.array(self.X_test)
+        self.y_train =  np.array(y_train.tolist())
+        self.y_test = np.array(y_test.tolist())
         self.train_indexes = X_train.index
         self.test_indexes = X_test.index
         self.train_idx_arr = train_idx_arr
         self.val_idx_arr = val_idx_arr
 
-        # Dataframe to numpy array conversion
-        self.X_train = np.array(self.X_train)
-        self.y_train = np.array(self.y_train.tolist())
-        self.X_test = np.array(self.X_test)
-        self.y_test = np.array(self.y_test.tolist())
-
+       
         if save_indexes:
             # train_pairs = [row['id1'].join(',').row['id2'] for index, row in X_train.iterrows()]
             self.__save_indexes__(

ddi_fw/ml/ml_helper.py

@@ -32,7 +32,7 @@ import ddi_fw.utils as utils
 
 class MultiModalRunner:
     # todo model related parameters to config
-    def __init__(self, library, multi_modal, use_mlflow=True):
+    def __init__(self, library, multi_modal, use_mlflow=False):
         self.library = library
         self.multi_modal = multi_modal
         self.use_mlflow = use_mlflow
@@ -111,44 +111,6 @@ class MultiModalRunner:
         if self.use_mlflow:
             with mlflow.start_run(run_name=self.prefix, description="***") as run:
                 self.__predict(single_results)
-                # self.level_0_run_id = run.info.run_id
-                # item_dict = {t[0]: t for t in self.items}
-                # print("multi_modal")
-                # print(self.multi_modal)
-                # print(item_dict.keys())
-
-                # for m in self.multi_modal:
-                #     name = m.get('name')
-                #     input_type = m.get('input_type')
-                #     input = m.get('input')
-                #     inputs = m.get('inputs')
-                #     model_type = get_import(m.get("model_type"))
-                #     kwargs = m.get('params')
-                #     T = self.__create_model(self.library)
-                #     single_modal = T(self.date, name, model_type, **kwargs)
-                #     if input_type == '1D':
-                #         item = item_dict[input]
-                #         single_modal.set_data(
-                #             self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
-                #     elif input_type == '2D':
-                #         # check keys
-                #         filtered_dict = {k: item_dict[k]
-                #                          for k in inputs if k in item_dict}
-                #         print(filtered_dict.keys())
-                #         first_input = next(iter(filtered_dict.values()))
-                #         train_data_list = [f[1] for f in filtered_dict.values()]
-                #         test_data_list = [f[3] for f in filtered_dict.values()]
-                #         train_data = np.stack(train_data_list, axis=1)
-                #         test_data = np.stack(test_data_list, axis=1)
-                #         train_label = first_input[2]
-                #         test_label = first_input[4]
-                #         single_modal.set_data(
-                #             self.train_idx_arr, self.val_idx_arr, train_data, train_label, test_data, test_label)
-                #     else:
-                #         raise Exception("check configurations")
-                #     logs, metrics, prediction = single_modal.fit_and_evaluate()
-                #     self.result.add_metric(name, metrics)
-                #     single_results[name] = prediction
         else:
             self.__predict(single_results)
         if combinations:

ddi_fw/ml/tensorflow_wrapper.py

@@ -120,7 +120,7 @@ class TFModelWrapper(ModelWrapper):
         print(self.train_data.shape)
         models = {}
         models_val_acc = {}
-        if self.train_idx_arr is not None and self.val_idx_arr is not None:
+        if self.train_idx_arr and self.val_idx_arr:
             for i, (train_idx, val_idx) in enumerate(zip(self.train_idx_arr, self.val_idx_arr)):
                 print(f"Validation {i}")
 
@@ -155,7 +155,8 @@ class TFModelWrapper(ModelWrapper):
                     self.train_data, self.train_label, None, None)
                 models[self.descriptor] = model
                 models_val_acc[self.descriptor] = checkpoint.best
-
+        if models_val_acc == {}:
+            return model, None
         best_model_key = max(models_val_acc, key=lambda k: models_val_acc[k])
         # best_model_key = max(models_val_acc, key=models_val_acc.get)
         best_model = models[best_model_key]

ddi_fw/pipeline/ner_pipeline.py

@@ -106,7 +106,7 @@ class NerParameterSearch:
                     **kwargs)
 
                 # train_idx_arr, val_idx_arr  bir kez hesaplanması yeterli aslında
-                X_train, X_test, y_train, y_test, X_train.index, X_test.index, train_idx_arr, val_idx_arr = dataset.load()
+                dataset.load()
                 group_items = dataset.produce_inputs()
                 for item in group_items:
                     # item[0] = f'threshold_{threshold}_{item[0]}'
@@ -115,8 +115,8 @@ class NerParameterSearch:
 
                 self.items.extend(group_items)
         self.y_test_label = self.items[0][4]
-        self.train_idx_arr = train_idx_arr
-        self.val_idx_arr = val_idx_arr
+        self.train_idx_arr = dataset.train_idx_arr
+        self.val_idx_arr = dataset.val_idx_arr
 
     def run(self, model_func, batch_size=128, epochs=100):
         mlflow.set_tracking_uri(self.tracking_uri)

ddi_fw/pipeline/pipeline.py

@@ -36,7 +36,7 @@ class Pipeline(BaseModel):
     combinations: Optional[List[str]] = None
     model: Optional[Any] = None
     multi_modal:  Optional[Any] = None
-    use_mlflow: bool = True
+    use_mlflow: bool = False
     _items:List=[]
     _train_idx_arr:List|None=[]
     _val_idx_arr:List|None=[]
@@ -53,45 +53,7 @@ class Pipeline(BaseModel):
 
     class Config:
         arbitrary_types_allowed = True
-
-# class Pipeline:
-#     def __init__(self,
-#                  library='tensorflow',
-#                  experiment_name=None,
-#                  experiment_description=None,
-#                  experiment_tags=None,
-#                  artifact_location=None,
-#                  tracking_uri=None,
-#                  dataset_type: BaseDataset = None,
-#                  columns=None,
-#                  embedding_dict=None,
-#                  column_embedding_configs=None,
-#                  vector_db_persist_directory=None,
-#                  vector_db_collection_name=None,
-#                  embedding_pooling_strategy_type: PoolingStrategy = None,
-#                  ner_data_file=None,
-#                  ner_threshold=None,
-#                  combinations=None,
-#                  model=None,
-#                  multi_modal = None ):
-#         self.library = library
-#         self.experiment_name = experiment_name
-#         self.experiment_description = experiment_description
-#         self.experiment_tags = experiment_tags
-#         self.artifact_location = artifact_location
-#         self.tracking_uri = tracking_uri
-#         self.dataset_type = dataset_type
-#         self.columns = columns
-#         self.embedding_dict = embedding_dict
-#         self.column_embedding_configs = column_embedding_configs
-#         self.vector_db_persist_directory = vector_db_persist_directory
-#         self.vector_db_collection_name = vector_db_collection_name
-#         self.embedding_pooling_strategy_type = embedding_pooling_strategy_type
-#         self.ner_data_file = ner_data_file
-#         self.ner_threshold = ner_threshold
-#         self.combinations = combinations
-#         self.model = model
-#         self.multi_modal = multi_modal
+ 
 
     def __create_or_update_embeddings__(self, embedding_dict, vector_db_persist_directory, vector_db_collection_name, column=None):
         """
@@ -194,35 +156,27 @@ class Pipeline(BaseModel):
                 embedding_dict=embedding_dict,
                 embedding_size=embedding_size,
                 embeddings_pooling_strategy=pooling_strategy,
-                dataset_splitter = dataset_splitter,
+                dataset_splitter_type = self.dataset_splitter_type,
+                **kwargs)
+        elif self.dataset_type == BaseDataset:
+             dataset = self.dataset_type(
+                dataset_splitter_type = self.dataset_splitter_type,
                 **kwargs)
         else:
             dataset = self.dataset_type(**kwargs)
 
         # X_train, X_test, y_train, y_test, train_indexes, test_indexes, train_idx_arr, val_idx_arr = dataset.load()
         dataset.load()
-
         dataframe = dataset.dataframe
+        b = not( dataset.X_train and dataset.y_train and dataset.X_test and dataset.y_test)
+        c = dataframe is None or dataframe.empty 
 
-        if dataframe is None:  # if the dataframe is None, it means that the dataset is not loaded
+        if b or c:
             raise ValueError("The dataset is not loaded")
-
-        # dataframe.dropna()
-        # X_train = dataset.X_train
-        # X_test = dataset.X_test
-        # y_train = dataset.y_train
-        # y_test = dataset.y_test
-        # self._train_idx_arr = dataset.train_idx_arr
-        # self._val_idx_arr = dataset.val_idx_arr
-        # Logic to set up the experiment
+ 
         # column name, train data, train label, test data, test label
         self._items = dataset.produce_inputs()
-
-        # unique_classes = pd.unique(dataframe[dataset.class_column])
-        # event_num = len(unique_classes)
-        # droprate = 0.3
-        # vector_size = self.dataset.drugs_df.shape[0]
-
+ 
         print("Building the experiment with the following settings:")
         print(
             f"Name: {self.experiment_name}, Dataset: {dataset}, Model: {self.model}")
@@ -244,7 +198,7 @@ class Pipeline(BaseModel):
 
         y_test_label = self.items[0][4]
         multi_modal_runner = MultiModalRunner(
-            library=self.library, multi_modal=self.multi_modal)
+            library=self.library, multi_modal=self.multi_modal, use_mlflow=self.use_mlflow)
         # multi_modal_runner = MultiModalRunner(
         #     library=self.library, model_func=model_func, batch_size=batch_size,  epochs=epochs)
         # multi_modal = TFMultiModal(

{ddi_fw-0.0.185.dist-info → ddi_fw-0.0.187.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.185
+Version: 0.0.187
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.185.dist-info → ddi_fw-0.0.187.dist-info}/RECORD

@@ -1,5 +1,5 @@
 ddi_fw/datasets/__init__.py,sha256=_I3iDHARwzmg7_EL5XKtB_TgG1yAkLSOVTujLL9Wz9Q,280
-ddi_fw/datasets/core.py,sha256=FYFKzKpaPqRXl5UqLoD6DNZEaJEAgxLlfCPtHULjc_s,10865
+ddi_fw/datasets/core.py,sha256=mZcGqP3Ukx5FbYSMi08uq4vYDr7jbHR3xg1qOPJmU0s,10640
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
@@ -74,18 +74,18 @@ ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSz
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=tIxiW0g6q1VsmDYVXR_ovvHQR3SCir8g2bKxx_CrS7s,221
 ddi_fw/ml/evaluation_helper.py,sha256=2-7CLSgGTqLEk4HkgCVIOt-GxfLAn6SBozJghAtHb5M,11581
-ddi_fw/ml/ml_helper.py,sha256=vT_k-m0wGWQOufBYP4qLGdJThMkFh9046UqjoaEJ3Pc,8549
+ddi_fw/ml/ml_helper.py,sha256=l1ZLYL3x5bHxD2bh2ezEgWDlV0ni8zGZGgj07x7KR40,6310
 ddi_fw/ml/model_wrapper.py,sha256=kabPXuo7S8tGkp9a00V04n4rXDmv7dD8wYGMjotISRc,1050
 ddi_fw/ml/pytorch_wrapper.py,sha256=pe6UsjP2XeTgLxDnIUiodoyhJTGCxV27wD4Cjxysu2Q,8553
-ddi_fw/ml/tensorflow_wrapper.py,sha256=xX_rP6nzB2yQiNvGP9_PbbQt1bXiPPWEozIzpbV6Ens,12911
+ddi_fw/ml/tensorflow_wrapper.py,sha256=-zcbd0LBg9QNMF9K1I-JC379cS3rTO7ibgsDIOnMsoc,12951
 ddi_fw/ner/__init__.py,sha256=JwhGXrepomxPSsGsg2b_xPRC72AjvxOIn2CW5Mvscn0,26
 ddi_fw/ner/mmlrestclient.py,sha256=NZta7m2Qm6I_qtVguMZhqtAUjVBmmXn0-TMnsNp0jpg,6859
 ddi_fw/ner/ner.py,sha256=FHyyX53Xwpdw8Hec261dyN88yD7Z9LmJua2mIrQLguI,17967
 ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,212
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
 ddi_fw/pipeline/multi_pipeline.py,sha256=NfcH4Ze5U-JRiH3lrxEDWj-VPxYQYtp7tq6bLCImBzs,5550
-ddi_fw/pipeline/ner_pipeline.py,sha256=kNGtkg5rNX5MDywzvRxmvyk-DxXAjEbYzZkp8pNlAZo,6023
-ddi_fw/pipeline/pipeline.py,sha256=11CgBgNxzo1KqKWudezSM2iFruoUVG-JMNbwznvt1KA,11362
+ddi_fw/pipeline/ner_pipeline.py,sha256=Bp6BA6nozfWFaMHH6jKlzesnCGO6qiMkzdGy_ed6nh0,5947
+ddi_fw/pipeline/pipeline.py,sha256=VSILkxot_O1DJMWPavzFUH3le4zVKQydcH32SbuHZlQ,9355
 ddi_fw/utils/__init__.py,sha256=bqIC0YjbD0YSHtO0nWUkRs4w5nu7qBV0yU72sRzwCj8,475
 ddi_fw/utils/categorical_data_encoding_checker.py,sha256=gzb_vUDBrCMUhBxY1fBYTe8hmK72p0_uw3DTga8cqP8,1580
 ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
@@ -98,7 +98,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=Z1A_DOBqDFPqLN4YB-3oYlOQWJK-X6Oes6UFjpzR47Q,4760
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.185.dist-info/METADATA,sha256=cafBi3CO83LDvyYHTP5qXVRTkBYXTdHKhDu_r5Fki5E,2542
-ddi_fw-0.0.185.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddi_fw-0.0.185.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.185.dist-info/RECORD,,
+ddi_fw-0.0.187.dist-info/METADATA,sha256=dzH9YAqsPxQcvuS9h0JRNx5qtd8vGNr-1c5f0uE3c7M,2542
+ddi_fw-0.0.187.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddi_fw-0.0.187.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.187.dist-info/RECORD,,