ddi-fw 0.0.171__py3-none-any.whl → 0.0.173__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -21,7 +21,6 @@ except ImportError:
         "Failed to import langchain.embeddings module. ")
 
 
-
 def stack(df_column):
     return np.stack(df_column.values)
 
@@ -61,10 +60,10 @@ class BaseDataset(BaseModel):
     dataset_splitter_type: Type[DatasetSplitter]
     class_column: str = 'class'
     dataframe: Optional[pd.DataFrame] = None
-    X_train: Optional[pd.DataFrame] = None
-    X_test: Optional[pd.DataFrame] = None
-    y_train: Optional[pd.Series] = None
-    y_test: Optional[pd.Series] = None
+    X_train: Optional[pd.DataFrame | np.ndarray] = None
+    X_test: Optional[pd.DataFrame | np.ndarray] = None
+    y_train: Optional[pd.Series | np.ndarray] = None
+    y_test: Optional[pd.Series | np.ndarray] = None
     train_indexes: Optional[pd.Index] = None
     test_indexes: Optional[pd.Index] = None
     train_idx_arr: Optional[List[np.ndarray]] = None
@@ -89,7 +88,7 @@ class BaseDataset(BaseModel):
             # items.append([f'{column}_embedding', train_data,
             #             y_train_label, test_data, y_test_label])
         return items
-    
+
     @computed_field
     @property
     def dataset_splitter(self) -> DatasetSplitter:
@@ -109,17 +108,20 @@ class BaseDataset(BaseModel):
         """
         if self.X_train is not None and self.y_train is not None and self.X_test is not None and self.y_test is not None:
             # Data is already provided, no need to calculate
-            logging.info("X_train, y_train, X_test, and y_test are already provided. Skipping calculation.")
+            logging.info(
+                "X_train, y_train, X_test, and y_test are already provided. Skipping calculation.")
             return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
 
         if self.index_path is None:
-            raise Exception("There is no index path. Please call split_dataset or provide indices.")
+            raise Exception(
+                "There is no index path. Please call split_dataset or provide indices.")
 
         if self.dataframe is None:
             raise Exception("There is no dataframe to derive data from.")
 
         try:
-            train_idx_all, test_idx_all, train_idx_arr, val_idx_arr = self.__get_indexes__(self.index_path)
+            train_idx_all, test_idx_all, train_idx_arr, val_idx_arr = self.__get_indexes__(
+                self.index_path)
         except FileNotFoundError as e:
             raise FileNotFoundError(f"Index files not found: {e.filename}")
 
@@ -176,13 +178,15 @@ class BaseDataset(BaseModel):
         raise an error.
         """
         if self.X_train is not None or self.X_test is not None:
-            raise Exception("X_train and X_test are already present. Splitting is not allowed.")
-
+            raise Exception(
+                "X_train and X_test are already present. Splitting is not allowed.")
+            
+        self.prep()
         if self.dataframe is None:
             raise Exception("There is no dataframe to split.")
 
         save_path = self.index_path
-        self.prep()
+        
 
         X = self.dataframe.drop(self.class_column, axis=1)
         y = self.dataframe[self.class_column]
@@ -215,8 +219,9 @@ class BaseDataset(BaseModel):
 
 
 class TextDatasetMixin(BaseDataset):
-    embedding_size: Optional[int] = None 
-    embedding_dict: Dict[str, Any] = Field(default_factory=dict, description="Dictionary for embeddings")
+    embedding_size: Optional[int] = None
+    embedding_dict: Dict[str, Any] = Field(
+        default_factory=dict, description="Dictionary for embeddings")
     embeddings_pooling_strategy: PoolingStrategy | None = None
 
     def process_text(self):

ddi_fw/ml/evaluation_helper.py

@@ -11,6 +11,7 @@ from sklearn.metrics import auc
 from sklearn.metrics import classification_report
 from sklearn.preprocessing import OneHotEncoder
 
+
 def __format__(d: Union[Dict[str, Union[List[float], float]], float], floating_number_precision=4) -> Union[Dict[str, Union[List[float], float]], float]:
     if isinstance(d, dict):
         d = {k: __round__(v, floating_number_precision) for k, v in d.items()}
@@ -18,25 +19,25 @@ def __format__(d: Union[Dict[str, Union[List[float], float]], float], floating_n
         d = round(d, floating_number_precision)
     return d
 
-def __round__(v,floating_number_precision = 4) -> Union[List[float], float]:
+
+def __round__(v, floating_number_precision=4) -> Union[List[float], float]:
     if type(v) is list or type(v) is set:
-       return [round(item,floating_number_precision) for item in v]
+        return [round(item, floating_number_precision) for item in v]
     else:
-        return round(v,floating_number_precision)
+        return round(v, floating_number_precision)
 
 
 class Metrics(BaseModel):
     label: str
     accuracy: float = 0.0
-    precision: Any= None
-    recall: Any= None
-    f1_score: Any= None
-    roc_auc: Any= None
-    roc_aupr: Any= None
+    precision: Any = None
+    recall: Any = None
+    f1_score: Any = None
+    roc_auc: Any = None
+    roc_aupr: Any = None
     classification_report: Any = None
 
-
-    def set_classification_report(self,classification_report):
+    def set_classification_report(self, classification_report):
         self.classification_report = classification_report
 
     def set_accuracy(self, accuracy):
@@ -57,15 +58,14 @@ class Metrics(BaseModel):
     def set_roc_aupr(self, roc_aupr):
         self.roc_aupr = roc_aupr
 
-    def format_float(self, floating_number_precision = 4):
-        self.accuracy = round(self.accuracy,floating_number_precision)
-        self.precision = __format__( self.precision ,floating_number_precision)
-        self.recall = __format__( self.recall ,floating_number_precision)
-        self.f1_score = __format__( self.f1_score ,floating_number_precision)
-        self.roc_auc = __format__( self.roc_auc ,floating_number_precision)
-        self.roc_aupr = __format__( self.roc_aupr ,floating_number_precision)
+    def format_float(self, floating_number_precision=4):
+        self.accuracy = round(self.accuracy, floating_number_precision)
+        self.precision = __format__(self.precision, floating_number_precision)
+        self.recall = __format__(self.recall, floating_number_precision)
+        self.f1_score = __format__(self.f1_score, floating_number_precision)
+        self.roc_auc = __format__(self.roc_auc, floating_number_precision)
+        self.roc_aupr = __format__(self.roc_aupr, floating_number_precision)
 
- 
 
 # taken from https://github.com/YifanDengWHU/DDIMDL/blob/master/DDIMDL.py#L214
 def roc_aupr_score(y_true, y_score, average="macro"):
@@ -96,19 +96,119 @@ def roc_aupr_score(y_true, y_score, average="macro"):
     return _average_binary_score(_binary_roc_aupr_score, y_true, y_score, average)
 
 
-def evaluate(actual, pred, info='', print_detail=False):
+def evaluate(actual: np.ndarray, pred: np.ndarray, info='', print_detail=False):
+    y_true = actual
+    y_pred = pred
+
+    # Generate classification report
+    c_report = classification_report(y_true, y_pred, output_dict=True)
+
+    # Metrics initialization
+    metrics = Metrics(label=info)
+
+    n_classes = actual.shape[1]
+    # n_classes = len(np.unique(actual))
+
+    precision = {}
+    recall = {}
+    f_score = {}
+    roc_aupr = {}
+    roc_auc = {
+        "weighted": 0.0,
+        "macro": 0.0,
+        "micro": 0.0
+    }
+
+    # Preallocate lists
+    precision_vals: List[np.ndarray] = [np.array([]) for _ in range(n_classes)]
+    recall_vals: List[np.ndarray] = [np.array([]) for _ in range(n_classes)]
+
+    # Compute metrics for each class
+    for i in range(n_classes):
+        precision_vals[i], recall_vals[i], _ = precision_recall_curve(
+            actual[:, i], pred[:, i])
+        roc_aupr[i] = auc(recall_vals[i], precision_vals[i])
+
+    # Calculate ROC AUC scores
+    roc_auc["weighted"] = float(roc_auc_score(
+        actual, pred, multi_class='ovr', average='weighted'))
+    roc_auc["macro"] = float(roc_auc_score(
+        actual, pred, multi_class='ovr', average='macro'))
+    roc_auc["micro"] = float(roc_auc_score(
+        actual, pred, multi_class='ovr', average='micro'))
+
+    # Micro-average Precision-Recall curve and ROC-AUPR
+    precision["micro_event"], recall["micro_event"], _ = precision_recall_curve(
+        actual.ravel(), pred.ravel())
+    roc_aupr["micro"] = auc(recall["micro_event"], precision["micro_event"])
+
+    # Convert lists to numpy arrays for better performance
+    precision["micro_event"] = precision["micro_event"].tolist()
+    recall["micro_event"] = recall["micro_event"].tolist()
+
+    # Overall accuracy
+    acc = accuracy_score(y_true, y_pred)
+
+    # Aggregate precision, recall, and f_score
+    # for avg_type in ['weighted', 'macro', 'micro']:
+    for avg_type in Literal['weighted', 'macro', 'micro'].__args__:
+        precision[avg_type] = precision_score(y_true, y_pred, average=avg_type)
+        recall[avg_type] = recall_score(y_true, y_pred, average=avg_type)
+        f_score[avg_type] = f1_score(y_true, y_pred, average=avg_type)
+
+    if print_detail:
+        print(
+            f'''Accuracy: {acc}
+            , Precision:{precision['weighted']}
+            , Recall: {recall['weighted']}
+            , F1-score: {f_score['weighted']}
+            ''')
+
+    logs = {'accuracy': acc,
+            'weighted_precision': precision['weighted'],
+            'macro_precision': precision['macro'],
+            'micro_precision': precision['micro'],
+            'weighted_recall_score': recall['weighted'],
+            'macro_recall_score': recall['macro'],
+            'micro_recall_score': recall['micro'],
+            'weighted_f1_score': f_score['weighted'],
+            'macro_f1_score': f_score['macro'],
+            'micro_f1_score': f_score['micro'],
+            # 'weighted_roc_auc_score': weighted_roc_auc_score,
+            # 'macro_roc_auc_score': macro_roc_auc_score,
+            # 'micro_roc_auc_score': micro_roc_auc_score,
+            # 'macro_aupr_score': macro_aupr_score,
+            # 'micro_aupr_score': micro_aupr_score
+            "micro_roc_aupr": roc_aupr['micro'],
+            # "micro_precision_from_precision_recall_curve":precision["micro"],
+            # "micro_recall_from_precision_recall_curve":recall["micro"],
+            "weighted_roc_auc": roc_auc['weighted'],
+            "macro_roc_auc": roc_auc['macro'],
+            "micro_roc_auc": roc_auc['micro']
+            }
+    metrics.set_accuracy(acc)
+    metrics.set_precision(precision)
+    metrics.set_recall(recall)
+    metrics.set_f1_score(f_score)
+    metrics.set_roc_auc(roc_auc)
+    metrics.set_roc_aupr(roc_aupr)
+    metrics.set_classification_report(c_report)
+    return logs, metrics
+
+
+def evaluate_ex(actual, pred, info='', print_detail=False):
     # Precompute y_true and y_pred
     y_true = np.argmax(actual, axis=1)
     y_pred = np.argmax(pred, axis=1)
-    
+
     # Generate classification report
     c_report = classification_report(y_true, y_pred, output_dict=True)
-    
+
     # Metrics initialization
-    metrics = Metrics(label= info)
-    
+    metrics = Metrics(label=info)
+
     n_classes = actual.shape[1]
-    
+
     precision = {}
     recall = {}
     f_score = {}
@@ -123,7 +223,6 @@ def evaluate(actual, pred, info='', print_detail=False):
     precision_vals: List[np.ndarray] = [np.array([]) for _ in range(n_classes)]
     recall_vals: List[np.ndarray] = [np.array([]) for _ in range(n_classes)]
 
-
     # Compute metrics for each class
     for i in range(n_classes):
         precision_vals[i], recall_vals[i], _ = precision_recall_curve(
@@ -131,12 +230,16 @@ def evaluate(actual, pred, info='', print_detail=False):
         roc_aupr[i] = auc(recall_vals[i], precision_vals[i])
 
     # Calculate ROC AUC scores
-    roc_auc["weighted"] = float(roc_auc_score(actual, pred, multi_class='ovr', average='weighted'))
-    roc_auc["macro"] = float(roc_auc_score(actual, pred, multi_class='ovr', average='macro'))
-    roc_auc["micro"] = float(roc_auc_score(actual, pred, multi_class='ovr', average='micro'))
+    roc_auc["weighted"] = float(roc_auc_score(
+        actual, pred, multi_class='ovr', average='weighted'))
+    roc_auc["macro"] = float(roc_auc_score(
+        actual, pred, multi_class='ovr', average='macro'))
+    roc_auc["micro"] = float(roc_auc_score(
+        actual, pred, multi_class='ovr', average='micro'))
 
     # Micro-average Precision-Recall curve and ROC-AUPR
-    precision["micro_event"], recall["micro_event"], _ = precision_recall_curve(actual.ravel(), pred.ravel())
+    precision["micro_event"], recall["micro_event"], _ = precision_recall_curve(
+        actual.ravel(), pred.ravel())
     roc_aupr["micro"] = auc(recall["micro_event"], precision["micro_event"])
 
     # Convert lists to numpy arrays for better performance
@@ -191,5 +294,3 @@ def evaluate(actual, pred, info='', print_detail=False):
     metrics.set_roc_aupr(roc_aupr)
     metrics.set_classification_report(c_report)
     return logs, metrics
-
-  

ddi_fw/ml/tensorflow_wrapper.py

@@ -30,6 +30,7 @@ class TFModelWrapper(ModelWrapper):
 
     def fit_model(self, X_train, y_train, X_valid, y_valid):
         self.kwargs['input_shape'] = self.train_data.shape
+        self.num_classes = len(np.unique(y_train))
         model = self.model_func(**self.kwargs)
         checkpoint = ModelCheckpoint(
             filepath=f'{self.descriptor}_validation.weights.h5',
@@ -135,6 +136,9 @@ class TFModelWrapper(ModelWrapper):
                 print(best_model_key)
                 self.best_model: Model = best_model
                 pred = self.predict()
+                pred = tf.keras.utils.to_categorical(np.argmax(pred,axis=1), num_classes=self.num_classes)
+                actual = tf.keras.utils.to_categorical(self.test_label, num_classes=self.num_classes)
+
                 logs, metrics = evaluate(
                     actual=self.test_label, pred=pred, info=self.descriptor, print_detail=print_detail)
                 metrics.format_float()

{ddi_fw-0.0.171.dist-info → ddi_fw-0.0.173.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.171
+Version: 0.0.173
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.171.dist-info → ddi_fw-0.0.173.dist-info}/RECORD

@@ -1,5 +1,5 @@
 ddi_fw/datasets/__init__.py,sha256=_I3iDHARwzmg7_EL5XKtB_TgG1yAkLSOVTujLL9Wz9Q,280
-ddi_fw/datasets/core.py,sha256=vnbaqDRcnlKd7TX1emkc3lNEEcD_PACbeu72p4y-4Ok,9223
+ddi_fw/datasets/core.py,sha256=j6YpH6IqPQ2va1cC26xT-Jn3fIPsF43xD3GuluJRJb4,9372
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
@@ -73,11 +73,11 @@ ddi_fw/langchain/embeddings.py,sha256=eEWy4okcjdhUJHi4N48Wd8XauPXyeaQVLUdNWEvtEc
 ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=tIxiW0g6q1VsmDYVXR_ovvHQR3SCir8g2bKxx_CrS7s,221
-ddi_fw/ml/evaluation_helper.py,sha256=JFATMquaQVa2gckxmEivCztZmivWBAAP7EpJ8PVeI3c,7626
+ddi_fw/ml/evaluation_helper.py,sha256=2-7CLSgGTqLEk4HkgCVIOt-GxfLAn6SBozJghAtHb5M,11581
 ddi_fw/ml/ml_helper.py,sha256=E6ef7f1UnQl6JBUdGDbbbI4FIS-904VGypT7tI0a598,8545
 ddi_fw/ml/model_wrapper.py,sha256=kabPXuo7S8tGkp9a00V04n4rXDmv7dD8wYGMjotISRc,1050
 ddi_fw/ml/pytorch_wrapper.py,sha256=pe6UsjP2XeTgLxDnIUiodoyhJTGCxV27wD4Cjxysu2Q,8553
-ddi_fw/ml/tensorflow_wrapper.py,sha256=jt6h9Q-wF0mkbnvV6yCCl1SpUd2paHK70Bu6EFrkmd0,10112
+ddi_fw/ml/tensorflow_wrapper.py,sha256=IQq0KSU-WuRI90b3DcZ8vhxATfZgdymkAqiiz4a1D6g,10377
 ddi_fw/ner/__init__.py,sha256=JwhGXrepomxPSsGsg2b_xPRC72AjvxOIn2CW5Mvscn0,26
 ddi_fw/ner/mmlrestclient.py,sha256=NZta7m2Qm6I_qtVguMZhqtAUjVBmmXn0-TMnsNp0jpg,6859
 ddi_fw/ner/ner.py,sha256=FHyyX53Xwpdw8Hec261dyN88yD7Z9LmJua2mIrQLguI,17967
@@ -97,7 +97,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=Z1A_DOBqDFPqLN4YB-3oYlOQWJK-X6Oes6UFjpzR47Q,4760
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.171.dist-info/METADATA,sha256=qAlq05fNm5PaC-jJ-kCZz7oH-1yZZdxq7eAbRuNmM6U,2542
-ddi_fw-0.0.171.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddi_fw-0.0.171.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.171.dist-info/RECORD,,
+ddi_fw-0.0.173.dist-info/METADATA,sha256=4HVYwgrsyel7JO4cJ3pZTtw5G_YwmRsrNyIClsmJaFo,2542
+ddi_fw-0.0.173.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddi_fw-0.0.173.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.173.dist-info/RECORD,,