balancr 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

balancr/cli/commands.py

@@ -1535,7 +1535,7 @@ def run_comparison(args):
         )
 
         # Process each classifier and save its results in a separate directory
-        standard_start_time = time.time()
+        orig_start_time = time.time()
         for classifier_name in current_config.get("classifiers", {}):
             logging.info(f"Processing results for classifier: {classifier_name}")
 
@@ -1543,18 +1543,18 @@ def run_comparison(args):
             classifier_dir = output_dir / classifier_name
             classifier_dir.mkdir(exist_ok=True)
 
-            # Create standard metrics directory
-            std_metrics_dir = classifier_dir / "standard_metrics"
-            std_metrics_dir.mkdir(exist_ok=True)
+            # Create original dataset metrics directory
+            orig_metrics_dir = classifier_dir / "metrics_on_original_test"
+            orig_metrics_dir.mkdir(exist_ok=True)
 
-            # Save standard metrics in requested formats
+            # Save original dataset metrics in requested formats
             for format_type in save_metrics_formats:
                 if format_type == "none":
                     continue
 
-                results_file = std_metrics_dir / f"comparison_results.{format_type}"
+                results_file = orig_metrics_dir / f"comparison_results.{format_type}"
                 logging.info(
-                    f"Saving standard metrics for {classifier_name} to {results_file}"
+                    f"Saving metrics from testing against original test data for {classifier_name} to {results_file}"
                 )
 
                 # We need a modified save_results method that can extract a specific classifier's results
@@ -1565,15 +1565,16 @@ def run_comparison(args):
                     file_type=format_type,
                 )
 
-            # Generate and save standard metrics visualisations
+            # Generate and save original test data metrics visualisations
             for format_type in save_vis_formats:
                 if format_type == "none":
                     continue
 
                 if "metrics" in vis_types_to_generate or "all" in visualisations:
-                    metrics_path = std_metrics_dir / f"metrics_comparison.{format_type}"
+                    metrics_path = orig_metrics_dir / f"metrics_comparison.{format_type}"
                     logging.info(
-                        f"Generating metrics comparison for {classifier_name} in {format_type} format..."
+                        f"Generating metrics comparison for {classifier_name}, against original test data,"
+                        f"in {format_type} format..."
                     )
 
                     metrics_to_plot = current_config.get("output", {}).get(
@@ -1591,7 +1592,7 @@ def run_comparison(args):
 
                     if "radar" in vis_types_to_generate or "all" in visualisations:
                         std_radar_path = (
-                            classifier_dir / f"standard_metrics_radar.{format_type}"
+                            classifier_dir / f"metrics_on_original_test_radar.{format_type}"
                         )
                         plot_radar_chart(
                             results,
@@ -1603,7 +1604,7 @@ def run_comparison(args):
                         )
 
                     if "3d" in vis_types_to_generate or "all" in visualisations:
-                        std_3d_path = output_dir / "standard_metrics_3d.html"
+                        std_3d_path = output_dir / "metrics_on_original_test_3d.html"
                         plot_3d_scatter(
                             results,
                             metric_type="standard_metrics",
@@ -1612,9 +1613,47 @@ def run_comparison(args):
                             display=display_visualisations,
                         )
 
-        standard_total_time = time.time() - standard_start_time
+                if (
+                    "learning_curves" in vis_types_to_generate
+                    or "all" in visualisations
+                ):
+                    orig_learning_curve_path = (
+                        orig_metrics_dir / f"learning_curves.{format_type}"
+                    )
+
+                    start_time = time.time()
+                    logging.info(
+                        f"Generating learning curves for {classifier_name}, against original test data "
+                        f"in {format_type} format..."
+                    )
+
+                    # Get learning curve parameters from config
+                    learning_curve_points = eval_config.get(
+                        "learning_curve_points", 10
+                    )
+                    learning_curve_folds = eval_config.get(
+                        "learning_curve_folds", 5
+                    )
+                    train_sizes = np.linspace(0.1, 1.0, learning_curve_points)
+
+                    learning_curve_type = "Original Dataset"
+                    framework.generate_learning_curves(
+                        classifier_name=classifier_name,
+                        learning_curve_type=learning_curve_type,
+                        train_sizes=train_sizes,
+                        n_folds=learning_curve_folds,
+                        save_path=str(orig_learning_curve_path),
+                        display=display_visualisations,
+                    )
+                    cv_learning_curves_time = time.time() - start_time
+                    logging.info(
+                        f"Successfully generated cv learning curves for {classifier_name}, against original test data "
+                        f"(Time Taken: {format_time(cv_learning_curves_time)})"
+                    )
+
+        orig_total_time = time.time() - orig_start_time
         logging.info(
-            f"Standard metrics evaluation total time: {format_time(standard_total_time)}"
+            f"Metrics evaluation against original test data total time: {format_time(orig_total_time)}"
         )
 
         # If cross-validation is enabled, create CV metrics directory and save results
@@ -1625,7 +1664,7 @@ def run_comparison(args):
                 classifier_dir = output_dir / classifier_name
                 classifier_dir.mkdir(exist_ok=True)
 
-                cv_metrics_dir = classifier_dir / "cv_metrics"
+                cv_metrics_dir = classifier_dir / "metrics_on_balanced_cv"
                 cv_metrics_dir.mkdir(exist_ok=True)
 
                 # Save CV metrics in requested formats
@@ -1717,8 +1756,10 @@ def run_comparison(args):
                         )
                         train_sizes = np.linspace(0.1, 1.0, learning_curve_points)
 
+                        learning_curve_type = "Balanced Datasets"
                         framework.generate_learning_curves(
                             classifier_name=classifier_name,
+                            learning_curve_type=learning_curve_type,
                             train_sizes=train_sizes,
                             n_folds=learning_curve_folds,
                             save_path=str(cv_learning_curve_path),
@@ -1739,13 +1780,13 @@ def run_comparison(args):
 
         # Print summary of timing results
         print("\nExecution Time Summary:\n")
-        print(f"  Data Loading:        {format_time(load_time)}")
-        print(f"  Balancing:           {format_time(balancing_time)}")
-        print(f"  Training Classifiers: {format_time(training_time)}")
-        print(f"  Standard Metrics Evaluation: {format_time(standard_total_time)}")
+        print(f"  Data Loading:                                               {format_time(load_time)}")
+        print(f"  Balancing:                                                  {format_time(balancing_time)}")
+        print(f"  Training Classifiers:                                       {format_time(training_time)}")
+        print(f"  Metrics From Testing Against Original Test Dataset:         {format_time(orig_total_time)}")
         if cv_enabled:
-            print(f"  CV Metrics Evaluation: {format_time(cv_total_time)}")
-        print(f"  Total Time:          {format_time(total_time)}")
+            print(f"  CV Metrics From Testing Against Balanced Test Datasets:     {format_time(cv_total_time)}")
+        print(f"  Total Time:                                                 {format_time(total_time)}")
 
         print("\nResults Summary:")
 
@@ -1758,7 +1799,7 @@ def run_comparison(args):
         )
 
         if has_standard_metrics:
-            print("\nStandard Metrics:")
+            print("\nMetrics From Testing Against Original Test Dataset:")
             for classifier_name, classifier_results in results.items():
                 print(f"\n{classifier_name}:")
                 for technique_name, technique_metrics in classifier_results.items():
@@ -1783,7 +1824,7 @@ def run_comparison(args):
         )
 
         if has_cv_metrics:
-            print("\nCross Validation Metrics:")
+            print("\nCross Validation Metrics From Testing Against Balanced Test Datasets:")
             for classifier_name, classifier_results in results.items():
                 print(f"\n{classifier_name}:")
                 for technique_name, technique_metrics in classifier_results.items():

balancr/evaluation/__init__.py

@@ -7,6 +7,7 @@ from .metrics import (
     get_cv_scores,
     get_learning_curve_data,
     get_learning_curve_data_multiple_techniques,
+    get_learning_curve_data_against_imbalanced_multiple_techniques
 )
 
 from .visualisation import (

balancr/evaluation/metrics.py

@@ -4,6 +4,8 @@ import logging
 import time
 from typing import Dict
 import numpy as np
+import pandas as pd
+from sklearn.base import clone
 from sklearn.metrics import (
     accuracy_score,
     precision_score,
@@ -13,7 +15,7 @@ from sklearn.metrics import (
     average_precision_score,
     confusion_matrix,
 )
-from sklearn.model_selection import learning_curve
+from sklearn.model_selection import StratifiedKFold, learning_curve
 
 
 def format_time(seconds):
@@ -415,8 +417,8 @@ def get_learning_curve_data_multiple_techniques(
 
         start_time = time.time()
         logging.info(
-            f"Generating learning curve for {classifier_name} trained on data"
-            f"balanced by {technique_name}..."
+            f"Generating learning curve for {classifier_name} trained on data "
+            f"balanced by {technique_name}, against balanced data..."
         )
         train_sizes_abs, train_scores, val_scores = learning_curve(
             estimator=classifier,
@@ -429,8 +431,9 @@ def get_learning_curve_data_multiple_techniques(
         )
         curve_generating_time = time.time() - start_time
         logging.info(
-            f"Generated learning curve for {classifier_name} trained on data"
-            f"balanced by {technique_name} (Time Taken: {format_time(curve_generating_time)})"
+            f"Generated learning curve for {classifier_name} trained on data "
+            f"balanced by {technique_name}, against balanced data"
+            f"(Time Taken: {format_time(curve_generating_time)})"
         )
 
         learning_curve_data[technique_name] = {
@@ -440,3 +443,82 @@ def get_learning_curve_data_multiple_techniques(
         }
 
     return learning_curve_data
+
+
+def get_learning_curve_data_against_imbalanced_multiple_techniques(
+    classifier_name: str,
+    classifier,
+    techniques_data: Dict[str, Dict[str, np.ndarray]],
+    X_test,
+    y_test,
+    train_sizes: np.ndarray = np.linspace(0.1, 1.0, 10),
+    n_folds: int = 5,
+) -> Dict[str, Dict[str, np.ndarray]]:
+    """
+    Custom learning curve function that trains on balanced data
+    and evaluates on original imbalanced test set.
+
+    Returns training and validation scores at each learning curve point.
+    Validation scores are computed on X_test / y_test.
+    """
+
+    def safe_index(X, indices):
+        return X.iloc[indices] if isinstance(X, (pd.DataFrame, pd.Series)) else X[indices]
+
+    learning_curve_data = {}
+
+    for technique_name, data in techniques_data.items():
+        X_balanced = data["X_balanced"]
+        y_balanced = data["y_balanced"]
+
+        n_samples = X_balanced.shape[0]
+        train_sizes_abs = (train_sizes * n_samples).astype(int)
+
+        train_scores = []
+        val_scores = []
+
+        start_time = time.time()
+        logging.info(
+            f"Generating learning curve for {classifier_name} trained on data "
+            f"balanced by {technique_name}, against original test data..."
+        )
+
+        for train_size in train_sizes_abs:
+            X_subset = safe_index(X_balanced, np.arange(train_size))
+            y_subset = safe_index(y_balanced, np.arange(train_size))
+
+            fold_train_scores = []
+            fold_val_scores = []
+
+            kf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=42)
+
+            for train_idx, _ in kf.split(X_subset, y_subset):
+                X_fold_train = safe_index(X_subset, train_idx)
+                y_fold_train = safe_index(y_subset, train_idx)
+
+                clf = clone(classifier)
+                clf.fit(X_fold_train, y_fold_train)
+
+                train_acc = clf.score(X_fold_train, y_fold_train)
+                val_acc = clf.score(X_test, y_test)
+
+                fold_train_scores.append(train_acc)
+                fold_val_scores.append(val_acc)
+
+            train_scores.append(fold_train_scores)
+            val_scores.append(fold_val_scores)
+
+        curve_generating_time = time.time() - start_time
+        logging.info(
+            f"Generated learning curve for {classifier_name} trained on data "
+            f"balanced by {technique_name}, against original test data "
+            f"(Time Taken: {format_time(curve_generating_time)})"
+        )
+
+        learning_curve_data[technique_name] = {
+            "train_sizes": train_sizes_abs,
+            "train_scores": np.array(train_scores),
+            "val_scores": np.array(val_scores),
+        }
+
+    return learning_curve_data

balancr/evaluation/visualisation.py

@@ -290,6 +290,11 @@ def plot_learning_curves(
     if num_techniques == 1:
         axes = [axes]
 
+    fig.suptitle(
+        title,
+        size=16,
+    )
+
     for idx, (technique_name, data) in enumerate(learning_curve_data.items()):
         # Extract the train_sizes, train_scores, and val_scores from the dictionary
         train_sizes = data["train_sizes"]
@@ -317,7 +322,7 @@ def plot_learning_curves(
             train_sizes, val_mean - val_std, val_mean + val_std, alpha=0.1, color="red"
         )
 
-        ax.set_title(f"{technique_name} - Learning Curves")
+        ax.set_title(f"{technique_name}")
         ax.set_xlabel("Training Examples")
         ax.set_ylabel("Score")
         ax.legend(loc="best")

balancr/imbalance_analyser.py

@@ -15,6 +15,7 @@ from .evaluation import (
     get_metrics,
     get_cv_scores,
     get_learning_curve_data_multiple_techniques,
+    get_learning_curve_data_against_imbalanced_multiple_techniques,
 )
 from .evaluation import (
     plot_class_distribution,
@@ -558,6 +559,7 @@ class BalancingFramework:
     def generate_learning_curves(
         self,
         classifier_name: str,
+        learning_curve_type: str,
         train_sizes: np.ndarray = np.linspace(0.1, 1.0, 10),
         n_folds: int = 5,
         save_path: Optional[str] = None,
@@ -598,19 +600,39 @@ class BalancingFramework:
             # Create classifier instance with the same parameters used in training
             classifier = clf_class(**clf_params)
 
-            learning_curve_data = get_learning_curve_data_multiple_techniques(
-                classifier_name=classifier_name,
-                classifier=classifier,
-                techniques_data=self.current_balanced_datasets,
-                train_sizes=train_sizes,
-                n_folds=n_folds,
-            )
+            if learning_curve_type == "Balanced Datasets":
+                learning_curve_data = get_learning_curve_data_multiple_techniques(
+                    classifier_name=classifier_name,
+                    classifier=classifier,
+                    techniques_data=self.current_balanced_datasets,
+                    train_sizes=train_sizes,
+                    n_folds=n_folds,
+                )
 
-            title = f"{classifier_name} - Learning Curves"
+                title = f"Learning Curves for {classifier_name} Evaluated Against {learning_curve_type}"
 
-            plot_learning_curves(
-                learning_curve_data, title=title, save_path=save_path, display=display
-            )
+                plot_learning_curves(
+                    learning_curve_data,
+                    title=title,
+                    save_path=save_path,
+                    display=display
+                )
+            elif learning_curve_type == "Original Dataset":
+                learning_curve_data = get_learning_curve_data_against_imbalanced_multiple_techniques(
+                    classifier_name=classifier_name,
+                    classifier=classifier,
+                    techniques_data=self.current_balanced_datasets,
+                    X_test=self.X_test,
+                    y_test=self.y_test,
+                    train_sizes=train_sizes,
+                    n_folds=n_folds,
+                )
+
+                title = f"Learning Curves for {classifier_name} Evaluated Against {learning_curve_type}"
+
+                plot_learning_curves(
+                    learning_curve_data, title=title, save_path=save_path, display=display
+                )
 
         except Exception as e:
             logging.warning(

{balancr-0.1.0.dist-info → balancr-0.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: balancr
-Version: 0.1.0
+Version: 0.1.1
 Summary: A unified framework for analysing and comparing techniques for handling imbalanced datasets
 Home-page: https://github.com/Ruaskill/balancr
 Author: Conor Doherty
@@ -38,6 +38,20 @@ Provides-Extra: dev
 Requires-Dist: pytest>=6.0.0; extra == "dev"
 Requires-Dist: black>=21.0.0; extra == "dev"
 Requires-Dist: flake8>=3.9.0; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Balancr: A Unified Framework for Analysing Data Balancing Techniques
 

{balancr-0.1.0.dist-info → balancr-0.1.1.dist-info}/RECORD

@@ -1,25 +1,25 @@
 balancr/__init__.py,sha256=L1Ko0WvrjehAhjuQJxtEJLQkva8zzW1BS6SalbZJneY,254
 balancr/base.py,sha256=3bALV9IdBm2cng_pKZ91zR3W9OktnD0OYpaZnCDHb0o,355
 balancr/classifier_registry.py,sha256=FljlaB4XqZeXqFme1zTEFgGQDzfFkZxUOBFdtrO7Xow,12074
-balancr/imbalance_analyser.py,sha256=sPVA0NuOXRffsAwHX9jVWAskBmosYQftZjdLahjqHu8,25671
+balancr/imbalance_analyser.py,sha256=MNwXbIkyqdA1QEDZYr7ccFa-wTO25qZbC6cqsB6GvDs,26738
 balancr/technique_registry.py,sha256=mr-mAeZ4mJi8IaetpyYCKAn0A8tTeq3G_dX0RxMYx7U,11501
 balancr/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-balancr/cli/commands.py,sha256=DXlFIDiMmv9yFS77ivUHTWoawLdIlqfVMQWoEBbh7w8,69419
+balancr/cli/commands.py,sha256=Ez8EJW4dYAyTdnOF9tZULNQH-aFQCWtTu4Dqy8E9QTk,71789
 balancr/cli/config.py,sha256=YNpIXg90HttoMXXtSByndsXMyeyYhE1E-PDb1iT4Ge0,4292
 balancr/cli/main.py,sha256=Iiwi6EQ0Dr0DmPtcwb7jDKkn1-kr0wowr82k2Ami03s,26388
 balancr/cli/utils.py,sha256=kFxSWzjtrqebCiVrzulZVA8-ci0f8lxUXurANVftIZg,3202
 balancr/data/__init__.py,sha256=QuQ5saA7miRQRhLYoyQMDIac_noMjbqRXw5DcP7xnps,121
 balancr/data/loader.py,sha256=QlWKB60hQfNEoLPIxS_rEKI5gEPh5Id4TZT3Y42HnDQ,2027
 balancr/data/preprocessor.py,sha256=UDBU3S3EGqXPeZnMZzCbDfNE8xLrtQ5M_AnP_bk6gvI,22648
-balancr/evaluation/__init__.py,sha256=x0_cs1bmzDgxX9AQrAAk6jBSU9lJbpgCnjQImAYEOaA,408
-balancr/evaluation/metrics.py,sha256=91pbq_my-QHQ6O061hse3NkQewLrDogGUe2MmPSS5gM,15706
-balancr/evaluation/visualisation.py,sha256=hK_e7zto_LDwugUk4XrzPXOhrZo1iadSeeQHrysul-M,22616
+balancr/evaluation/__init__.py,sha256=FbA5Q2HaDCtAndNf5BM3BVQaE3jEmA2LU2MwdxI7OGY,475
+balancr/evaluation/metrics.py,sha256=OgJh1JYlMlcuz_hhz3z43QEXXp1WeMldEwsrmjh8hUg,18553
+balancr/evaluation/visualisation.py,sha256=oND6VvjOqVnhTmgVHGfymju0CP4ixNreLdC2ZT7zrA4,22655
 balancr/techniques/__init__.py,sha256=w1rHhvc4ZkHyg-ZS-g9ZTk_vlhSv9ANuF4aNMb4Ok_Y,119
 balancr/techniques/custom/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 balancr/techniques/custom/example_custom_technique.py,sha256=TDo6Js5grmy-7ZozxTO4N4Hee3dQGXL2jPY187v0vz4,732
-balancr-0.1.0.dist-info/LICENSE,sha256=ImeeOOsQ56Rv9q6dkuScrpyBn6e01e6xgO8qFMJ1fGs,1069
-balancr-0.1.0.dist-info/METADATA,sha256=GknbYve4lCOAWeGoXMEJuwB-PZIQR0LJqylH7VhUI2M,16626
-balancr-0.1.0.dist-info/WHEEL,sha256=A3WOREP4zgxI0fKrHUG8DC8013e3dK3n7a6HDbcEIwE,91
-balancr-0.1.0.dist-info/entry_points.txt,sha256=OVchXsA0rcBF58V3hhC_zbrBm-RWUzRYSjjg2xahLV4,50
-balancr-0.1.0.dist-info/top_level.txt,sha256=RcWjXBQWsz9XBKIMgSvC_hU1yjhR-quGQ5jxqJOiUq0,8
-balancr-0.1.0.dist-info/RECORD,,
+balancr-0.1.1.dist-info/licenses/LICENSE,sha256=ImeeOOsQ56Rv9q6dkuScrpyBn6e01e6xgO8qFMJ1fGs,1069
+balancr-0.1.1.dist-info/METADATA,sha256=W1oML5Cby0U_5Q-uVEs1VaLx7oqwmljSiKzo-PfrrJY,16925
+balancr-0.1.1.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+balancr-0.1.1.dist-info/entry_points.txt,sha256=OVchXsA0rcBF58V3hhC_zbrBm-RWUzRYSjjg2xahLV4,50
+balancr-0.1.1.dist-info/top_level.txt,sha256=RcWjXBQWsz9XBKIMgSvC_hU1yjhR-quGQ5jxqJOiUq0,8
+balancr-0.1.1.dist-info/RECORD,,

{balancr-0.1.0.dist-info → balancr-0.1.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.7.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any