imbreg 0.1.0__py3-none-any.whl

imbreg/__init__.py

@@ -0,0 +1,79 @@
+"""
+imbreg - Imbalanced Regression library.
+
+Public API
+----------
+Phi relevance
+    phi_control   - build relevance control structure
+    phi           - evaluate relevance for target values
+
+Resampling
+    dibs_regress  - DIBS strategy resampling (SmoteR + GaussNoise)
+
+Stratification / CV
+    cv_partitions - repeated K-fold CV with optional SMOGN + imputation
+    make_folds    - generate fold indices (stratified or random)
+
+Data I/O
+    read_dataset      - read KEEL-style .dat, csv and arff datasets
+    write_dataset     - write datasets (CSV/KEEL)
+    get_percentages   - compute % of rare cases per dataset
+    split_features_target - convenience X / y split
+"""
+
+from .stratification import phi_control, phi, cv_partitions, make_folds
+from .resampling import dibs_regress, safe_dibs_regress
+from .data_loader import (
+    read_dataset,
+    write_dataset,
+    get_percentages,
+    split_features_target,
+    encode_categoricals,
+    impute_train,
+    impute_test,
+)
+from .plots import (
+    plot_target_distribution,
+    plot_scatter_2d,
+    plot_scatter_3d,
+    plot_prediction_error,
+)
+from .metrics import utility_f1_score, sera_score
+from .validation import evaluate_folds, export_experiment_summaries, evaluate_predictions_from_files
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Phi relevance
+    "phi_control",
+    "phi",
+    # Resampling
+    "dibs_regress",
+    "safe_dibs_regress",
+    # CV partitioning
+    "cv_partitions",
+    "make_folds",
+    # Data I/O
+    "train_extra_trees",
+    "train_xgboost",
+    "predict_model",
+    "read_dataset",
+    "write_dataset",
+    "get_percentages",
+    "split_features_target",
+    "encode_categoricals",
+    # Imputation
+    "impute_train",
+    "impute_test",
+    # Visualization
+    "plot_target_distribution",
+    "plot_scatter_2d",
+    "plot_scatter_3d",
+    "plot_prediction_error",
+    # Metrics / Validation
+    "utility_f1_score",
+    "sera_score",
+    "evaluate_folds",
+    "export_experiment_summaries",
+    "evaluate_predictions_from_files",
+]

imbreg/data_loader.py

@@ -0,0 +1,427 @@
+"""
+data_loader.py - KEEL dataset I/O, imputation helpers, and dataset utilities.
+"""
+import re
+from io import StringIO
+from pathlib import Path
+
+import pandas as pd
+
+from .utils import (
+    align_categories,
+    apply_decimal_map,
+    apply_range_cap,
+    decimal_map_from_reference,
+    range_map_from_reference,
+)
+
+
+# ── KEEL reader ───────────────────────────────────────────────────────────────
+
+def read_dataset(name, location="", infer_categoricals=True, max_levels=5, max_decimals=6):
+    """
+    Reads a dataset (.dat KEEL, .arff Weka, or .csv).
+
+    Conventions:
+    - For .dat/.arff: Lines starting with '@' or '%' are skipped. Missing values are '?'.
+    - For .csv: The first row is assumed to be the header. Missing values can be empty or '?'.
+    - The LAST column is treated as the regression target and renamed to 'y'.
+    - Low-cardinality numeric columns are cast to `pd.Categorical` if `infer_categoricals` is True.
+
+    Parameters
+    ----------
+    name : str
+        The name of the dataset file (e.g., 'abalone.dat', 'data.csv', 'data.arff').
+    location : str or Path, default=''
+        The sub-path or directory containing the file.
+    infer_categoricals : bool, default=True
+        If True, automatically converts low cardinality numeric columns into categoricals.
+    max_levels : int, default=5
+        Maximum unique numeric values a column can have before being promoted to 
+        categorical.
+    max_decimals : int or None, default=6
+        If set, numeric columns are capped (rounded) at this precision.
+
+    Returns
+    -------
+    pandas.DataFrame
+        The data loaded and structurally formatted.
+
+    Raises
+    ------
+    ValueError
+        If no data lines are found.
+    """
+    path = Path(".") / location / name
+    
+    is_csv = path.suffix.lower() == '.csv'
+
+    if is_csv:
+        raw = pd.read_csv(path, na_values=["?", "NA"], skipinitialspace=True)
+        # Rename target column (assume last column is target)
+        cols = list(raw.columns)
+        cols[-1] = "y"
+        raw.columns = cols
+    else:
+        with open(path, "r", encoding="utf-8", errors="replace") as fh:
+            lines = fh.readlines()
+
+        data_lines = [
+            ln for ln in lines if not ln.strip().startswith(("@", "%")) and ln.strip()
+        ]
+        if not data_lines:
+            raise ValueError(f"No data lines found in {path}")
+
+        raw = pd.read_csv(
+            StringIO("".join(data_lines)),
+            header=None,
+            na_values=["?"],
+            skipinitialspace=True,
+        )
+
+        # Rename columns: predictors V1..V(n-1), target 'y'
+        n_cols = raw.shape[1]
+        raw.columns = [f"V{i + 1}" for i in range(n_cols - 1)] + ["y"]
+
+    # 1) String predictors → Categorical
+    for i in range(raw.shape[1] - 1):
+        if pd.api.types.is_object_dtype(raw.iloc[:, i]) or pd.api.types.is_string_dtype(raw.iloc[:, i]) or raw.iloc[:, i].dtype == str:
+            col_name = raw.columns[i]
+            raw[col_name] = raw.iloc[:, i].astype(str).str.strip().astype("category")
+
+    # 2) Low-cardinality numeric predictors → Categorical (mirrors R factor)
+    if infer_categoricals:
+        for col in raw.columns[:-1]:
+            if pd.api.types.is_numeric_dtype(raw[col]) and not isinstance(raw[col].dtype, pd.CategoricalDtype):
+                n_uniq = raw[col].dropna().nunique()
+                if 1 < n_uniq <= max_levels:
+                    raw[col] = raw[col].astype("category")
+
+    # 3) Cap decimals on remaining numeric columns (including y)
+    if max_decimals is not None:
+        for col in raw.columns:
+            if pd.api.types.is_numeric_dtype(raw[col]) and not isinstance(raw[col].dtype, pd.CategoricalDtype):
+                raw[col] = raw[col].round(max_decimals)
+
+    return raw
+
+
+# ── KEEL writer ───────────────────────────────────────────────────────────────
+
+def write_dataset(df, file_path, template_path=None, na_symbol="<null>", sep=",", dec=".", out_format="dat"):
+    """
+    Writes a DataFrame as a dataset file (.dat, .arff, or .csv).
+
+    For .dat and .arff, the header is preserved up to and including the 
+    '@data' identifier from the template file prior to appending row values, 
+    or generated dynamically if no template is found.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        The data payload intended for serialization.
+    file_path : str or pathlib.Path
+        The path where the resulting file will be written.
+    template_path : str or pathlib.Path, optional
+        A path pointing to the original partition containing the original meta-headers.
+    na_symbol : str, default='<null>'
+        The token representation for missing values.
+    sep : str, default=','
+        The separator / delimiter string used in the writing process.
+    dec : str, default='.'
+        The decimal separator to encode floating point data properly.
+    out_format : str, default='dat'
+        The format of the output ('dat', 'arff', 'csv').
+
+    Returns
+    -------
+    None
+    """
+    file_path = Path(file_path)
+    
+    # Make sure we enforce the requested extension
+    if file_path.suffix != f".{out_format}":
+        file_path = file_path.with_suffix(f".{out_format}")
+
+    # If format is CSV, use simple Pandas exporter
+    if out_format.lower() == "csv":
+        df.to_csv(file_path, index=False, na_rep=na_symbol)
+        return
+
+    # For KEEL/ARFF formats, extract or build the block header
+    header = None
+    if template_path and Path(template_path).exists():
+        with open(template_path, "r", encoding="utf-8", errors="replace") as fh:
+            tpl_lines = fh.readlines()
+
+        data_idx = None
+        for i, line in enumerate(tpl_lines):
+            if re.match(r"^\s*@data\s*$", line, re.IGNORECASE):
+                data_idx = i
+                break
+                
+        if data_idx is not None:
+            header = tpl_lines[: data_idx + 1]
+
+    if header is None:
+        # Fallback for CSV templates: Generate a valid basic KEEL/ARFF header dynamically
+        header = [f"@relation {file_path.stem}\n"]
+        for col in df.columns:
+            header.append(f"@attribute {col} real\n")
+        header.append("@data\n")
+
+    # Build output rows
+    out = df.copy()
+    rows = []
+    for _, row in out.iterrows():
+        vals = []
+        for v in row:
+            if pd.isna(v) if not isinstance(v, str) else v in ("nan", "None", "<NA>"):
+                vals.append(na_symbol)
+            elif isinstance(v, float):
+                # No scientific notation, replace decimal separator if needed
+                s = f"{v:.10g}"
+                if dec != ".":
+                    s = s.replace(".", dec)
+                vals.append(s)
+            else:
+                vals.append(str(v))
+        rows.append(sep.join(vals))
+
+    with open(file_path, "w", encoding="utf-8", newline="") as fh:
+        fh.writelines(header)
+        fh.write("\n".join(rows) + "\n")
+
+
+# ── Scikit-learn Iterative Imputation ──────────────────────────────────────────
+
+def impute_train(train_df, target_col="y", max_decimals_cap=6, random_state=None):
+    """
+    Fits a scikit-learn iterative imputation kernel exclusively on training predictors.
+    
+    Fits the kernel dynamically on purely continuous or purely categorical data,
+    while masking out the regression target to avoid data leakage metrics.
+
+    Parameters
+    ----------
+    train_df : pandas.DataFrame
+        The full training dataset layout.
+    target_col : str, default='y'
+        The target column to explicitly exclude from the MICE calculation.
+    max_decimals_cap : int or None, default=6
+        Rounds outputs structurally ensuring the precision logic remains stable across 
+        iterations of mice imputation.
+    random_state : int or None, default=None
+        Ensures imputation reproducibility across instances and permutations.
+
+    Returns
+    -------
+    dict
+        A mapping containing:
+        - 'kernel' (`sklearn.impute.IterativeImputer`): the fitted model block used for downstream targets.
+        - 'train_imp' (`pandas.DataFrame`): the iteratively imputed layout identical to `train_df`.
+        - 'dec_map' (dict): numeric precision mappings computed from the un-imputed references.
+        - 'range_map' (dict): feature scale caps defining numeric thresholds mappings.
+
+    Raises
+    ------
+    ImportError
+        If `scikit-learn` is missing internally.
+    """
+    try:
+        from sklearn.experimental import enable_iterative_imputer  # noqa: F401
+        from sklearn.impute import IterativeImputer  # noqa: F401
+        from sklearn.ensemble import ExtraTreesRegressor
+        import pandas as pd
+    except ImportError:
+        raise ImportError(
+            "scikit-learn is required for imputation. Install with: pip install scikit-learn"
+        )
+
+    train_x = train_df.drop(columns=[target_col])
+
+    dec_map   = decimal_map_from_reference(train_x, max_decimals_cap=max_decimals_cap)
+    range_map = range_map_from_reference(train_x)
+
+    kernel = IterativeImputer(
+        estimator=ExtraTreesRegressor(n_estimators=10, random_state=random_state),
+        max_iter=3,
+        random_state=random_state
+    )
+    
+    train_x_imp_array = kernel.fit_transform(train_x)
+    train_x_imp = pd.DataFrame(train_x_imp_array, columns=train_x.columns, index=train_x.index)
+
+    train_x_imp = apply_range_cap(train_x_imp, range_map)
+    train_x_imp = apply_decimal_map(train_x_imp, dec_map)
+
+    train_imp = train_x_imp.copy()
+    train_imp[target_col] = train_df[target_col].values
+    # Restore original column order
+    train_imp = train_imp[list(train_df.columns)]
+
+    return {
+        "kernel":    kernel,
+        "train_imp": train_imp,
+        "dec_map":   dec_map,
+        "range_map": range_map,
+    }
+
+
+def impute_test(train_imp_df, test_df, dec_map, range_map, kernel, target_col="y"):
+    """
+    Applies the pre-fitted imputation kernel block explicitly onto the test predictors.
+
+    This ensures no data leakage occurs from test feature spaces onto the training
+    features. Applies previous categorical alignments mapping values independently.
+
+    Parameters
+    ----------
+    train_imp_df : pandas.DataFrame
+        The imputed layout of the corresponding train set.
+    test_df : pandas.DataFrame
+        The missing value-prone test layout containing identical structures.
+    dec_map : dict
+        A map defining categorical rounding sizes.
+    range_map : dict
+        A map describing limit scaling boundaries.
+    kernel : sklearn.impute.IterativeImputer
+        The underlying iterative mice engine holding the target parameters.
+    target_col : str, default='y'
+        String column identifier to avoid metric imputation leaks.
+
+    Returns
+    -------
+    pandas.DataFrame
+        The fully iteratively processed `test_df` preserving its baseline structure.
+
+    Raises
+    ------
+    ImportError
+        When unable to import `scikit-learn`.
+    """
+    try:
+        from sklearn.experimental import enable_iterative_imputer  # noqa: F401
+        from sklearn.impute import IterativeImputer  # noqa: F401
+        import pandas as pd
+    except ImportError:
+        raise ImportError(
+            "scikit-learn is required for imputation. Install with: pip install scikit-learn"
+        )
+
+    train_x_imp = train_imp_df.drop(columns=[target_col])
+    test_x = test_df.drop(columns=[target_col])
+    test_x = align_categories(train_x_imp, test_x)
+
+    # Modification to prevent failure when the test set has no NaN values during imputation
+    
+    # Columns containing NaN values in the test set
+    test_nan_cols = set(test_x.columns[test_x.isna().any()])
+    
+    # Only perform imputation if there is a real intersection
+    if len(test_nan_cols) > 0:
+        test_x_imp_array = kernel.transform(test_x)
+        test_x_imp = pd.DataFrame(test_x_imp_array, columns=test_x.columns, index=test_x.index)
+    else:
+        test_x_imp = test_x.copy()
+
+    test_x_imp = apply_range_cap(test_x_imp, range_map)
+    test_x_imp = apply_decimal_map(test_x_imp, dec_map)
+
+    test_imp = test_x_imp.copy()
+    test_imp[target_col] = test_df[target_col].values
+    test_imp = test_imp[list(test_df.columns)]
+
+    return test_imp
+
+
+# ── Utilities ─────────────────────────────────────────────────────────────────
+
+def split_features_target(df, target_col="y"):
+    """
+    Separates a DataFrame into its features (X) and target variable (y).
+    
+    Convenience functional wrapper that logically splits data.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        The full dataset containing predictors and target.
+    target_col : str, default='y'
+        The name of the column explicitly denoting the regression target.
+
+    Returns
+    -------
+    tuple of (pandas.DataFrame, pandas.Series)
+        A structural tuple mapping `(X, y)` subsets sequentially.
+    """
+    return df.drop(columns=[target_col]), df[target_col]
+
+
+def encode_categoricals(df, target_col="y", drop_first=True):
+    """
+    Identifies and one-hot encodes all categorical/string predictors in a DataFrame.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        The dataset to process.
+    target_col : str, default='y'
+        The name of the target column, which should be excluded from encoding.
+    drop_first : bool, default=True
+        Whether to drop the first category column to avoid the collinearity trap.
+
+    Returns
+    -------
+    pandas.DataFrame
+        The DataFrame with encoded features.
+    """
+    import pandas as pd
+    cat_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()
+    if target_col in cat_cols:
+        cat_cols.remove(target_col)
+        
+    if cat_cols:
+        print(f"[Encoding] Categorical variables detected. Applying pd.get_dummies to: {cat_cols}")
+        df = pd.get_dummies(df, columns=cat_cols, drop_first=drop_first, dtype=int)
+        
+    return df
+
+
+def get_percentages(ds_names, ds_location, phi_thr=0.8):
+    """
+    Computes the percentage of 'rare' (extreme) cases per dataset.
+    
+    This function analyzes the target variable distribution and checks what percentage
+    of instances exceed the relevance threshold. It is essential for quantifying the 
+    imbalance severity.
+
+    Parameters
+    ----------
+    ds_names : list of str
+        A list containing the names of the dataset files (e.g., ['abalone.dat', 'bos.dat']).
+    ds_location : str or Path
+        The directory path where the dataset files are located.
+    phi_thr : float, default=0.8
+        The relevance threshold [0, 1] above which an instance is considered 'rare' or extreme.
+
+    Returns
+    -------
+    pd.DataFrame
+        A DataFrame with two columns: 'dataset' (the name of the file) and 
+        'relevant_pct' (the percentage of rare cases, float).
+
+    Notes
+    -----
+    Uses `phi_control` with method "extremes" and `phi` internally.
+    """
+    from .stratification import phi_control, phi  # local import avoids circular dep
+
+    rows = []
+    for nm in ds_names:
+        d = read_dataset(nm, ds_location)
+        ctrl = phi_control(d["y"].values, method="extremes")
+        p = phi(d["y"].values, ctrl)
+        pct = float((p >= phi_thr).sum()) / len(d) * 100
+        rows.append({"dataset": nm, "relevant_pct": pct})
+    return pd.DataFrame(rows)