mlpipe-cli 0.1.0__py3-none-any.whl

mlpipe/data/validation.py

@@ -0,0 +1,248 @@
+"""
+Data validation and automatic task detection for MLPipe.
+
+Provides rigorous pre-training checks separating fatal errors from warnings,
+and infers task type (classification vs. regression) from target distribution.
+"""
+
+from dataclasses import asdict, dataclass, field
+import json
+from typing import Any, Dict, List, Optional, Union
+
+import numpy as np
+import pandas as pd
+
+from mlpipe.core.exceptions import ValidationError
+from mlpipe.data.ingestion import Dataset
+
+
+def detect_task(target_series: pd.Series) -> str:
+    """
+    Automatically determine if the problem is classification or regression.
+
+    Heuristic:
+    - Object, string, category, boolean dtypes -> classification.
+    - Low-cardinality integers (<= 10 unique or <= 2% of rows) -> classification.
+    - Floating point numbers with high cardinality -> regression.
+    """
+    clean_target = target_series.dropna()
+    n_unique = clean_target.nunique()
+    n_total = len(clean_target)
+
+    if n_total == 0 or n_unique <= 1:
+        return "classification"
+
+    if pd.api.types.is_bool_dtype(clean_target):
+        return "classification"
+
+    if clean_target.dtype == object or pd.api.types.is_string_dtype(clean_target):
+        return "classification"
+
+    if pd.api.types.is_float_dtype(clean_target):
+        # Float targets are regression unless they only contain a tiny set of integers like 0.0, 1.0
+        unique_vals = clean_target.unique()
+        if len(unique_vals) <= 5 and all(float(v).is_integer() for v in unique_vals):
+            return "classification"
+        return "regression"
+
+    if pd.api.types.is_integer_dtype(clean_target):
+        # If very few unique values (e.g. 2 to 10 classes) and small ratio compared to rows -> classification
+        if n_unique <= 10 or (n_unique <= 20 and (n_unique / n_total) < 0.05):
+            return "classification"
+        return "regression"
+
+    return "classification"
+
+
+@dataclass
+class ValidationReport:
+    """Detailed outcome of pre-training dataset validation."""
+
+    is_valid: bool
+    errors: List[str] = field(default_factory=list)
+    warnings: List[str] = field(default_factory=list)
+    detected_task: str = "classification"
+    target_info: Dict[str, Any] = field(default_factory=dict)
+    can_train: bool = True
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    def to_json(self, indent: int = 2) -> str:
+        return json.dumps(self.to_dict(), indent=indent)
+
+    def raise_if_invalid(self) -> None:
+        """Raise ValidationError if any fatal check failed."""
+        if not self.can_train or not self.is_valid:
+            error_details = "\n• " + "\n• ".join(self.errors)
+            raise ValidationError(
+                f"Dataset validation failed with {len(self.errors)} fatal error(s):{error_details}",
+                "Fix the fatal issues indicated above before proceeding with model training."
+            )
+
+
+def validate_dataset(
+    dataset: Union[Dataset, pd.DataFrame],
+    target_column: str,
+    task_override: Optional[str] = None,
+    test_size: float = 0.20,
+    cv_folds: int = 5,
+) -> ValidationReport:
+    """
+    Perform comprehensive pre-training validation.
+
+    Checks:
+    - Target existence
+    - Target usable values and missingness
+    - Target variation (non-constant)
+    - Minimum sample count
+    - Minimum class counts (for classification)
+    - Available feature columns
+    - Train/test and CV fold feasibility
+    - Class imbalance
+    """
+    df = dataset.df if isinstance(dataset, Dataset) else dataset
+    errors: List[str] = []
+    warnings: List[str] = []
+
+    # 1. Target column existence
+    if target_column not in df.columns:
+        errors.append(
+            f"Target column '{target_column}' does not exist in the dataset. "
+            f"Available columns: {list(df.columns)}"
+        )
+        return ValidationReport(
+            is_valid=False,
+            errors=errors,
+            warnings=warnings,
+            detected_task="unknown",
+            target_info={},
+            can_train=False,
+        )
+
+    target = df[target_column]
+    n_rows = len(df)
+    target_missing = int(target.isna().sum())
+    target_valid_rows = n_rows - target_missing
+
+    # 2. Target missingness
+    if target_valid_rows == 0:
+        errors.append(f"Target column '{target_column}' contains only missing (NaN/null) values.")
+    elif target_missing > 0:
+        missing_pct = round((target_missing / n_rows) * 100, 2)
+        if missing_pct > 50.0:
+            errors.append(
+                f"Target column '{target_column}' has {missing_pct}% missing values (more than 50%)."
+            )
+        else:
+            warnings.append(
+                f"Target column '{target_column}' contains {target_missing} missing values ({missing_pct}%). "
+                f"These {target_missing} rows will be dropped prior to training."
+            )
+
+    # 3. Available feature columns
+    feature_cols = [c for c in df.columns if c != target_column]
+    if len(feature_cols) == 0:
+        errors.append("Dataset contains only the target column; at least one feature column is required.")
+
+    # 4. Minimum rows check
+    if target_valid_rows < 10:
+        errors.append(
+            f"Dataset has only {target_valid_rows} usable rows. MLPipe requires at least 10 rows."
+        )
+    elif target_valid_rows < 50:
+        warnings.append(
+            f"Dataset has only {target_valid_rows} usable rows. Results may have high variance."
+        )
+
+    # 5. Task detection or override
+    inferred_task = detect_task(target)
+    if task_override and task_override != "auto":
+        task = task_override.lower()
+    else:
+        task = inferred_task
+
+    target_info: Dict[str, Any] = {
+        "column": target_column,
+        "task": task,
+        "inferred_task": inferred_task,
+        "total_rows": n_rows,
+        "usable_rows": target_valid_rows,
+        "missing_count": target_missing,
+    }
+
+    # 6. Task-specific checks
+    clean_target = target.dropna()
+    unique_targets = clean_target.unique()
+    n_classes = len(unique_targets)
+    target_info["unique_values"] = n_classes
+
+    if task == "classification":
+        if n_classes < 2:
+            errors.append(
+                f"Target column '{target_column}' has only {n_classes} unique class. "
+                "Classification requires at least 2 distinct classes."
+            )
+        else:
+            target_info["classes"] = [str(c) for c in unique_targets[:10]]
+            # Check class distribution
+            val_counts = clean_target.value_counts()
+            min_class_count = int(val_counts.min())
+            min_class_pct = round((min_class_count / len(clean_target)) * 100, 2)
+            target_info["min_class_count"] = min_class_count
+            target_info["min_class_pct"] = min_class_pct
+
+            # Check if smallest class has enough samples for train/test and CV
+            if min_class_count < cv_folds:
+                warnings.append(
+                    f"Minority class '{val_counts.idxmin()}' has only {min_class_count} samples, "
+                    f"which is fewer than cv_folds ({cv_folds}). Stratified folds may be adjusted."
+                )
+
+            if min_class_pct < 15.0:
+                warnings.append(
+                    f"Target is imbalanced: minority class represents only {min_class_pct}% of samples."
+                )
+
+    elif task == "regression":
+        if n_classes <= 1:
+            errors.append(
+                f"Target column '{target_column}' has only 1 distinct numeric value. "
+                "Regression requires variation in the target."
+            )
+        else:
+            try:
+                target_numeric = pd.to_numeric(clean_target, errors="raise")
+                target_info["mean"] = float(round(target_numeric.mean(), 4))
+                target_info["std"] = float(round(target_numeric.std(), 4))
+            except Exception:
+                errors.append(
+                    f"Target column '{target_column}' contains non-numeric values that cannot be used for regression."
+                )
+
+    # 7. Train/Test feasibility
+    test_rows = int(target_valid_rows * test_size)
+    train_rows = target_valid_rows - test_rows
+    if train_rows < 5:
+        errors.append(f"Training set would have only {train_rows} rows; increase dataset size or decrease test_size.")
+    if test_rows < 2:
+        errors.append(f"Test set would have only {test_rows} rows; evaluation cannot be reliably performed.")
+
+    # 8. Feature quality warnings
+    for col in feature_cols:
+        col_series = df[col]
+        if col_series.nunique(dropna=True) <= 1:
+            warnings.append(f"Feature column '{col}' is constant (<=1 unique value) and will be dropped.")
+        elif col_series.isna().mean() > 0.90:
+            warnings.append(f"Feature column '{col}' has >90% missing values and will be dropped.")
+
+    is_valid = len(errors) == 0
+
+    return ValidationReport(
+        is_valid=is_valid,
+        errors=errors,
+        warnings=warnings,
+        detected_task=task,
+        target_info=target_info,
+        can_train=is_valid,
+    )

mlpipe/evaluation/__init__.py

@@ -0,0 +1,11 @@
+"""Evaluation metrics and leaderboard computation for MLPipe."""
+
+from mlpipe.evaluation.evaluator import build_leaderboard, evaluate_pipeline_on_test
+from mlpipe.evaluation.metrics import get_display_metric_name, select_primary_metric
+
+__all__ = [
+    "build_leaderboard",
+    "evaluate_pipeline_on_test",
+    "get_display_metric_name",
+    "select_primary_metric",
+]

mlpipe/evaluation/evaluator.py

@@ -0,0 +1,146 @@
+"""
+Model evaluation engine for MLPipe.
+
+Calculates unbiased test-set metrics and compiles the model comparison leaderboard.
+"""
+
+import math
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import (
+    accuracy_score,
+    balanced_accuracy_score,
+    confusion_matrix,
+    f1_score,
+    mean_absolute_error,
+    mean_squared_error,
+    precision_score,
+    r2_score,
+    recall_score,
+    roc_auc_score,
+)
+from sklearn.pipeline import Pipeline
+
+from mlpipe.evaluation.metrics import get_display_metric_name
+from mlpipe.tuning.search import TuningResult
+from mlpipe.utils.logging import get_logger
+
+logger = get_logger("evaluation")
+
+
+def evaluate_pipeline_on_test(
+    pipeline: Pipeline,
+    X_test: pd.DataFrame,
+    y_test: pd.Series,
+    task_type: str,
+) -> Dict[str, Any]:
+    """
+    Evaluate a fitted pipeline on held-out test data.
+
+    All metrics are directly computed from actual model predictions.
+    """
+    y_pred = pipeline.predict(X_test)
+    metrics: Dict[str, Any] = {}
+
+    if task_type == "classification":
+        acc = float(accuracy_score(y_test, y_pred))
+        bal_acc = float(balanced_accuracy_score(y_test, y_pred))
+        prec = float(precision_score(y_test, y_pred, average="weighted", zero_division=0))
+        rec = float(recall_score(y_test, y_pred, average="weighted", zero_division=0))
+        f1 = float(f1_score(y_test, y_pred, average="weighted", zero_division=0))
+
+        metrics["accuracy"] = round(acc, 4)
+        metrics["balanced_accuracy"] = round(bal_acc, 4)
+        metrics["precision"] = round(prec, 4)
+        metrics["recall"] = round(rec, 4)
+        metrics["f1_weighted"] = round(f1, 4)
+
+        # Calculate ROC-AUC if predict_proba is supported
+        if hasattr(pipeline, "predict_proba"):
+            try:
+                y_prob = pipeline.predict_proba(X_test)
+                classes = np.unique(y_test)
+                if len(classes) == 2:
+                    auc = float(roc_auc_score(y_test, y_prob[:, 1]))
+                    metrics["roc_auc"] = round(auc, 4)
+                elif len(classes) > 2:
+                    auc = float(roc_auc_score(y_test, y_prob, multi_class="ovr", average="weighted"))
+                    metrics["roc_auc"] = round(auc, 4)
+            except Exception as e:
+                logger.debug("ROC-AUC computation skipped: %s", e)
+
+        # Confusion Matrix
+        cm = confusion_matrix(y_test, y_pred)
+        metrics["confusion_matrix"] = cm.tolist()
+
+    else:  # regression
+        mae = float(mean_absolute_error(y_test, y_pred))
+        mse = float(mean_squared_error(y_test, y_pred))
+        rmse = float(math.sqrt(mse))
+        r2 = float(r2_score(y_test, y_pred))
+
+        metrics["mae"] = round(mae, 4)
+        metrics["mse"] = round(mse, 4)
+        metrics["rmse"] = round(rmse, 4)
+        metrics["r2"] = round(r2, 4)
+
+    return metrics
+
+
+def build_leaderboard(
+    tuning_results: List[TuningResult],
+    X_test: pd.DataFrame,
+    y_test: pd.Series,
+    task_type: str,
+    primary_metric: str,
+) -> List[Dict[str, Any]]:
+    """
+    Compile and rank candidate models into a leaderboard.
+
+    MODELS ARE RANKED STRICTLY BY CV SCORE ON TRAINING DATA.
+    Test scores are calculated for reporting but never used for model selection.
+    """
+    leaderboard = []
+
+    for res in tuning_results:
+        if res.best_pipeline is None or res.error:
+            leaderboard.append({
+                "model": res.candidate_name,
+                "cv_score": None,
+                "test_score": None,
+                "training_time_s": res.training_time_s,
+                "status": "failed",
+                "error": res.error,
+                "best_params": res.best_params,
+            })
+            continue
+
+        # Evaluate on test set
+        test_metrics = evaluate_pipeline_on_test(res.best_pipeline, X_test, y_test, task_type)
+
+        # Extract primary metric test value
+        if task_type == "regression":
+            test_score = test_metrics.get("r2", 0.0)
+        else:
+            test_score = test_metrics.get(primary_metric, test_metrics.get("f1_weighted", 0.0))
+
+        leaderboard.append({
+            "model": res.candidate_name,
+            "cv_score": res.best_cv_score,
+            "test_score": test_score,
+            "training_time_s": res.training_time_s,
+            "status": "success",
+            "error": None,
+            "best_params": res.best_params,
+            "test_metrics": test_metrics,
+        })
+
+    # Sort descending by CV score (failed models with None placed at the end)
+    leaderboard.sort(
+        key=lambda x: x["cv_score"] if x["cv_score"] is not None else -999999.0,
+        reverse=True,
+    )
+
+    return leaderboard

mlpipe/evaluation/metrics.py

@@ -0,0 +1,53 @@
+"""
+Metric definitions and primary metric selection logic for MLPipe.
+"""
+
+from typing import Dict, Optional, Tuple
+
+import pandas as pd
+
+
+def select_primary_metric(task_type: str, y_train: pd.Series) -> str:
+    """
+    Select the optimal primary optimization metric.
+
+    Documented Rationale:
+    - Regression:
+        Uses 'r2' (Coefficient of Determination) to measure variance explained.
+    - Classification:
+        - Checks class distribution in y_train.
+        - If binary and balanced (minority >= 20%): uses 'roc_auc' or 'f1' depending on class balance.
+        - If imbalanced (minority < 20%): uses 'f1_weighted' to penalize poor minority performance
+          rather than misleading accuracy.
+        - If multiclass (> 2 classes): uses 'f1_weighted' to account for class representation.
+    """
+    if task_type == "regression":
+        return "r2"
+
+    val_counts = y_train.value_counts(normalize=True)
+    n_classes = len(val_counts)
+
+    if n_classes == 2:
+        minority_prop = float(val_counts.min())
+        if minority_prop < 0.20:
+            # Imbalanced binary
+            return "f1_weighted"
+        return "roc_auc"
+    else:
+        # Multiclass
+        return "f1_weighted"
+
+
+def get_display_metric_name(metric_key: str) -> str:
+    """Map sklearn scoring key to friendly display name."""
+    mapping = {
+        "f1_weighted": "F1 (Weighted)",
+        "f1": "F1",
+        "roc_auc": "ROC-AUC",
+        "accuracy": "Accuracy",
+        "balanced_accuracy": "Balanced Accuracy",
+        "r2": "R²",
+        "neg_mean_squared_error": "MSE",
+        "neg_mean_absolute_error": "MAE",
+    }
+    return mapping.get(metric_key, metric_key.upper())

mlpipe/explainability/__init__.py

@@ -0,0 +1,5 @@
+"""Explainability module for MLPipe."""
+
+from mlpipe.explainability.importance import extract_feature_importance
+
+__all__ = ["extract_feature_importance"]

mlpipe/explainability/importance.py

@@ -0,0 +1,65 @@
+"""
+Feature importance extraction and model explainability for MLPipe.
+
+Recovers post-transformation feature names to provide transparent explanations
+for tree-based and linear models.
+"""
+
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+from sklearn.pipeline import Pipeline
+
+from mlpipe.preprocessing.builder import get_transformed_feature_names
+from mlpipe.utils.logging import get_logger
+
+logger = get_logger("explainability")
+
+
+def extract_feature_importance(pipeline: Pipeline) -> List[Dict[str, Any]]:
+    """
+    Extract feature importance or coefficients from a fitted sklearn Pipeline.
+
+    Uses post-transformation feature names from the preprocessor step.
+    Returns sorted list of features with importance scores.
+    """
+    if "preprocessor" not in pipeline.named_steps or "estimator" not in pipeline.named_steps:
+        return []
+
+    preprocessor = pipeline.named_steps["preprocessor"]
+    estimator = pipeline.named_steps["estimator"]
+
+    feature_names = get_transformed_feature_names(preprocessor)
+    n_features = len(feature_names)
+
+    scores: Optional[np.ndarray] = None
+
+    # 1. Tree-based models (Random Forest, Decision Tree)
+    if hasattr(estimator, "feature_importances_"):
+        raw_imp = estimator.feature_importances_
+        if len(raw_imp) == n_features:
+            scores = raw_imp
+
+    # 2. Linear models (LogisticRegression, Ridge)
+    elif hasattr(estimator, "coef_"):
+        coef = estimator.coef_
+        if coef.ndim == 2:
+            # Multiclass: average absolute coefficients across classes
+            scores = np.mean(np.abs(coef), axis=0)
+        else:
+            scores = np.abs(coef)
+
+        if len(scores) != n_features:
+            scores = None
+
+    if scores is None or len(scores) == 0:
+        return []
+
+    # Pair with feature names and sort descending
+    paired = [
+        {"feature": str(name), "importance": float(round(abs(score), 4))}
+        for name, score in zip(feature_names, scores)
+    ]
+    paired.sort(key=lambda x: x["importance"], reverse=True)
+
+    return paired

mlpipe/models/__init__.py

@@ -0,0 +1,13 @@
+"""Model candidates and registries for MLPipe."""
+
+from mlpipe.models.classification import get_classification_candidates
+from mlpipe.models.regression import get_regression_candidates
+from mlpipe.models.registry import ModelCandidate
+from mlpipe.models.selection import get_candidates_for_task
+
+__all__ = [
+    "ModelCandidate",
+    "get_classification_candidates",
+    "get_regression_candidates",
+    "get_candidates_for_task",
+]