modeldiff-ml 0.1.0__py3-none-any.whl

modeldiff/__init__.py

@@ -0,0 +1,14 @@
+"""ModelDiff-ML: behavioral regression testing for machine-learning models."""
+
+from .api import compare, compare_predictions
+from .config import ModelDiffConfig
+from .result import ModelDiffReport, RegionResult
+
+__all__ = [
+    "ModelDiffConfig",
+    "ModelDiffReport",
+    "RegionResult",
+    "compare",
+    "compare_predictions",
+]
+__version__ = "0.1.0"

modeldiff/adapters/__init__.py

@@ -0,0 +1,5 @@
+"""Model adapters."""
+
+from .sklearn import predict_model
+
+__all__ = ["predict_model"]

modeldiff/adapters/sklearn.py

@@ -0,0 +1,31 @@
+"""Small model protocol adapter compatible with sklearn-style estimators."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+
+def predict_model(model: Any, X: pd.DataFrame, task: str) -> np.ndarray:
+    """Obtain one-dimensional predictions without transforming model scores."""
+    if task == "regression":
+        if not hasattr(model, "predict"):
+            raise TypeError("regression models must provide predict(X)")
+        predictions = np.asarray(model.predict(X))
+    else:
+        if not hasattr(model, "predict_proba"):
+            if hasattr(model, "decision_function"):
+                raise TypeError(
+                    "probability-based classification metrics require predict_proba(X); "
+                    "decision_function scores are not silently transformed"
+                )
+            raise TypeError("binary classification models must provide predict_proba(X)")
+        probabilities = np.asarray(model.predict_proba(X))
+        if probabilities.ndim != 2 or probabilities.shape[1] != 2:
+            raise ValueError("predict_proba(X) must return shape (n_samples, 2)")
+        predictions = probabilities[:, 1]
+    if predictions.ndim != 1:
+        raise ValueError("model predictions must be one-dimensional")
+    return predictions.astype(float)

modeldiff/analysis/__init__.py

@@ -0,0 +1,5 @@
+"""Analysis helpers."""
+
+from .disagreement import classification_disagreement, regression_disagreement
+
+__all__ = ["classification_disagreement", "regression_disagreement"]

modeldiff/analysis/disagreement.py

@@ -0,0 +1,40 @@
+"""Prediction-level behavioral diagnostics."""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def classification_disagreement(
+    baseline: np.ndarray, candidate: np.ndarray, threshold: float = 0.5
+) -> dict[str, float]:
+    """Summarize probability shifts and hard-label disagreement."""
+    shift = np.abs(candidate - baseline)
+    baseline_confidence = np.abs(baseline - 0.5) * 2
+    candidate_confidence = np.abs(candidate - 0.5) * 2
+    return {
+        "mean_absolute_probability_shift": float(np.mean(shift)),
+        "median_absolute_probability_shift": float(np.median(shift)),
+        "p95_absolute_probability_shift": float(np.quantile(shift, 0.95)),
+        "hard_classification_disagreement_rate": float(
+            np.mean((baseline >= threshold) != (candidate >= threshold))
+        ),
+        "mean_candidate_confidence_change": float(
+            np.mean(candidate_confidence - baseline_confidence)
+        ),
+    }
+
+
+def regression_disagreement(baseline: np.ndarray, candidate: np.ndarray) -> dict[str, float]:
+    """Summarize regression prediction shifts."""
+    difference = np.abs(candidate - baseline)
+    if np.std(baseline) == 0 or np.std(candidate) == 0:
+        correlation = float("nan")
+    else:
+        correlation = float(np.corrcoef(baseline, candidate)[0, 1])
+    return {
+        "mean_absolute_prediction_difference": float(np.mean(difference)),
+        "median_absolute_prediction_difference": float(np.median(difference)),
+        "p95_absolute_prediction_difference": float(np.quantile(difference, 0.95)),
+        "prediction_correlation": correlation,
+    }

modeldiff/analysis/distributions.py

@@ -0,0 +1,26 @@
+"""Aggregate metric comparisons."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from modeldiff.metrics.classification import classification_metrics
+from modeldiff.metrics.regression import regression_metrics
+
+
+def aggregate_metrics(
+    task: str,
+    y: np.ndarray,
+    baseline: np.ndarray,
+    candidate: np.ndarray,
+    threshold: float,
+    epsilon: float,
+) -> dict[str, dict[str, float]]:
+    """Compute full aggregate metric sets for both models."""
+    if task == "binary_classification":
+        baseline_metrics = classification_metrics(y, baseline, threshold, epsilon)
+        candidate_metrics = classification_metrics(y, candidate, threshold, epsilon)
+    else:
+        baseline_metrics = regression_metrics(y, baseline)
+        candidate_metrics = regression_metrics(y, candidate)
+    return {"baseline": baseline_metrics, "candidate": candidate_metrics}

modeldiff/analysis/regions.py

@@ -0,0 +1,210 @@
+"""Interpretable condition generation and beam-search region discovery."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Literal
+
+import numpy as np
+import pandas as pd
+
+from modeldiff.config import ModelDiffConfig
+from modeldiff.utils import python_value
+
+Direction = Literal["regression", "improvement", "both"]
+
+
+@dataclass(frozen=True)
+class Condition:
+    """A single human-readable feature predicate."""
+
+    feature: str
+    operator: str
+    value: Any
+
+    def apply(self, X: pd.DataFrame) -> np.ndarray:
+        """Evaluate the condition, treating missing values as non-members."""
+        series = X[self.feature]
+        present = series.notna().to_numpy()
+        if self.operator == "<=":
+            result = series.le(self.value).to_numpy()
+        elif self.operator == ">":
+            result = series.gt(self.value).to_numpy()
+        elif self.operator == "==":
+            result = series.eq(self.value).to_numpy()
+        elif self.operator == "!=":
+            result = series.ne(self.value).to_numpy()
+        else:
+            raise ValueError(f"unsupported condition operator: {self.operator}")
+        return np.asarray(result, dtype=bool) & present
+
+    def __str__(self) -> str:
+        value = repr(self.value) if isinstance(self.value, str) else f"{self.value:.6g}"
+        return f"{self.feature} {self.operator} {value}"
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a JSON-compatible representation."""
+        return {
+            "feature": self.feature,
+            "operator": self.operator,
+            "value": python_value(self.value),
+        }
+
+
+@dataclass(frozen=True)
+class Region:
+    """A conjunction of conditions found on discovery data."""
+
+    conditions: tuple[Condition, ...]
+    discovery_support: int
+    discovery_fraction: float
+    discovery_delta: float
+    score: float
+
+    def apply(self, X: pd.DataFrame) -> np.ndarray:
+        """Evaluate all conditions against a feature frame."""
+        mask = np.ones(len(X), dtype=bool)
+        for condition in self.conditions:
+            mask &= condition.apply(X)
+        return mask
+
+    def __str__(self) -> str:
+        return " AND ".join(str(condition) for condition in self.conditions)
+
+
+def generate_conditions(
+    X: pd.DataFrame, config: ModelDiffConfig, minimum_support: int
+) -> list[Condition]:
+    """Generate bounded numeric and categorical candidate predicates."""
+    conditions: list[Condition] = []
+    n = len(X)
+    maximum_support = int(np.floor(config.max_region_fraction * n))
+    for feature in X.columns:
+        series = X[feature]
+        if pd.api.types.is_numeric_dtype(series) and not pd.api.types.is_bool_dtype(series):
+            values = series.dropna().to_numpy(dtype=float)
+            if values.size == 0:
+                continue
+            thresholds = np.unique(np.quantile(values, config.quantiles))
+            for threshold in thresholds:
+                for operator in ("<=", ">"):
+                    condition = Condition(feature, operator, float(threshold))
+                    support = int(condition.apply(X).sum())
+                    if minimum_support <= support <= maximum_support:
+                        conditions.append(condition)
+        else:
+            counts = series.dropna().value_counts()
+            for category in counts.head(config.max_categories).index:
+                for operator in ("==", "!="):
+                    condition = Condition(feature, operator, python_value(category))
+                    support = int(condition.apply(X).sum())
+                    if minimum_support <= support <= maximum_support:
+                        conditions.append(condition)
+    return conditions
+
+
+def _direction_score(mean_delta: float, support_fraction: float, direction: Direction) -> float:
+    if direction == "regression":
+        effect = mean_delta
+    elif direction == "improvement":
+        effect = -mean_delta
+    else:
+        effect = abs(mean_delta)
+    return max(effect, 0.0) * float(np.sqrt(support_fraction))
+
+
+def _region_key(conditions: tuple[Condition, ...]) -> tuple[str, ...]:
+    return tuple(sorted(str(condition) for condition in conditions))
+
+
+def discover_regions(
+    X: pd.DataFrame,
+    delta: np.ndarray,
+    config: ModelDiffConfig,
+    direction: Direction = "both",
+) -> list[Region]:
+    """Find high-effect regions using deterministic breadth-limited beam search."""
+    n = len(X)
+    minimum_support = max(config.min_region_size or 0, int(np.ceil(config.min_region_fraction * n)))
+    candidates = generate_conditions(X, config, minimum_support)
+    candidate_masks = {condition: condition.apply(X) for condition in candidates}
+    beam: list[tuple[Region, np.ndarray]] = []
+    all_regions: list[tuple[Region, np.ndarray]] = []
+    seen: set[tuple[str, ...]] = set()
+
+    for depth in range(1, config.max_depth + 1):
+        expanded: list[tuple[Region, np.ndarray]] = []
+        parents: list[tuple[tuple[Condition, ...], np.ndarray]]
+        if depth == 1:
+            parents = [((), np.ones(n, dtype=bool))]
+        else:
+            parents = [(region.conditions, mask) for region, mask in beam]
+        for parent_conditions, parent_mask in parents:
+            used_features = {condition.feature for condition in parent_conditions}
+            for condition in candidates:
+                # One predicate per feature is simple, readable, and avoids contradictions.
+                if condition.feature in used_features:
+                    continue
+                conditions = (*parent_conditions, condition)
+                key = _region_key(conditions)
+                if key in seen:
+                    continue
+                seen.add(key)
+                mask = parent_mask & candidate_masks[condition]
+                support = int(mask.sum())
+                fraction = support / n
+                if support < minimum_support or fraction > config.max_region_fraction:
+                    continue
+                mean_delta = float(np.mean(delta[mask]))
+                score = _direction_score(mean_delta, fraction, direction)
+                if score <= 0:
+                    continue
+                expanded.append(
+                    (
+                        Region(conditions, support, fraction, mean_delta, score),
+                        mask,
+                    )
+                )
+        expanded.sort(
+            key=lambda item: (
+                -item[0].score,
+                -item[0].discovery_support,
+                len(item[0].conditions),
+                str(item[0]),
+            )
+        )
+        beam = expanded[: config.beam_width]
+        all_regions.extend(beam)
+        if not beam:
+            break
+
+    return deduplicate_regions(all_regions, config.jaccard_threshold, config.max_regions)
+
+
+def jaccard(mask_a: np.ndarray, mask_b: np.ndarray) -> float:
+    """Return Jaccard similarity between two boolean membership masks."""
+    union = int(np.sum(mask_a | mask_b))
+    return float(np.sum(mask_a & mask_b) / union) if union else 1.0
+
+
+def deduplicate_regions(
+    regions_with_masks: list[tuple[Region, np.ndarray]], threshold: float, maximum: int
+) -> list[Region]:
+    """Keep the strongest region among highly overlapping candidates."""
+    ranked = sorted(
+        regions_with_masks,
+        key=lambda item: (
+            -item[0].score,
+            -item[0].discovery_support,
+            len(item[0].conditions),
+            str(item[0]),
+        ),
+    )
+    selected: list[tuple[Region, np.ndarray]] = []
+    for region, mask in ranked:
+        if any(jaccard(mask, existing_mask) > threshold for _, existing_mask in selected):
+            continue
+        selected.append((region, mask))
+        if len(selected) >= maximum:
+            break
+    return [region for region, _ in selected]

modeldiff/api.py

@@ -0,0 +1,249 @@
+"""Public comparison pipeline."""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from typing import Any, Literal
+
+import numpy as np
+
+from modeldiff.adapters import predict_model
+from modeldiff.analysis.disagreement import (
+    classification_disagreement,
+    regression_disagreement,
+)
+from modeldiff.analysis.distributions import aggregate_metrics
+from modeldiff.analysis.regions import Region, discover_regions
+from modeldiff.config import ModelDiffConfig
+from modeldiff.metrics.base import get_loss_function, primary_metric_value
+from modeldiff.result import ModelDiffReport, RegionResult
+from modeldiff.statistics import benjamini_hochberg, bootstrap_mean_ci, sign_flip_test
+from modeldiff.utils import normalize_X, validate_inputs
+
+
+def _observation_losses(
+    task: str,
+    metric: str,
+    y: np.ndarray,
+    predictions: np.ndarray,
+    config: ModelDiffConfig,
+) -> np.ndarray:
+    function = get_loss_function(task, metric)
+    if task == "binary_classification" and metric == "log_loss":
+        return function(y, predictions, epsilon=config.epsilon)
+    if task == "binary_classification" and metric == "accuracy":
+        return function(y, predictions, threshold=config.threshold)
+    return function(y, predictions)
+
+
+def _validate_regions(
+    regions: list[Region],
+    direction: str,
+    X_validation: Any,
+    baseline_loss: np.ndarray,
+    candidate_loss: np.ndarray,
+    config: ModelDiffConfig,
+    rng: np.random.Generator,
+    warnings: list[str],
+) -> list[RegionResult]:
+    results: list[RegionResult] = []
+    delta = candidate_loss - baseline_loss
+    for region in regions:
+        mask = region.apply(X_validation)
+        support = int(mask.sum())
+        if support == 0:
+            warnings.append(f"Region disappeared in validation: {region}")
+            continue
+        if support < 5:
+            warnings.append(f"Region has fewer than five validation observations: {region}")
+        region_delta = delta[mask]
+        alternative: Literal["greater", "less"] = "greater" if direction == "regression" else "less"
+        ci = bootstrap_mean_ci(
+            region_delta,
+            n_bootstrap=config.n_bootstrap,
+            confidence_level=config.confidence_level,
+            random_state=rng,
+        )
+        p_value = sign_flip_test(
+            region_delta,
+            n_permutations=config.n_permutations,
+            alternative=alternative,
+            random_state=rng,
+        )
+        results.append(
+            RegionResult(
+                conditions=region.conditions,
+                direction=direction,
+                discovery_support=region.discovery_support,
+                discovery_fraction=region.discovery_fraction,
+                validation_support=support,
+                validation_fraction=support / len(X_validation),
+                discovery_delta=region.discovery_delta,
+                validation_delta=float(np.mean(region_delta)),
+                confidence_interval=ci,
+                p_value=p_value,
+                q_value=1.0,
+                score=region.score,
+                baseline_mean_loss=float(np.mean(baseline_loss[mask])),
+                candidate_mean_loss=float(np.mean(candidate_loss[mask])),
+            )
+        )
+    return results
+
+
+def compare_predictions(
+    baseline_predictions: Any,
+    candidate_predictions: Any,
+    X: Any,
+    y: Any,
+    task: str,
+    metric: str,
+    *,
+    config: ModelDiffConfig | None = None,
+    random_state: int | None = None,
+) -> ModelDiffReport:
+    """Compare precomputed model predictions and discover validated behavior changes.
+
+    Positive loss deltas mean the candidate is worse. Regions are discovered on
+    one random partition and statistically evaluated on the independent holdout.
+    """
+    effective_config = config or ModelDiffConfig()
+    if random_state is not None:
+        effective_config = replace(effective_config, random_state=random_state)
+    frame, target, baseline, candidate = validate_inputs(
+        X, y, baseline_predictions, candidate_predictions, task, metric
+    )
+    baseline_loss = _observation_losses(task, metric, target, baseline, effective_config)
+    candidate_loss = _observation_losses(task, metric, target, candidate, effective_config)
+    delta = candidate_loss - baseline_loss
+
+    baseline_primary = primary_metric_value(baseline_loss, metric)
+    candidate_primary = primary_metric_value(candidate_loss, metric)
+    absolute_delta = candidate_primary - baseline_primary
+    relative_delta = absolute_delta / baseline_primary if baseline_primary != 0 else None
+    overall = {
+        "baseline": baseline_primary,
+        "candidate": candidate_primary,
+        "absolute_delta": absolute_delta,
+        "relative_delta": relative_delta,
+    }
+    aggregates = aggregate_metrics(
+        task, target, baseline, candidate, effective_config.threshold, effective_config.epsilon
+    )
+    if task == "binary_classification":
+        diagnostics = classification_disagreement(baseline, candidate, effective_config.threshold)
+    else:
+        diagnostics = regression_disagreement(baseline, candidate)
+
+    rng = np.random.default_rng(effective_config.random_state)
+    order = rng.permutation(len(frame))
+    discovery_count = int(np.floor(effective_config.discovery_fraction * len(frame)))
+    discovery_indices = order[:discovery_count]
+    validation_indices = order[discovery_count:]
+    X_discovery = frame.iloc[discovery_indices].reset_index(drop=True)
+    X_validation = frame.iloc[validation_indices].reset_index(drop=True)
+    discovery_delta = delta[discovery_indices]
+
+    warnings: list[str] = []
+    if len(validation_indices) < 30:
+        warnings.append("Validation set has fewer than 30 observations; inference may be unstable.")
+    if np.array_equal(baseline, candidate):
+        warnings.append("Candidate predictions are identical to baseline predictions.")
+    if np.std(baseline) == 0 or np.std(candidate) == 0:
+        warnings.append("At least one model has constant predictions.")
+    for feature in frame.columns:
+        if frame[feature].isna().mean() > 0.5:
+            warnings.append(f"Feature {feature!r} contains more than 50% missing values.")
+
+    discovered_regressions = discover_regions(
+        X_discovery, discovery_delta, effective_config, "regression"
+    )
+    discovered_improvements = discover_regions(
+        X_discovery, discovery_delta, effective_config, "improvement"
+    )
+    validated = _validate_regions(
+        discovered_regressions,
+        "regression",
+        X_validation,
+        baseline_loss[validation_indices],
+        candidate_loss[validation_indices],
+        effective_config,
+        rng,
+        warnings,
+    )
+    validated += _validate_regions(
+        discovered_improvements,
+        "improvement",
+        X_validation,
+        baseline_loss[validation_indices],
+        candidate_loss[validation_indices],
+        effective_config,
+        rng,
+        warnings,
+    )
+    q_values = benjamini_hochberg([region.p_value for region in validated])
+    validated = [
+        replace(
+            region,
+            q_value=float(q_value),
+            significant=float(q_value) <= effective_config.significance_level,
+        )
+        for region, q_value in zip(validated, q_values, strict=True)
+    ]
+    regressions = sorted(
+        [
+            region
+            for region in validated
+            if region.direction == "regression" and region.validation_delta > 0
+        ],
+        key=lambda region: (not region.significant, -region.validation_delta, region.q_value),
+    )
+    improvements = sorted(
+        [
+            region
+            for region in validated
+            if region.direction == "improvement" and region.validation_delta < 0
+        ],
+        key=lambda region: (not region.significant, region.validation_delta, region.q_value),
+    )
+    return ModelDiffReport(
+        task=task,
+        metric=metric,
+        sample_count=len(frame),
+        discovery_count=len(discovery_indices),
+        validation_count=len(validation_indices),
+        overall=overall,
+        aggregate_metrics=aggregates,
+        disagreement=diagnostics,
+        regression_regions=regressions,
+        improvement_regions=improvements,
+        warnings=warnings,
+        configuration=effective_config,
+    )
+
+
+def compare(
+    baseline: Any,
+    candidate: Any,
+    X: Any,
+    y: Any,
+    task: str,
+    metric: str,
+    *,
+    config: ModelDiffConfig | None = None,
+    random_state: int | None = None,
+) -> ModelDiffReport:
+    """Compare two fitted sklearn-style models on a shared evaluation dataset."""
+    frame = normalize_X(X)
+    baseline_predictions = predict_model(baseline, frame, task)
+    candidate_predictions = predict_model(candidate, frame, task)
+    return compare_predictions(
+        baseline_predictions,
+        candidate_predictions,
+        frame,
+        y,
+        task,
+        metric,
+        config=config,
+        random_state=random_state,
+    )

modeldiff/cli.py

@@ -0,0 +1,74 @@
+"""Command-line interface for precomputed-prediction comparisons."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+import pandas as pd
+
+from modeldiff import ModelDiffConfig, compare_predictions
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the command-line argument parser."""
+    parser = argparse.ArgumentParser(prog="modeldiff-ml", description=__doc__)
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    compare_parser = subparsers.add_parser("compare", help="compare predictions in a CSV file")
+    compare_parser.add_argument("--data", required=True, type=Path)
+    compare_parser.add_argument("--target", required=True)
+    compare_parser.add_argument("--baseline-pred", required=True)
+    compare_parser.add_argument("--candidate-pred", required=True)
+    compare_parser.add_argument(
+        "--task", required=True, choices=["binary_classification", "regression"]
+    )
+    compare_parser.add_argument("--metric", required=True)
+    compare_parser.add_argument("--exclude", action="append", default=[])
+    compare_parser.add_argument("--html", type=Path)
+    compare_parser.add_argument("--json", type=Path)
+    compare_parser.add_argument("--random-state", type=int, default=42)
+    compare_parser.add_argument("--fail-on-significant-regression", action="store_true")
+    compare_parser.add_argument("--max-overall-regression", type=float)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the ModelDiff-ML CLI and return a CI-friendly exit status."""
+    args = build_parser().parse_args(argv)
+    data = pd.read_csv(args.data)
+    required = {args.target, args.baseline_pred, args.candidate_pred}
+    missing = required - set(data.columns)
+    if missing:
+        raise SystemExit(f"missing columns: {', '.join(sorted(missing))}")
+    excluded = required | set(args.exclude)
+    features = data.drop(columns=list(excluded), errors="ignore")
+    if features.shape[1] == 0:
+        raise SystemExit("no feature columns remain after exclusions")
+    config = ModelDiffConfig(random_state=args.random_state)
+    report = compare_predictions(
+        data[args.baseline_pred],
+        data[args.candidate_pred],
+        features,
+        data[args.target],
+        args.task,
+        args.metric,
+        config=config,
+    )
+    print(report.summary())
+    if args.html:
+        report.to_html(args.html)
+    if args.json:
+        report.to_json(args.json)
+    failed = args.fail_on_significant_regression and bool(report.regressions(significant_only=True))
+    if (
+        args.max_overall_regression is not None
+        and report.overall["absolute_delta"] > args.max_overall_regression
+    ):
+        failed = True
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())