SKSurrogate 0.1.0__py3-none-any.whl

SKSurrogate/monitoring.py

@@ -0,0 +1,360 @@
+"""Data-quality and drift monitoring for tabular model inputs."""
+
+from collections import Counter
+
+import numpy as np
+import pandas as pd
+
+from sklearn.metrics import accuracy_score, mean_absolute_error, mean_squared_error
+
+
+def _schema(frame):
+    return {
+        column: {
+            "dtype": str(frame[column].dtype),
+            "nullable": bool(frame[column].isna().any()),
+        }
+        for column in frame.columns
+    }
+
+
+def _distribution(values, categories):
+    counts = Counter(str(value) for value in values.dropna())
+    total = float(sum(counts.values()))
+    return {
+        category: counts.get(category, 0) / total if total else 0.0
+        for category in categories
+    }
+
+
+def _numeric_drift(reference, current, bins):
+    reference = reference.dropna().to_numpy(dtype=float)
+    current = current.dropna().to_numpy(dtype=float)
+    if not len(reference) or not len(current):
+        return {"method": "psi", "value": None, "bins": []}
+    quantiles = np.linspace(0.0, 1.0, bins + 1)
+    edges = np.unique(np.quantile(reference, quantiles))
+    if len(edges) < 2:
+        value = 0.0 if np.all(current == reference[0]) else 1.0
+        return {"method": "psi", "value": value, "bins": [float(edges[0])]}
+    reference_counts, _ = np.histogram(reference, bins=edges)
+    current_counts, _ = np.histogram(current, bins=edges)
+    reference_share = np.clip(reference_counts / len(reference), 1.0e-12, None)
+    current_share = np.clip(current_counts / len(current), 1.0e-12, None)
+    psi = float(np.sum((current_share - reference_share) * np.log(current_share / reference_share)))
+    return {"method": "psi", "value": psi, "bins": edges.tolist()}
+
+
+def drift_report(
+    reference,
+    current,
+    *,
+    model_version=None,
+    reference_dataset_fingerprint=None,
+    current_dataset_fingerprint=None,
+    psi_threshold=0.2,
+    categorical_threshold=0.2,
+    missingness_threshold=0.1,
+    range_threshold=0.1,
+    bins=10,
+):
+    """Compare reference and current tabular data and return a JSON-safe report.
+
+    Schema and data-quality changes are reported independently from statistical drift.
+    Numeric drift uses population stability index (PSI); categorical drift uses total
+    variation distance. Threshold crossings are listed in ``alerts``.
+    """
+    if not isinstance(reference, pd.DataFrame) or not isinstance(current, pd.DataFrame):
+        raise TypeError("reference and current must be pandas DataFrames")
+    if bins < 2:
+        raise ValueError("bins must be at least 2")
+    reference_schema = _schema(reference)
+    current_schema = _schema(current)
+    reference_columns = list(reference.columns)
+    current_columns = list(current.columns)
+    missing = [column for column in reference_columns if column not in current_columns]
+    extra = [column for column in current_columns if column not in reference_columns]
+    reordered = not missing and not extra and reference_columns != current_columns
+    schema_changes = {
+        "missing_columns": missing,
+        "extra_columns": extra,
+        "reordered": reordered,
+        "dtype_changes": {
+            column: {"reference": reference_schema[column]["dtype"], "current": current_schema[column]["dtype"]}
+            for column in reference_columns
+            if column in current_schema
+            and reference_schema[column]["dtype"] != current_schema[column]["dtype"]
+        },
+    }
+    quality = {
+        "reference_rows": int(len(reference)),
+        "current_rows": int(len(current)),
+        "missingness": {},
+    }
+    drift = {"numeric": {}, "categorical": {}, "range": {}}
+    alerts = []
+    for column in reference_columns:
+        if column not in current:
+            continue
+        reference_missingness = float(reference[column].isna().mean())
+        current_missingness = float(current[column].isna().mean())
+        missingness_delta = current_missingness - reference_missingness
+        quality["missingness"][column] = {
+            "reference": reference_missingness,
+            "current": current_missingness,
+            "delta": missingness_delta,
+        }
+        if abs(missingness_delta) >= missingness_threshold:
+            alerts.append({"type": "missingness", "column": column, "value": missingness_delta})
+        if not pd.api.types.is_numeric_dtype(reference[column]):
+            categories = sorted(
+                set(str(value) for value in reference[column].dropna())
+                | set(str(value) for value in current[column].dropna())
+            )
+            reference_distribution = _distribution(reference[column], categories)
+            current_distribution = _distribution(current[column], categories)
+            distance = 0.5 * sum(
+                abs(reference_distribution[category] - current_distribution[category])
+                for category in categories
+            )
+            drift["categorical"][column] = {
+                "metric": "total_variation_distance",
+                "value": float(distance),
+                "reference_distribution": reference_distribution,
+                "current_distribution": current_distribution,
+            }
+            if distance >= categorical_threshold:
+                alerts.append({"type": "categorical_drift", "column": column, "value": float(distance)})
+        else:
+            numeric = _numeric_drift(reference[column], current[column], bins)
+            drift["numeric"][column] = numeric
+            if numeric["value"] is not None and numeric["value"] >= psi_threshold:
+                alerts.append({"type": "numeric_drift", "column": column, "value": numeric["value"]})
+            reference_min = float(reference[column].min()) if not reference[column].dropna().empty else None
+            reference_max = float(reference[column].max()) if not reference[column].dropna().empty else None
+            if reference_min is not None and reference_max is not None:
+                values = current[column].dropna()
+                out_of_range = (
+                    float(((values < reference_min) | (values > reference_max)).mean())
+                    if not values.empty
+                    else 0.0
+                )
+                drift["range"][column] = {
+                    "reference_min": reference_min,
+                    "reference_max": reference_max,
+                    "current_min": float(values.min()) if not values.empty else None,
+                    "current_max": float(values.max()) if not values.empty else None,
+                    "out_of_range_rate": out_of_range,
+                }
+                if out_of_range >= range_threshold:
+                    alerts.append({"type": "range_drift", "column": column, "value": out_of_range})
+    return {
+        "model_version": model_version,
+        "reference_dataset_fingerprint": reference_dataset_fingerprint,
+        "current_dataset_fingerprint": current_dataset_fingerprint,
+        "schema_changes": schema_changes,
+        "data_quality": quality,
+        "drift": drift,
+        "thresholds": {
+            "psi": psi_threshold,
+            "categorical": categorical_threshold,
+            "missingness": missingness_threshold,
+            "range": range_threshold,
+        },
+        "alerts": alerts,
+    }
+
+
+def prediction_distribution_report(
+    reference_predictions,
+    current_predictions,
+    *,
+    model_version=None,
+    threshold=0.2,
+    bins=10,
+):
+    """Report distribution drift in model predictions."""
+    reference = pd.DataFrame({"prediction": reference_predictions})
+    current = pd.DataFrame({"prediction": current_predictions})
+    report = drift_report(
+        reference,
+        current,
+        model_version=model_version,
+        psi_threshold=threshold,
+        categorical_threshold=threshold,
+        missingness_threshold=1.0,
+        range_threshold=1.0,
+        bins=bins,
+    )
+    report["prediction_column"] = "prediction"
+    report["alerts"] = [
+        alert for alert in report["alerts"]
+        if alert["type"] in {"numeric_drift", "categorical_drift"}
+    ]
+    return report
+
+
+def delayed_label_performance(predictions, labels, *, model_version=None, task_type="classification"):
+    """Compute performance after delayed labels arrive for recorded predictions."""
+    predictions = np.asarray(predictions)
+    labels = np.asarray(labels)
+    if len(predictions) != len(labels):
+        raise ValueError("predictions and labels must contain the same number of rows")
+    if not len(labels):
+        raise ValueError("predictions and labels cannot be empty")
+    if task_type == "classification":
+        metrics = {"accuracy": float(accuracy_score(labels, predictions))}
+    elif task_type == "regression":
+        metrics = {
+            "mae": float(mean_absolute_error(labels, predictions)),
+            "mse": float(mean_squared_error(labels, predictions)),
+        }
+    else:
+        raise ValueError("task_type must be 'classification' or 'regression'")
+    return {"model_version": model_version, "rows": int(len(labels)), "metrics": metrics}
+
+
+def fairness_report(y_true, y_pred, groups, *, positive_label=1):
+    """Summarize demographic parity and equal-opportunity gaps across groups."""
+    y_true = np.asarray(y_true)
+    y_pred = np.asarray(y_pred)
+    groups = np.asarray(groups)
+    if len(y_true) != len(y_pred) or len(y_true) != len(groups):
+        raise ValueError("y_true, y_pred, and groups must have the same length")
+    if not len(y_true):
+        raise ValueError("inputs cannot be empty")
+    by_group = {}
+    selection_rates = []
+    opportunity_rates = []
+    for group in sorted(np.unique(groups)):
+        mask = groups == group
+        predicted_positive = y_pred[mask] == positive_label
+        actual_positive = y_true[mask] == positive_label
+        selection_rate = float(np.mean(predicted_positive)) if np.any(mask) else 0.0
+        true_positive_rate = float(np.mean(predicted_positive[actual_positive])) if np.any(actual_positive) else 0.0
+        by_group[str(group)] = {
+            "count": int(np.sum(mask)),
+            "selection_rate": selection_rate,
+            "true_positive_rate": true_positive_rate,
+            "accuracy": float(np.mean(y_true[mask] == y_pred[mask])) if np.any(mask) else 0.0,
+        }
+        selection_rates.append(selection_rate)
+        opportunity_rates.append(true_positive_rate)
+    fairness = {
+        "demographic_parity_gap": float(max(selection_rates) - min(selection_rates)) if selection_rates else 0.0,
+        "equal_opportunity_gap": float(max(opportunity_rates) - min(opportunity_rates)) if opportunity_rates else 0.0,
+    }
+    return {"groups": by_group, "fairness": fairness}
+
+
+def subgroup_performance_report(y_true, y_pred, groups, *, metric="accuracy", positive_label=1):
+    """Report group-level metrics and the maximum performance gap between subgroups."""
+    y_true = np.asarray(y_true)
+    y_pred = np.asarray(y_pred)
+    groups = np.asarray(groups)
+    if len(y_true) != len(y_pred) or len(y_true) != len(groups):
+        raise ValueError("y_true, y_pred, and groups must have the same length")
+    if metric not in {"accuracy", "precision", "recall", "f1", "loss"}:
+        raise ValueError("metric must be one of accuracy, precision, recall, f1, or loss")
+    results = {}
+    values = []
+    for group in sorted(np.unique(groups)):
+        mask = groups == group
+        group_true = y_true[mask]
+        group_pred = y_pred[mask]
+        if metric == "accuracy":
+            value = float(np.mean(group_true == group_pred)) if len(group_true) else 0.0
+        elif metric == "precision":
+            positive_pred = group_pred == positive_label
+            actual_positive = group_true == positive_label
+            value = float(np.mean(positive_pred[actual_positive])) if np.any(actual_positive) else 0.0
+        elif metric == "recall":
+            actual_positive = group_true == positive_label
+            value = float(np.mean(group_pred[actual_positive] == positive_label)) if np.any(actual_positive) else 0.0
+        elif metric == "f1":
+            actual_positive = group_true == positive_label
+            actual_negative = group_true != positive_label
+            tp = np.sum((group_pred == positive_label) & actual_positive)
+            fp = np.sum((group_pred == positive_label) & actual_negative)
+            precision = tp / (tp + fp) if (tp + fp) else 0.0
+            recall = tp / np.sum(actual_positive) if np.any(actual_positive) else 0.0
+            value = 0.0 if (precision + recall) == 0.0 else (2.0 * precision * recall) / (precision + recall)
+        else:
+            value = 1.0 - float(np.mean(group_true == group_pred)) if len(group_true) else 0.0
+        results[str(group)] = {metric: value, "count": int(len(group_true))}
+        values.append(value)
+    return {"groups": results, "metric": metric, "fairness_gap": float(max(values) - min(values)) if values else 0.0}
+
+
+def loss_report(y_true, y_pred, groups=None, *, positive_label=1):
+    """Convenience wrapper for group-level loss reporting."""
+    if groups is None:
+        groups = np.zeros_like(np.asarray(y_true), dtype=object)
+    return subgroup_performance_report(y_true, y_pred, groups, metric="loss", positive_label=positive_label)
+
+
+def sensitive_feature_report(frame, *, sensitive_features=None):
+    """Flag sensitive and PII-like columns before model training or serving."""
+    if not isinstance(frame, pd.DataFrame):
+        raise TypeError("frame must be a pandas DataFrame")
+    explicit_sensitive = set(sensitive_features or [])
+    pii_tokens = (
+        "email",
+        "phone",
+        "ssn",
+        "social_security",
+        "address",
+        "name",
+        "passport",
+        "tax_id",
+        "dob",
+        "date_of_birth",
+        "ip_address",
+    )
+    pii_columns = []
+    sensitive_columns = []
+    for column in frame.columns:
+        lowered = str(column).lower()
+        if any(token in lowered for token in pii_tokens):
+            pii_columns.append(column)
+        if column in explicit_sensitive or any(token in lowered for token in ("ssn", "secret", "password", "token", "api_key")):
+            sensitive_columns.append(column)
+    pii_columns = sorted(set(pii_columns))
+    sensitive_columns = sorted(set(sensitive_columns))
+    warnings = []
+    if pii_columns:
+        warnings.append("PII-like columns detected: %s" % ", ".join(pii_columns))
+    if sensitive_columns:
+        warnings.append("Sensitive columns detected: %s" % ", ".join(sensitive_columns))
+    return {
+        "pii_columns": pii_columns,
+        "sensitive_columns": sensitive_columns,
+        "warnings": warnings,
+    }
+
+
+class InferenceMonitor:
+    """Accumulate latency, error-rate, and throughput metrics for requests."""
+
+    def __init__(self, model_version=None):
+        self.model_version = model_version
+        self._requests = []
+
+    def record(self, latency_ms, rows, *, success=True):
+        if latency_ms < 0 or rows < 0:
+            raise ValueError("latency_ms and rows must be non-negative")
+        self._requests.append({"latency_ms": float(latency_ms), "rows": int(rows), "success": bool(success)})
+
+    def summary(self):
+        total = len(self._requests)
+        rows = sum(item["rows"] for item in self._requests)
+        latency = sum(item["latency_ms"] for item in self._requests)
+        successes = sum(item["success"] for item in self._requests)
+        return {
+            "model_version": self.model_version,
+            "requests": total,
+            "rows": rows,
+            "error_rate": (total - successes) / total if total else 0.0,
+            "mean_latency_ms": latency / total if total else 0.0,
+            "throughput_rows_per_second": rows / (latency / 1000.0) if latency else 0.0,
+        }

SKSurrogate/retraining.py

@@ -0,0 +1,49 @@
+"""Scheduler-agnostic retraining entry points."""
+
+from .modelbundle import ModelBundle
+
+
+class RetrainingJob:
+    """Run a trainer when a scheduled or data-driven trigger is satisfied."""
+
+    def __init__(self, trainer, registry, task_name, promotion_state=None):
+        if not callable(trainer):
+            raise TypeError("trainer must be callable")
+        self.trainer = trainer
+        self.registry = registry
+        self.task_name = task_name
+        self.promotion_state = promotion_state
+
+    def run(self, *, trigger=None, context=None):
+        context = {} if context is None else dict(context)
+        if trigger is not None and not trigger(context):
+            return {"status": "skipped", "reason": "trigger_not_satisfied", "context": context}
+
+        bundle = self.trainer(context)
+        if not isinstance(bundle, ModelBundle):
+            raise TypeError("trainer must return a ModelBundle")
+        if bundle.task_name is None:
+            bundle.task_name = self.task_name
+        elif bundle.task_name != self.task_name:
+            raise ValueError("trainer returned a bundle for a different task")
+        model_version = self.registry.register(bundle)
+        if self.promotion_state is not None:
+            self.registry.promote(self.task_name, model_version, self.promotion_state)
+        return {
+            "status": "completed",
+            "model_version": model_version,
+            "task_name": self.task_name,
+            "promotion_state": self.promotion_state,
+            "context": context,
+        }
+
+    def run_scheduled(self, is_due, *, context=None):
+        """Run when a scheduler-specific due predicate returns true."""
+        return self.run(trigger=is_due, context=context)
+
+    def run_on_data(self, has_new_data, *, context=None):
+        """Run when a data watcher-specific predicate returns true."""
+        return self.run(trigger=has_new_data, context=context)
+
+
+__all__ = ["RetrainingJob"]

SKSurrogate/sensapprx.py

@@ -0,0 +1,250 @@
+"""
+Sensitivity Analysis
+===========================
+
+Sensitivity analysis of a dataset based on a fit, sklearn style.
+The core functionality is provided by `SALib <https://github.com/SALib/SALib>`_ .
+"""
+
+try:
+    from sklearn.base import BaseEstimator, TransformerMixin
+except ModuleNotFoundError:
+    BaseEstimator = type("BaseEstimator", (object,), dict())
+    TransformerMixin = type("TransformerMixin", (object,), dict())
+
+
+class SensAprx(BaseEstimator, TransformerMixin):
+    """
+    Transform data to select the most secretive factors according to a regressor that fits the data.
+
+    :param n_features_to_select: `int` number of top features to be selected
+    :param regressor: a sklearn style regressor to fit the data for sensitivity analysis
+    :param method: `str` the sensitivity analysis method; defalt 'sobol', other options are 'morris' and 'delta-mmnt'
+    :param margin: domain margine, default: .2
+    :param num_smpl: number of samples to perform the analysis, default: 512
+    :param num_levels: number of levels for morris analysis, default: 6
+    :param grid_jump: grid jump for morris analysis, default: 1
+    :param num_resmpl: number of resamples for moment independent analysis, default: 10
+    :param reduce: whether to reduce the data points to uniques and calculate the averages of the target or not,
+        default: False
+    :param domain: pre-calculated unique points, if none, and reduce is `True` then unique points will be found
+    :param probs: pre-calculated values associated to `domain` points
+    """
+
+    def __init__(
+        self,
+        n_features_to_select=10,
+        regressor=None,
+        method="sobol",
+        margin=0.2,
+        num_smpl=512,
+        num_levels=6,
+        grid_jump=1,
+        num_resmpl=8,
+        reduce=False,
+        domain=None,
+        probs=None,
+    ):
+        self.n_features_to_select = n_features_to_select
+        self.regressor = regressor
+        self.method = method
+        self.margin = margin
+        self.num_smpl = num_smpl
+        self.num_levels = num_levels
+        self.grid_jump = grid_jump
+        self.num_resmpl = num_resmpl
+        self.reduce = reduce
+        self.domain = domain
+        self.probs = probs
+        self.weights_ = None
+        self.top_features_ = []
+
+    def _avg_fucn(self, X, y):
+        from numpy import bincount, unique
+
+        if self.reduce:
+            self.domain, inverse = unique(X, axis=0, return_inverse=True)
+            sums = bincount(inverse, weights=y)
+            counts = bincount(inverse)
+            self.probs = sums / counts
+        else:
+            self.domain = X
+            self.probs = y
+
+    def fit(self, X, y):
+        """
+        Fits the regressor to the data `(X, y)` and performs a sensitivity analysis on the result of the regression.
+
+        :param X: Training data
+        :param y: Target values
+        :return: `self`
+        """
+        from numpy import argpartition
+
+        self.n_features_to_select = min(self.n_features_to_select, X.shape[1])
+
+        N = len(X[0])
+        if (self.domain is None) or (self.probs is None):
+            self._avg_fucn(X, y)
+        if self.regressor is None:
+            from sklearn.svm import SVR
+            self.regressor = SVR()
+        self.regressor.fit(self.domain, self.probs)
+        bounds = [
+            [
+                min(self.domain[:, idx]) - self.margin,
+                max(self.domain[:, idx]) + self.margin,
+            ]
+            for idx in range(N)
+        ]
+        problem = dict(
+            num_vars=N, names=["x%d" % idx for idx in range(N)], bounds=bounds
+        )
+        res = []
+        if self.method == "sobol":
+            from SALib.sample import saltelli
+            from SALib.analyze import sobol
+
+            param_values = saltelli.sample(problem, self.num_smpl)
+            y_ = self.regressor.predict(param_values)
+            res = sobol.analyze(problem, y_)["ST"]
+            self.weights_ = res
+        elif self.method == "morris":
+            from SALib.sample import morris as mrs
+            from SALib.analyze import morris
+
+            param_values = mrs.sample(
+                problem, self.num_smpl, num_levels=self.num_levels
+            )
+            y_ = self.regressor.predict(param_values)
+            res = morris.analyze(problem, param_values, y_, num_levels=self.num_levels)[
+                "mu_star"
+            ]
+            self.weights_ = res
+        elif self.method == "delta-mmnt":
+            from SALib.sample import latin
+            from SALib.analyze import delta
+
+            param_values = latin.sample(problem, self.num_smpl)
+            y_ = self.regressor.predict(param_values)
+            res = delta.analyze(
+                problem, param_values, y_, num_resamples=self.num_resmpl
+            )["delta"]
+            self.weights_ = res
+        self.top_features_ = argpartition(res, -self.n_features_to_select)[
+            -self.n_features_to_select:
+        ]
+        return self
+
+    def transform(self, X):
+        return X[:, self.top_features_[: self.n_features_to_select]]
+
+    def fit_transform(self, X, y=None, **fit_params):
+        """
+        Fit to data, then transform it.
+
+        Fits transformer to X and y with optional parameters fit_params and returns a transformed version of X.
+
+        :param X: numpy array of shape [n_samples, n_features]; Training set.
+        :param y: numpy array of shape [n_samples]; Target values.
+        :return: Transformed array
+        """
+        self.fit(X, y)
+        return self.transform(X)
+
+
+class CorrelationThreshold(BaseEstimator, TransformerMixin):
+    """
+    Selects a minimal set of features based on a given (Pearson) correlation threshold.
+    The transformer omits the maximum number features with a high correlation and makes sure that the remaining
+    features are not correlated behind the given threshold.
+
+    :param threshold: the threshold for selecting correlated pairs.
+    """
+
+    def __init__(self, threshold=0.7):
+        self.threshold = threshold
+        self.indices = []
+
+    def _high_corr(self, crln):
+        pairs = []
+        nodes = set()
+        n = crln.shape[0]
+        for i in range(1, n):
+            for j in range(i):
+                if abs(crln[i, j]) >= self.threshold:
+                    pairs.append((i, j))
+                    nodes.add(i)
+                    nodes.add(j)
+        return pairs, nodes
+
+    @staticmethod
+    def _node_degrees(pairs):
+        nodes = set()
+        degs = {}
+        for p in pairs:
+            nodes.add(p[0])
+            nodes.add(p[1])
+            if p[0] not in degs:
+                degs[p[0]] = 1
+            else:
+                degs[p[0]] += 1
+            if p[1] not in degs:
+                degs[p[1]] = 1
+            else:
+                degs[p[1]] += 1
+        node_deg = [(k, degs[k]) for k in degs]
+        node_deg.sort(key=lambda x: -x[1])
+        return node_deg
+
+    def _minimal_selection(self, crln):
+        from copy import copy
+
+        pairs = copy(crln)
+        selection = []
+        while pairs != []:
+            temp_pairs = []
+            degs = self._node_degrees(pairs)
+            candid = degs[0][0]
+            selection.append(candid)
+            for p in pairs:
+                if candid not in p:
+                    temp_pairs.append(p)
+            pairs = temp_pairs
+        return set(selection)
+
+    def fit(self, X, y=None):
+        """
+        Finds the Pearson correlation among all features, selects the pairs with absolute value of correlation above
+        the given threshold and selects a minimal set of features with low correlation
+
+        :param X: Training data
+        :param y: Target values (default: `None`)
+        :return: `self`
+        """
+
+        from numpy import corrcoef
+
+        n = X.shape[1]
+        crln = corrcoef(X.transpose())
+        pairs, nodes = self._high_corr(crln)
+        sel_features = self._minimal_selection(pairs)
+        self.indices = list(set(range(n)).difference(nodes).union(sel_features))
+        self.indices.sort()
+        return self
+
+    def transform(self, X):
+        return X[:, self.indices]
+
+    def fit_transform(self, X, y=None, **fit_params):
+        """
+        Fit to data, then transform it.
+
+        Fits transformer to X and y with optional parameters fit_params and returns a transformed version of X.
+
+        :param X: numpy array of shape [n_samples, n_features]; Training set.
+        :param y: numpy array of shape [n_samples]; Target values.
+        :return: Transformed array
+        """
+        self.fit(X, y)
+        return self.transform(X)