evalkit-ml 0.1.1__py3-none-any.whl

evalkit/__init__.py

@@ -0,0 +1,9 @@
+from .tracker import ModelTracker, track
+from .metrics import register_metric, unregister_metric
+
+__all__ = [
+	"ModelTracker",
+	"track",
+	"register_metric",
+	"unregister_metric",
+]

evalkit/ci.py

@@ -0,0 +1,37 @@
+import json
+
+from .rules import evaluate_rules, rules_passed
+
+
+def check_report(report, thresholds=None):
+    result = report.check_regression(
+        thresholds=thresholds
+    )
+
+    if result["status"] == "regression_detected":
+        return False
+
+    return True
+
+
+def check_report_with_rules(report, thresholds=None, rules=None):
+    regression_passed = check_report(report, thresholds)
+    rule_results = evaluate_rules(report.metrics, rules)
+    report.rule_results = rule_results
+    return regression_passed and rules_passed(rule_results)
+
+
+def ci_output(report, passed, fmt="text"):
+    payload = {
+        "run_id": report.run_id,
+        "model": report.model_name,
+        "passed": passed,
+        "metrics": report.metrics,
+        "rules": report.rule_results or {},
+    }
+    if fmt == "json":
+        return json.dumps(payload, indent=4)
+    if fmt == "github":
+        level = "notice" if passed else "error"
+        return f"::{level} title=EvalKit::CI {'PASSED' if passed else 'FAILED'}"
+    return f"CI Status: {'PASSED' if passed else 'FAILED'}"

evalkit/cli.py

@@ -0,0 +1,192 @@
+import argparse
+import json
+import sys
+
+from .ci import check_report_with_rules, ci_output
+from .comparator import ReportComparator
+from .config import load_config, merge_thresholds
+from .report import EvaluationReport
+
+
+def load_json(filepath):
+    with open(filepath, "r") as file:
+        return json.load(file)
+
+
+def load_report(filepath):
+    data = load_json(filepath)
+
+    if isinstance(data, list):
+        if not data:
+            raise ValueError(f"Report file is empty: {filepath}")
+        data = data[-1]
+
+    return EvaluationReport(
+        model_name=data["model"],
+        model_version=data.get("model_version"),
+        timestamp=data.get("timestamp"),
+        run_id=data.get("run_id"),
+        metrics=data["metrics"],
+        diagnostics=data.get("diagnostics", {}),
+        data_profile=data.get("data_profile", {}),
+        performance=data.get("performance", {}),
+        previous_report=data.get("previous_report"),
+        drift=data.get("drift"),
+        model_metadata=data.get("model_metadata", {}),
+        dataset_metadata=data.get("dataset_metadata", {}),
+        feature_monitoring=data.get("feature_monitoring", {}),
+        prediction_distribution=data.get("prediction_distribution", {}),
+        explainability=data.get("explainability", {}),
+        rule_results=data.get("rule_results", {}),
+    )
+
+
+def parse_thresholds(values):
+    thresholds = {}
+
+    for value in values:
+        try:
+            metric, threshold = value.split("=", 1)
+            thresholds[metric] = float(threshold)
+        except ValueError as error:
+            raise argparse.ArgumentTypeError(
+                "thresholds must use METRIC=VALUE format"
+            ) from error
+
+    return thresholds
+
+
+def add_threshold_argument(parser):
+    parser.add_argument(
+        "--threshold",
+        action="append",
+        default=[],
+        metavar="METRIC=VALUE",
+        help="Allowed metric change; may be repeated",
+    )
+
+
+def print_comparison(results):
+    for metric, result in results.items():
+        if metric == "_model_version":
+            continue
+
+        print(
+            f"{metric}: "
+            f"{result['previous']:.4f} -> "
+            f"{result['current']:.4f} "
+            f"({result['status']})"
+        )
+
+    version = results.get("_model_version")
+    if version:
+        print(
+            f"model_version: {version['previous']} -> "
+            f"{version['current']} "
+            f"({'changed' if version['changed'] else 'unchanged'})"
+        )
+
+
+def command_check(args):
+    config = load_config(args.config)
+    report = load_report(args.report)
+
+    if args.previous:
+        report.previous_report = load_json(args.previous)
+        if isinstance(report.previous_report, list):
+            report.previous_report = report.previous_report[-1]
+
+    thresholds = merge_thresholds(config, parse_thresholds(args.threshold))
+    passed = check_report_with_rules(
+        report,
+        thresholds=thresholds,
+        rules=config.get("rules", {}),
+    )
+    print(ci_output(report, passed, args.output))
+
+    return 0 if passed else 1
+
+
+def command_compare(args):
+    config = load_config(args.config)
+    previous = load_json(args.previous)
+    current = load_json(args.current)
+
+    if isinstance(previous, list):
+        previous = previous[-1]
+    if isinstance(current, list):
+        current = current[-1]
+
+    results = ReportComparator(previous, current).compare(
+        thresholds=merge_thresholds(config, parse_thresholds(args.threshold))
+    )
+    print_comparison(results)
+    return 0
+
+
+def command_history(args):
+    config = load_config(args.config)
+    data = load_json(args.file or config["history_file"])
+    runs = data if isinstance(data, list) else [data]
+
+    if args.limit:
+        runs = runs[-args.limit:]
+
+    start_index = len(data) - len(runs) + 1 if isinstance(data, list) else 1
+    for index, run in enumerate(runs, start=start_index):
+        print(
+            f"{index}: {run.get('model')} "
+            f"{run.get('model_version', 'unknown')} "
+            f"{run.get('timestamp', 'unknown')}"
+        )
+
+    return 0
+
+
+def build_parser():
+    parser = argparse.ArgumentParser(
+        prog="evalkit",
+        description="ML model evaluation toolkit",
+    )
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    check = commands.add_parser("check", help="Check a report for regression")
+    check.add_argument("--report", required=True)
+    check.add_argument("--previous")
+    check.add_argument("--config", default="evalkit.yaml")
+    check.add_argument(
+        "--output",
+        choices=["text", "json", "github"],
+        default="text",
+    )
+    add_threshold_argument(check)
+    check.set_defaults(handler=command_check)
+
+    compare = commands.add_parser("compare", help="Compare two reports")
+    compare.add_argument("--current", required=True)
+    compare.add_argument("--previous", required=True)
+    compare.add_argument("--config", default="evalkit.yaml")
+    add_threshold_argument(compare)
+    compare.set_defaults(handler=command_compare)
+
+    history = commands.add_parser("history", help="List saved evaluation reports")
+    history.add_argument("--file")
+    history.add_argument("--config", default="evalkit.yaml")
+    history.add_argument("--limit", type=int)
+    history.set_defaults(handler=command_history)
+
+    return parser
+
+
+def main():
+    parser = build_parser()
+    args = parser.parse_args()
+
+    try:
+        return args.handler(args)
+    except (FileNotFoundError, KeyError, ValueError) as error:
+        parser.error(str(error))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

evalkit/comparator.py

@@ -0,0 +1,87 @@
+METRIC_DIRECTION = {
+    "accuracy": "higher",
+    "precision": "higher",
+    "recall": "higher",
+    "f1": "higher",
+
+    "mae": "lower",
+    "mse": "lower",
+    "rmse": "lower",
+
+    "r2": "higher",
+
+    "fit_time": "lower",
+    "prediction_time": "lower",
+}
+
+
+class ReportComparator:
+
+    def __init__(self, previous, current):
+        self.previous = previous
+        self.current = current
+
+    def compare(self, thresholds=None):
+        if thresholds is None:
+            thresholds = {}
+
+        results = {}
+
+        previous_version = self.previous.get("model_version")
+        current_version = self.current.get("model_version")
+
+        version_changed = (
+            previous_version != current_version
+        )
+
+        previous_metrics = self.previous["metrics"]
+        current_metrics = self.current["metrics"]
+
+        for metric, current_value in current_metrics.items():
+
+            if metric not in previous_metrics:
+                continue
+
+            previous_value = previous_metrics[metric]
+            change = current_value - previous_value
+
+            direction = METRIC_DIRECTION.get(metric)
+            threshold = thresholds.get(metric, 0)
+
+            if direction == "higher":
+
+                if change < -threshold:
+                    status = "degraded"
+                elif change > threshold:
+                    status = "improved"
+                else:
+                    status = "unchanged"
+
+            elif direction == "lower":
+
+                if change > threshold:
+                    status = "degraded"
+                elif change < -threshold:
+                    status = "improved"
+                else:
+                    status = "unchanged"
+
+            else:
+                status = "unknown"
+
+            results[metric] = {
+                "previous": previous_value,
+                "current": current_value,
+                "change": change,
+                "threshold": threshold,
+                "status": status,
+            }
+            
+        results["_model_version"] = {
+            "previous": previous_version,
+            "current": current_version,
+            "changed": version_changed,
+        }
+
+        return results
+    

evalkit/config.py

@@ -0,0 +1,38 @@
+from pathlib import Path
+
+
+DEFAULT_CONFIG = {
+    "history_file": "evalkit_history.json",
+    "thresholds": {},
+    "report_format": "text",
+}
+
+
+def load_config(filepath="evalkit.yaml"):
+    config = dict(DEFAULT_CONFIG)
+    path = Path(filepath)
+
+    if not path.exists():
+        return config
+
+    try:
+        import yaml
+    except ImportError as error:
+        raise RuntimeError(
+            "YAML configuration requires PyYAML. Install it with 'pip install pyyaml'."
+        ) from error
+
+    with path.open("r") as file:
+        loaded = yaml.safe_load(file) or {}
+
+    if not isinstance(loaded, dict):
+        raise ValueError("Configuration must contain a YAML mapping")
+
+    config.update(loaded)
+    return config
+
+
+def merge_thresholds(config, cli_thresholds):
+    thresholds = dict(config.get("thresholds", {}) or {})
+    thresholds.update(cli_thresholds)
+    return thresholds

evalkit/diagnostics.py

@@ -0,0 +1,23 @@
+import numpy as np
+from sklearn.metrics import confusion_matrix
+
+
+def classification_diagnostics(y_true, y_pred):
+
+    matrix = confusion_matrix(y_true, y_pred)
+
+    classes, counts = np.unique(
+        y_true,
+        return_counts=True
+    )
+
+    class_distribution = {
+        str(cls): int(count)
+        for cls, count in zip(classes, counts)
+    }
+
+    return {
+        "confusion_matrix": matrix.tolist(),
+        "class_distribution": class_distribution,
+        "number_of_classes": len(classes),
+    }

evalkit/drift.py

@@ -0,0 +1,240 @@
+import numpy as np
+import pandas as pd
+from scipy.stats import ks_2samp
+
+
+def calculate_psi(expected, actual, bins=10):
+    """
+    Calculate Population Stability Index (PSI)
+    between two numeric distributions.
+    """
+
+    expected = np.asarray(expected)
+    actual = np.asarray(actual)
+
+    breakpoints = np.percentile(
+        expected,
+        np.linspace(0, 100, bins + 1)
+    )
+
+    breakpoints[0] = -np.inf
+    breakpoints[-1] = np.inf
+
+    expected_counts, _ = np.histogram(
+        expected,
+        bins=breakpoints
+    )
+
+    actual_counts, _ = np.histogram(
+        actual,
+        bins=breakpoints
+    )
+
+    expected_pct = expected_counts / len(expected)
+    actual_pct = actual_counts / len(actual)
+
+    # Avoid division by zero
+    expected_pct = np.where(
+        expected_pct == 0,
+        0.0001,
+        expected_pct
+    )
+
+    actual_pct = np.where(
+        actual_pct == 0,
+        0.0001,
+        actual_pct
+    )
+
+    psi = np.sum(
+        (actual_pct - expected_pct)
+        * np.log(actual_pct / expected_pct)
+    )
+
+    return float(psi)
+
+
+def detect_numeric_drift(
+    reference,
+    current,
+    threshold=0.2
+):
+    """
+    Detect drift between reference and current
+    numeric datasets.
+    """
+
+    reference = pd.DataFrame(reference)
+    current = pd.DataFrame(current)
+
+    results = {}
+
+    common_columns = reference.columns.intersection(
+        current.columns
+    )
+
+    for column in common_columns:
+
+        if not (
+            pd.api.types.is_numeric_dtype(
+                reference[column]
+            )
+            and
+            pd.api.types.is_numeric_dtype(
+                current[column]
+            )
+        ):
+            continue
+
+        reference_values = reference[column].dropna()
+        current_values = current[column].dropna()
+
+        if len(reference_values) == 0 or len(current_values) == 0:
+            continue
+
+        psi = calculate_psi(
+            reference_values,
+            current_values
+        )
+        ks_statistic, ks_pvalue = ks_2samp(
+            reference_values,
+            current_values
+        )
+
+        if psi >= threshold:
+            status = "drifted"
+        else:
+            status = "stable"
+
+        results[column] = {
+            "psi": psi,
+            "ks_statistic": float(ks_statistic),
+            "ks_pvalue": float(ks_pvalue),
+            "threshold": threshold,
+            "status": status
+        }
+
+    return results
+
+def detect_categorical_drift(
+    reference,
+    current,
+    threshold=0.2
+):
+    """
+    Detect drift between reference and current
+    categorical datasets.
+    """
+
+    reference = pd.DataFrame(reference)
+    current = pd.DataFrame(current)
+
+    results = {}
+
+    common_columns = reference.columns.intersection(
+        current.columns
+    )
+
+    for column in common_columns:
+
+        if (
+            pd.api.types.is_numeric_dtype(reference[column])
+            or
+            pd.api.types.is_numeric_dtype(current[column])
+        ):
+            continue
+
+        reference_counts = (
+            reference[column]
+            .value_counts(normalize=True)
+        )
+
+        current_counts = (
+            current[column]
+            .value_counts(normalize=True)
+        )
+
+        categories = set(reference_counts.index).union(
+            current_counts.index
+        )
+
+        psi = 0.0
+
+        for category in categories:
+
+            expected = reference_counts.get(
+                category,
+                0
+            )
+
+            actual = current_counts.get(
+                category,
+                0
+            )
+
+            expected = max(expected, 0.0001)
+            actual = max(actual, 0.0001)
+
+            psi += (
+                (actual - expected)
+                * np.log(actual / expected)
+            )
+
+        reference_probability = np.array([
+            max(reference_counts.get(category, 0), 0.0001)
+            for category in categories
+        ])
+        current_probability = np.array([
+            max(current_counts.get(category, 0), 0.0001)
+            for category in categories
+        ])
+        reference_probability /= reference_probability.sum()
+        current_probability /= current_probability.sum()
+        midpoint = (reference_probability + current_probability) / 2
+        js_divergence = 0.5 * np.sum(
+            reference_probability * np.log(reference_probability / midpoint)
+        ) + 0.5 * np.sum(
+            current_probability * np.log(current_probability / midpoint)
+        )
+
+        if psi >= threshold:
+            status = "drifted"
+        else:
+            status = "stable"
+
+        results[column] = {
+            "psi": float(psi),
+            "js_divergence": float(js_divergence),
+            "threshold": threshold,
+            "status": status
+        }
+
+    return results
+
+
+def detect_drift(
+    reference,
+    current,
+    threshold=0.2
+):
+    """
+    Automatically detect numeric and categorical
+    drift between reference and current datasets.
+    """
+
+    numeric_results = detect_numeric_drift(
+        reference,
+        current,
+        threshold=threshold
+    )
+
+    categorical_results = detect_categorical_drift(
+        reference,
+        current,
+        threshold=threshold
+    )
+
+    return {
+        "numeric": numeric_results,
+        "categorical": categorical_results
+    }

evalkit/evaluator.py

@@ -0,0 +1,35 @@
+from sklearn.base import is_classifier, is_regressor
+
+from .metrics import (
+    classification_metrics,
+    regression_metrics,
+)
+
+
+class Evaluator:
+
+    def __init__(self, model, metrics=None):
+        self.model = model
+        self.metrics = metrics or {}
+
+    def evaluate(self, y_true, y_pred):
+        if is_classifier(self.model):
+            results = classification_metrics(
+                y_true,
+                y_pred
+            )
+        elif is_regressor(self.model):
+            results = regression_metrics(
+                y_true,
+                y_pred
+            )
+        else:
+            raise NotImplementedError(
+                "This model type is not supported yet."
+            )
+
+        results.update({
+            name: float(function(y_true, y_pred))
+            for name, function in self.metrics.items()
+        })
+        return results