ml-benchmark-toolkit 0.1.0__tar.gz

ml_benchmark_toolkit-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,91 @@
+name: Build and Publish to PyPI
+
+# Trigger on version tags like v0.1.0, v1.2.3, etc.
+on:
+  push:
+    tags:
+      - "v*.*.*"
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build tooling
+        run: python -m pip install --upgrade pip build
+
+      - name: Build sdist and wheel
+        run: python -m build
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  test:
+    name: Smoke test the built wheel
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Install wheel and run example
+        run: |
+          pip install dist/*.whl
+          python examples/quickstart.py
+
+  publish-testpypi:
+    name: Publish to TestPyPI
+    needs: test
+    runs-on: ubuntu-latest
+    environment: testpypi
+    permissions:
+      id-token: write  # required for OIDC Trusted Publishing
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true
+
+  publish-pypi:
+    name: Publish to PyPI
+    needs: publish-testpypi
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # required for OIDC Trusted Publishing
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

ml_benchmark_toolkit-0.1.0/PKG-INFO

@@ -0,0 +1,151 @@
+Metadata-Version: 2.5
+Name: ml-benchmark-toolkit
+Version: 0.1.0
+Summary: Lightweight toolkit to benchmark, compare and visually report on classification models.
+Project-URL: Homepage, https://github.com/yourorg/ml-benchmark-toolkit
+Project-URL: Issues, https://github.com/yourorg/ml-benchmark-toolkit/issues
+Author-email: Your Name <you@example.com>
+License: MIT
+Keywords: benchmarking,classification,machine-learning,reporting,scikit-learn
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Requires-Dist: matplotlib>=3.6
+Requires-Dist: numpy>=1.23
+Requires-Dist: pandas>=1.5
+Requires-Dist: scikit-learn>=1.1
+Requires-Dist: seaborn>=0.12
+Requires-Dist: tabulate>=0.9
+Provides-Extra: dev
+Requires-Dist: build>=1.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: twine>=4.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# ml-benchmark-toolkit
+
+Lightweight, dependency-minimal Python toolkit (`mlbenchmark`) for evaluating,
+comparing, and visually reporting on classification models — single-model or
+side-by-side multi-model benchmarks.
+
+## Features
+
+- **Metrics**: accuracy, precision/recall/F1 (macro & weighted), log-loss, ROC-AUC
+  (binary and multi-class One-vs-Rest), computed with graceful fallback when
+  `y_proba` is unavailable.
+- **Visualizations**: annotated confusion-matrix heatmaps (raw or normalized),
+  ROC curves (multi-class OvR with per-class + micro-average AUC), and
+  train-vs-validation training curves from a Keras-style `history` dict.
+- **Multi-model comparison**: register any number of models and get a ranked
+  `pandas.DataFrame` / Markdown comparison table.
+- **One-line reporting**: `Report.to_html()` compiles all tables and figures
+  into a single, standalone, self-contained HTML dashboard (figures are
+  base64-embedded, no external assets required).
+
+## Installation
+
+```bash
+pip install ml-benchmark-toolkit
+```
+
+Or, for local development:
+
+```bash
+git clone https://github.com/yourorg/ml-benchmark-toolkit.git
+cd ml-benchmark-toolkit
+python -m venv .venv && source .venv/bin/activate
+pip install -e ".[dev]"
+```
+
+## Quickstart
+
+```python
+from mlbenchmark import ModelComparator, Report
+
+comparator = ModelComparator()
+comparator.add_model("random_forest", y_true, y_pred_rf, y_proba_rf)
+comparator.add_model("logistic_regression", y_true, y_pred_lr, y_proba_lr)
+
+# Ranked comparison table
+table = comparator.comparison_table(rank_by="f1_macro")
+print(table)
+
+# Standalone HTML dashboard: tables + confusion matrices + ROC curves
+report = Report(comparator, title="My Benchmark")
+report.to_html("report.html")
+report.to_markdown("report.md")
+```
+
+See [`examples/quickstart.py`](examples/quickstart.py) for a full,
+runnable end-to-end example (including training curves).
+
+## API overview
+
+| Module | Purpose |
+|---|---|
+| `mlbenchmark.metrics` | `compute_metrics(y_true, y_pred, y_proba=None)` -> dict |
+| `mlbenchmark.plots` | `plot_confusion_matrix`, `plot_roc_curve`, `plot_training_curves` -> `matplotlib.figure.Figure` |
+| `mlbenchmark.comparator` | `ModelComparator` — register models, build ranked comparison tables and per-model figure dicts |
+| `mlbenchmark.report` | `Report` — compile a `ModelComparator` into a standalone HTML/Markdown dashboard |
+
+## Development & Distribution
+
+### 1. Local editable install & test
+
+```bash
+python -m venv .venv
+source .venv/bin/activate        # Windows: .venv\Scripts\activate
+pip install -e ".[dev]"
+python examples/quickstart.py
+pytest
+```
+
+### 2. Build distribution artifacts
+
+```bash
+pip install --upgrade build
+python -m build                  # produces dist/*.whl and dist/*.tar.gz
+```
+
+### 3. Publish to TestPyPI, then PyPI
+
+```bash
+pip install --upgrade twine
+
+# TestPyPI first (use an API token, not your password)
+twine upload --repository testpypi dist/* \
+  -u __token__ -p "$TEST_PYPI_API_TOKEN"
+
+# Verify install from TestPyPI
+pip install --index-url https://test.pypi.org/simple/ \
+  --extra-index-url https://pypi.org/simple/ ml-benchmark-toolkit
+
+# Production PyPI
+twine upload dist/* -u __token__ -p "$PYPI_API_TOKEN"
+```
+
+Store tokens as environment variables or in `~/.pypirc` — never commit them.
+
+### 4. Automated releases via GitHub Actions
+
+See [`.github/workflows/publish.yml`](.github/workflows/publish.yml): the
+workflow builds and publishes to PyPI automatically whenever a tag matching
+`v*.*.*` is pushed, using PyPI's [Trusted Publishing](https://docs.pypi.org/trusted-publishers/)
+(OIDC — no long-lived API token stored in the repo).
+
+```bash
+git tag v0.1.0
+git push origin v0.1.0
+```
+
+## License
+
+MIT

ml_benchmark_toolkit-0.1.0/README.md

@@ -0,0 +1,119 @@
+# ml-benchmark-toolkit
+
+Lightweight, dependency-minimal Python toolkit (`mlbenchmark`) for evaluating,
+comparing, and visually reporting on classification models — single-model or
+side-by-side multi-model benchmarks.
+
+## Features
+
+- **Metrics**: accuracy, precision/recall/F1 (macro & weighted), log-loss, ROC-AUC
+  (binary and multi-class One-vs-Rest), computed with graceful fallback when
+  `y_proba` is unavailable.
+- **Visualizations**: annotated confusion-matrix heatmaps (raw or normalized),
+  ROC curves (multi-class OvR with per-class + micro-average AUC), and
+  train-vs-validation training curves from a Keras-style `history` dict.
+- **Multi-model comparison**: register any number of models and get a ranked
+  `pandas.DataFrame` / Markdown comparison table.
+- **One-line reporting**: `Report.to_html()` compiles all tables and figures
+  into a single, standalone, self-contained HTML dashboard (figures are
+  base64-embedded, no external assets required).
+
+## Installation
+
+```bash
+pip install ml-benchmark-toolkit
+```
+
+Or, for local development:
+
+```bash
+git clone https://github.com/yourorg/ml-benchmark-toolkit.git
+cd ml-benchmark-toolkit
+python -m venv .venv && source .venv/bin/activate
+pip install -e ".[dev]"
+```
+
+## Quickstart
+
+```python
+from mlbenchmark import ModelComparator, Report
+
+comparator = ModelComparator()
+comparator.add_model("random_forest", y_true, y_pred_rf, y_proba_rf)
+comparator.add_model("logistic_regression", y_true, y_pred_lr, y_proba_lr)
+
+# Ranked comparison table
+table = comparator.comparison_table(rank_by="f1_macro")
+print(table)
+
+# Standalone HTML dashboard: tables + confusion matrices + ROC curves
+report = Report(comparator, title="My Benchmark")
+report.to_html("report.html")
+report.to_markdown("report.md")
+```
+
+See [`examples/quickstart.py`](examples/quickstart.py) for a full,
+runnable end-to-end example (including training curves).
+
+## API overview
+
+| Module | Purpose |
+|---|---|
+| `mlbenchmark.metrics` | `compute_metrics(y_true, y_pred, y_proba=None)` -> dict |
+| `mlbenchmark.plots` | `plot_confusion_matrix`, `plot_roc_curve`, `plot_training_curves` -> `matplotlib.figure.Figure` |
+| `mlbenchmark.comparator` | `ModelComparator` — register models, build ranked comparison tables and per-model figure dicts |
+| `mlbenchmark.report` | `Report` — compile a `ModelComparator` into a standalone HTML/Markdown dashboard |
+
+## Development & Distribution
+
+### 1. Local editable install & test
+
+```bash
+python -m venv .venv
+source .venv/bin/activate        # Windows: .venv\Scripts\activate
+pip install -e ".[dev]"
+python examples/quickstart.py
+pytest
+```
+
+### 2. Build distribution artifacts
+
+```bash
+pip install --upgrade build
+python -m build                  # produces dist/*.whl and dist/*.tar.gz
+```
+
+### 3. Publish to TestPyPI, then PyPI
+
+```bash
+pip install --upgrade twine
+
+# TestPyPI first (use an API token, not your password)
+twine upload --repository testpypi dist/* \
+  -u __token__ -p "$TEST_PYPI_API_TOKEN"
+
+# Verify install from TestPyPI
+pip install --index-url https://test.pypi.org/simple/ \
+  --extra-index-url https://pypi.org/simple/ ml-benchmark-toolkit
+
+# Production PyPI
+twine upload dist/* -u __token__ -p "$PYPI_API_TOKEN"
+```
+
+Store tokens as environment variables or in `~/.pypirc` — never commit them.
+
+### 4. Automated releases via GitHub Actions
+
+See [`.github/workflows/publish.yml`](.github/workflows/publish.yml): the
+workflow builds and publishes to PyPI automatically whenever a tag matching
+`v*.*.*` is pushed, using PyPI's [Trusted Publishing](https://docs.pypi.org/trusted-publishers/)
+(OIDC — no long-lived API token stored in the repo).
+
+```bash
+git tag v0.1.0
+git push origin v0.1.0
+```
+
+## License
+
+MIT

ml_benchmark_toolkit-0.1.0/examples/quickstart.py

@@ -0,0 +1,71 @@
+"""End-to-end quickstart: train two models, benchmark them, export a report.
+
+Run with:
+    python examples/quickstart.py
+"""
+
+from __future__ import annotations
+
+from sklearn.datasets import make_classification
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.model_selection import train_test_split
+
+from mlbenchmark import ModelComparator, Report
+
+# 1. Synthetic multi-class dataset
+X, y = make_classification(
+    n_samples=1500,
+    n_features=20,
+    n_informative=10,
+    n_classes=3,
+    n_clusters_per_class=1,
+    random_state=42,
+)
+X_train, X_test, y_train, y_test = train_test_split(
+    X, y, test_size=0.25, random_state=42, stratify=y
+)
+
+# 2. Train a couple of candidate models
+rf = RandomForestClassifier(n_estimators=200, random_state=42).fit(X_train, y_train)
+logreg = LogisticRegression(max_iter=1000).fit(X_train, y_train)
+
+# A fabricated training history, e.g. for a neural-net-style model, to
+# demonstrate the training-curve plot on one of the models.
+fake_history = {
+    "loss": [1.10, 0.85, 0.67, 0.52, 0.41, 0.34, 0.29, 0.26],
+    "val_loss": [1.15, 0.95, 0.80, 0.70, 0.66, 0.64, 0.63, 0.64],
+    "accuracy": [0.45, 0.58, 0.68, 0.75, 0.80, 0.84, 0.87, 0.89],
+    "val_accuracy": [0.42, 0.55, 0.63, 0.68, 0.71, 0.72, 0.73, 0.72],
+}
+
+# 3. Register results with the comparator
+comparator = ModelComparator()
+comparator.add_model(
+    name="random_forest",
+    y_true=y_test,
+    y_pred=rf.predict(X_test),
+    y_proba=rf.predict_proba(X_test),
+)
+comparator.add_model(
+    name="logistic_regression",
+    y_true=y_test,
+    y_pred=logreg.predict(X_test),
+    y_proba=logreg.predict_proba(X_test),
+    history=fake_history,
+)
+
+# 4. Ranked comparison table (DataFrame + Markdown)
+table = comparator.comparison_table(rank_by="f1_macro")
+print(table)
+print()
+print(comparator.comparison_markdown(rank_by="f1_macro"))
+
+# 5. Standalone HTML dashboard with embedded confusion matrices, ROC
+#    curves, and training curves
+report = Report(comparator, title="Synthetic Classification Benchmark", rank_by="f1_macro")
+html_path = report.to_html("benchmark_report.html")
+report.to_markdown("benchmark_report.md")  # also writes the file to disk
+
+print(f"\nHTML report written to: {html_path.resolve()}")
+print("Markdown summary written to: benchmark_report.md")

ml_benchmark_toolkit-0.1.0/pyproject.toml

@@ -0,0 +1,54 @@
+[build-system]
+requires = ["hatchling>=1.21.0"]
+build-backend = "hatchling.build"
+
+[project]
+name = "ml-benchmark-toolkit"
+version = "0.1.0"
+description = "Lightweight toolkit to benchmark, compare and visually report on classification models."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [
+    { name = "Your Name", email = "you@example.com" }
+]
+keywords = ["machine-learning", "benchmarking", "classification", "reporting", "scikit-learn"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+dependencies = [
+    "numpy>=1.23",
+    "pandas>=1.5",
+    "scikit-learn>=1.1",
+    "matplotlib>=3.6",
+    "seaborn>=0.12",
+    "tabulate>=0.9",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "build>=1.0",
+    "twine>=4.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/yourorg/ml-benchmark-toolkit"
+Issues = "https://github.com/yourorg/ml-benchmark-toolkit/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/mlbenchmark"]
+
+[tool.hatch.version]
+path = "src/mlbenchmark/__init__.py"
+pattern = "__version__ = ['\"](?P<version>[^'\"]+)['\"]"

ml_benchmark_toolkit-0.1.0/src/mlbenchmark/__init__.py

@@ -0,0 +1,30 @@
+"""ml-benchmark-toolkit: lightweight evaluation and reporting for classifiers.
+
+Public API
+----------
+- compute_metrics: single-model metric computation
+- plot_confusion_matrix, plot_roc_curve, plot_training_curves: figure builders
+- ModelComparator: register multiple models and compare them side-by-side
+- Report: compile a ModelComparator into a standalone HTML/Markdown dashboard
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from .comparator import ModelComparator, ModelResult
+from .metrics import compute_metrics, is_binary
+from .plots import plot_confusion_matrix, plot_roc_curve, plot_training_curves
+from .report import Report
+
+__all__ = [
+    "__version__",
+    "compute_metrics",
+    "is_binary",
+    "plot_confusion_matrix",
+    "plot_roc_curve",
+    "plot_training_curves",
+    "ModelComparator",
+    "ModelResult",
+    "Report",
+]

ml_benchmark_toolkit-0.1.0/src/mlbenchmark/comparator.py

@@ -0,0 +1,160 @@
+"""Multi-model registration and side-by-side comparison utilities."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence
+
+import numpy as np
+import pandas as pd
+from matplotlib.figure import Figure
+
+from .metrics import compute_metrics
+from .plots import plot_confusion_matrix, plot_roc_curve, plot_training_curves
+
+ArrayLike = Sequence[Any]
+
+
+@dataclass
+class ModelResult:
+    """Container for a single model's evaluation inputs and computed metrics."""
+
+    name: str
+    y_true: np.ndarray
+    y_pred: np.ndarray
+    y_proba: Optional[np.ndarray] = None
+    history: Optional[Dict[str, List[float]]] = None
+    metrics: Dict[str, float] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        self.y_true = np.asarray(self.y_true)
+        self.y_pred = np.asarray(self.y_pred)
+        if self.y_proba is not None:
+            self.y_proba = np.asarray(self.y_proba)
+        self.metrics = compute_metrics(self.y_true, self.y_pred, self.y_proba)
+
+
+class ModelComparator:
+    """Registers evaluation results for multiple models and compares them.
+
+    Examples
+    --------
+    >>> comparator = ModelComparator()
+    >>> comparator.add_model("random_forest", y_true, y_pred, y_proba)
+    >>> comparator.add_model("xgboost", y_true, y_pred2, y_proba2)
+    >>> table = comparator.comparison_table(rank_by="f1_macro")
+    """
+
+    def __init__(self) -> None:
+        self._results: Dict[str, ModelResult] = {}
+
+    def add_model(
+        self,
+        name: str,
+        y_true: ArrayLike,
+        y_pred: ArrayLike,
+        y_proba: Optional[ArrayLike] = None,
+        history: Optional[Dict[str, List[float]]] = None,
+    ) -> "ModelComparator":
+        """Register a model's predictions for comparison. Returns self for chaining."""
+        if name in self._results:
+            raise ValueError(f"A model named '{name}' has already been added.")
+        self._results[name] = ModelResult(
+            name=name, y_true=y_true, y_pred=y_pred, y_proba=y_proba, history=history
+        )
+        return self
+
+    @property
+    def model_names(self) -> List[str]:
+        return list(self._results.keys())
+
+    def get_result(self, name: str) -> ModelResult:
+        if name not in self._results:
+            raise KeyError(f"No model named '{name}' has been registered.")
+        return self._results[name]
+
+    def comparison_table(
+        self,
+        rank_by: str = "f1_macro",
+        ascending: bool = False,
+        metrics: Optional[Sequence[str]] = None,
+    ) -> pd.DataFrame:
+        """Build a ranked Pandas DataFrame comparing all registered models.
+
+        Parameters
+        ----------
+        rank_by : str, default "f1_macro"
+            Metric column name to sort the table by.
+        ascending : bool, default False
+            Sort order; metrics like log_loss are usually best ascending.
+        metrics : sequence of str, optional
+            Subset of metric columns to include. Defaults to all computed
+            metrics.
+        """
+        if not self._results:
+            raise ValueError("No models have been added yet. Call add_model() first.")
+
+        rows = []
+        for name, result in self._results.items():
+            row = {"model": name}
+            row.update(result.metrics)
+            rows.append(row)
+
+        df = pd.DataFrame(rows).set_index("model")
+
+        if metrics is not None:
+            missing = set(metrics) - set(df.columns)
+            if missing:
+                raise ValueError(f"Unknown metric(s) requested: {sorted(missing)}")
+            df = df[list(metrics)]
+
+        if rank_by not in df.columns:
+            raise ValueError(
+                f"rank_by='{rank_by}' is not a computed metric. "
+                f"Available metrics: {list(df.columns)}"
+            )
+
+        df = df.sort_values(by=rank_by, ascending=ascending)
+        df.insert(0, "rank", range(1, len(df) + 1))
+        return df
+
+    def comparison_markdown(
+        self, rank_by: str = "f1_macro", ascending: bool = False, decimals: int = 4
+    ) -> str:
+        """Return the comparison table formatted as a Markdown string."""
+        df = self.comparison_table(rank_by=rank_by, ascending=ascending)
+        return df.round(decimals).to_markdown()
+
+    def confusion_matrices(self, normalize: bool = False) -> Dict[str, Figure]:
+        """Return a dict of model name -> confusion-matrix Figure."""
+        return {
+            name: plot_confusion_matrix(
+                result.y_true,
+                result.y_pred,
+                normalize=normalize,
+                title=f"Confusion Matrix — {name}",
+            )
+            for name, result in self._results.items()
+        }
+
+    def roc_curves(self) -> Dict[str, Figure]:
+        """Return a dict of model name -> ROC-curve Figure (skips models without y_proba)."""
+        figures = {}
+        for name, result in self._results.items():
+            if result.y_proba is None:
+                continue
+            figures[name] = plot_roc_curve(
+                result.y_true, result.y_proba, title=f"ROC Curve — {name}"
+            )
+        return figures
+
+    def training_curves(self) -> Dict[str, Figure]:
+        """Return a dict of model name -> training-curve Figure (skips models without history)."""
+        figures = {}
+        for name, result in self._results.items():
+            if result.history is None:
+                continue
+            figures[name] = plot_training_curves(
+                result.history, title=f"Training History — {name}"
+            )
+        return figures

ml_benchmark_toolkit-0.1.0/src/mlbenchmark/metrics.py

@@ -0,0 +1,161 @@
+"""Classification metric computation utilities.
+
+This module provides a single entry point, :func:`compute_metrics`, that
+takes true labels, predicted labels and (optionally) predicted
+probabilities and returns a flat dictionary of standard classification
+metrics. It defensively handles binary vs. multi-class problems and
+missing probability estimates.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Sequence
+
+import numpy as np
+from sklearn.metrics import (
+    accuracy_score,
+    f1_score,
+    log_loss,
+    precision_score,
+    recall_score,
+    roc_auc_score,
+)
+from sklearn.preprocessing import label_binarize
+
+
+ArrayLike = Sequence[Any]
+
+
+def _to_numpy(arr: Optional[ArrayLike]) -> Optional[np.ndarray]:
+    """Convert list-like/pandas input to a numpy array, passing through None."""
+    if arr is None:
+        return None
+    return np.asarray(arr)
+
+
+def is_binary(y_true: ArrayLike) -> bool:
+    """Return True if `y_true` contains exactly two distinct classes."""
+    return len(np.unique(_to_numpy(y_true))) <= 2
+
+
+def compute_metrics(
+    y_true: ArrayLike,
+    y_pred: ArrayLike,
+    y_proba: Optional[ArrayLike] = None,
+    *,
+    average: str = "macro",
+    labels: Optional[Sequence[Any]] = None,
+) -> Dict[str, float]:
+    """Compute a standard suite of classification metrics.
+
+    Parameters
+    ----------
+    y_true : array-like
+        Ground-truth class labels.
+    y_pred : array-like
+        Predicted class labels.
+    y_proba : array-like, optional
+        Predicted probabilities. For binary problems this may be a 1D
+        array of positive-class probabilities or a 2D array of shape
+        (n_samples, 2). For multi-class problems it must be a 2D array
+        of shape (n_samples, n_classes). If omitted, log-loss and
+        ROC-AUC are reported as NaN.
+    average : str, default "macro"
+        Averaging strategy for the secondary precision/recall/F1 metric
+        (in addition to the always-computed macro and weighted variants).
+    labels : sequence, optional
+        Explicit ordering of class labels. Inferred from `y_true`/`y_pred`
+        when not provided.
+
+    Returns
+    -------
+    dict
+        Dictionary of metric name -> float value. Metrics that cannot be
+        computed (e.g. ROC-AUC without probabilities) are set to NaN
+        rather than raising, so downstream reporting degrades gracefully.
+    """
+    y_true_arr = _to_numpy(y_true)
+    y_pred_arr = _to_numpy(y_pred)
+    y_proba_arr = _to_numpy(y_proba)
+
+    if y_true_arr is None or y_pred_arr is None:
+        raise ValueError("y_true and y_pred are required and cannot be None.")
+    if len(y_true_arr) != len(y_pred_arr):
+        raise ValueError(
+            f"y_true (n={len(y_true_arr)}) and y_pred (n={len(y_pred_arr)}) "
+            "must have the same length."
+        )
+
+    resolved_labels = list(labels) if labels is not None else sorted(
+        set(np.unique(y_true_arr).tolist()) | set(np.unique(y_pred_arr).tolist())
+    )
+    binary = len(resolved_labels) <= 2
+
+    metrics: Dict[str, float] = {
+        "accuracy": float(accuracy_score(y_true_arr, y_pred_arr)),
+        "precision_macro": float(
+            precision_score(y_true_arr, y_pred_arr, average="macro", zero_division=0)
+        ),
+        "precision_weighted": float(
+            precision_score(y_true_arr, y_pred_arr, average="weighted", zero_division=0)
+        ),
+        "recall_macro": float(
+            recall_score(y_true_arr, y_pred_arr, average="macro", zero_division=0)
+        ),
+        "recall_weighted": float(
+            recall_score(y_true_arr, y_pred_arr, average="weighted", zero_division=0)
+        ),
+        "f1_macro": float(
+            f1_score(y_true_arr, y_pred_arr, average="macro", zero_division=0)
+        ),
+        "f1_weighted": float(
+            f1_score(y_true_arr, y_pred_arr, average="weighted", zero_division=0)
+        ),
+    }
+
+    if average not in ("macro", "weighted"):
+        metrics[f"precision_{average}"] = float(
+            precision_score(y_true_arr, y_pred_arr, average=average, zero_division=0)
+        )
+        metrics[f"recall_{average}"] = float(
+            recall_score(y_true_arr, y_pred_arr, average=average, zero_division=0)
+        )
+        metrics[f"f1_{average}"] = float(
+            f1_score(y_true_arr, y_pred_arr, average=average, zero_division=0)
+        )
+
+    metrics["log_loss"] = np.nan
+    metrics["roc_auc"] = np.nan
+
+    if y_proba_arr is not None:
+        try:
+            proba_for_logloss = y_proba_arr
+            if binary and proba_for_logloss.ndim == 1:
+                proba_for_logloss = np.column_stack(
+                    [1 - proba_for_logloss, proba_for_logloss]
+                )
+            metrics["log_loss"] = float(
+                log_loss(y_true_arr, proba_for_logloss, labels=resolved_labels)
+            )
+        except (ValueError, IndexError):
+            metrics["log_loss"] = np.nan
+
+        try:
+            if binary:
+                pos_proba = (
+                    y_proba_arr
+                    if y_proba_arr.ndim == 1
+                    else y_proba_arr[:, 1]
+                )
+                metrics["roc_auc"] = float(roc_auc_score(y_true_arr, pos_proba))
+            else:
+                y_bin = label_binarize(y_true_arr, classes=resolved_labels)
+                metrics["roc_auc"] = float(
+                    roc_auc_score(
+                        y_bin, y_proba_arr, average="macro", multi_class="ovr"
+                    )
+                )
+        except (ValueError, IndexError):
+            metrics["roc_auc"] = np.nan
+
+    return metrics

ml_benchmark_toolkit-0.1.0/src/mlbenchmark/plots.py

@@ -0,0 +1,206 @@
+"""Visualization utilities: confusion matrices, ROC curves, training curves.
+
+All plotting functions return a `matplotlib.figure.Figure` instance rather
+than calling `plt.show()`, so callers (including the reporting module) can
+embed, save, or further customize the figures.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Sequence
+
+import matplotlib
+import numpy as np
+
+matplotlib.use("Agg")  # headless-safe backend for servers/CI
+import matplotlib.pyplot as plt  # noqa: E402
+import seaborn as sns  # noqa: E402
+from matplotlib.figure import Figure  # noqa: E402
+from sklearn.metrics import auc, confusion_matrix, roc_curve  # noqa: E402
+from sklearn.preprocessing import label_binarize  # noqa: E402
+
+ArrayLike = Sequence[Any]
+
+sns.set_theme(style="whitegrid")
+
+
+def plot_confusion_matrix(
+    y_true: ArrayLike,
+    y_pred: ArrayLike,
+    *,
+    labels: Optional[Sequence[Any]] = None,
+    normalize: bool = False,
+    title: str = "Confusion Matrix",
+    cmap: str = "Blues",
+    figsize: tuple = (6, 5),
+) -> Figure:
+    """Plot an annotated confusion-matrix heatmap.
+
+    Parameters
+    ----------
+    normalize : bool, default False
+        If True, rows are normalized to show proportions instead of raw
+        counts.
+    """
+    y_true_arr = np.asarray(y_true)
+    y_pred_arr = np.asarray(y_pred)
+    resolved_labels = (
+        list(labels)
+        if labels is not None
+        else sorted(set(np.unique(y_true_arr).tolist()) | set(np.unique(y_pred_arr).tolist()))
+    )
+
+    cm = confusion_matrix(y_true_arr, y_pred_arr, labels=resolved_labels)
+    fmt = "d"
+    display_cm = cm
+    if normalize:
+        with np.errstate(all="ignore"):
+            row_sums = cm.sum(axis=1, keepdims=True)
+            display_cm = np.divide(
+                cm, row_sums, out=np.zeros_like(cm, dtype=float), where=row_sums != 0
+            )
+        fmt = ".2f"
+
+    fig, ax = plt.subplots(figsize=figsize)
+    sns.heatmap(
+        display_cm,
+        annot=True,
+        fmt=fmt,
+        cmap=cmap,
+        xticklabels=resolved_labels,
+        yticklabels=resolved_labels,
+        cbar=True,
+        square=True,
+        ax=ax,
+    )
+    ax.set_xlabel("Predicted label")
+    ax.set_ylabel("True label")
+    ax.set_title(title)
+    fig.tight_layout()
+    return fig
+
+
+def plot_roc_curve(
+    y_true: ArrayLike,
+    y_proba: ArrayLike,
+    *,
+    labels: Optional[Sequence[Any]] = None,
+    title: str = "ROC Curve",
+    figsize: tuple = (6, 5),
+) -> Figure:
+    """Plot ROC curve(s) with AUC scores.
+
+    For binary targets a single curve is drawn. For multi-class targets,
+    a One-vs-Rest curve (with individual AUC) is drawn per class, plus a
+    micro-average curve.
+    """
+    y_true_arr = np.asarray(y_true)
+    y_proba_arr = np.asarray(y_proba)
+    resolved_labels = (
+        list(labels) if labels is not None else sorted(np.unique(y_true_arr).tolist())
+    )
+
+    fig, ax = plt.subplots(figsize=figsize)
+
+    if len(resolved_labels) <= 2:
+        pos_proba = y_proba_arr if y_proba_arr.ndim == 1 else y_proba_arr[:, 1]
+        fpr, tpr, _ = roc_curve(y_true_arr, pos_proba, pos_label=resolved_labels[-1])
+        roc_auc = auc(fpr, tpr)
+        ax.plot(fpr, tpr, lw=2, label=f"ROC (AUC = {roc_auc:.3f})")
+    else:
+        y_bin = label_binarize(y_true_arr, classes=resolved_labels)
+        for i, class_label in enumerate(resolved_labels):
+            fpr, tpr, _ = roc_curve(y_bin[:, i], y_proba_arr[:, i])
+            roc_auc = auc(fpr, tpr)
+            ax.plot(fpr, tpr, lw=1.5, label=f"Class {class_label} (AUC = {roc_auc:.3f})")
+
+        fpr_micro, tpr_micro, _ = roc_curve(y_bin.ravel(), y_proba_arr.ravel())
+        auc_micro = auc(fpr_micro, tpr_micro)
+        ax.plot(
+            fpr_micro,
+            tpr_micro,
+            lw=2.5,
+            linestyle="--",
+            color="black",
+            label=f"Micro-average (AUC = {auc_micro:.3f})",
+        )
+
+    ax.plot([0, 1], [0, 1], linestyle=":", color="gray", lw=1)
+    ax.set_xlim([0.0, 1.0])
+    ax.set_ylim([0.0, 1.05])
+    ax.set_xlabel("False Positive Rate")
+    ax.set_ylabel("True Positive Rate")
+    ax.set_title(title)
+    ax.legend(loc="lower right", fontsize="small")
+    fig.tight_layout()
+    return fig
+
+
+def plot_training_curves(
+    history: Dict[str, List[float]],
+    *,
+    title: str = "Training History",
+    figsize: tuple = (10, 4),
+) -> Figure:
+    """Plot train vs. validation loss/accuracy curves from a history dict.
+
+    Parameters
+    ----------
+    history : dict
+        Dictionary with any of the keys: "loss", "val_loss", "accuracy",
+        "val_accuracy" (Keras-style naming is also accepted, e.g. "acc",
+        "val_acc"). Each value is a sequence of per-epoch metric values.
+    """
+    normalized = dict(history)
+    if "acc" in normalized and "accuracy" not in normalized:
+        normalized["accuracy"] = normalized.pop("acc")
+    if "val_acc" in normalized and "val_accuracy" not in normalized:
+        normalized["val_accuracy"] = normalized.pop("val_acc")
+
+    has_loss = "loss" in normalized or "val_loss" in normalized
+    has_acc = "accuracy" in normalized or "val_accuracy" in normalized
+    n_panels = max(sum([has_loss, has_acc]), 1)
+
+    fig, axes = plt.subplots(1, n_panels, figsize=figsize)
+    if n_panels == 1:
+        axes = [axes]
+
+    panel_idx = 0
+    if has_loss:
+        ax = axes[panel_idx]
+        if "loss" in normalized:
+            epochs = range(1, len(normalized["loss"]) + 1)
+            ax.plot(epochs, normalized["loss"], label="Train Loss", marker="o", markersize=3)
+        if "val_loss" in normalized:
+            epochs = range(1, len(normalized["val_loss"]) + 1)
+            ax.plot(epochs, normalized["val_loss"], label="Val Loss", marker="o", markersize=3)
+        ax.set_xlabel("Epoch")
+        ax.set_ylabel("Loss")
+        ax.set_title("Loss")
+        ax.legend(fontsize="small")
+        panel_idx += 1
+
+    if has_acc:
+        ax = axes[panel_idx]
+        if "accuracy" in normalized:
+            epochs = range(1, len(normalized["accuracy"]) + 1)
+            ax.plot(
+                epochs, normalized["accuracy"], label="Train Accuracy", marker="o", markersize=3
+            )
+        if "val_accuracy" in normalized:
+            epochs = range(1, len(normalized["val_accuracy"]) + 1)
+            ax.plot(
+                epochs,
+                normalized["val_accuracy"],
+                label="Val Accuracy",
+                marker="o",
+                markersize=3,
+            )
+        ax.set_xlabel("Epoch")
+        ax.set_ylabel("Accuracy")
+        ax.set_title("Accuracy")
+        ax.legend(fontsize="small")
+
+    fig.suptitle(title)
+    fig.tight_layout()
+    return fig

ml_benchmark_toolkit-0.1.0/src/mlbenchmark/report.py

@@ -0,0 +1,183 @@
+"""Standalone HTML/Markdown report generation for model comparisons."""
+
+from __future__ import annotations
+
+import base64
+import io
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, Optional
+
+from matplotlib.figure import Figure
+
+from .comparator import ModelComparator
+
+
+def _figure_to_base64(fig: Figure) -> str:
+    """Encode a matplotlib Figure as a base64 PNG data URI for HTML embedding."""
+    buffer = io.BytesIO()
+    fig.savefig(buffer, format="png", dpi=130, bbox_inches="tight")
+    buffer.seek(0)
+    encoded = base64.b64encode(buffer.read()).decode("utf-8")
+    buffer.close()
+    return f"data:image/png;base64,{encoded}"
+
+
+_HTML_TEMPLATE = """<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<title>{title}</title>
+<style>
+  body {{
+    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+    margin: 0; padding: 2rem; background: #f6f7f9; color: #1a1a1a;
+  }}
+  .container {{ max-width: 1100px; margin: 0 auto; }}
+  h1 {{ font-size: 1.8rem; margin-bottom: 0.2rem; }}
+  .subtitle {{ color: #666; margin-bottom: 2rem; font-size: 0.9rem; }}
+  h2 {{ border-bottom: 2px solid #e0e0e0; padding-bottom: 0.4rem; margin-top: 2.5rem; }}
+  table {{ border-collapse: collapse; width: 100%; background: white; box-shadow: 0 1px 3px rgba(0,0,0,0.08); }}
+  th, td {{ border: 1px solid #e0e0e0; padding: 8px 12px; text-align: right; font-size: 0.9rem; }}
+  th {{ background: #2b3a55; color: white; text-align: center; }}
+  td:first-child, th:first-child {{ text-align: left; }}
+  tr:nth-child(even) {{ background: #fafbfc; }}
+  tr:first-child td {{ font-weight: 600; background: #eef4ff; }}
+  .fig-grid {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(420px, 1fr)); gap: 1.5rem; margin-top: 1rem; }}
+  .fig-card {{ background: white; border-radius: 8px; padding: 1rem; box-shadow: 0 1px 3px rgba(0,0,0,0.08); }}
+  .fig-card img {{ width: 100%; height: auto; border-radius: 4px; }}
+  .fig-card h3 {{ margin: 0 0 0.5rem 0; font-size: 1rem; color: #2b3a55; }}
+  footer {{ margin-top: 3rem; color: #999; font-size: 0.8rem; text-align: center; }}
+</style>
+</head>
+<body>
+<div class="container">
+  <h1>{title}</h1>
+  <p class="subtitle">Generated {timestamp} &middot; ml-benchmark-toolkit</p>
+
+  <h2>Model Comparison</h2>
+  {comparison_table}
+
+  <h2>Confusion Matrices</h2>
+  <div class="fig-grid">{confusion_figs}</div>
+
+  {roc_section}
+
+  {training_section}
+
+  <footer>Report generated with mlbenchmark.Report</footer>
+</div>
+</body>
+</html>
+"""
+
+
+class Report:
+    """Compiles a :class:`ModelComparator`'s tables and figures into a
+    standalone HTML or Markdown summary dashboard.
+
+    Examples
+    --------
+    >>> report = Report(comparator, title="Model Benchmark")
+    >>> report.to_html("report.html")
+    >>> report.to_markdown("report.md")
+    """
+
+    def __init__(
+        self,
+        comparator: ModelComparator,
+        title: str = "Model Benchmark Report",
+        rank_by: str = "f1_macro",
+        normalize_confusion: bool = False,
+    ) -> None:
+        self.comparator = comparator
+        self.title = title
+        self.rank_by = rank_by
+        self.normalize_confusion = normalize_confusion
+
+    def _confusion_figs(self) -> Dict[str, Figure]:
+        return self.comparator.confusion_matrices(normalize=self.normalize_confusion)
+
+    def to_html(self, path: str) -> Path:
+        """Render the full dashboard (tables + embedded figures) to a standalone HTML file."""
+        table_df = self.comparator.comparison_table(rank_by=self.rank_by).round(4)
+        table_html = table_df.to_html(classes="comparison-table", border=0)
+
+        confusion_figs = self._confusion_figs()
+        confusion_html = "".join(
+            f'<div class="fig-card"><h3>{name}</h3>'
+            f'<img src="{_figure_to_base64(fig)}" alt="Confusion matrix for {name}"></div>'
+            for name, fig in confusion_figs.items()
+        )
+
+        roc_figs = self.comparator.roc_curves()
+        roc_section = ""
+        if roc_figs:
+            roc_html = "".join(
+                f'<div class="fig-card"><h3>{name}</h3>'
+                f'<img src="{_figure_to_base64(fig)}" alt="ROC curve for {name}"></div>'
+                for name, fig in roc_figs.items()
+            )
+            roc_section = f'<h2>ROC Curves</h2><div class="fig-grid">{roc_html}</div>'
+
+        training_figs = self.comparator.training_curves()
+        training_section = ""
+        if training_figs:
+            training_html = "".join(
+                f'<div class="fig-card"><h3>{name}</h3>'
+                f'<img src="{_figure_to_base64(fig)}" alt="Training curves for {name}"></div>'
+                for name, fig in training_figs.items()
+            )
+            training_section = (
+                f'<h2>Training Curves</h2><div class="fig-grid">{training_html}</div>'
+            )
+
+        html = _HTML_TEMPLATE.format(
+            title=self.title,
+            timestamp=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"),
+            comparison_table=table_html,
+            confusion_figs=confusion_html,
+            roc_section=roc_section,
+            training_section=training_section,
+        )
+
+        out_path = Path(path)
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        out_path.write_text(html, encoding="utf-8")
+        return out_path
+
+    def to_markdown(self, path: Optional[str] = None) -> str:
+        """Render the comparison table (and notes on available figures) as Markdown.
+
+        Parameters
+        ----------
+        path : str, optional
+            If provided, the Markdown is also written to this file path.
+        """
+        table_df = self.comparator.comparison_table(rank_by=self.rank_by).round(4)
+        lines = [
+            f"# {self.title}",
+            "",
+            f"_Generated {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}_",
+            "",
+            "## Model Comparison",
+            "",
+            table_df.to_markdown(),
+            "",
+            "## Figures",
+            "",
+            (
+                "Confusion matrices, ROC curves, and training curves are available "
+                "via `Report.to_html()` or by calling the plotting functions in "
+                "`mlbenchmark.plots` / `ModelComparator` directly (Markdown does not "
+                "support embedded raster images inline in this export)."
+            ),
+        ]
+        markdown = "\n".join(lines)
+
+        if path is not None:
+            out_path = Path(path)
+            out_path.parent.mkdir(parents=True, exist_ok=True)
+            out_path.write_text(markdown, encoding="utf-8")
+
+        return markdown