trustbench 0.2.2__tar.gz → 0.2.3__tar.gz

{trustbench-0.2.2 → trustbench-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustbench
-Version: 0.2.2
+Version: 0.2.3
 Summary: TrustBench: Evaluation framework for ML model reliability, robustness, and uncertainty.
 Keywords: machine-learning,evaluation,reliability,robustness,uncertainty,calibration,ood-detection,fairness
 Author: Uni-Creator
@@ -56,12 +56,11 @@ Description-Content-Type: text/markdown
 
 **ML Reliability, Robustness, Uncertainty & Fairness Evaluation Framework**
 
-[![PyPI version](https://img.shields.io/badge/pypi-v0.2.2-blue.svg)](https://pypi.org/project/trustbench/)
-[![Python Versions](https://img.shields.io/badge/python-3.11%20%7C%203.12-blue.svg)](https://pypi.org/project/trustbench/)
-[![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](https://github.com/Uni-Creator/trustbench/blob/main/LICENSE)
-[![CI Status](https://img.shields.io/badge/build-passing-brightgreen.svg)](https://github.com/Uni-Creator/trustbench/actions/workflows/ci.yml)
-[![Documentation](https://img.shields.io/badge/docs-mkdocs-blue.svg)](https://uni-creator.github.io/trustbench/)
-[![Tests](https://img.shields.io/badge/tests-204%20passed-success.svg)](https://github.com/Uni-Creator/trustbench/tree/main/tests)
+[![PyPI version](https://img.shields.io/pypi/v/trustbench.svg)](https://pypi.org/project/trustbench/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/trustbench.svg)](https://pypi.org/project/trustbench/)
+[![License](https://img.shields.io/github/license/Uni-Creator/trustbench.svg)](https://github.com/Uni-Creator/trustbench/blob/main/LICENSE)
+[![CI Status](https://github.com/Uni-Creator/trustbench/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/Uni-Creator/trustbench/actions/workflows/ci.yml)
+[![Documentation](https://github.com/Uni-Creator/trustbench/actions/workflows/docs.yml/badge.svg)](https://uni-creator.github.io/trustbench/)
 
 </div>
 
@@ -243,6 +242,23 @@ print(f"Pixel Accuracy: {result.metrics['pixel_accuracy']:.2%}")
 
 See [Metrics Documentation](https://github.com/Uni-Creator/trustbench/blob/main/docs/metrics.md) for mathematical derivations.
 
+### Metric Verification & Standards Compliance
+
+To guarantee mathematical correctness and numerical fidelity, TrustBench metrics are rigorously tested directly against established reference implementations and peer-reviewed literature benchmarks on identical inputs:
+
+| TrustBench Metric Domain | Reference Implementation / Peer-Reviewed Standard | Verification Test Suite |
+| :--- | :--- | :--- |
+| **Classification** (Acc, Balanced Acc, Macro-F1, Precision, Recall, Confusion Matrix) | `scikit-learn` (`sklearn.metrics`) | `tests/unit/test_metric_reference_benchmarks.py` |
+| **Regression** (MAE, MSE, RMSE, $R^2$, MedAE, Max Error) | `scikit-learn` (`sklearn.metrics`) | `tests/unit/test_metric_reference_benchmarks.py` |
+| **Calibration** (Brier Score, Negative Log-Likelihood) | `scikit-learn` (`brier_score_loss`, `log_loss`) | `tests/unit/test_metric_reference_benchmarks.py` |
+| **Calibration** (Equal-width ECE, Adaptive Quantile ECE) | Guo et al. (2017), Naeini et al. (2015), Ding et al. (2020) | `tests/unit/test_metric_reference_benchmarks.py` |
+| **Error Detection** (AUROC, AUPRC, FPR@95%TPR) | `scikit-learn` (`roc_auc_score`, `precision_recall_curve`, `auc`) | `tests/unit/test_metric_reference_benchmarks.py` |
+| **Fairness & Bias** (Demographic Parity, Equalized Odds, Disparate Impact) | Fairlearn Mathematical Specification | `tests/unit/test_metric_reference_benchmarks.py` |
+| **Selective Prediction** (Risk-Coverage, Optimal AURC, Excess AURC) | Geifman & El-Yaniv (2017) Analytical Bounds | `tests/unit/test_metric_reference_benchmarks.py` |
+
+> [!NOTE]
+> **Task Maturity**: Classification, Calibration, Robustness, Uncertainty, and Error Detection are core, production-verified pipelines with end-to-end integration tests. Specialized domain pipelines (Information Retrieval, Semantic Segmentation, and Algorithmic Fairness) and non-PyTorch adapters (Keras, JAX) are provided with protocol compliance tests and marked as experimental extensions.
+
 ---
 
 ## Tutorials & Examples
@@ -292,7 +308,7 @@ pytest
 ruff check .
 
 # Static type analysis
-mypy src/trustbench/core src/trustbench/reporting src/trustbench/metrics
+mypy src/trustbench --ignore-missing-imports
 ```
 
 ---

{trustbench-0.2.2 → trustbench-0.2.3}/README.md

@@ -4,12 +4,11 @@
 
 **ML Reliability, Robustness, Uncertainty & Fairness Evaluation Framework**
 
-[![PyPI version](https://img.shields.io/badge/pypi-v0.2.2-blue.svg)](https://pypi.org/project/trustbench/)
-[![Python Versions](https://img.shields.io/badge/python-3.11%20%7C%203.12-blue.svg)](https://pypi.org/project/trustbench/)
-[![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](https://github.com/Uni-Creator/trustbench/blob/main/LICENSE)
-[![CI Status](https://img.shields.io/badge/build-passing-brightgreen.svg)](https://github.com/Uni-Creator/trustbench/actions/workflows/ci.yml)
-[![Documentation](https://img.shields.io/badge/docs-mkdocs-blue.svg)](https://uni-creator.github.io/trustbench/)
-[![Tests](https://img.shields.io/badge/tests-204%20passed-success.svg)](https://github.com/Uni-Creator/trustbench/tree/main/tests)
+[![PyPI version](https://img.shields.io/pypi/v/trustbench.svg)](https://pypi.org/project/trustbench/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/trustbench.svg)](https://pypi.org/project/trustbench/)
+[![License](https://img.shields.io/github/license/Uni-Creator/trustbench.svg)](https://github.com/Uni-Creator/trustbench/blob/main/LICENSE)
+[![CI Status](https://github.com/Uni-Creator/trustbench/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/Uni-Creator/trustbench/actions/workflows/ci.yml)
+[![Documentation](https://github.com/Uni-Creator/trustbench/actions/workflows/docs.yml/badge.svg)](https://uni-creator.github.io/trustbench/)
 
 </div>
 
@@ -191,6 +190,23 @@ print(f"Pixel Accuracy: {result.metrics['pixel_accuracy']:.2%}")
 
 See [Metrics Documentation](https://github.com/Uni-Creator/trustbench/blob/main/docs/metrics.md) for mathematical derivations.
 
+### Metric Verification & Standards Compliance
+
+To guarantee mathematical correctness and numerical fidelity, TrustBench metrics are rigorously tested directly against established reference implementations and peer-reviewed literature benchmarks on identical inputs:
+
+| TrustBench Metric Domain | Reference Implementation / Peer-Reviewed Standard | Verification Test Suite |
+| :--- | :--- | :--- |
+| **Classification** (Acc, Balanced Acc, Macro-F1, Precision, Recall, Confusion Matrix) | `scikit-learn` (`sklearn.metrics`) | `tests/unit/test_metric_reference_benchmarks.py` |
+| **Regression** (MAE, MSE, RMSE, $R^2$, MedAE, Max Error) | `scikit-learn` (`sklearn.metrics`) | `tests/unit/test_metric_reference_benchmarks.py` |
+| **Calibration** (Brier Score, Negative Log-Likelihood) | `scikit-learn` (`brier_score_loss`, `log_loss`) | `tests/unit/test_metric_reference_benchmarks.py` |
+| **Calibration** (Equal-width ECE, Adaptive Quantile ECE) | Guo et al. (2017), Naeini et al. (2015), Ding et al. (2020) | `tests/unit/test_metric_reference_benchmarks.py` |
+| **Error Detection** (AUROC, AUPRC, FPR@95%TPR) | `scikit-learn` (`roc_auc_score`, `precision_recall_curve`, `auc`) | `tests/unit/test_metric_reference_benchmarks.py` |
+| **Fairness & Bias** (Demographic Parity, Equalized Odds, Disparate Impact) | Fairlearn Mathematical Specification | `tests/unit/test_metric_reference_benchmarks.py` |
+| **Selective Prediction** (Risk-Coverage, Optimal AURC, Excess AURC) | Geifman & El-Yaniv (2017) Analytical Bounds | `tests/unit/test_metric_reference_benchmarks.py` |
+
+> [!NOTE]
+> **Task Maturity**: Classification, Calibration, Robustness, Uncertainty, and Error Detection are core, production-verified pipelines with end-to-end integration tests. Specialized domain pipelines (Information Retrieval, Semantic Segmentation, and Algorithmic Fairness) and non-PyTorch adapters (Keras, JAX) are provided with protocol compliance tests and marked as experimental extensions.
+
 ---
 
 ## Tutorials & Examples
@@ -240,7 +256,7 @@ pytest
 ruff check .
 
 # Static type analysis
-mypy src/trustbench/core src/trustbench/reporting src/trustbench/metrics
+mypy src/trustbench --ignore-missing-imports
 ```
 
 ---

{trustbench-0.2.2 → trustbench-0.2.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "trustbench"
-version = "0.2.2"
+version = "0.2.3"
 description = "TrustBench: Evaluation framework for ML model reliability, robustness, and uncertainty."
 readme = "README.md"
 license = "Apache-2.0"
@@ -78,6 +78,7 @@ dev = [
     "httpx>=0.28.1",
     "mypy>=2.4.0",
     "pytest>=9.1.1",
+    "pytest-cov>=4.0.0",
     "ruff>=0.17.2",
     "fastapi>=0.100.0",
     "plotly>=5.0.0",

{trustbench-0.2.2 → trustbench-0.2.3}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "trustbench"
-version = "0.2.2"
+version = "0.2.3"
 description = "TrustBench: Evaluation framework for ML model reliability, robustness, and uncertainty."
 readme = "README.md"
 license = "Apache-2.0"
@@ -97,6 +97,7 @@ dev = [
     "httpx>=0.28.1",
     "mypy>=2.4.0",
     "pytest>=9.1.1",
+    "pytest-cov>=4.0.0",
     "ruff>=0.17.2",
 
     # Dependencies required by the test suite

{trustbench-0.2.2 → trustbench-0.2.3}/src/trustbench/__init__.py

@@ -1,6 +1,6 @@
 """TrustBench: Evaluation framework for machine learning model reliability."""
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"
 
 from trustbench.benchmark import TrustBench
 from trustbench.core.evaluator import evaluate

{trustbench-0.2.2 → trustbench-0.2.3}/src/trustbench/datasets/cifar10.py

@@ -84,7 +84,7 @@ def get_cifar10_validation_split(
 
     # Deterministic split indices
     generator = torch.Generator().manual_seed(seed)
-    train_indices, val_indices = random_split(range(total_len), [train_len, val_size], generator=generator)
+    train_indices, val_indices = random_split(range(total_len), [train_len, val_size], generator=generator)  # type: ignore[arg-type,var-annotated]
 
     train_subset = Subset(train_base, train_indices.indices)
     val_subset = Subset(val_base, val_indices.indices)

{trustbench-0.2.2 → trustbench-0.2.3}/src/trustbench/evaluation/__init__.py

@@ -119,11 +119,10 @@ def __getattr__(name: str):
             save_experiment_artifacts,
         )
 
-        exports = {
+        return {
             "load_experiment_artifacts": load_experiment_artifacts,
             "save_experiment_artifacts": save_experiment_artifacts,
-        }
-        return exports[name]
+        }[name]
     if name in {
         "IndexedDataset",
         "load_model_from_checkpoint",
@@ -137,13 +136,12 @@ def __getattr__(name: str):
             run_inference,
         )
 
-        exports = {
+        return {
             "IndexedDataset": IndexedDataset,
             "load_model_from_checkpoint": load_model_from_checkpoint,
             "resolve_model_architecture": resolve_model_architecture,
             "run_inference": run_inference,
-        }
-        return exports[name]
+        }[name]
     if name in {"get_environment_info", "get_git_commit", "set_seed"}:
         from trustbench.core.reproducibility import (
             get_environment_info,

{trustbench-0.2.2 → trustbench-0.2.3}/src/trustbench/evaluation/distribution_shift.py

@@ -83,8 +83,8 @@ def compute_label_distribution_shift(
     counts_s = np.bincount(labels_s[labels_s >= 0], minlength=n_c)[:n_c]
     counts_t = np.bincount(labels_t[labels_t >= 0], minlength=n_c)[:n_c]
 
-    priors_s = counts_s / max(1, np.sum(counts_s))
-    priors_t = counts_t / max(1, np.sum(counts_t))
+    priors_s = counts_s / max(1.0, float(np.sum(counts_s)))
+    priors_t = counts_t / max(1.0, float(np.sum(counts_t)))
 
     tv = compute_tv_distance(priors_s, priors_t)
     js = float(jensenshannon(priors_s, priors_t))
@@ -177,8 +177,8 @@ def evaluate_distribution_shift(
     n_classes = clean_probs.shape[1] if clean_probs.ndim == 2 else int(max(np.max(clean_preds), np.max(shifted_preds)) + 1)
     counts_c = np.bincount(clean_preds[clean_preds >= 0], minlength=n_classes)[:n_classes]
     counts_s = np.bincount(shifted_preds[shifted_preds >= 0], minlength=n_classes)[:n_classes]
-    marg_clean = counts_c / max(1, np.sum(counts_c))
-    marg_shifted = counts_s / max(1, np.sum(counts_s))
+    marg_clean = counts_c / max(1.0, float(np.sum(counts_c)))
+    marg_shifted = counts_s / max(1.0, float(np.sum(counts_s)))
     pred_tv = compute_tv_distance(marg_clean, marg_shifted)
 
     # 2. Confidence & Entropy Shifts

{trustbench-0.2.2 → trustbench-0.2.3}/src/trustbench/experiments/comparison.py

@@ -148,13 +148,13 @@ class ExperimentComparator:
         Accepts experiment IDs (resolved via store or database session),
         or ExperimentRecord / dict objects directly.
         """
-        records: list[ExperimentRecord | dict[str, Any]] = []
+        records: list[Any] = []
 
         for item in experiments:
             if isinstance(item, (ExperimentRecord, dict)):
                 records.append(item)
             elif isinstance(item, str):
-                rec = None
+                rec: Any = None
                 if self.store is not None:
                     rec = self.store.get(item)
                 if rec is None and (self._session is not None or self.session_provided):

{trustbench-0.2.2 → trustbench-0.2.3}/src/trustbench/experiments/registry.py

@@ -110,39 +110,39 @@ class ExperimentRegistry:
             repo.update_status(exp_model.id, status="completed", completed=True)
 
         # 3. Index Artifact Files
-        for f in exp_path.iterdir():
-            if f.is_file():
-                art_type = f.stem
-                csum = compute_file_sha256(f)
+        for file_item in exp_path.iterdir():
+            if file_item.is_file():
+                art_type = file_item.stem
+                csum = compute_file_sha256(file_item)
                 repo.add_artifact(
                     experiment_id=exp_model.id,
                     artifact_type=art_type,
-                    file_path=str(f.resolve()),
+                    file_path=str(file_item.resolve()),
                     checksum=csum,
                 )
 
         # Index corruption_results subfolder if present
         corruptions_dir = exp_path / "corruption_results"
         if corruptions_dir.is_dir():
-            for f in corruptions_dir.iterdir():
-                if f.is_file() and f.suffix == ".json":
+            for file_item in corruptions_dir.iterdir():
+                if file_item.is_file() and file_item.suffix == ".json":
                     repo.add_artifact(
                         experiment_id=exp_model.id,
-                        artifact_type=f"corruption_{f.stem}",
-                        file_path=str(f.resolve()),
-                        checksum=compute_file_sha256(f),
+                        artifact_type=f"corruption_{file_item.stem}",
+                        file_path=str(file_item.resolve()),
+                        checksum=compute_file_sha256(file_item),
                     )
 
         # Index predictions subfolder if present
         preds_dir = exp_path / "predictions"
         if preds_dir.is_dir():
-            for f in preds_dir.iterdir():
-                if f.is_file() and f.suffix == ".npz":
+            for file_item in preds_dir.iterdir():
+                if file_item.is_file() and file_item.suffix == ".npz":
                     repo.add_artifact(
                         experiment_id=exp_model.id,
-                        artifact_type=f"prediction_{f.stem}",
-                        file_path=str(f.resolve()),
-                        checksum=compute_file_sha256(f),
+                        artifact_type=f"prediction_{file_item.stem}",
+                        file_path=str(file_item.resolve()),
+                        checksum=compute_file_sha256(file_item),
                     )
 
         # 4. Extract and index scalar metrics

{trustbench-0.2.2 → trustbench-0.2.3}/src/trustbench/registry/metrics.py

@@ -133,10 +133,10 @@ class MetricRegistry:
                     raise MetricRegistrationError("Metric name must be a non-empty string.")
                 metric_name = name.strip()
             else:
-                metric_name = getattr(func, "__name__", None)
-                if not metric_name or not isinstance(metric_name, str) or not metric_name.strip():
+                raw_name = getattr(func, "__name__", None)
+                if not raw_name or not isinstance(raw_name, str) or not raw_name.strip():
                     raise MetricRegistrationError("Metric name must be a non-empty string.")
-                metric_name = metric_name.strip()
+                metric_name = raw_name.strip()
 
             if not callable(func):
                 raise MetricRegistrationError(f"Target metric {func!r} is not callable.")

{trustbench-0.2.2 → trustbench-0.2.3}/src/trustbench/registry/plugins.py

@@ -9,7 +9,7 @@ from typing import Any
 try:
     CURRENT_VERSION = importlib.metadata.version("trustbench")
 except importlib.metadata.PackageNotFoundError:
-    CURRENT_VERSION = "0.2.2"
+    CURRENT_VERSION = "0.2.3"
 
 logger = logging.getLogger(__name__)
 
@@ -164,7 +164,7 @@ class PluginManager:
             if hasattr(eps, "select"):
                 matching_eps = eps.select(group=group)
             else:
-                matching_eps = eps.get(group, [])
+                matching_eps = eps.get(group, [])  # type: ignore[attr-defined,arg-type]
         except Exception as exc:  # noqa: BLE001
             logger.warning("Error querying entry points for group %r: %s", group, exc)
             return dict(self._plugins)

{trustbench-0.2.2 → trustbench-0.2.3}/src/trustbench/robustness/evaluator.py

@@ -109,8 +109,8 @@ class RobustnessEvaluator:
             base_seed=self.seed,
         )
 
-        loader = DataLoader(
-            corrupted_ds,
+        loader: Any = DataLoader(
+            corrupted_ds,  # type: ignore[arg-type]
             batch_size=self.batch_size,
             shuffle=False,
             num_workers=self.num_workers,
@@ -261,7 +261,7 @@ class RobustnessEvaluator:
             # Save predictions NPZ: predictions/<c_type>.npz (if enabled)
             if save_predictions:
                 npz_file = predictions_dir / f"{c_type}.npz"
-                np.savez_compressed(npz_file, **npz_arrays)
+                np.savez_compressed(npz_file, **npz_arrays)  # type: ignore[arg-type]
             del npz_arrays
 
         # Save config.json

{trustbench-0.2.2 → trustbench-0.2.3}/src/trustbench/storage/filesystem.py

@@ -55,7 +55,7 @@ def _atomic_write_npz(path: Path, data: dict[str, np.ndarray]) -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
     tmp_path = path.with_suffix(path.suffix + f".tmp.{uuid.uuid4().hex[:8]}")
     with open(tmp_path, "wb") as f:
-        np.savez_compressed(f, **data)
+        np.savez_compressed(f, **data)  # type: ignore[arg-type]
         f.flush()
         os.fsync(f.fileno())
     os.replace(tmp_path, path)
@@ -184,7 +184,7 @@ class FilesystemExperimentStore:
         checkpoint_sha = meta.get("checkpoint_sha256") or cfg.get("checkpoint_sha256")
         dataset_fp = meta.get("dataset_fingerprint") or cfg.get("dataset_fingerprint")
 
-        manifest = {
+        manifest: dict[str, Any] = {
             "id": exp_id,
             "name": name,
             "experiment_type": experiment_type,