trustbench 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {trustbench-0.2.0 → trustbench-0.2.2}/PKG-INFO +39 -33
- {trustbench-0.2.0 → trustbench-0.2.2}/README.md +33 -32
- {trustbench-0.2.0 → trustbench-0.2.2}/pyproject.toml +14 -1
- {trustbench-0.2.0 → trustbench-0.2.2}/pyproject.toml.orig +13 -1
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/__init__.py +1 -1
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/registry/plugins.py +1 -1
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/html_report.py +11 -8
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/plots.py +5 -2
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/api/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/api/app.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/api/routes/experiments.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/benchmark.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/cli/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/cli/main.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/config/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/config/loader.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/config/schema.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/evaluator.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/exceptions.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/inference.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/protocols.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/reproducibility.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/results.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/datasets/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/datasets/base.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/datasets/cifar10.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/datasets/memory.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/datasets/torch_dataset.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/artifacts.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/calibration.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/classification.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/distribution_shift.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/error_detection.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/fairness.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/inference.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/regression.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/reproducibility.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/retrieval.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/runner.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/segmentation.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/selective.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/streaming.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/uncertainty.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/experiments/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/experiments/comparison.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/experiments/lineage.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/experiments/manager.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/experiments/registry.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/experiments/schemas.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/calibration.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/classification.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/error_detection.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/regression.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/retrieval.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/segmentation.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/selective.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/uncertainty.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/base.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/checkpoint.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/densenet.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/dla.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/dla_simple.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/dpn.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/efficientnet.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/googlenet.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/jax.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/keras.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/lenet.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/mobilenet.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/mobilenetv2.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/onnx.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/pnasnet.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/preact_resnet.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/pytorch.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/regnet.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/resnet.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/resnext.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/senet.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/shufflenet.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/shufflenetv2.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/sklearn.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/vgg.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/registry/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/registry/corruptions.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/registry/datasets.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/registry/metrics.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/registry/models.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/artifacts.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/hashing.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/result_io.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/schema.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/tables.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/robustness/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/robustness/aggregation.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/robustness/corruptions.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/robustness/evaluator.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/robustness/report.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/robustness/severity.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/__init__.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/base.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/database.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/filesystem.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/models.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/repositories.py +0 -0
- {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/sqlite.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: trustbench
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: TrustBench: Evaluation framework for ML model reliability, robustness, and uncertainty.
|
|
5
5
|
Keywords: machine-learning,evaluation,reliability,robustness,uncertainty,calibration,ood-detection,fairness
|
|
6
6
|
Author: Uni-Creator
|
|
@@ -34,6 +34,11 @@ Requires-Dist: tensorflow>=2.15.0 ; extra == 'tensorflow'
|
|
|
34
34
|
Requires-Dist: torch>=2.0.0 ; extra == 'torch'
|
|
35
35
|
Requires-Dist: torchvision>=0.15.0 ; extra == 'torch'
|
|
36
36
|
Requires-Python: >=3.11
|
|
37
|
+
Project-URL: Homepage, https://uni-creator.github.io/trustbench/
|
|
38
|
+
Project-URL: Documentation, https://uni-creator.github.io/trustbench/
|
|
39
|
+
Project-URL: Repository, https://github.com/Uni-Creator/trustbench
|
|
40
|
+
Project-URL: Issues, https://github.com/Uni-Creator/trustbench/issues
|
|
41
|
+
Project-URL: Changelog, https://github.com/Uni-Creator/trustbench/blob/main/CHANGELOG.md
|
|
37
42
|
Provides-Extra: all
|
|
38
43
|
Provides-Extra: api
|
|
39
44
|
Provides-Extra: jax
|
|
@@ -51,11 +56,12 @@ Description-Content-Type: text/markdown
|
|
|
51
56
|
|
|
52
57
|
**ML Reliability, Robustness, Uncertainty & Fairness Evaluation Framework**
|
|
53
58
|
|
|
54
|
-
[](https://pypi.org/project/trustbench/)
|
|
55
60
|
[](https://pypi.org/project/trustbench/)
|
|
56
|
-
[](LICENSE)
|
|
57
|
-
[](.
|
|
58
|
-
[](https://github.com/Uni-Creator/trustbench/blob/main/LICENSE)
|
|
62
|
+
[](https://github.com/Uni-Creator/trustbench/actions/workflows/ci.yml)
|
|
63
|
+
[](https://uni-creator.github.io/trustbench/)
|
|
64
|
+
[](https://github.com/Uni-Creator/trustbench/tree/main/tests)
|
|
59
65
|
|
|
60
66
|
</div>
|
|
61
67
|
|
|
@@ -131,7 +137,7 @@ pip install "trustbench[onnx]"
|
|
|
131
137
|
pip install "trustbench[all]"
|
|
132
138
|
```
|
|
133
139
|
|
|
134
|
-
See the [Installation Guide](docs/installation.md) for detailed platform requirements.
|
|
140
|
+
See the [Installation Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/installation.md) for detailed platform requirements.
|
|
135
141
|
|
|
136
142
|
---
|
|
137
143
|
|
|
@@ -235,48 +241,48 @@ print(f"Pixel Accuracy: {result.metrics['pixel_accuracy']:.2%}")
|
|
|
235
241
|
| **Segmentation** | Mean IoU (mIoU), Dice (F1), Pixel Accuracy, Frequency-Weighted IoU (FWIoU) |
|
|
236
242
|
| **Fairness** | Demographic Parity Difference, Disparate Impact Ratio, Equalized Odds Difference |
|
|
237
243
|
|
|
238
|
-
See [Metrics Documentation](docs/metrics.md) for mathematical derivations.
|
|
244
|
+
See [Metrics Documentation](https://github.com/Uni-Creator/trustbench/blob/main/docs/metrics.md) for mathematical derivations.
|
|
239
245
|
|
|
240
246
|
---
|
|
241
247
|
|
|
242
248
|
## Tutorials & Examples
|
|
243
249
|
|
|
244
|
-
Ready-to-run examples are available in the [`examples/`](examples
|
|
250
|
+
Ready-to-run examples are available in the [`examples/`](https://github.com/Uni-Creator/trustbench/tree/main/examples) directory:
|
|
245
251
|
|
|
246
|
-
1. **[01_evaluate_pytorch_model.py](examples/01_evaluate_pytorch_model.py)**: Evaluate custom PyTorch `nn.Module`.
|
|
247
|
-
2. **[02_evaluate_cifar10.py](examples/02_evaluate_cifar10.py)**: End-to-end CIFAR-10 ResNet evaluation via configuration.
|
|
248
|
-
3. **[03_robustness_benchmark.py](examples/03_robustness_benchmark.py)**: Evaluate accuracy degradation across noise & corruptions.
|
|
249
|
-
4. **[04_analyze_calibration.py](examples/04_analyze_calibration.py)**: Compute ECE and fit validation temperature scaling.
|
|
250
|
-
5. **[05_evaluate_uncertainty.py](examples/05_evaluate_uncertainty.py)**: Predict error detection AUROC and risk-coverage curves.
|
|
251
|
-
6. **[06_compare_checkpoints.py](examples/06_compare_checkpoints.py)**: Quantify statistical differentials between model checkpoints.
|
|
252
|
-
7. **[07_save_and_reproduce_experiment.py](examples/07_save_and_reproduce_experiment.py)**: Atomic experiment persistence and checksum verification.
|
|
253
|
-
8. **[08_create_custom_metric.py](examples/08_create_custom_metric.py)**: Register custom evaluation scoring functions.
|
|
254
|
-
9. **[09_create_custom_model_adapter.py](examples/09_create_custom_model_adapter.py)**: Implement the runtime `ModelAdapter` protocol.
|
|
252
|
+
1. **[01_evaluate_pytorch_model.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/01_evaluate_pytorch_model.py)**: Evaluate custom PyTorch `nn.Module`.
|
|
253
|
+
2. **[02_evaluate_cifar10.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/02_evaluate_cifar10.py)**: End-to-end CIFAR-10 ResNet evaluation via configuration.
|
|
254
|
+
3. **[03_robustness_benchmark.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/03_robustness_benchmark.py)**: Evaluate accuracy degradation across noise & corruptions.
|
|
255
|
+
4. **[04_analyze_calibration.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/04_analyze_calibration.py)**: Compute ECE and fit validation temperature scaling.
|
|
256
|
+
5. **[05_evaluate_uncertainty.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/05_evaluate_uncertainty.py)**: Predict error detection AUROC and risk-coverage curves.
|
|
257
|
+
6. **[06_compare_checkpoints.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/06_compare_checkpoints.py)**: Quantify statistical differentials between model checkpoints.
|
|
258
|
+
7. **[07_save_and_reproduce_experiment.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/07_save_and_reproduce_experiment.py)**: Atomic experiment persistence and checksum verification.
|
|
259
|
+
8. **[08_create_custom_metric.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/08_create_custom_metric.py)**: Register custom evaluation scoring functions.
|
|
260
|
+
9. **[09_create_custom_model_adapter.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/09_create_custom_model_adapter.py)**: Implement the runtime `ModelAdapter` protocol.
|
|
255
261
|
|
|
256
262
|
---
|
|
257
263
|
|
|
258
264
|
## Documentation
|
|
259
265
|
|
|
260
|
-
Full documentation is available in the [`docs/`](docs
|
|
261
|
-
- [Installation Guide](docs/installation.md)
|
|
262
|
-
- [Quickstart Guide](docs/quickstart.md)
|
|
263
|
-
- [Python API Reference](docs/api.md)
|
|
264
|
-
- [CLI Reference](docs/cli.md)
|
|
265
|
-
- [Configuration Guide](docs/configuration.md)
|
|
266
|
-
- [Model Adapters](docs/model-adapters.md)
|
|
267
|
-
- [Dataset Adapters](docs/dataset-adapters.md)
|
|
268
|
-
- [Metrics Documentation](docs/metrics.md)
|
|
269
|
-
- [Robustness Testing](docs/robustness.md)
|
|
270
|
-
- [Experiment Tracking](docs/experiments.md)
|
|
271
|
-
- [Reporting & Visualization](docs/reporting.md)
|
|
272
|
-
- [Plugin Development](docs/plugins.md)
|
|
273
|
-
- [Troubleshooting & FAQ](docs/troubleshooting.md)
|
|
266
|
+
Full documentation is available on the [Documentation Website](https://uni-creator.github.io/trustbench/) or in the [`docs/`](https://github.com/Uni-Creator/trustbench/tree/main/docs) directory:
|
|
267
|
+
- [Installation Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/installation.md)
|
|
268
|
+
- [Quickstart Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/quickstart.md)
|
|
269
|
+
- [Python API Reference](https://github.com/Uni-Creator/trustbench/blob/main/docs/api.md)
|
|
270
|
+
- [CLI Reference](https://github.com/Uni-Creator/trustbench/blob/main/docs/cli.md)
|
|
271
|
+
- [Configuration Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/configuration.md)
|
|
272
|
+
- [Model Adapters](https://github.com/Uni-Creator/trustbench/blob/main/docs/model-adapters.md)
|
|
273
|
+
- [Dataset Adapters](https://github.com/Uni-Creator/trustbench/blob/main/docs/dataset-adapters.md)
|
|
274
|
+
- [Metrics Documentation](https://github.com/Uni-Creator/trustbench/blob/main/docs/metrics.md)
|
|
275
|
+
- [Robustness Testing](https://github.com/Uni-Creator/trustbench/blob/main/docs/robustness.md)
|
|
276
|
+
- [Experiment Tracking](https://github.com/Uni-Creator/trustbench/blob/main/docs/experiments.md)
|
|
277
|
+
- [Reporting & Visualization](https://github.com/Uni-Creator/trustbench/blob/main/docs/reporting.md)
|
|
278
|
+
- [Plugin Development](https://github.com/Uni-Creator/trustbench/blob/main/docs/plugins.md)
|
|
279
|
+
- [Troubleshooting & FAQ](https://github.com/Uni-Creator/trustbench/blob/main/docs/troubleshooting.md)
|
|
274
280
|
|
|
275
281
|
---
|
|
276
282
|
|
|
277
283
|
## Contributing
|
|
278
284
|
|
|
279
|
-
We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) and our [Code of Conduct](CODE_OF_CONDUCT.md).
|
|
285
|
+
We welcome contributions! Please see [CONTRIBUTING.md](https://github.com/Uni-Creator/trustbench/blob/main/CONTRIBUTING.md) and our [Code of Conduct](https://github.com/Uni-Creator/trustbench/blob/main/CODE_OF_CONDUCT.md).
|
|
280
286
|
|
|
281
287
|
```bash
|
|
282
288
|
# Run tests
|
|
@@ -293,4 +299,4 @@ mypy src/trustbench/core src/trustbench/reporting src/trustbench/metrics
|
|
|
293
299
|
|
|
294
300
|
## License
|
|
295
301
|
|
|
296
|
-
TrustBench is open-source software licensed under the [Apache License, Version 2.0](LICENSE).
|
|
302
|
+
TrustBench is open-source software licensed under the [Apache License, Version 2.0](https://github.com/Uni-Creator/trustbench/blob/main/LICENSE).
|
|
@@ -4,11 +4,12 @@
|
|
|
4
4
|
|
|
5
5
|
**ML Reliability, Robustness, Uncertainty & Fairness Evaluation Framework**
|
|
6
6
|
|
|
7
|
-
[](https://pypi.org/project/trustbench/)
|
|
8
8
|
[](https://pypi.org/project/trustbench/)
|
|
9
|
-
[](LICENSE)
|
|
10
|
-
[](.
|
|
11
|
-
[](https://github.com/Uni-Creator/trustbench/blob/main/LICENSE)
|
|
10
|
+
[](https://github.com/Uni-Creator/trustbench/actions/workflows/ci.yml)
|
|
11
|
+
[](https://uni-creator.github.io/trustbench/)
|
|
12
|
+
[](https://github.com/Uni-Creator/trustbench/tree/main/tests)
|
|
12
13
|
|
|
13
14
|
</div>
|
|
14
15
|
|
|
@@ -84,7 +85,7 @@ pip install "trustbench[onnx]"
|
|
|
84
85
|
pip install "trustbench[all]"
|
|
85
86
|
```
|
|
86
87
|
|
|
87
|
-
See the [Installation Guide](docs/installation.md) for detailed platform requirements.
|
|
88
|
+
See the [Installation Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/installation.md) for detailed platform requirements.
|
|
88
89
|
|
|
89
90
|
---
|
|
90
91
|
|
|
@@ -188,48 +189,48 @@ print(f"Pixel Accuracy: {result.metrics['pixel_accuracy']:.2%}")
|
|
|
188
189
|
| **Segmentation** | Mean IoU (mIoU), Dice (F1), Pixel Accuracy, Frequency-Weighted IoU (FWIoU) |
|
|
189
190
|
| **Fairness** | Demographic Parity Difference, Disparate Impact Ratio, Equalized Odds Difference |
|
|
190
191
|
|
|
191
|
-
See [Metrics Documentation](docs/metrics.md) for mathematical derivations.
|
|
192
|
+
See [Metrics Documentation](https://github.com/Uni-Creator/trustbench/blob/main/docs/metrics.md) for mathematical derivations.
|
|
192
193
|
|
|
193
194
|
---
|
|
194
195
|
|
|
195
196
|
## Tutorials & Examples
|
|
196
197
|
|
|
197
|
-
Ready-to-run examples are available in the [`examples/`](examples
|
|
198
|
+
Ready-to-run examples are available in the [`examples/`](https://github.com/Uni-Creator/trustbench/tree/main/examples) directory:
|
|
198
199
|
|
|
199
|
-
1. **[01_evaluate_pytorch_model.py](examples/01_evaluate_pytorch_model.py)**: Evaluate custom PyTorch `nn.Module`.
|
|
200
|
-
2. **[02_evaluate_cifar10.py](examples/02_evaluate_cifar10.py)**: End-to-end CIFAR-10 ResNet evaluation via configuration.
|
|
201
|
-
3. **[03_robustness_benchmark.py](examples/03_robustness_benchmark.py)**: Evaluate accuracy degradation across noise & corruptions.
|
|
202
|
-
4. **[04_analyze_calibration.py](examples/04_analyze_calibration.py)**: Compute ECE and fit validation temperature scaling.
|
|
203
|
-
5. **[05_evaluate_uncertainty.py](examples/05_evaluate_uncertainty.py)**: Predict error detection AUROC and risk-coverage curves.
|
|
204
|
-
6. **[06_compare_checkpoints.py](examples/06_compare_checkpoints.py)**: Quantify statistical differentials between model checkpoints.
|
|
205
|
-
7. **[07_save_and_reproduce_experiment.py](examples/07_save_and_reproduce_experiment.py)**: Atomic experiment persistence and checksum verification.
|
|
206
|
-
8. **[08_create_custom_metric.py](examples/08_create_custom_metric.py)**: Register custom evaluation scoring functions.
|
|
207
|
-
9. **[09_create_custom_model_adapter.py](examples/09_create_custom_model_adapter.py)**: Implement the runtime `ModelAdapter` protocol.
|
|
200
|
+
1. **[01_evaluate_pytorch_model.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/01_evaluate_pytorch_model.py)**: Evaluate custom PyTorch `nn.Module`.
|
|
201
|
+
2. **[02_evaluate_cifar10.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/02_evaluate_cifar10.py)**: End-to-end CIFAR-10 ResNet evaluation via configuration.
|
|
202
|
+
3. **[03_robustness_benchmark.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/03_robustness_benchmark.py)**: Evaluate accuracy degradation across noise & corruptions.
|
|
203
|
+
4. **[04_analyze_calibration.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/04_analyze_calibration.py)**: Compute ECE and fit validation temperature scaling.
|
|
204
|
+
5. **[05_evaluate_uncertainty.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/05_evaluate_uncertainty.py)**: Predict error detection AUROC and risk-coverage curves.
|
|
205
|
+
6. **[06_compare_checkpoints.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/06_compare_checkpoints.py)**: Quantify statistical differentials between model checkpoints.
|
|
206
|
+
7. **[07_save_and_reproduce_experiment.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/07_save_and_reproduce_experiment.py)**: Atomic experiment persistence and checksum verification.
|
|
207
|
+
8. **[08_create_custom_metric.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/08_create_custom_metric.py)**: Register custom evaluation scoring functions.
|
|
208
|
+
9. **[09_create_custom_model_adapter.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/09_create_custom_model_adapter.py)**: Implement the runtime `ModelAdapter` protocol.
|
|
208
209
|
|
|
209
210
|
---
|
|
210
211
|
|
|
211
212
|
## Documentation
|
|
212
213
|
|
|
213
|
-
Full documentation is available in the [`docs/`](docs
|
|
214
|
-
- [Installation Guide](docs/installation.md)
|
|
215
|
-
- [Quickstart Guide](docs/quickstart.md)
|
|
216
|
-
- [Python API Reference](docs/api.md)
|
|
217
|
-
- [CLI Reference](docs/cli.md)
|
|
218
|
-
- [Configuration Guide](docs/configuration.md)
|
|
219
|
-
- [Model Adapters](docs/model-adapters.md)
|
|
220
|
-
- [Dataset Adapters](docs/dataset-adapters.md)
|
|
221
|
-
- [Metrics Documentation](docs/metrics.md)
|
|
222
|
-
- [Robustness Testing](docs/robustness.md)
|
|
223
|
-
- [Experiment Tracking](docs/experiments.md)
|
|
224
|
-
- [Reporting & Visualization](docs/reporting.md)
|
|
225
|
-
- [Plugin Development](docs/plugins.md)
|
|
226
|
-
- [Troubleshooting & FAQ](docs/troubleshooting.md)
|
|
214
|
+
Full documentation is available on the [Documentation Website](https://uni-creator.github.io/trustbench/) or in the [`docs/`](https://github.com/Uni-Creator/trustbench/tree/main/docs) directory:
|
|
215
|
+
- [Installation Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/installation.md)
|
|
216
|
+
- [Quickstart Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/quickstart.md)
|
|
217
|
+
- [Python API Reference](https://github.com/Uni-Creator/trustbench/blob/main/docs/api.md)
|
|
218
|
+
- [CLI Reference](https://github.com/Uni-Creator/trustbench/blob/main/docs/cli.md)
|
|
219
|
+
- [Configuration Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/configuration.md)
|
|
220
|
+
- [Model Adapters](https://github.com/Uni-Creator/trustbench/blob/main/docs/model-adapters.md)
|
|
221
|
+
- [Dataset Adapters](https://github.com/Uni-Creator/trustbench/blob/main/docs/dataset-adapters.md)
|
|
222
|
+
- [Metrics Documentation](https://github.com/Uni-Creator/trustbench/blob/main/docs/metrics.md)
|
|
223
|
+
- [Robustness Testing](https://github.com/Uni-Creator/trustbench/blob/main/docs/robustness.md)
|
|
224
|
+
- [Experiment Tracking](https://github.com/Uni-Creator/trustbench/blob/main/docs/experiments.md)
|
|
225
|
+
- [Reporting & Visualization](https://github.com/Uni-Creator/trustbench/blob/main/docs/reporting.md)
|
|
226
|
+
- [Plugin Development](https://github.com/Uni-Creator/trustbench/blob/main/docs/plugins.md)
|
|
227
|
+
- [Troubleshooting & FAQ](https://github.com/Uni-Creator/trustbench/blob/main/docs/troubleshooting.md)
|
|
227
228
|
|
|
228
229
|
---
|
|
229
230
|
|
|
230
231
|
## Contributing
|
|
231
232
|
|
|
232
|
-
We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) and our [Code of Conduct](CODE_OF_CONDUCT.md).
|
|
233
|
+
We welcome contributions! Please see [CONTRIBUTING.md](https://github.com/Uni-Creator/trustbench/blob/main/CONTRIBUTING.md) and our [Code of Conduct](https://github.com/Uni-Creator/trustbench/blob/main/CODE_OF_CONDUCT.md).
|
|
233
234
|
|
|
234
235
|
```bash
|
|
235
236
|
# Run tests
|
|
@@ -246,4 +247,4 @@ mypy src/trustbench/core src/trustbench/reporting src/trustbench/metrics
|
|
|
246
247
|
|
|
247
248
|
## License
|
|
248
249
|
|
|
249
|
-
TrustBench is open-source software licensed under the [Apache License, Version 2.0](LICENSE).
|
|
250
|
+
TrustBench is open-source software licensed under the [Apache License, Version 2.0](https://github.com/Uni-Creator/trustbench/blob/main/LICENSE).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "trustbench"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.2"
|
|
4
4
|
description = "TrustBench: Evaluation framework for ML model reliability, robustness, and uncertainty."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -39,6 +39,13 @@ dependencies = [
|
|
|
39
39
|
name = "Uni-Creator"
|
|
40
40
|
email = "abhayr24564@gmail.com"
|
|
41
41
|
|
|
42
|
+
[project.urls]
|
|
43
|
+
Homepage = "https://uni-creator.github.io/trustbench/"
|
|
44
|
+
Documentation = "https://uni-creator.github.io/trustbench/"
|
|
45
|
+
Repository = "https://github.com/Uni-Creator/trustbench"
|
|
46
|
+
Issues = "https://github.com/Uni-Creator/trustbench/issues"
|
|
47
|
+
Changelog = "https://github.com/Uni-Creator/trustbench/blob/main/CHANGELOG.md"
|
|
48
|
+
|
|
42
49
|
[project.optional-dependencies]
|
|
43
50
|
torch = [
|
|
44
51
|
"torch>=2.0.0",
|
|
@@ -73,10 +80,16 @@ dev = [
|
|
|
73
80
|
"pytest>=9.1.1",
|
|
74
81
|
"ruff>=0.17.2",
|
|
75
82
|
"fastapi>=0.100.0",
|
|
83
|
+
"plotly>=5.0.0",
|
|
76
84
|
"sqlalchemy>=2.0.0",
|
|
77
85
|
"torch>=2.0.0",
|
|
78
86
|
"torchvision>=0.15.0",
|
|
79
87
|
]
|
|
88
|
+
docs = [
|
|
89
|
+
"mkdocs>=1.6.0",
|
|
90
|
+
"mkdocs-material>=9.5.0",
|
|
91
|
+
"pymdown-extensions>=10.7",
|
|
92
|
+
]
|
|
80
93
|
|
|
81
94
|
[tool.ruff.lint]
|
|
82
95
|
ignore = ["B008"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "trustbench"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.2"
|
|
4
4
|
description = "TrustBench: Evaluation framework for ML model reliability, robustness, and uncertainty."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -38,6 +38,12 @@ dependencies = [
|
|
|
38
38
|
"jinja2>=3.1.6",
|
|
39
39
|
"pillow>=10.0.0",
|
|
40
40
|
]
|
|
41
|
+
[project.urls]
|
|
42
|
+
Homepage = "https://uni-creator.github.io/trustbench/"
|
|
43
|
+
Documentation = "https://uni-creator.github.io/trustbench/"
|
|
44
|
+
Repository = "https://github.com/Uni-Creator/trustbench"
|
|
45
|
+
Issues = "https://github.com/Uni-Creator/trustbench/issues"
|
|
46
|
+
Changelog = "https://github.com/Uni-Creator/trustbench/blob/main/CHANGELOG.md"
|
|
41
47
|
|
|
42
48
|
[project.optional-dependencies]
|
|
43
49
|
# PyTorch inference
|
|
@@ -95,10 +101,16 @@ dev = [
|
|
|
95
101
|
|
|
96
102
|
# Dependencies required by the test suite
|
|
97
103
|
"fastapi>=0.100.0",
|
|
104
|
+
"plotly>=5.0.0",
|
|
98
105
|
"sqlalchemy>=2.0.0",
|
|
99
106
|
"torch>=2.0.0",
|
|
100
107
|
"torchvision>=0.15.0",
|
|
101
108
|
]
|
|
109
|
+
docs = [
|
|
110
|
+
"mkdocs>=1.6.0",
|
|
111
|
+
"mkdocs-material>=9.5.0",
|
|
112
|
+
"pymdown-extensions>=10.7",
|
|
113
|
+
]
|
|
102
114
|
|
|
103
115
|
[tool.ruff.lint]
|
|
104
116
|
ignore = ["B008"]
|
|
@@ -50,6 +50,7 @@ _TEMPLATE_SRC = """\
|
|
|
50
50
|
<meta charset="UTF-8">
|
|
51
51
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
52
52
|
<title>TrustBench Report — {{ experiment_name }}</title>
|
|
53
|
+
<script src="https://cdn.plot.ly/plotly-2.35.2.min.js"></script>
|
|
53
54
|
<style>
|
|
54
55
|
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
|
55
56
|
body { font-family: 'Segoe UI', system-ui, sans-serif; background: #f5f7fa; color: #1a202c; line-height: 1.6; }
|
|
@@ -77,8 +78,10 @@ _TEMPLATE_SRC = """\
|
|
|
77
78
|
footer { text-align: center; color: #a0aec0; font-size: 0.8rem; padding: 2rem; }
|
|
78
79
|
pre { background: #f7fafc; border-radius: 0.5rem; padding: 1rem; font-size: 0.8rem; overflow-x: auto; white-space: pre-wrap; word-break: break-all; }
|
|
79
80
|
.charts { display: grid; grid-template-columns: repeat(auto-fill, minmax(500px, 1fr)); gap: 1rem; margin-top: 1rem; }
|
|
80
|
-
.chart-box { border: 1px solid #e2e8f0; border-radius: 0.5rem; padding: 0.5rem; display: flex; align-items: center; justify-content: center; min-height:
|
|
81
|
+
.chart-box { border: 1px solid #e2e8f0; border-radius: 0.5rem; padding: 0.5rem; display: flex; align-items: center; justify-content: center; min-height: 400px; width: 100%; overflow: hidden; }
|
|
82
|
+
.chart-box > div { width: 100%; }
|
|
81
83
|
.chart-placeholder { color: #a0aec0; font-size: 0.8rem; text-align: center; padding: 1rem; }
|
|
84
|
+
.js-plotly-plot { width: 100% !important; margin: 0 auto; }
|
|
82
85
|
.two-col { display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; }
|
|
83
86
|
@media (max-width: 700px) { .two-col { grid-template-columns: 1fr; } .charts { grid-template-columns: 1fr; } }
|
|
84
87
|
</style>
|
|
@@ -109,7 +112,7 @@ _TEMPLATE_SRC = """\
|
|
|
109
112
|
<h2>Classification</h2>
|
|
110
113
|
<div class="charts">
|
|
111
114
|
{% if confusion_matrix_svg %}
|
|
112
|
-
<div class="chart-box">{{ confusion_matrix_svg }}</div>
|
|
115
|
+
<div class="chart-box">{{ confusion_matrix_svg|safe }}</div>
|
|
113
116
|
{% else %}
|
|
114
117
|
<div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
|
|
115
118
|
{% endif %}
|
|
@@ -155,7 +158,7 @@ _TEMPLATE_SRC = """\
|
|
|
155
158
|
<h2>Calibration</h2>
|
|
156
159
|
<div class="charts">
|
|
157
160
|
{% if reliability_diagram_svg %}
|
|
158
|
-
<div class="chart-box">{{ reliability_diagram_svg }}</div>
|
|
161
|
+
<div class="chart-box">{{ reliability_diagram_svg|safe }}</div>
|
|
159
162
|
{% else %}
|
|
160
163
|
<div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
|
|
161
164
|
{% endif %}
|
|
@@ -177,7 +180,7 @@ _TEMPLATE_SRC = """\
|
|
|
177
180
|
<h2>Uncertainty</h2>
|
|
178
181
|
<div class="charts">
|
|
179
182
|
{% if confidence_histogram_svg %}
|
|
180
|
-
<div class="chart-box">{{ confidence_histogram_svg }}</div>
|
|
183
|
+
<div class="chart-box">{{ confidence_histogram_svg|safe }}</div>
|
|
181
184
|
{% else %}
|
|
182
185
|
<div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
|
|
183
186
|
{% endif %}
|
|
@@ -199,7 +202,7 @@ _TEMPLATE_SRC = """\
|
|
|
199
202
|
<h2>Error Detection</h2>
|
|
200
203
|
<div class="charts">
|
|
201
204
|
{% if auroc_curve_svg %}
|
|
202
|
-
<div class="chart-box">{{ auroc_curve_svg }}</div>
|
|
205
|
+
<div class="chart-box">{{ auroc_curve_svg|safe }}</div>
|
|
203
206
|
{% else %}
|
|
204
207
|
<div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
|
|
205
208
|
{% endif %}
|
|
@@ -227,7 +230,7 @@ _TEMPLATE_SRC = """\
|
|
|
227
230
|
<h2>Selective Prediction</h2>
|
|
228
231
|
<div class="charts">
|
|
229
232
|
{% if risk_coverage_svg %}
|
|
230
|
-
<div class="chart-box">{{ risk_coverage_svg }}</div>
|
|
233
|
+
<div class="chart-box">{{ risk_coverage_svg|safe }}</div>
|
|
231
234
|
{% else %}
|
|
232
235
|
<div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
|
|
233
236
|
{% endif %}
|
|
@@ -255,12 +258,12 @@ _TEMPLATE_SRC = """\
|
|
|
255
258
|
<h2>Robustness Analysis</h2>
|
|
256
259
|
<div class="charts">
|
|
257
260
|
{% if robustness_degradation_svg %}
|
|
258
|
-
<div class="chart-box">{{ robustness_degradation_svg }}</div>
|
|
261
|
+
<div class="chart-box">{{ robustness_degradation_svg|safe }}</div>
|
|
259
262
|
{% else %}
|
|
260
263
|
<div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
|
|
261
264
|
{% endif %}
|
|
262
265
|
{% if severity_comparison_svg %}
|
|
263
|
-
<div class="chart-box">{{ severity_comparison_svg }}</div>
|
|
266
|
+
<div class="chart-box">{{ severity_comparison_svg|safe }}</div>
|
|
264
267
|
{% else %}
|
|
265
268
|
<div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
|
|
266
269
|
{% endif %}
|
|
@@ -279,8 +279,11 @@ _PLOTLY_COMMON = {
|
|
|
279
279
|
|
|
280
280
|
|
|
281
281
|
def _fig_to_svg(fig) -> str:
|
|
282
|
-
"""Convert a plotly Figure to an SVG string."""
|
|
283
|
-
|
|
282
|
+
"""Convert a plotly Figure to an inline SVG string or interactive HTML div."""
|
|
283
|
+
try:
|
|
284
|
+
return fig.to_image(format="svg").decode("utf-8")
|
|
285
|
+
except Exception: # noqa: BLE001
|
|
286
|
+
return fig.to_html(full_html=False, include_plotlyjs=False)
|
|
284
287
|
|
|
285
288
|
|
|
286
289
|
def render_confusion_matrix(data: dict[str, Any]) -> str:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|