trustbench 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. {trustbench-0.2.0 → trustbench-0.2.2}/PKG-INFO +39 -33
  2. {trustbench-0.2.0 → trustbench-0.2.2}/README.md +33 -32
  3. {trustbench-0.2.0 → trustbench-0.2.2}/pyproject.toml +14 -1
  4. {trustbench-0.2.0 → trustbench-0.2.2}/pyproject.toml.orig +13 -1
  5. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/__init__.py +1 -1
  6. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/registry/plugins.py +1 -1
  7. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/html_report.py +11 -8
  8. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/plots.py +5 -2
  9. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/api/__init__.py +0 -0
  10. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/api/app.py +0 -0
  11. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/api/routes/experiments.py +0 -0
  12. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/benchmark.py +0 -0
  13. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/cli/__init__.py +0 -0
  14. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/cli/main.py +0 -0
  15. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/config/__init__.py +0 -0
  16. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/config/loader.py +0 -0
  17. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/config/schema.py +0 -0
  18. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/__init__.py +0 -0
  19. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/evaluator.py +0 -0
  20. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/exceptions.py +0 -0
  21. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/inference.py +0 -0
  22. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/protocols.py +0 -0
  23. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/reproducibility.py +0 -0
  24. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/core/results.py +0 -0
  25. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/datasets/__init__.py +0 -0
  26. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/datasets/base.py +0 -0
  27. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/datasets/cifar10.py +0 -0
  28. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/datasets/memory.py +0 -0
  29. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/datasets/torch_dataset.py +0 -0
  30. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/__init__.py +0 -0
  31. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/artifacts.py +0 -0
  32. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/calibration.py +0 -0
  33. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/classification.py +0 -0
  34. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/distribution_shift.py +0 -0
  35. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/error_detection.py +0 -0
  36. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/fairness.py +0 -0
  37. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/inference.py +0 -0
  38. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/regression.py +0 -0
  39. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/reproducibility.py +0 -0
  40. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/retrieval.py +0 -0
  41. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/runner.py +0 -0
  42. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/segmentation.py +0 -0
  43. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/selective.py +0 -0
  44. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/streaming.py +0 -0
  45. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/evaluation/uncertainty.py +0 -0
  46. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/experiments/__init__.py +0 -0
  47. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/experiments/comparison.py +0 -0
  48. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/experiments/lineage.py +0 -0
  49. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/experiments/manager.py +0 -0
  50. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/experiments/registry.py +0 -0
  51. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/experiments/schemas.py +0 -0
  52. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/__init__.py +0 -0
  53. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/calibration.py +0 -0
  54. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/classification.py +0 -0
  55. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/error_detection.py +0 -0
  56. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/regression.py +0 -0
  57. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/retrieval.py +0 -0
  58. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/segmentation.py +0 -0
  59. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/selective.py +0 -0
  60. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/metrics/uncertainty.py +0 -0
  61. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/__init__.py +0 -0
  62. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/base.py +0 -0
  63. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/checkpoint.py +0 -0
  64. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/densenet.py +0 -0
  65. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/dla.py +0 -0
  66. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/dla_simple.py +0 -0
  67. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/dpn.py +0 -0
  68. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/efficientnet.py +0 -0
  69. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/googlenet.py +0 -0
  70. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/jax.py +0 -0
  71. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/keras.py +0 -0
  72. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/lenet.py +0 -0
  73. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/mobilenet.py +0 -0
  74. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/mobilenetv2.py +0 -0
  75. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/onnx.py +0 -0
  76. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/pnasnet.py +0 -0
  77. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/preact_resnet.py +0 -0
  78. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/pytorch.py +0 -0
  79. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/regnet.py +0 -0
  80. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/resnet.py +0 -0
  81. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/resnext.py +0 -0
  82. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/senet.py +0 -0
  83. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/shufflenet.py +0 -0
  84. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/shufflenetv2.py +0 -0
  85. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/sklearn.py +0 -0
  86. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/models/vgg.py +0 -0
  87. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/registry/__init__.py +0 -0
  88. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/registry/corruptions.py +0 -0
  89. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/registry/datasets.py +0 -0
  90. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/registry/metrics.py +0 -0
  91. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/registry/models.py +0 -0
  92. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/__init__.py +0 -0
  93. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/artifacts.py +0 -0
  94. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/hashing.py +0 -0
  95. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/result_io.py +0 -0
  96. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/schema.py +0 -0
  97. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/reporting/tables.py +0 -0
  98. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/robustness/__init__.py +0 -0
  99. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/robustness/aggregation.py +0 -0
  100. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/robustness/corruptions.py +0 -0
  101. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/robustness/evaluator.py +0 -0
  102. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/robustness/report.py +0 -0
  103. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/robustness/severity.py +0 -0
  104. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/__init__.py +0 -0
  105. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/base.py +0 -0
  106. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/database.py +0 -0
  107. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/filesystem.py +0 -0
  108. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/models.py +0 -0
  109. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/repositories.py +0 -0
  110. {trustbench-0.2.0 → trustbench-0.2.2}/src/trustbench/storage/sqlite.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: trustbench
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: TrustBench: Evaluation framework for ML model reliability, robustness, and uncertainty.
5
5
  Keywords: machine-learning,evaluation,reliability,robustness,uncertainty,calibration,ood-detection,fairness
6
6
  Author: Uni-Creator
@@ -34,6 +34,11 @@ Requires-Dist: tensorflow>=2.15.0 ; extra == 'tensorflow'
34
34
  Requires-Dist: torch>=2.0.0 ; extra == 'torch'
35
35
  Requires-Dist: torchvision>=0.15.0 ; extra == 'torch'
36
36
  Requires-Python: >=3.11
37
+ Project-URL: Homepage, https://uni-creator.github.io/trustbench/
38
+ Project-URL: Documentation, https://uni-creator.github.io/trustbench/
39
+ Project-URL: Repository, https://github.com/Uni-Creator/trustbench
40
+ Project-URL: Issues, https://github.com/Uni-Creator/trustbench/issues
41
+ Project-URL: Changelog, https://github.com/Uni-Creator/trustbench/blob/main/CHANGELOG.md
37
42
  Provides-Extra: all
38
43
  Provides-Extra: api
39
44
  Provides-Extra: jax
@@ -51,11 +56,12 @@ Description-Content-Type: text/markdown
51
56
 
52
57
  **ML Reliability, Robustness, Uncertainty & Fairness Evaluation Framework**
53
58
 
54
- [![PyPI version](https://img.shields.io/badge/pypi-v0.2.0-blue.svg)](https://pypi.org/project/trustbench/)
59
+ [![PyPI version](https://img.shields.io/badge/pypi-v0.2.2-blue.svg)](https://pypi.org/project/trustbench/)
55
60
  [![Python Versions](https://img.shields.io/badge/python-3.11%20%7C%203.12-blue.svg)](https://pypi.org/project/trustbench/)
56
- [![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](LICENSE)
57
- [![CI Status](https://img.shields.io/badge/build-passing-brightgreen.svg)](.github/workflows/ci.yml)
58
- [![Tests](https://img.shields.io/badge/tests-194%20passed-success.svg)](tests/)
61
+ [![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](https://github.com/Uni-Creator/trustbench/blob/main/LICENSE)
62
+ [![CI Status](https://img.shields.io/badge/build-passing-brightgreen.svg)](https://github.com/Uni-Creator/trustbench/actions/workflows/ci.yml)
63
+ [![Documentation](https://img.shields.io/badge/docs-mkdocs-blue.svg)](https://uni-creator.github.io/trustbench/)
64
+ [![Tests](https://img.shields.io/badge/tests-204%20passed-success.svg)](https://github.com/Uni-Creator/trustbench/tree/main/tests)
59
65
 
60
66
  </div>
61
67
 
@@ -131,7 +137,7 @@ pip install "trustbench[onnx]"
131
137
  pip install "trustbench[all]"
132
138
  ```
133
139
 
134
- See the [Installation Guide](docs/installation.md) for detailed platform requirements.
140
+ See the [Installation Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/installation.md) for detailed platform requirements.
135
141
 
136
142
  ---
137
143
 
@@ -235,48 +241,48 @@ print(f"Pixel Accuracy: {result.metrics['pixel_accuracy']:.2%}")
235
241
  | **Segmentation** | Mean IoU (mIoU), Dice (F1), Pixel Accuracy, Frequency-Weighted IoU (FWIoU) |
236
242
  | **Fairness** | Demographic Parity Difference, Disparate Impact Ratio, Equalized Odds Difference |
237
243
 
238
- See [Metrics Documentation](docs/metrics.md) for mathematical derivations.
244
+ See [Metrics Documentation](https://github.com/Uni-Creator/trustbench/blob/main/docs/metrics.md) for mathematical derivations.
239
245
 
240
246
  ---
241
247
 
242
248
  ## Tutorials & Examples
243
249
 
244
- Ready-to-run examples are available in the [`examples/`](examples/) directory:
250
+ Ready-to-run examples are available in the [`examples/`](https://github.com/Uni-Creator/trustbench/tree/main/examples) directory:
245
251
 
246
- 1. **[01_evaluate_pytorch_model.py](examples/01_evaluate_pytorch_model.py)**: Evaluate custom PyTorch `nn.Module`.
247
- 2. **[02_evaluate_cifar10.py](examples/02_evaluate_cifar10.py)**: End-to-end CIFAR-10 ResNet evaluation via configuration.
248
- 3. **[03_robustness_benchmark.py](examples/03_robustness_benchmark.py)**: Evaluate accuracy degradation across noise & corruptions.
249
- 4. **[04_analyze_calibration.py](examples/04_analyze_calibration.py)**: Compute ECE and fit validation temperature scaling.
250
- 5. **[05_evaluate_uncertainty.py](examples/05_evaluate_uncertainty.py)**: Predict error detection AUROC and risk-coverage curves.
251
- 6. **[06_compare_checkpoints.py](examples/06_compare_checkpoints.py)**: Quantify statistical differentials between model checkpoints.
252
- 7. **[07_save_and_reproduce_experiment.py](examples/07_save_and_reproduce_experiment.py)**: Atomic experiment persistence and checksum verification.
253
- 8. **[08_create_custom_metric.py](examples/08_create_custom_metric.py)**: Register custom evaluation scoring functions.
254
- 9. **[09_create_custom_model_adapter.py](examples/09_create_custom_model_adapter.py)**: Implement the runtime `ModelAdapter` protocol.
252
+ 1. **[01_evaluate_pytorch_model.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/01_evaluate_pytorch_model.py)**: Evaluate custom PyTorch `nn.Module`.
253
+ 2. **[02_evaluate_cifar10.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/02_evaluate_cifar10.py)**: End-to-end CIFAR-10 ResNet evaluation via configuration.
254
+ 3. **[03_robustness_benchmark.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/03_robustness_benchmark.py)**: Evaluate accuracy degradation across noise & corruptions.
255
+ 4. **[04_analyze_calibration.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/04_analyze_calibration.py)**: Compute ECE and fit validation temperature scaling.
256
+ 5. **[05_evaluate_uncertainty.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/05_evaluate_uncertainty.py)**: Predict error detection AUROC and risk-coverage curves.
257
+ 6. **[06_compare_checkpoints.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/06_compare_checkpoints.py)**: Quantify statistical differentials between model checkpoints.
258
+ 7. **[07_save_and_reproduce_experiment.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/07_save_and_reproduce_experiment.py)**: Atomic experiment persistence and checksum verification.
259
+ 8. **[08_create_custom_metric.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/08_create_custom_metric.py)**: Register custom evaluation scoring functions.
260
+ 9. **[09_create_custom_model_adapter.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/09_create_custom_model_adapter.py)**: Implement the runtime `ModelAdapter` protocol.
255
261
 
256
262
  ---
257
263
 
258
264
  ## Documentation
259
265
 
260
- Full documentation is available in the [`docs/`](docs/) directory:
261
- - [Installation Guide](docs/installation.md)
262
- - [Quickstart Guide](docs/quickstart.md)
263
- - [Python API Reference](docs/api.md)
264
- - [CLI Reference](docs/cli.md)
265
- - [Configuration Guide](docs/configuration.md)
266
- - [Model Adapters](docs/model-adapters.md)
267
- - [Dataset Adapters](docs/dataset-adapters.md)
268
- - [Metrics Documentation](docs/metrics.md)
269
- - [Robustness Testing](docs/robustness.md)
270
- - [Experiment Tracking](docs/experiments.md)
271
- - [Reporting & Visualization](docs/reporting.md)
272
- - [Plugin Development](docs/plugins.md)
273
- - [Troubleshooting & FAQ](docs/troubleshooting.md)
266
+ Full documentation is available on the [Documentation Website](https://uni-creator.github.io/trustbench/) or in the [`docs/`](https://github.com/Uni-Creator/trustbench/tree/main/docs) directory:
267
+ - [Installation Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/installation.md)
268
+ - [Quickstart Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/quickstart.md)
269
+ - [Python API Reference](https://github.com/Uni-Creator/trustbench/blob/main/docs/api.md)
270
+ - [CLI Reference](https://github.com/Uni-Creator/trustbench/blob/main/docs/cli.md)
271
+ - [Configuration Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/configuration.md)
272
+ - [Model Adapters](https://github.com/Uni-Creator/trustbench/blob/main/docs/model-adapters.md)
273
+ - [Dataset Adapters](https://github.com/Uni-Creator/trustbench/blob/main/docs/dataset-adapters.md)
274
+ - [Metrics Documentation](https://github.com/Uni-Creator/trustbench/blob/main/docs/metrics.md)
275
+ - [Robustness Testing](https://github.com/Uni-Creator/trustbench/blob/main/docs/robustness.md)
276
+ - [Experiment Tracking](https://github.com/Uni-Creator/trustbench/blob/main/docs/experiments.md)
277
+ - [Reporting & Visualization](https://github.com/Uni-Creator/trustbench/blob/main/docs/reporting.md)
278
+ - [Plugin Development](https://github.com/Uni-Creator/trustbench/blob/main/docs/plugins.md)
279
+ - [Troubleshooting & FAQ](https://github.com/Uni-Creator/trustbench/blob/main/docs/troubleshooting.md)
274
280
 
275
281
  ---
276
282
 
277
283
  ## Contributing
278
284
 
279
- We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) and our [Code of Conduct](CODE_OF_CONDUCT.md).
285
+ We welcome contributions! Please see [CONTRIBUTING.md](https://github.com/Uni-Creator/trustbench/blob/main/CONTRIBUTING.md) and our [Code of Conduct](https://github.com/Uni-Creator/trustbench/blob/main/CODE_OF_CONDUCT.md).
280
286
 
281
287
  ```bash
282
288
  # Run tests
@@ -293,4 +299,4 @@ mypy src/trustbench/core src/trustbench/reporting src/trustbench/metrics
293
299
 
294
300
  ## License
295
301
 
296
- TrustBench is open-source software licensed under the [Apache License, Version 2.0](LICENSE).
302
+ TrustBench is open-source software licensed under the [Apache License, Version 2.0](https://github.com/Uni-Creator/trustbench/blob/main/LICENSE).
@@ -4,11 +4,12 @@
4
4
 
5
5
  **ML Reliability, Robustness, Uncertainty & Fairness Evaluation Framework**
6
6
 
7
- [![PyPI version](https://img.shields.io/badge/pypi-v0.2.0-blue.svg)](https://pypi.org/project/trustbench/)
7
+ [![PyPI version](https://img.shields.io/badge/pypi-v0.2.2-blue.svg)](https://pypi.org/project/trustbench/)
8
8
  [![Python Versions](https://img.shields.io/badge/python-3.11%20%7C%203.12-blue.svg)](https://pypi.org/project/trustbench/)
9
- [![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](LICENSE)
10
- [![CI Status](https://img.shields.io/badge/build-passing-brightgreen.svg)](.github/workflows/ci.yml)
11
- [![Tests](https://img.shields.io/badge/tests-194%20passed-success.svg)](tests/)
9
+ [![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](https://github.com/Uni-Creator/trustbench/blob/main/LICENSE)
10
+ [![CI Status](https://img.shields.io/badge/build-passing-brightgreen.svg)](https://github.com/Uni-Creator/trustbench/actions/workflows/ci.yml)
11
+ [![Documentation](https://img.shields.io/badge/docs-mkdocs-blue.svg)](https://uni-creator.github.io/trustbench/)
12
+ [![Tests](https://img.shields.io/badge/tests-204%20passed-success.svg)](https://github.com/Uni-Creator/trustbench/tree/main/tests)
12
13
 
13
14
  </div>
14
15
 
@@ -84,7 +85,7 @@ pip install "trustbench[onnx]"
84
85
  pip install "trustbench[all]"
85
86
  ```
86
87
 
87
- See the [Installation Guide](docs/installation.md) for detailed platform requirements.
88
+ See the [Installation Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/installation.md) for detailed platform requirements.
88
89
 
89
90
  ---
90
91
 
@@ -188,48 +189,48 @@ print(f"Pixel Accuracy: {result.metrics['pixel_accuracy']:.2%}")
188
189
  | **Segmentation** | Mean IoU (mIoU), Dice (F1), Pixel Accuracy, Frequency-Weighted IoU (FWIoU) |
189
190
  | **Fairness** | Demographic Parity Difference, Disparate Impact Ratio, Equalized Odds Difference |
190
191
 
191
- See [Metrics Documentation](docs/metrics.md) for mathematical derivations.
192
+ See [Metrics Documentation](https://github.com/Uni-Creator/trustbench/blob/main/docs/metrics.md) for mathematical derivations.
192
193
 
193
194
  ---
194
195
 
195
196
  ## Tutorials & Examples
196
197
 
197
- Ready-to-run examples are available in the [`examples/`](examples/) directory:
198
+ Ready-to-run examples are available in the [`examples/`](https://github.com/Uni-Creator/trustbench/tree/main/examples) directory:
198
199
 
199
- 1. **[01_evaluate_pytorch_model.py](examples/01_evaluate_pytorch_model.py)**: Evaluate custom PyTorch `nn.Module`.
200
- 2. **[02_evaluate_cifar10.py](examples/02_evaluate_cifar10.py)**: End-to-end CIFAR-10 ResNet evaluation via configuration.
201
- 3. **[03_robustness_benchmark.py](examples/03_robustness_benchmark.py)**: Evaluate accuracy degradation across noise & corruptions.
202
- 4. **[04_analyze_calibration.py](examples/04_analyze_calibration.py)**: Compute ECE and fit validation temperature scaling.
203
- 5. **[05_evaluate_uncertainty.py](examples/05_evaluate_uncertainty.py)**: Predict error detection AUROC and risk-coverage curves.
204
- 6. **[06_compare_checkpoints.py](examples/06_compare_checkpoints.py)**: Quantify statistical differentials between model checkpoints.
205
- 7. **[07_save_and_reproduce_experiment.py](examples/07_save_and_reproduce_experiment.py)**: Atomic experiment persistence and checksum verification.
206
- 8. **[08_create_custom_metric.py](examples/08_create_custom_metric.py)**: Register custom evaluation scoring functions.
207
- 9. **[09_create_custom_model_adapter.py](examples/09_create_custom_model_adapter.py)**: Implement the runtime `ModelAdapter` protocol.
200
+ 1. **[01_evaluate_pytorch_model.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/01_evaluate_pytorch_model.py)**: Evaluate custom PyTorch `nn.Module`.
201
+ 2. **[02_evaluate_cifar10.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/02_evaluate_cifar10.py)**: End-to-end CIFAR-10 ResNet evaluation via configuration.
202
+ 3. **[03_robustness_benchmark.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/03_robustness_benchmark.py)**: Evaluate accuracy degradation across noise & corruptions.
203
+ 4. **[04_analyze_calibration.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/04_analyze_calibration.py)**: Compute ECE and fit validation temperature scaling.
204
+ 5. **[05_evaluate_uncertainty.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/05_evaluate_uncertainty.py)**: Predict error detection AUROC and risk-coverage curves.
205
+ 6. **[06_compare_checkpoints.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/06_compare_checkpoints.py)**: Quantify statistical differentials between model checkpoints.
206
+ 7. **[07_save_and_reproduce_experiment.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/07_save_and_reproduce_experiment.py)**: Atomic experiment persistence and checksum verification.
207
+ 8. **[08_create_custom_metric.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/08_create_custom_metric.py)**: Register custom evaluation scoring functions.
208
+ 9. **[09_create_custom_model_adapter.py](https://github.com/Uni-Creator/trustbench/blob/main/examples/09_create_custom_model_adapter.py)**: Implement the runtime `ModelAdapter` protocol.
208
209
 
209
210
  ---
210
211
 
211
212
  ## Documentation
212
213
 
213
- Full documentation is available in the [`docs/`](docs/) directory:
214
- - [Installation Guide](docs/installation.md)
215
- - [Quickstart Guide](docs/quickstart.md)
216
- - [Python API Reference](docs/api.md)
217
- - [CLI Reference](docs/cli.md)
218
- - [Configuration Guide](docs/configuration.md)
219
- - [Model Adapters](docs/model-adapters.md)
220
- - [Dataset Adapters](docs/dataset-adapters.md)
221
- - [Metrics Documentation](docs/metrics.md)
222
- - [Robustness Testing](docs/robustness.md)
223
- - [Experiment Tracking](docs/experiments.md)
224
- - [Reporting & Visualization](docs/reporting.md)
225
- - [Plugin Development](docs/plugins.md)
226
- - [Troubleshooting & FAQ](docs/troubleshooting.md)
214
+ Full documentation is available on the [Documentation Website](https://uni-creator.github.io/trustbench/) or in the [`docs/`](https://github.com/Uni-Creator/trustbench/tree/main/docs) directory:
215
+ - [Installation Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/installation.md)
216
+ - [Quickstart Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/quickstart.md)
217
+ - [Python API Reference](https://github.com/Uni-Creator/trustbench/blob/main/docs/api.md)
218
+ - [CLI Reference](https://github.com/Uni-Creator/trustbench/blob/main/docs/cli.md)
219
+ - [Configuration Guide](https://github.com/Uni-Creator/trustbench/blob/main/docs/configuration.md)
220
+ - [Model Adapters](https://github.com/Uni-Creator/trustbench/blob/main/docs/model-adapters.md)
221
+ - [Dataset Adapters](https://github.com/Uni-Creator/trustbench/blob/main/docs/dataset-adapters.md)
222
+ - [Metrics Documentation](https://github.com/Uni-Creator/trustbench/blob/main/docs/metrics.md)
223
+ - [Robustness Testing](https://github.com/Uni-Creator/trustbench/blob/main/docs/robustness.md)
224
+ - [Experiment Tracking](https://github.com/Uni-Creator/trustbench/blob/main/docs/experiments.md)
225
+ - [Reporting & Visualization](https://github.com/Uni-Creator/trustbench/blob/main/docs/reporting.md)
226
+ - [Plugin Development](https://github.com/Uni-Creator/trustbench/blob/main/docs/plugins.md)
227
+ - [Troubleshooting & FAQ](https://github.com/Uni-Creator/trustbench/blob/main/docs/troubleshooting.md)
227
228
 
228
229
  ---
229
230
 
230
231
  ## Contributing
231
232
 
232
- We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) and our [Code of Conduct](CODE_OF_CONDUCT.md).
233
+ We welcome contributions! Please see [CONTRIBUTING.md](https://github.com/Uni-Creator/trustbench/blob/main/CONTRIBUTING.md) and our [Code of Conduct](https://github.com/Uni-Creator/trustbench/blob/main/CODE_OF_CONDUCT.md).
233
234
 
234
235
  ```bash
235
236
  # Run tests
@@ -246,4 +247,4 @@ mypy src/trustbench/core src/trustbench/reporting src/trustbench/metrics
246
247
 
247
248
  ## License
248
249
 
249
- TrustBench is open-source software licensed under the [Apache License, Version 2.0](LICENSE).
250
+ TrustBench is open-source software licensed under the [Apache License, Version 2.0](https://github.com/Uni-Creator/trustbench/blob/main/LICENSE).
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "trustbench"
3
- version = "0.2.0"
3
+ version = "0.2.2"
4
4
  description = "TrustBench: Evaluation framework for ML model reliability, robustness, and uncertainty."
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -39,6 +39,13 @@ dependencies = [
39
39
  name = "Uni-Creator"
40
40
  email = "abhayr24564@gmail.com"
41
41
 
42
+ [project.urls]
43
+ Homepage = "https://uni-creator.github.io/trustbench/"
44
+ Documentation = "https://uni-creator.github.io/trustbench/"
45
+ Repository = "https://github.com/Uni-Creator/trustbench"
46
+ Issues = "https://github.com/Uni-Creator/trustbench/issues"
47
+ Changelog = "https://github.com/Uni-Creator/trustbench/blob/main/CHANGELOG.md"
48
+
42
49
  [project.optional-dependencies]
43
50
  torch = [
44
51
  "torch>=2.0.0",
@@ -73,10 +80,16 @@ dev = [
73
80
  "pytest>=9.1.1",
74
81
  "ruff>=0.17.2",
75
82
  "fastapi>=0.100.0",
83
+ "plotly>=5.0.0",
76
84
  "sqlalchemy>=2.0.0",
77
85
  "torch>=2.0.0",
78
86
  "torchvision>=0.15.0",
79
87
  ]
88
+ docs = [
89
+ "mkdocs>=1.6.0",
90
+ "mkdocs-material>=9.5.0",
91
+ "pymdown-extensions>=10.7",
92
+ ]
80
93
 
81
94
  [tool.ruff.lint]
82
95
  ignore = ["B008"]
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "trustbench"
3
- version = "0.2.0"
3
+ version = "0.2.2"
4
4
  description = "TrustBench: Evaluation framework for ML model reliability, robustness, and uncertainty."
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -38,6 +38,12 @@ dependencies = [
38
38
  "jinja2>=3.1.6",
39
39
  "pillow>=10.0.0",
40
40
  ]
41
+ [project.urls]
42
+ Homepage = "https://uni-creator.github.io/trustbench/"
43
+ Documentation = "https://uni-creator.github.io/trustbench/"
44
+ Repository = "https://github.com/Uni-Creator/trustbench"
45
+ Issues = "https://github.com/Uni-Creator/trustbench/issues"
46
+ Changelog = "https://github.com/Uni-Creator/trustbench/blob/main/CHANGELOG.md"
41
47
 
42
48
  [project.optional-dependencies]
43
49
  # PyTorch inference
@@ -95,10 +101,16 @@ dev = [
95
101
 
96
102
  # Dependencies required by the test suite
97
103
  "fastapi>=0.100.0",
104
+ "plotly>=5.0.0",
98
105
  "sqlalchemy>=2.0.0",
99
106
  "torch>=2.0.0",
100
107
  "torchvision>=0.15.0",
101
108
  ]
109
+ docs = [
110
+ "mkdocs>=1.6.0",
111
+ "mkdocs-material>=9.5.0",
112
+ "pymdown-extensions>=10.7",
113
+ ]
102
114
 
103
115
  [tool.ruff.lint]
104
116
  ignore = ["B008"]
@@ -1,6 +1,6 @@
1
1
  """TrustBench: Evaluation framework for machine learning model reliability."""
2
2
 
3
- __version__ = "0.2.0"
3
+ __version__ = "0.2.2"
4
4
 
5
5
  from trustbench.benchmark import TrustBench
6
6
  from trustbench.core.evaluator import evaluate
@@ -9,7 +9,7 @@ from typing import Any
9
9
  try:
10
10
  CURRENT_VERSION = importlib.metadata.version("trustbench")
11
11
  except importlib.metadata.PackageNotFoundError:
12
- CURRENT_VERSION = "0.2.0"
12
+ CURRENT_VERSION = "0.2.2"
13
13
 
14
14
  logger = logging.getLogger(__name__)
15
15
 
@@ -50,6 +50,7 @@ _TEMPLATE_SRC = """\
50
50
  <meta charset="UTF-8">
51
51
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
52
52
  <title>TrustBench Report — {{ experiment_name }}</title>
53
+ <script src="https://cdn.plot.ly/plotly-2.35.2.min.js"></script>
53
54
  <style>
54
55
  *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
55
56
  body { font-family: 'Segoe UI', system-ui, sans-serif; background: #f5f7fa; color: #1a202c; line-height: 1.6; }
@@ -77,8 +78,10 @@ _TEMPLATE_SRC = """\
77
78
  footer { text-align: center; color: #a0aec0; font-size: 0.8rem; padding: 2rem; }
78
79
  pre { background: #f7fafc; border-radius: 0.5rem; padding: 1rem; font-size: 0.8rem; overflow-x: auto; white-space: pre-wrap; word-break: break-all; }
79
80
  .charts { display: grid; grid-template-columns: repeat(auto-fill, minmax(500px, 1fr)); gap: 1rem; margin-top: 1rem; }
80
- .chart-box { border: 1px solid #e2e8f0; border-radius: 0.5rem; padding: 0.5rem; display: flex; align-items: center; justify-content: center; min-height: 200px; }
81
+ .chart-box { border: 1px solid #e2e8f0; border-radius: 0.5rem; padding: 0.5rem; display: flex; align-items: center; justify-content: center; min-height: 400px; width: 100%; overflow: hidden; }
82
+ .chart-box > div { width: 100%; }
81
83
  .chart-placeholder { color: #a0aec0; font-size: 0.8rem; text-align: center; padding: 1rem; }
84
+ .js-plotly-plot { width: 100% !important; margin: 0 auto; }
82
85
  .two-col { display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; }
83
86
  @media (max-width: 700px) { .two-col { grid-template-columns: 1fr; } .charts { grid-template-columns: 1fr; } }
84
87
  </style>
@@ -109,7 +112,7 @@ _TEMPLATE_SRC = """\
109
112
  <h2>Classification</h2>
110
113
  <div class="charts">
111
114
  {% if confusion_matrix_svg %}
112
- <div class="chart-box">{{ confusion_matrix_svg }}</div>
115
+ <div class="chart-box">{{ confusion_matrix_svg|safe }}</div>
113
116
  {% else %}
114
117
  <div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
115
118
  {% endif %}
@@ -155,7 +158,7 @@ _TEMPLATE_SRC = """\
155
158
  <h2>Calibration</h2>
156
159
  <div class="charts">
157
160
  {% if reliability_diagram_svg %}
158
- <div class="chart-box">{{ reliability_diagram_svg }}</div>
161
+ <div class="chart-box">{{ reliability_diagram_svg|safe }}</div>
159
162
  {% else %}
160
163
  <div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
161
164
  {% endif %}
@@ -177,7 +180,7 @@ _TEMPLATE_SRC = """\
177
180
  <h2>Uncertainty</h2>
178
181
  <div class="charts">
179
182
  {% if confidence_histogram_svg %}
180
- <div class="chart-box">{{ confidence_histogram_svg }}</div>
183
+ <div class="chart-box">{{ confidence_histogram_svg|safe }}</div>
181
184
  {% else %}
182
185
  <div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
183
186
  {% endif %}
@@ -199,7 +202,7 @@ _TEMPLATE_SRC = """\
199
202
  <h2>Error Detection</h2>
200
203
  <div class="charts">
201
204
  {% if auroc_curve_svg %}
202
- <div class="chart-box">{{ auroc_curve_svg }}</div>
205
+ <div class="chart-box">{{ auroc_curve_svg|safe }}</div>
203
206
  {% else %}
204
207
  <div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
205
208
  {% endif %}
@@ -227,7 +230,7 @@ _TEMPLATE_SRC = """\
227
230
  <h2>Selective Prediction</h2>
228
231
  <div class="charts">
229
232
  {% if risk_coverage_svg %}
230
- <div class="chart-box">{{ risk_coverage_svg }}</div>
233
+ <div class="chart-box">{{ risk_coverage_svg|safe }}</div>
231
234
  {% else %}
232
235
  <div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
233
236
  {% endif %}
@@ -255,12 +258,12 @@ _TEMPLATE_SRC = """\
255
258
  <h2>Robustness Analysis</h2>
256
259
  <div class="charts">
257
260
  {% if robustness_degradation_svg %}
258
- <div class="chart-box">{{ robustness_degradation_svg }}</div>
261
+ <div class="chart-box">{{ robustness_degradation_svg|safe }}</div>
259
262
  {% else %}
260
263
  <div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
261
264
  {% endif %}
262
265
  {% if severity_comparison_svg %}
263
- <div class="chart-box">{{ severity_comparison_svg }}</div>
266
+ <div class="chart-box">{{ severity_comparison_svg|safe }}</div>
264
267
  {% else %}
265
268
  <div class="chart-box"><div class="chart-placeholder">Install trustbench[reporting] for charts</div></div>
266
269
  {% endif %}
@@ -279,8 +279,11 @@ _PLOTLY_COMMON = {
279
279
 
280
280
 
281
281
  def _fig_to_svg(fig) -> str:
282
- """Convert a plotly Figure to an SVG string."""
283
- return fig.to_image(format="svg").decode("utf-8")
282
+ """Convert a plotly Figure to an inline SVG string or interactive HTML div."""
283
+ try:
284
+ return fig.to_image(format="svg").decode("utf-8")
285
+ except Exception: # noqa: BLE001
286
+ return fig.to_html(full_html=False, include_plotlyjs=False)
284
287
 
285
288
 
286
289
  def render_confusion_matrix(data: dict[str, Any]) -> str: