oodeel 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. oodeel/__init__.py +28 -0
  2. oodeel/aggregator/__init__.py +26 -0
  3. oodeel/aggregator/base.py +70 -0
  4. oodeel/aggregator/fisher.py +259 -0
  5. oodeel/aggregator/mean.py +72 -0
  6. oodeel/aggregator/std.py +86 -0
  7. oodeel/datasets/__init__.py +24 -0
  8. oodeel/datasets/data_handler.py +334 -0
  9. oodeel/datasets/deprecated/DEPRECATED_data_handler.py +236 -0
  10. oodeel/datasets/deprecated/DEPRECATED_ooddataset.py +330 -0
  11. oodeel/datasets/deprecated/DEPRECATED_tf_data_handler.py +671 -0
  12. oodeel/datasets/deprecated/DEPRECATED_torch_data_handler.py +769 -0
  13. oodeel/datasets/deprecated/__init__.py +31 -0
  14. oodeel/datasets/tf_data_handler.py +600 -0
  15. oodeel/datasets/torch_data_handler.py +672 -0
  16. oodeel/eval/__init__.py +22 -0
  17. oodeel/eval/metrics.py +218 -0
  18. oodeel/eval/plots/__init__.py +27 -0
  19. oodeel/eval/plots/features.py +345 -0
  20. oodeel/eval/plots/metrics.py +118 -0
  21. oodeel/eval/plots/plotly.py +162 -0
  22. oodeel/extractor/__init__.py +35 -0
  23. oodeel/extractor/feature_extractor.py +187 -0
  24. oodeel/extractor/hf_torch_feature_extractor.py +184 -0
  25. oodeel/extractor/keras_feature_extractor.py +409 -0
  26. oodeel/extractor/torch_feature_extractor.py +506 -0
  27. oodeel/methods/__init__.py +47 -0
  28. oodeel/methods/base.py +570 -0
  29. oodeel/methods/dknn.py +185 -0
  30. oodeel/methods/energy.py +119 -0
  31. oodeel/methods/entropy.py +113 -0
  32. oodeel/methods/gen.py +113 -0
  33. oodeel/methods/gram.py +274 -0
  34. oodeel/methods/mahalanobis.py +209 -0
  35. oodeel/methods/mls.py +113 -0
  36. oodeel/methods/odin.py +109 -0
  37. oodeel/methods/rmds.py +172 -0
  38. oodeel/methods/she.py +159 -0
  39. oodeel/methods/vim.py +273 -0
  40. oodeel/preprocess/__init__.py +31 -0
  41. oodeel/preprocess/tf_preprocess.py +95 -0
  42. oodeel/preprocess/torch_preprocess.py +97 -0
  43. oodeel/types/__init__.py +75 -0
  44. oodeel/utils/__init__.py +38 -0
  45. oodeel/utils/general_utils.py +97 -0
  46. oodeel/utils/operator.py +253 -0
  47. oodeel/utils/tf_operator.py +269 -0
  48. oodeel/utils/tf_training_tools.py +219 -0
  49. oodeel/utils/torch_operator.py +292 -0
  50. oodeel/utils/torch_training_tools.py +303 -0
  51. oodeel-0.4.0.dist-info/METADATA +409 -0
  52. oodeel-0.4.0.dist-info/RECORD +63 -0
  53. oodeel-0.4.0.dist-info/WHEEL +5 -0
  54. oodeel-0.4.0.dist-info/licenses/LICENSE +21 -0
  55. oodeel-0.4.0.dist-info/top_level.txt +2 -0
  56. tests/__init__.py +22 -0
  57. tests/tests_tensorflow/__init__.py +37 -0
  58. tests/tests_tensorflow/tf_methods_utils.py +140 -0
  59. tests/tests_tensorflow/tools_tf.py +86 -0
  60. tests/tests_torch/__init__.py +38 -0
  61. tests/tests_torch/tools_torch.py +151 -0
  62. tests/tests_torch/torch_methods_utils.py +148 -0
  63. tests/tools_operator.py +153 -0
@@ -0,0 +1,409 @@
1
+ Metadata-Version: 2.4
2
+ Name: oodeel
3
+ Version: 0.4.0
4
+ Summary: Simple, compact, and hackable post-hoc deep OOD detection for alreadytrained tensorflow or pytorch image classifiers.
5
+ Author: DEEL Core Team
6
+ Author-email: paul.novello@irt-saintexupery.com
7
+ Classifier: Programming Language :: Python
8
+ Classifier: Programming Language :: Python :: 3
9
+ Classifier: Programming Language :: Python :: 3.7
10
+ Classifier: Programming Language :: Python :: 3.8
11
+ Classifier: Programming Language :: Python :: 3.9
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Description-Content-Type: text/markdown
14
+ License-File: LICENSE
15
+ Requires-Dist: numpy<=1.26.4
16
+ Requires-Dist: scikit_learn
17
+ Requires-Dist: scipy
18
+ Requires-Dist: setuptools
19
+ Requires-Dist: matplotlib
20
+ Requires-Dist: pandas
21
+ Requires-Dist: seaborn
22
+ Requires-Dist: plotly
23
+ Requires-Dist: tqdm
24
+ Requires-Dist: datasets
25
+ Provides-Extra: dev
26
+ Requires-Dist: mypy; extra == "dev"
27
+ Requires-Dist: ipywidgets; extra == "dev"
28
+ Requires-Dist: mkdocs-jupyter; extra == "dev"
29
+ Requires-Dist: mkdocstrings-python; extra == "dev"
30
+ Requires-Dist: flake8; extra == "dev"
31
+ Requires-Dist: setuptools; extra == "dev"
32
+ Requires-Dist: pre-commit; extra == "dev"
33
+ Requires-Dist: tox; extra == "dev"
34
+ Requires-Dist: black; extra == "dev"
35
+ Requires-Dist: ruff; extra == "dev"
36
+ Requires-Dist: ipython; extra == "dev"
37
+ Requires-Dist: ipykernel; extra == "dev"
38
+ Requires-Dist: pytest; extra == "dev"
39
+ Requires-Dist: pylint; extra == "dev"
40
+ Requires-Dist: mypy; extra == "dev"
41
+ Requires-Dist: mkdocs; extra == "dev"
42
+ Requires-Dist: mkdocs-material; extra == "dev"
43
+ Requires-Dist: mkdocstrings; extra == "dev"
44
+ Requires-Dist: mknotebooks; extra == "dev"
45
+ Requires-Dist: mike; extra == "dev"
46
+ Requires-Dist: bump2version; extra == "dev"
47
+ Requires-Dist: docsig; extra == "dev"
48
+ Requires-Dist: no_implicit_optional; extra == "dev"
49
+ Requires-Dist: numpy==1.26.4; extra == "dev"
50
+ Requires-Dist: tensorflow==2.11.0; extra == "dev"
51
+ Requires-Dist: tensorflow_datasets; extra == "dev"
52
+ Requires-Dist: tensorflow_probability==0.19.0; extra == "dev"
53
+ Requires-Dist: timm; extra == "dev"
54
+ Requires-Dist: torch==1.13.1; extra == "dev"
55
+ Requires-Dist: torchvision==0.14.1; extra == "dev"
56
+ Provides-Extra: tensorflow-dev
57
+ Requires-Dist: mypy; extra == "tensorflow-dev"
58
+ Requires-Dist: ipywidgets; extra == "tensorflow-dev"
59
+ Requires-Dist: mkdocs-jupyter; extra == "tensorflow-dev"
60
+ Requires-Dist: mkdocstrings-python; extra == "tensorflow-dev"
61
+ Requires-Dist: flake8; extra == "tensorflow-dev"
62
+ Requires-Dist: setuptools; extra == "tensorflow-dev"
63
+ Requires-Dist: pre-commit; extra == "tensorflow-dev"
64
+ Requires-Dist: tox; extra == "tensorflow-dev"
65
+ Requires-Dist: black; extra == "tensorflow-dev"
66
+ Requires-Dist: ruff; extra == "tensorflow-dev"
67
+ Requires-Dist: ipython; extra == "tensorflow-dev"
68
+ Requires-Dist: ipykernel; extra == "tensorflow-dev"
69
+ Requires-Dist: pytest; extra == "tensorflow-dev"
70
+ Requires-Dist: pylint; extra == "tensorflow-dev"
71
+ Requires-Dist: mypy; extra == "tensorflow-dev"
72
+ Requires-Dist: mkdocs; extra == "tensorflow-dev"
73
+ Requires-Dist: mkdocs-material; extra == "tensorflow-dev"
74
+ Requires-Dist: mkdocstrings; extra == "tensorflow-dev"
75
+ Requires-Dist: mknotebooks; extra == "tensorflow-dev"
76
+ Requires-Dist: mike; extra == "tensorflow-dev"
77
+ Requires-Dist: bump2version; extra == "tensorflow-dev"
78
+ Requires-Dist: docsig; extra == "tensorflow-dev"
79
+ Requires-Dist: no_implicit_optional; extra == "tensorflow-dev"
80
+ Requires-Dist: numpy==1.26.4; extra == "tensorflow-dev"
81
+ Requires-Dist: tensorflow==2.11.0; extra == "tensorflow-dev"
82
+ Requires-Dist: tensorflow_datasets; extra == "tensorflow-dev"
83
+ Requires-Dist: tensorflow_probability==0.19.0; extra == "tensorflow-dev"
84
+ Provides-Extra: torch-dev
85
+ Requires-Dist: mypy; extra == "torch-dev"
86
+ Requires-Dist: ipywidgets; extra == "torch-dev"
87
+ Requires-Dist: mkdocs-jupyter; extra == "torch-dev"
88
+ Requires-Dist: mkdocstrings-python; extra == "torch-dev"
89
+ Requires-Dist: flake8; extra == "torch-dev"
90
+ Requires-Dist: setuptools; extra == "torch-dev"
91
+ Requires-Dist: pre-commit; extra == "torch-dev"
92
+ Requires-Dist: tox; extra == "torch-dev"
93
+ Requires-Dist: black; extra == "torch-dev"
94
+ Requires-Dist: ruff; extra == "torch-dev"
95
+ Requires-Dist: ipython; extra == "torch-dev"
96
+ Requires-Dist: ipykernel; extra == "torch-dev"
97
+ Requires-Dist: pytest; extra == "torch-dev"
98
+ Requires-Dist: pylint; extra == "torch-dev"
99
+ Requires-Dist: mypy; extra == "torch-dev"
100
+ Requires-Dist: mkdocs; extra == "torch-dev"
101
+ Requires-Dist: mkdocs-material; extra == "torch-dev"
102
+ Requires-Dist: mkdocstrings; extra == "torch-dev"
103
+ Requires-Dist: mknotebooks; extra == "torch-dev"
104
+ Requires-Dist: mike; extra == "torch-dev"
105
+ Requires-Dist: bump2version; extra == "torch-dev"
106
+ Requires-Dist: docsig; extra == "torch-dev"
107
+ Requires-Dist: no_implicit_optional; extra == "torch-dev"
108
+ Requires-Dist: numpy==1.26.4; extra == "torch-dev"
109
+ Requires-Dist: timm; extra == "torch-dev"
110
+ Requires-Dist: torch==1.13.1; extra == "torch-dev"
111
+ Requires-Dist: torchvision==0.14.1; extra == "torch-dev"
112
+ Provides-Extra: tensorflow
113
+ Requires-Dist: tensorflow==2.11.0; extra == "tensorflow"
114
+ Requires-Dist: tensorflow_datasets; extra == "tensorflow"
115
+ Requires-Dist: tensorflow_probability==0.19.0; extra == "tensorflow"
116
+ Provides-Extra: torch
117
+ Requires-Dist: timm; extra == "torch"
118
+ Requires-Dist: torch==1.13.1; extra == "torch"
119
+ Requires-Dist: torchvision==0.14.1; extra == "torch"
120
+ Provides-Extra: docs
121
+ Requires-Dist: mkdocs; extra == "docs"
122
+ Requires-Dist: mkdocs-material; extra == "docs"
123
+ Requires-Dist: mkdocstrings; extra == "docs"
124
+ Requires-Dist: mknotebooks; extra == "docs"
125
+ Requires-Dist: ipython; extra == "docs"
126
+ Dynamic: author
127
+ Dynamic: author-email
128
+ Dynamic: classifier
129
+ Dynamic: description
130
+ Dynamic: description-content-type
131
+ Dynamic: license-file
132
+ Dynamic: provides-extra
133
+ Dynamic: requires-dist
134
+ Dynamic: summary
135
+
136
+
137
+ <!-- Banner section -->
138
+ <div align="center">
139
+ <picture>
140
+ <source media="(prefers-color-scheme: dark)" srcset="docs/assets/oodeel_dark.png">
141
+ <source media="(prefers-color-scheme: light)" srcset="docs/assets/oodeel_light.png">
142
+ <img src="docs/assets/oodeel_light.png" alt="Library Banner">
143
+ </picture>
144
+ </div>
145
+ <br>
146
+
147
+ <!-- Badge section -->
148
+ <div align="center">
149
+ <a href="#">
150
+ <img src="https://img.shields.io/badge/python-3.8%2B-blue"></a>
151
+ <a href="https://github.com/deel-ai/oodeel/actions/workflows/python-linters.yml">
152
+ <img alt="Flake8" src="https://github.com/deel-ai/oodeel/actions/workflows/python-linters.yml/badge.svg"></a>
153
+ <a href="https://github.com/deel-ai/oodeel/actions/workflows/python-tests-tf.yml">
154
+ <img alt="Tests tf" src="https://github.com/deel-ai/oodeel/actions/workflows/python-tests-tf.yml/badge.svg"></a>
155
+ <a href="https://github.com/deel-ai/oodeel/actions/workflows/python-tests-torch.yml">
156
+ <img alt="Tests torch" src="https://github.com/deel-ai/oodeel/actions/workflows/python-tests-torch.yml/badge.svg"></a>
157
+ <a href="https://github.com/deel-ai/oodeel/actions/workflows/python-coverage-shield.yml">
158
+ <img alt="Coverage" src="https://github.com/deel-ai/oodeel/raw/gh-shields/coverage.svg"></a>
159
+ <a href="https://github.com/deel-ai/oodeel/blob/master/LICENSE">
160
+ <img alt="License MIT" src="https://img.shields.io/badge/License-MIT-efefef"></a>
161
+ </div>
162
+ <br>
163
+
164
+ <!-- Short description of your library -->
165
+
166
+ <b>Oodeel</b> is a library that performs post-hoc deep OOD (Out-of-Distribution) detection on already trained neural network image classifiers. The philosophy of the library is to favor quality over quantity and to foster easy adoption. As a result, we provide a simple, compact and easily customizable API and carefully integrate and test each proposed baseline into a coherent framework that is designed to enable their use in tensorflow **and** pytorch. You can find the documentation [here](https://deel-ai.github.io/oodeel/).
167
+
168
+ ```python
169
+ from oodeel.methods import MLS
170
+
171
+ mls = MLS()
172
+ mls.fit(model) # A tensorflow or torch model
173
+ scores, info = mls.score(ds) # ds is a tf.data.Dataset or a torch.DataLoader
174
+ ```
175
+
176
+ # Table of contents
177
+
178
+ - [Installation](#installation)
179
+ - [Quick Start](#quick-start)
180
+ - [Tutorials](#tutorials)
181
+ - [What's Included](#whats-included)
182
+ - [Development roadmap](#development-roadmap)
183
+ - [Contributing](#contributing)
184
+ - [See Also](#see-also)
185
+ - [Acknowledgments](#acknowledgments)
186
+ - [Creators](#creators)
187
+ - [Citation](#citation)
188
+ - [License](#license)
189
+
190
+ # Installation
191
+
192
+ Installation can be done using:
193
+
194
+ ```bash
195
+ pip install oodeel
196
+ ```
197
+
198
+ oodeel requires either `tensorflow` or `pytorch` to be already installed (it will not install them automatically not to mess-up with existing installations). It is regularly tested with:
199
+
200
+ |Python version|Pytorch version|Tensorflow version|
201
+ |---|---|---|
202
+ |`3.9`| `2.0`| `2.11`|
203
+ |`3.10`|`2.4` | `2.13`|
204
+ |`3.11`| `2.6` | `2.15`|
205
+
206
+ # Quick Start
207
+
208
+ Now that *oodeel* is installed, here are some basic examples of what you can do with the available modules. See also the notebooks directory for more advanced examples.
209
+
210
+ ### For benchmarking with one dataset as in-distribution and another as out-of-distribution
211
+
212
+ Load in-distribution and out-of-distribution datasets.
213
+
214
+ ```python
215
+ from oodeel.datasets import load_data_handler
216
+
217
+ data_handler = load_data_handler("torch")
218
+ # use backend="tensorflow" if using tensorflow
219
+ # Do not forget to adapt load_kwargs in that case
220
+
221
+ # Load in-distribution dataset: CIFAR-10
222
+ ds_in = data_handler.load_dataset(
223
+ "CIFAR10", load_kwargs={"root": data_path, "train": False, "download": True}
224
+ )
225
+ # Load out-of-distribution dataset: SVHN
226
+ ds_out = data_handler.load_dataset(
227
+ "SVHN", load_kwargs={"root": data_path, "split": "test", "download": True}
228
+ )
229
+
230
+ # Prepare datasets for forward (requires appropriate preprocess_fn e.g. input normalization)
231
+ ds_in = data_handler.prepare(
232
+ ds_in, batch_size, preprocess_fn, columns=["input", "label"]
233
+ )
234
+ ds_out = data_handler.prepare(
235
+ ds_out, batch_size, preprocess_fn, columns=["input", "label"]
236
+ )
237
+ ```
238
+
239
+ ### For benchmarking with a classes subset as in-distribution and another classes subset as out-of-distribution
240
+
241
+ Load a dataset and split it into an in-distribution dataset and ou-of-distribution dataset depending on its label values (a common practice of anomaly detection and open set recognition).
242
+
243
+ ```python
244
+ from oodeel.datasets import load_data_handler
245
+
246
+ data_handler = load_data_handler("torch") # use backend="tensorflow" if using tensorflow
247
+
248
+ # Load dataset to split: CIFAR-10
249
+ ds_test = data_handler.load_dataset(
250
+ "CIFAR10", load_kwargs={"root": data_path, "train": False, "download": True}
251
+ )
252
+
253
+ in_labels = [0, 1, 2, 3, 4]
254
+ ds_in, ds_out = data_handler.split_by_class(ds_test, in_labels)
255
+
256
+ # Prepare datasets for forward (requires appropriate preprocess_fn e.g. input normalization)
257
+ ds_in = data_handler.prepare(
258
+ ds_in, batch_size, preprocess_fn, columns=["input", "label"]
259
+ )
260
+ ds_out = data_handler.prepare(
261
+ ds_out, batch_size, preprocess_fn, columns=["input", "label"]
262
+ )
263
+ ```
264
+ ### Run an OOD method
265
+
266
+ Load an OOD method and use it on an already-trained model
267
+
268
+ ```python
269
+ from oodeel.methods import MLS
270
+
271
+ mls = MLS()
272
+ mls.fit(model) # Requires a pretrained model
273
+
274
+ # info_X contains model predictions and labels if available
275
+ scores_in, info_in = mls.score(ds_in)
276
+ scores_out, info_out = mls.score(ds_out)
277
+ ```
278
+
279
+ Evaluate the method
280
+
281
+ ```python
282
+ from oodeel.eval.metrics import bench_metrics
283
+
284
+ metrics = bench_metrics(
285
+ (scores_in, scores_out),
286
+ metrics = ["auroc", "fpr95tpr"],
287
+ )
288
+ ```
289
+
290
+ ### And visualize the results!
291
+
292
+ 2D t-SNE (3D is also available).
293
+
294
+ ```python
295
+ plot_2D_features(
296
+ model=model,
297
+ in_dataset=ds_in,
298
+ out_dataset=ds_out,
299
+ output_layer_id=-2,
300
+ )
301
+ ```
302
+ <p align="center">
303
+ <img src="docs/assets/tsne.png" alt="TSNE" />
304
+ </p>
305
+
306
+ Classical histograms and AUROC curve.
307
+ ```python
308
+ plot_ood_scores(scores_in, scores_out, log_scale=False)
309
+ plot_roc_curve(scores_in, scores_out)
310
+ ```
311
+ <p align="center">
312
+ <img src="docs/assets/auroc.png" alt="AUROC" />
313
+ </p>
314
+
315
+ # Tutorials
316
+
317
+ We propose some tutorials to get familiar with the library and its API. See the Tutorial section of the [doc](https://deel-ai.github.io/oodeel/)
318
+
319
+
320
+ # What's Included
321
+
322
+ The library is based on a class, `OODBaseDetector`, that fits a model and then scores new samples. Some baselines use extra data, so `OODBaseDetector` can also fit additional data if needed. The library uses `OODDataset` to properly load data from different sources and prepare it for OOD detection. It can perform OOD-specific operations like adding extra OOD data for tuning with Outlier Exposure or filters according to label values for anomaly detection or open set recognition benchmarks.
323
+
324
+ Currently, **oodeel** includes the following baselines:
325
+
326
+
327
+
328
+ | Name | Link | Venue | Status |
329
+ | --- | --- | --- | --- |
330
+ | MLS | [Open-Set Recognition: a Good Closed-Set Classifier is All You Need?](http://arxiv.org/abs/2110.06207) | ICLR 2022 | avail [tensorflow & torch](docs/pages/getting_started.ipynb) |
331
+ | MSP | [A Baseline for Detecting Misclassified and Out-of-Distribution Examples in Neural Networks](http://arxiv.org/abs/1610.02136) | ICLR 2017 | avail [tensorflow & torch](docs/pages/getting_started.ipynb)|
332
+ | Mahalanobis | [A Simple Unified Framework for Detecting Out-of-Distribution Samples and Adversarial Attacks](http://arxiv.org/abs/1807.03888) | NeurIPS 2018 | avail [tensorflow](docs/notebooks/tensorflow/demo_mahalanobis_tf.ipynb) or [torch](docs/notebooks/torch/demo_mahalanobis_torch.ipynb)|
333
+ | Energy | [Energy-based Out-of-distribution Detection](http://arxiv.org/abs/2010.03759) | NeurIPS 2020 |avail [tensorflow](docs/notebooks/tensorflow/demo_energy_tf.ipynb) or [torch](docs/notebooks/torch/demo_energy_torch.ipynb) |
334
+ | Odin | [Enhancing The Reliability of Out-of-distribution Image Detection in Neural Networks](http://arxiv.org/abs/1706.02690) | ICLR 2018 | avail [tensorflow](docs/notebooks/tensorflow/demo_odin_tf.ipynb) or [torch](docs/notebooks/torch/demo_odin_torch.ipynb) |
335
+ | DKNN | [Out-of-Distribution Detection with Deep Nearest Neighbors](http://arxiv.org/abs/2204.06507) | ICML 2022 | avail [tensorflow](docs/notebooks/tensorflow/demo_dknn_tf.ipynb) or [torch](docs/notebooks/torch/demo_dknn_torch.ipynb) |
336
+ | VIM | [ViM: Out-Of-Distribution with Virtual-logit Matching](http://arxiv.org/abs/2203.10807) | CVPR 2022 |avail [tensorflow](docs/notebooks/tensorflow/demo_vim_tf.ipynb) or [torch](docs/notebooks/torch/demo_vim_torch.ipynb) |
337
+ | Entropy | [Likelihood Ratios for Out-of-Distribution Detection](https://proceedings.neurips.cc/paper/2019/hash/1e79596878b2320cac26dd792a6c51c9-Abstract.html) | NeurIPS 2019 |avail [tensorflow](docs/notebooks/tensorflow/demo_entropy_tf.ipynb) or [torch](docs/notebooks/torch/demo_entropy_torch.ipynb) |
338
+ | ReAct | [ReAct: Out-of-distribution Detection With Rectified Activations](http://arxiv.org/abs/2111.12797) | NeurIPS 2021 | avail [tensorflow](docs/notebooks/tensorflow/demo_react_tf.ipynb) or [torch](docs/notebooks/torch/demo_react_torch.ipynb) |
339
+ | Gram | [Detecting Out-of-Distribution Examples with Gram Matrices](https://proceedings.mlr.press/v119/sastry20a.html) | ICML 2020 | avail [tensorflow](docs/notebooks/tensorflow/demo_gram_tf.ipynb) or [torch](docs/notebooks/torch/demo_gram_torch.ipynb) |
340
+ | GEN | [GEN: Pushing the Limits of Softmax-Based Out-of-Distribution Detection](https://openaccess.thecvf.com/content/CVPR2023/html/Liu_GEN_Pushing_the_Limits_of_Softmax-Based_Out-of-Distribution_Detection_CVPR_2023_paper.html) | CVPR 2023 | avail [tensorflow](docs/notebooks/tensorflow/demo_gen_tf.ipynb) or [torch](docs/notebooks/torch/demo_gen_torch.ipynb) |
341
+ | RMDS | [A Simple Fix to Mahalanobis Distance for Improving Near-OOD Detection](https://arxiv.org/abs/2106.09022) | preprint | avail [tensorflow](docs/notebooks/tensorflow/demo_rmds_tf.ipynb) or [torch](docs/notebooks/torch/demo_rmds_torch.ipynb) |
342
+ | SHE | [Out-of-Distribution Detection based on In-Distribution Data Patterns Memorization with Modern Hopfield Energy](https://openreview.net/forum?id=KkazG4lgKL) | ICLR 2023 | avail [tensorflow](docs/notebooks/tensorflow/demo_she_tf.ipynb) or [torch](docs/notebooks/torch/demo_she_torch.ipynb) |
343
+ | ASH | [Extremely Simple Activation Shaping for Out-of-Distribution Detection](https://arxiv.org/abs/2209.09858) | ICLR 2023 | avail [tensorflow](docs/notebooks/tensorflow/demo_ash_tf.ipynb) or [torch](docs/notebooks/torch/demo_ash_torch.ipynb) |
344
+ | SCALE | [Scaling for Training Time and Post-hoc Out-of-distribution Detection Enhancement](https://arxiv.org/abs/2310.00227) | ICLR 2024 | avail [tensorflow](docs/notebooks/tensorflow/demo_scale_tf.ipynb) or [torch](docs/notebooks/torch/demo_scale_torch.ipynb) |
345
+
346
+
347
+
348
+ **Oodeel** also includes standard training functions with data augmentation and learning rate scheduler for toy convnet models or models from `keras.applications` in [tf_training_tools.py](https://github.com/deel-ai/oodeel/tree/master/oodeel/utils/tf_training_tools.py) and `torchvision.models` in [torch_training_tools.py](https://github.com/deel-ai/oodeel/tree/master/oodeel/utils/torch_training_tools.py) files. These functions come in handy for benchmarks like *leave-k-classes-out* that requires retraining models on a subset of dataset classes.
349
+ # Development Roadmap
350
+
351
+ - [ ] More baselines!
352
+ - [x] A module for thorough visualizations (result plots and feature space visualizations)
353
+ - [ ] Integrate model loading and uploading with [hugginface's transformers](https://huggingface.co/docs/transformers/index) library for pretraining
354
+ - [ ] Extend the library to more diverse tasks like object detection, segmentation, NLP ...
355
+ - [ ] Towards OOD Generalization?
356
+
357
+ # Contributing
358
+
359
+ Feel free to propose your ideas or come and contribute with us on the oodeel toolbox! We have a specific document where we describe in a simple way how to make your first pull request: [just here](CONTRIBUTING.md).
360
+
361
+ # See Also
362
+
363
+ Other great tools in the field of OOD:
364
+
365
+ - [OpenOOD](https://github.com/Jingkang50/OpenOOD): Benchmarking Generalized Out-of-Distribution Detection
366
+ - [Pytorch-OOD](https://github.com/kkirchheim/pytorch-ood): Out-of-Distribution (OOD) Detection with Deep Neural Networks based on PyTorch.
367
+ - [ADBench](https://github.com/Minqi824/ADBench): Official Implement of "ADBench: Anomaly Detection Benchmark".
368
+ - [PyOD](https://github.com/yzhao062/pyod): A Comprehensive and Scalable Python Library for Outlier Detection (Anomaly Detection)
369
+ - [Anomalib](https://github.com/openvinotoolkit/anomalib): An anomaly detection library comprising state-of-the-art algorithms and features such as experiment management, hyper-parameter optimization, and edge inference.
370
+
371
+ More from the DEEL project:
372
+
373
+ - [Xplique](https://github.com/deel-ai/xplique) a Python library exclusively dedicated to explaining neural networks.
374
+ - [deel-lip](https://github.com/deel-ai/deel-lip) a Python library for training k-Lipschitz neural networks on TF.
375
+ - [Influenciae](https://github.com/deel-ai/influenciae) Python toolkit dedicated to computing influence values for the discovery of potentially problematic samples in a dataset.
376
+ - [deel-torchlip](https://github.com/deel-ai/deel-torchlip) a Python library for training k-Lipschitz neural networks on PyTorch.
377
+ - [DEEL White paper](https://arxiv.org/abs/2103.10529) a summary of the DEEL team on the challenges of certifiable AI and the role of data quality, representativity and explainability for this purpose.
378
+
379
+ # Acknowledgments
380
+
381
+ <div align="right">
382
+ <picture>
383
+ <source media="(prefers-color-scheme: dark)" srcset="docs/assets/deel_dark.png" width="25%" align="right">
384
+ <source media="(prefers-color-scheme: light)" srcset="docs/assets/deel_light.png" width="25%" align="right">
385
+ <img src="docs/assets/deel_dark.png" alt="DEEL Logo" width="25%" align="right">
386
+ </picture>
387
+ </div>
388
+ This project received funding from the French ”Investing for the Future – PIA3” program within the Artificial and Natural Intelligence Toulouse Institute (ANITI). The authors gratefully acknowledge the support of the <a href="https://www.deel.ai/"> DEEL </a>, a research project jointly conducted in France and Quebec.
389
+
390
+ # Creators
391
+
392
+ The library was created by Paul Novello to streamline DEEL research on post-hoc deep OOD methods and foster their adoption by DEEL industrial partners. He was soon joined by Yann Pequignot, Yannick Prudent, Corentin Friedrich and Matthieu Le Goff.
393
+
394
+ # Citation
395
+
396
+ If you use OODEEL for your research project, please consider citing:
397
+ ```
398
+ @misc{oodeel,
399
+ author = {Novello, Paul and Prudent, Yannick and Friedrich, Corentin and Pequignot, Yann and Le Goff, Matthieu},
400
+ title = {OODEEL, a simple, compact, and hackable post-hoc deep OOD detection for already trained tensorflow or pytorch image classifiers.},
401
+ year = {2023},
402
+ publisher = {GitHub},
403
+ journal = {GitHub repository},
404
+ howpublished = {\url{https://github.com/deel-ai/oodeel}},
405
+ }
406
+ ```
407
+ # License
408
+
409
+ The package is released under [MIT license](LICENSE).
@@ -0,0 +1,63 @@
1
+ oodeel/__init__.py,sha256=y0XhDQ0TUUFU9cNHkKHPY7LBvXel4lK-AilK-XvW3CA,1343
2
+ oodeel/aggregator/__init__.py,sha256=lIVjXJ7xb6038Ho6wtgUTg7O2F6FTHJYix1KmWf8smM,1451
3
+ oodeel/aggregator/base.py,sha256=fAL4Ox1uB_eR0WT2UnVmTzJMThLyWTALgSqe-jiyvFc,3138
4
+ oodeel/aggregator/fisher.py,sha256=IIl3kYMA2iu9zceDaHOzqAhE6Wdhp6FxjUk4kXT9xJM,11588
5
+ oodeel/aggregator/mean.py,sha256=NkmY4Xu5NdF69fTTs_NRsWjZDg8CCxz869eMX5BMIjc,3119
6
+ oodeel/aggregator/std.py,sha256=MtVrRAkfduBIPmgEVzytdCF2lhIOcjUTU9Bnms2uLDw,3921
7
+ oodeel/datasets/__init__.py,sha256=COhGN25e3TsYtIz5rJgc0aNr_k3JjGXE3cCYd5RajTQ,1398
8
+ oodeel/datasets/data_handler.py,sha256=JTt8dykeRtanSTRjX1aRXrA88WWhDk9b55wqx8goKYM,11368
9
+ oodeel/datasets/tf_data_handler.py,sha256=p3CtLGSiOQxyc0hLjo4nzlfg3RfUA79a1gNiRLO5bqg,22606
10
+ oodeel/datasets/torch_data_handler.py,sha256=Ns7QXpEQpzbowV-YzCR3ByM7klN_1-8Ux8Vx6rsXXdU,23625
11
+ oodeel/datasets/deprecated/DEPRECATED_data_handler.py,sha256=fGK3_YSbNdHuljBtrjt7xbi6ESfNpnQV_pUSn9Uot2k,7910
12
+ oodeel/datasets/deprecated/DEPRECATED_ooddataset.py,sha256=Ad2otHV_vClK7ZY2D3-gW4QO_B3ir6DzJM1kYeNCJpw,13294
13
+ oodeel/datasets/deprecated/DEPRECATED_tf_data_handler.py,sha256=heK9c1g829LZof8CkAmmxpTvE0EpoLZOLXuYKMrL8b4,24811
14
+ oodeel/datasets/deprecated/DEPRECATED_torch_data_handler.py,sha256=eWCOlSPbblS8KZZAlu-M10QgY2Hv7mOq1uawLcGs0FE,26986
15
+ oodeel/datasets/deprecated/__init__.py,sha256=VYVsYZ-_TgGyd5ls5rudytQGPocm9I7N5_M9rHcV91w,1506
16
+ oodeel/eval/__init__.py,sha256=lQIUQjczeiRtfIqH2uLNJGubKUN6vPM93mTfY1Qz3bc,1297
17
+ oodeel/eval/metrics.py,sha256=9fg7fVTT10_XIKrcsan6HQOlhUdFJhZ3f5VAprTKsjM,8839
18
+ oodeel/eval/plots/__init__.py,sha256=YmcFh8RUGvljc-vCew6EiIFMDn0YA_UOfDj4eAv5_Yk,1487
19
+ oodeel/eval/plots/features.py,sha256=6V5wdXa_pzN-uf1-PNS7ERGGs0FAWV1pFXWPzNebig8,12179
20
+ oodeel/eval/plots/metrics.py,sha256=RbypNvr14Ygv2z0NokCoso5Il6m1rlL88gZCN9TiPhA,4225
21
+ oodeel/eval/plots/plotly.py,sha256=XHPFuXARrRIGVrRB7d0Od36iCn3JBP387tDrwRvPhU0,6040
22
+ oodeel/extractor/__init__.py,sha256=nCB5TptpFvVfZTJwQ2jW2LUoi9hK0b36v_uyie0rH3c,1606
23
+ oodeel/extractor/feature_extractor.py,sha256=1HwPW6S5HJomQpai2NwytQ8SEN5d2eGT-0LLbzUgDoE,7325
24
+ oodeel/extractor/hf_torch_feature_extractor.py,sha256=cCmtscI-1LgTrn1VPe8c-24cK8z-kEYReEQKhpT2R-s,7542
25
+ oodeel/extractor/keras_feature_extractor.py,sha256=MSRYqYrJo-6bwQdqoY7d2CjiwLTYKe282n8LAaKV9qo,16722
26
+ oodeel/extractor/torch_feature_extractor.py,sha256=bNMAnUAoZLB0C7lvfeNIhlAOvOQlKYerAgoq9ylQpUg,19192
27
+ oodeel/methods/__init__.py,sha256=BkpwY0-N6SAvJ8iUxe1g0J8On-tbmLCCYTJIFI3M7i0,1721
28
+ oodeel/methods/base.py,sha256=VlNG8z_31UUBiImutSwMF2CSdCkDMBf-P0Lue8BB-xY,22421
29
+ oodeel/methods/dknn.py,sha256=4uUPevVGh8-SGVIW8vaoVMZxrwCqx96n11u2DnlDKo8,6873
30
+ oodeel/methods/energy.py,sha256=J8SxwTj-HLoB_p5ghkm7itux1QwuqfGWWVwN5UorNKU,4442
31
+ oodeel/methods/entropy.py,sha256=YwFplbr0jS9kj5kMeLG5HNBNnO-f9YIzciZ-uv2xlwU,4108
32
+ oodeel/methods/gen.py,sha256=CWPrIkfQDdzqMKqJCH_r8kuJmWT0BAMsUn-_4SKam_o,4441
33
+ oodeel/methods/gram.py,sha256=nWouBNeWoVaqpoUaLEQ8BKvpaw7zIEhMw5AV10I9ZDw,10940
34
+ oodeel/methods/mahalanobis.py,sha256=SOumiZPxKfdyymURKrwR6RRPbq4qT6Ld1bXsj8Totmo,8005
35
+ oodeel/methods/mls.py,sha256=MCqno_AoGvKEKrONWMpSyCAJE-gsIs6mMFyzvMyqUlU,4313
36
+ oodeel/methods/odin.py,sha256=lbdARTVesVNhK01ldf37IERBGk-w_h6PidG45ceEiyk,4079
37
+ oodeel/methods/rmds.py,sha256=UmYh49JLIvaSnI8ZFPh2E6LNE6Bh4dw5DqkQXpoPgmI,7011
38
+ oodeel/methods/she.py,sha256=whfbaeNJiSuGL7TSAKcwbgTsEU0fHPRxgSHtOzSwprI,6222
39
+ oodeel/methods/vim.py,sha256=0iiBeaw8Jh1pLTwiXtOuVvNxEXXXFYeSAkK_j5b6Z9U,10521
40
+ oodeel/preprocess/__init__.py,sha256=65M9hKYHYzZ6lACA6xo7ODKofaAoqfFZ4aC5ZSJeN2I,1484
41
+ oodeel/preprocess/tf_preprocess.py,sha256=TRaEA7KrVjWFB81vlnNk9hN-G0tGhDa97VvrBEaP9vM,4048
42
+ oodeel/preprocess/torch_preprocess.py,sha256=BTjOyEHPfqx_CSv6Lw3zBi2wKEjAIdPN1DuBptcebA0,3976
43
+ oodeel/types/__init__.py,sha256=9TTXjSBfbaDIVMRnclInHI-CBr4L6VZTi61rCJKcTw8,2484
44
+ oodeel/utils/__init__.py,sha256=Lue9BysqeJf5Ej0szafgC9na8IZZR7hWGInJxoEiHUg,1696
45
+ oodeel/utils/general_utils.py,sha256=VNH94b01Rgn4Oqo20TR_iRsT5XTyVHjD07f4GsjjmuA,3875
46
+ oodeel/utils/operator.py,sha256=ETAFJ_oYhiD1Rawjooueq5KDl4SNzJR5fQDUU05uMz8,8262
47
+ oodeel/utils/tf_operator.py,sha256=TnjwOKv-K5YV4YjQa7JQZcusS6cmQaLRiTgERiGDkjg,9543
48
+ oodeel/utils/tf_training_tools.py,sha256=31cPCpXKmO7lMRrefmY93m6cSn0DRaHxVFeYgUiE6kI,8090
49
+ oodeel/utils/torch_operator.py,sha256=1dP4w067-GgcOaMdtG7MYQIvidpk1ltH7mzbQhLrRjk,10341
50
+ oodeel/utils/torch_training_tools.py,sha256=ggL_iDwyquTw9CtQAg--IODVdS1BFsBw4U5IOVlAsK8,11192
51
+ oodeel-0.4.0.dist-info/licenses/LICENSE,sha256=XrlZ0uYNVeUAF-iEVX21J3CTJjYPgIZUagYSy3Hf0jk,1265
52
+ tests/__init__.py,sha256=lQIUQjczeiRtfIqH2uLNJGubKUN6vPM93mTfY1Qz3bc,1297
53
+ tests/tools_operator.py,sha256=YTU_FppXZPUyi3nldQsDkCMQ3Bvz1Hn_V7L45EEwftc,5328
54
+ tests/tests_tensorflow/__init__.py,sha256=VuiDSdOBB2jUeobzAW-XTtHJQrcyZpp8i2DyFu0A2RI,1710
55
+ tests/tests_tensorflow/tf_methods_utils.py,sha256=QHLGeL_7_x_HjrixqpmeDRizOhMjjXGxjTm2qnjWbOE,5455
56
+ tests/tests_tensorflow/tools_tf.py,sha256=Z_MEzhwCOF7E7uT-tnfRS3Po9hg0MFzbESP7sMAFBkY,3499
57
+ tests/tests_torch/__init__.py,sha256=3mVxix2Ecn2wUo9DxvzyJBmyOAkv4fVWHHp6uLQHoic,1738
58
+ tests/tests_torch/tools_torch.py,sha256=lS5_kctnyiB6eP-NTfVUsbj5Mn-AiRO0Jo75p8djmwg,4975
59
+ tests/tests_torch/torch_methods_utils.py,sha256=HFNINYdtnVFtpw_ghDwcQvl7vuHykAF-C24CNjfWHTs,5610
60
+ oodeel-0.4.0.dist-info/METADATA,sha256=7nHLENKRE4zrYCBh432HLwkqv3GtZs-iaDWc3avp6xA,20837
61
+ oodeel-0.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
62
+ oodeel-0.4.0.dist-info/top_level.txt,sha256=zkYRty1FGJ1dkpk-5MU_4uFfBFmcxXoqSwej73xELDs,13
63
+ oodeel-0.4.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (80.9.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,21 @@
1
+ Copyright 2020 © IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry, CRIAQ and ANITI - https://www.deel.ai/
2
+
3
+ The MIT License (MIT)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,2 @@
1
+ oodeel
2
+ tests
tests/__init__.py ADDED
@@ -0,0 +1,22 @@
1
+ # -*- coding: utf-8 -*-
2
+ # Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
3
+ # rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
4
+ # CRIAQ and ANITI - https://www.deel.ai/
5
+ #
6
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
7
+ # of this software and associated documentation files (the "Software"), to deal
8
+ # in the Software without restriction, including without limitation the rights
9
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
+ # copies of the Software, and to permit persons to whom the Software is
11
+ # furnished to do so, subject to the following conditions:
12
+ #
13
+ # The above copyright notice and this permission notice shall be included in all
14
+ # copies or substantial portions of the Software.
15
+ #
16
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22
+ # SOFTWARE.
@@ -0,0 +1,37 @@
1
+ # -*- coding: utf-8 -*-
2
+ # Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
3
+ # rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
4
+ # CRIAQ and ANITI - https://www.deel.ai/
5
+ #
6
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
7
+ # of this software and associated documentation files (the "Software"), to deal
8
+ # in the Software without restriction, including without limitation the rights
9
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
+ # copies of the Software, and to permit persons to whom the Software is
11
+ # furnished to do so, subject to the following conditions:
12
+ #
13
+ # The above copyright notice and this permission notice shall be included in all
14
+ # copies or substantial portions of the Software.
15
+ #
16
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22
+ # SOFTWARE.
23
+ from .tf_methods_utils import eval_detector_on_blobs
24
+ from .tools_tf import almost_equal
25
+ from .tools_tf import generate_data
26
+ from .tools_tf import generate_data_tf
27
+ from .tools_tf import generate_model
28
+ from .tools_tf import generate_regression_model
29
+
30
+ __all__ = [
31
+ "eval_detector_on_blobs",
32
+ "almost_equal",
33
+ "generate_data",
34
+ "generate_data_tf",
35
+ "generate_model",
36
+ "generate_regression_model",
37
+ ]