dataeval 0.61.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataeval/__init__.py +18 -0
- dataeval/_internal/detectors/__init__.py +0 -0
- dataeval/_internal/detectors/clusterer.py +469 -0
- dataeval/_internal/detectors/drift/__init__.py +0 -0
- dataeval/_internal/detectors/drift/base.py +265 -0
- dataeval/_internal/detectors/drift/cvm.py +97 -0
- dataeval/_internal/detectors/drift/ks.py +100 -0
- dataeval/_internal/detectors/drift/mmd.py +166 -0
- dataeval/_internal/detectors/drift/torch.py +310 -0
- dataeval/_internal/detectors/drift/uncertainty.py +149 -0
- dataeval/_internal/detectors/duplicates.py +49 -0
- dataeval/_internal/detectors/linter.py +78 -0
- dataeval/_internal/detectors/ood/__init__.py +0 -0
- dataeval/_internal/detectors/ood/ae.py +77 -0
- dataeval/_internal/detectors/ood/aegmm.py +69 -0
- dataeval/_internal/detectors/ood/base.py +199 -0
- dataeval/_internal/detectors/ood/llr.py +284 -0
- dataeval/_internal/detectors/ood/vae.py +86 -0
- dataeval/_internal/detectors/ood/vaegmm.py +79 -0
- dataeval/_internal/flags.py +47 -0
- dataeval/_internal/metrics/__init__.py +0 -0
- dataeval/_internal/metrics/base.py +92 -0
- dataeval/_internal/metrics/ber.py +124 -0
- dataeval/_internal/metrics/coverage.py +80 -0
- dataeval/_internal/metrics/divergence.py +94 -0
- dataeval/_internal/metrics/hash.py +79 -0
- dataeval/_internal/metrics/parity.py +180 -0
- dataeval/_internal/metrics/stats.py +332 -0
- dataeval/_internal/metrics/uap.py +45 -0
- dataeval/_internal/metrics/utils.py +158 -0
- dataeval/_internal/models/__init__.py +0 -0
- dataeval/_internal/models/pytorch/__init__.py +0 -0
- dataeval/_internal/models/pytorch/autoencoder.py +202 -0
- dataeval/_internal/models/pytorch/blocks.py +46 -0
- dataeval/_internal/models/pytorch/utils.py +67 -0
- dataeval/_internal/models/tensorflow/__init__.py +0 -0
- dataeval/_internal/models/tensorflow/autoencoder.py +317 -0
- dataeval/_internal/models/tensorflow/gmm.py +115 -0
- dataeval/_internal/models/tensorflow/losses.py +107 -0
- dataeval/_internal/models/tensorflow/pixelcnn.py +1106 -0
- dataeval/_internal/models/tensorflow/trainer.py +102 -0
- dataeval/_internal/models/tensorflow/utils.py +254 -0
- dataeval/_internal/workflows/sufficiency.py +555 -0
- dataeval/detectors/__init__.py +29 -0
- dataeval/flags/__init__.py +3 -0
- dataeval/metrics/__init__.py +7 -0
- dataeval/models/__init__.py +15 -0
- dataeval/models/tensorflow/__init__.py +6 -0
- dataeval/models/torch/__init__.py +8 -0
- dataeval/py.typed +0 -0
- dataeval/workflows/__init__.py +8 -0
- dataeval-0.61.0.dist-info/LICENSE.txt +21 -0
- dataeval-0.61.0.dist-info/METADATA +114 -0
- dataeval-0.61.0.dist-info/RECORD +55 -0
- dataeval-0.61.0.dist-info/WHEEL +4 -0
@@ -0,0 +1,114 @@
|
|
1
|
+
Metadata-Version: 2.1
|
2
|
+
Name: dataeval
|
3
|
+
Version: 0.61.0
|
4
|
+
Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
|
5
|
+
Home-page: https://dataeval.ai/
|
6
|
+
License: MIT
|
7
|
+
Author: Andrew Weng
|
8
|
+
Author-email: andrew.weng@ariacoustics.com
|
9
|
+
Maintainer: ARiA
|
10
|
+
Maintainer-email: dataeval@ariacoustics.com
|
11
|
+
Requires-Python: >=3.9,<3.12
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
15
|
+
Classifier: Operating System :: OS Independent
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
20
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
21
|
+
Classifier: Topic :: Scientific/Engineering
|
22
|
+
Provides-Extra: all
|
23
|
+
Provides-Extra: tensorflow
|
24
|
+
Provides-Extra: torch
|
25
|
+
Requires-Dist: hdbscan (>=0.8.36)
|
26
|
+
Requires-Dist: maite
|
27
|
+
Requires-Dist: matplotlib ; extra == "torch" or extra == "all"
|
28
|
+
Requires-Dist: numpy (>1.24.3)
|
29
|
+
Requires-Dist: nvidia-cudnn-cu11 (>=8.6.0.163) ; extra == "tensorflow" or extra == "torch" or extra == "all"
|
30
|
+
Requires-Dist: pillow (>=10.3.0)
|
31
|
+
Requires-Dist: scikit-learn (>=1.5.0)
|
32
|
+
Requires-Dist: scipy (>=1.10)
|
33
|
+
Requires-Dist: tensorflow (>=2.14.1,<2.16) ; extra == "tensorflow" or extra == "all"
|
34
|
+
Requires-Dist: tensorflow-io-gcs-filesystem (>=0.35.0,<0.37) ; extra == "tensorflow" or extra == "all"
|
35
|
+
Requires-Dist: tensorflow_probability (>=0.22.1,<0.24) ; extra == "tensorflow" or extra == "all"
|
36
|
+
Requires-Dist: torch (>=2.0.1,!=2.2.0) ; extra == "torch" or extra == "all"
|
37
|
+
Requires-Dist: xxhash (>=3.3)
|
38
|
+
Project-URL: Documentation, https://dataeval.readthedocs.io/
|
39
|
+
Project-URL: Repository, https://github.com/aria-ml/dataeval/
|
40
|
+
Description-Content-Type: text/markdown
|
41
|
+
|
42
|
+
# DataEval
|
43
|
+
|
44
|
+
## About DataEval
|
45
|
+
|
46
|
+
DataEval focuses on characterizing image data and its impact on model performance across classification and object-detection tasks.
|
47
|
+
|
48
|
+
<!-- start about -->
|
49
|
+
|
50
|
+
**Model-agnostic metrics that bound real-world performance**
|
51
|
+
- relevance/completeness/coverage
|
52
|
+
- metafeatures (data complexity)
|
53
|
+
|
54
|
+
**Model-specific metrics that guide model selection and training**
|
55
|
+
- dataset sufficiency
|
56
|
+
- data/model complexity mismatch
|
57
|
+
|
58
|
+
**Metrics for post-deployment monitoring of data with bounds on model performance to guide retraining**
|
59
|
+
- dataset-shift metrics
|
60
|
+
- model performance bounds under covariate shift
|
61
|
+
- guidance on sampling to assess model error and model retraining
|
62
|
+
|
63
|
+
<!-- end about -->
|
64
|
+
|
65
|
+
## Getting Started
|
66
|
+
|
67
|
+
### Requirements
|
68
|
+
- Python 3.9-3.11
|
69
|
+
|
70
|
+
### Installing DataEval
|
71
|
+
|
72
|
+
You can install DataEval directly from pypi.org using the following command. The optional dependencies of DataEval are `torch`, `tensorflow` and `all`. Using `torch` enables Sufficiency metrics, and `tensorflow` enables OOD Detection.
|
73
|
+
|
74
|
+
```
|
75
|
+
pip install dataeval[all]
|
76
|
+
```
|
77
|
+
|
78
|
+
### Installing DataEval from GitHub
|
79
|
+
|
80
|
+
To install DataEval from source locally on Ubuntu, you will need `git-lfs` to download larger, binary source files and `poetry` for project dependency management.
|
81
|
+
|
82
|
+
```
|
83
|
+
sudo apt-get install git-lfs
|
84
|
+
pip install poetry
|
85
|
+
```
|
86
|
+
|
87
|
+
Pull the source down and change to the DataEval project directory.
|
88
|
+
```
|
89
|
+
git clone https://github.com/aria-ml/dataeval.git
|
90
|
+
cd dataeval
|
91
|
+
```
|
92
|
+
|
93
|
+
|
94
|
+
|
95
|
+
Install DataEval with optional dependencies for development.
|
96
|
+
```
|
97
|
+
poetry install --all-extras --with dev
|
98
|
+
```
|
99
|
+
|
100
|
+
Now that DataEval is installed, you can run commands in the poetry virtual environment by prefixing shell commands with `poetry run`, or activate the virtual environment directly in the shell.
|
101
|
+
```
|
102
|
+
poetry shell
|
103
|
+
```
|
104
|
+
|
105
|
+
### Documentation and Tutorials
|
106
|
+
For more ideas on getting started using DataEval in your workflow, additional information and tutorials are in our Sphinx documentation hosted on [Read the Docs](https://dataeval.readthedocs.io/).
|
107
|
+
|
108
|
+
## Attribution
|
109
|
+
This project uses code from the [Alibi-Detect](https://github.com/SeldonIO/alibi-detect) python library developed by SeldonIO. Additional documentation from the developers are also available [here](https://docs.seldon.io/projects/alibi-detect/en/stable/).
|
110
|
+
|
111
|
+
## POCs
|
112
|
+
- **POC**: Scott Swan @scott.swan
|
113
|
+
- **DPOC**: Andrew Weng @aweng
|
114
|
+
|
@@ -0,0 +1,55 @@
|
|
1
|
+
dataeval/__init__.py,sha256=hoMLXpAqvfNQ6BtdLD49hOXWMUp5_4zBg_IPeuGPyZA,408
|
2
|
+
dataeval/_internal/detectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
3
|
+
dataeval/_internal/detectors/clusterer.py,sha256=zuWW7qfbQWQ777TqBOsDp2_fEbFmuUzNqOvTWv8xijo,19193
|
4
|
+
dataeval/_internal/detectors/drift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
5
|
+
dataeval/_internal/detectors/drift/base.py,sha256=b6_kUHGPtfDnN6S9gNkldpoaGwdRKw6ohGI4wGega24,8981
|
6
|
+
dataeval/_internal/detectors/drift/cvm.py,sha256=3YePeStvRSKVFqHEWshngXhJTCh3cn9-9B-Ou7FQHgM,3897
|
7
|
+
dataeval/_internal/detectors/drift/ks.py,sha256=PW4qB4XbDUhVxpKJeA39-0GYVgeAxWggipBh2bhKDKg,3924
|
8
|
+
dataeval/_internal/detectors/drift/mmd.py,sha256=ZJu28o6Brs0-pt3PVY2ysMKVfLYXaEz754I-lSLn1hM,6965
|
9
|
+
dataeval/_internal/detectors/drift/torch.py,sha256=NsQYfDVRcCGmU8k6oBG_aVzmML1zre-xUKBVK1W680o,10872
|
10
|
+
dataeval/_internal/detectors/drift/uncertainty.py,sha256=WeTQWZDL00-cs50neoJzs_9xBIdo-xxcj6bx0DApCQY,5263
|
11
|
+
dataeval/_internal/detectors/duplicates.py,sha256=6MVsopmgA4NccWtriHHvz_3y3tgWWu_vzFcxpXgS5DI,1469
|
12
|
+
dataeval/_internal/detectors/linter.py,sha256=Y8XIAbzMUe1UuYQLsZ1UyF1yVle2ExuzUmDQD2TF0z0,2792
|
13
|
+
dataeval/_internal/detectors/ood/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
14
|
+
dataeval/_internal/detectors/ood/ae.py,sha256=nM-uI-7h548yEc25iKlPYXTdG6RqDPA7Mup5IqPW7cg,2576
|
15
|
+
dataeval/_internal/detectors/ood/aegmm.py,sha256=05yOyP1RDOqVWW30weW_YAgciwBYsUJy35nPSuKQD10,2340
|
16
|
+
dataeval/_internal/detectors/ood/base.py,sha256=IxDhU4T8vlURvKkqakrwgUnWhl5a_u9_kXvrKkzWrog,6771
|
17
|
+
dataeval/_internal/detectors/ood/llr.py,sha256=EGgj1LV0qq-fE9RoWwL_6WXPaSTfhbnXkq25VtTiB3s,10029
|
18
|
+
dataeval/_internal/detectors/ood/vae.py,sha256=uh0QJ3b1_SQllHhb7BSt7IV369dmHjlU5PBwrOMesQg,2892
|
19
|
+
dataeval/_internal/detectors/ood/vaegmm.py,sha256=Yr5dKWSEcfP0xqqHJIM98QhS62oS8aYMj6edl26bKHI,2766
|
20
|
+
dataeval/_internal/flags.py,sha256=dRApeFkdSXFbYHSmvzgUP78zH8jUGtfzKFfLQtX0Q18,883
|
21
|
+
dataeval/_internal/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
22
|
+
dataeval/_internal/metrics/base.py,sha256=hUxJO-k0L7_nfach6xMW96YuMuu--A8Ypz7plYKGQA8,2623
|
23
|
+
dataeval/_internal/metrics/ber.py,sha256=HI2Cn-YWkqPz-mVZ8JgwubgjKYFyquq7L3yuKo-s8vc,3970
|
24
|
+
dataeval/_internal/metrics/coverage.py,sha256=eSOiCyqTN6t9OnhgFcPOp6HIReQvNL3tw6PzenrJ8aw,2700
|
25
|
+
dataeval/_internal/metrics/divergence.py,sha256=aihP84c8SUQ7IMUmBebFKtJxbH9VDfV_R-QtWEdUkvA,2823
|
26
|
+
dataeval/_internal/metrics/hash.py,sha256=5hdxMU2mFA9GxjF1MfhE_ztjJh7Ku1XpP82B_N54dwc,2782
|
27
|
+
dataeval/_internal/metrics/parity.py,sha256=9JunlLgrdbB2EPDKEtUDQnkBrsNiFJZD58dg3lSiYgE,6823
|
28
|
+
dataeval/_internal/metrics/stats.py,sha256=WnuO0tNnpz7mYi5_rmxdk8wZoEZFczo1DrV828lgcVk,12118
|
29
|
+
dataeval/_internal/metrics/uap.py,sha256=7QmJb-wM2B6c9ORa9kAar657KhDFScjNOGhpbuTv8YA,1149
|
30
|
+
dataeval/_internal/metrics/utils.py,sha256=u1kkGtS0irnx9dZTo9MahA-_4_uIorPDttQkBe8iU7U,4120
|
31
|
+
dataeval/_internal/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
32
|
+
dataeval/_internal/models/pytorch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
33
|
+
dataeval/_internal/models/pytorch/autoencoder.py,sha256=iK3Z9claesU_pJkRaiFJIZ9zKZg-Qj8ugzVYTTokDbE,6123
|
34
|
+
dataeval/_internal/models/pytorch/blocks.py,sha256=pm2xwsDZjZJYXrhhiz8husvh2vHmrkFMSYEn-EDUD5Q,1354
|
35
|
+
dataeval/_internal/models/pytorch/utils.py,sha256=Qgwym1PxGuwxbXCKUT-8r6Iyrxqm7x94oj45Vf5_CjE,1675
|
36
|
+
dataeval/_internal/models/tensorflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
37
|
+
dataeval/_internal/models/tensorflow/autoencoder.py,sha256=rErnOfDFTd7e4brSGQ2Lr1x1kNjSEHdbOREOtUfIhIM,9975
|
38
|
+
dataeval/_internal/models/tensorflow/gmm.py,sha256=wnqQKm3fURuvBROUd2fitCqzKViDo-g0-Djr3TBHZ3U,3640
|
39
|
+
dataeval/_internal/models/tensorflow/losses.py,sha256=3y6tHm7PTQ7hmasJDwTXjdARjCUWycoXqSyXJ1uT2mM,3766
|
40
|
+
dataeval/_internal/models/tensorflow/pixelcnn.py,sha256=B5cwB2IGPw-7b8klt82j_60g_IvqSiDELxvbiBYJtAo,48068
|
41
|
+
dataeval/_internal/models/tensorflow/trainer.py,sha256=2KHtMRniVselCaDXeb8QEfX-wMRsPfT1xiG2gUQgelg,4090
|
42
|
+
dataeval/_internal/models/tensorflow/utils.py,sha256=uK_fQ1JXUSVi0kgnhd9eRArlr36OzXUEdL4inJZCs-8,8579
|
43
|
+
dataeval/_internal/workflows/sufficiency.py,sha256=QZQdhERVr3EmPA2sAFFudSDL4KLM0eAvYgV38jhjzaI,18374
|
44
|
+
dataeval/detectors/__init__.py,sha256=I2e7YWb55RRlKQll85Z6KdN5wdBa53smn-_fcZIsCwA,1507
|
45
|
+
dataeval/flags/__init__.py,sha256=1-HmwmtfPkHWwqXUjDwWko396qAKBeaSvqVsQZLrzD0,170
|
46
|
+
dataeval/metrics/__init__.py,sha256=uVz0GtRvCsh_r08qaBM-jc4bjcfAb1yzlIyx6zfPc9Y,358
|
47
|
+
dataeval/models/__init__.py,sha256=onevPb5wznCggowBnVT0OUa8uBJXZCbrkFuek1UFvOs,293
|
48
|
+
dataeval/models/tensorflow/__init__.py,sha256=A1XRxVGHefuvh_WpaKE1x95pRD1FecuFp66iuNPA_5U,424
|
49
|
+
dataeval/models/torch/__init__.py,sha256=su7P9DF9LChlVCNHWG6d7s_yeIfWQbhCYWIkzJe0Qig,190
|
50
|
+
dataeval/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
51
|
+
dataeval/workflows/__init__.py,sha256=ObgS1cVYFRzFZWbNzGs2OcU02IVkJkAMHNnlnSNTMCE,208
|
52
|
+
dataeval-0.61.0.dist-info/LICENSE.txt,sha256=Kpzcfobf1HlqafF-EX6dQLw9TlJiaJzfgvLQFukyXYw,1060
|
53
|
+
dataeval-0.61.0.dist-info/METADATA,sha256=-d5akTEAHSqWXZ1JeCTkIvxDc-HdECfNxcLWQlFqhhs,4238
|
54
|
+
dataeval-0.61.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
55
|
+
dataeval-0.61.0.dist-info/RECORD,,
|