dataeval 0.61.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. dataeval/__init__.py +18 -0
  2. dataeval/_internal/detectors/__init__.py +0 -0
  3. dataeval/_internal/detectors/clusterer.py +469 -0
  4. dataeval/_internal/detectors/drift/__init__.py +0 -0
  5. dataeval/_internal/detectors/drift/base.py +265 -0
  6. dataeval/_internal/detectors/drift/cvm.py +97 -0
  7. dataeval/_internal/detectors/drift/ks.py +100 -0
  8. dataeval/_internal/detectors/drift/mmd.py +166 -0
  9. dataeval/_internal/detectors/drift/torch.py +310 -0
  10. dataeval/_internal/detectors/drift/uncertainty.py +149 -0
  11. dataeval/_internal/detectors/duplicates.py +49 -0
  12. dataeval/_internal/detectors/linter.py +78 -0
  13. dataeval/_internal/detectors/ood/__init__.py +0 -0
  14. dataeval/_internal/detectors/ood/ae.py +77 -0
  15. dataeval/_internal/detectors/ood/aegmm.py +69 -0
  16. dataeval/_internal/detectors/ood/base.py +199 -0
  17. dataeval/_internal/detectors/ood/llr.py +284 -0
  18. dataeval/_internal/detectors/ood/vae.py +86 -0
  19. dataeval/_internal/detectors/ood/vaegmm.py +79 -0
  20. dataeval/_internal/flags.py +47 -0
  21. dataeval/_internal/metrics/__init__.py +0 -0
  22. dataeval/_internal/metrics/base.py +92 -0
  23. dataeval/_internal/metrics/ber.py +124 -0
  24. dataeval/_internal/metrics/coverage.py +80 -0
  25. dataeval/_internal/metrics/divergence.py +94 -0
  26. dataeval/_internal/metrics/hash.py +79 -0
  27. dataeval/_internal/metrics/parity.py +180 -0
  28. dataeval/_internal/metrics/stats.py +332 -0
  29. dataeval/_internal/metrics/uap.py +45 -0
  30. dataeval/_internal/metrics/utils.py +158 -0
  31. dataeval/_internal/models/__init__.py +0 -0
  32. dataeval/_internal/models/pytorch/__init__.py +0 -0
  33. dataeval/_internal/models/pytorch/autoencoder.py +202 -0
  34. dataeval/_internal/models/pytorch/blocks.py +46 -0
  35. dataeval/_internal/models/pytorch/utils.py +67 -0
  36. dataeval/_internal/models/tensorflow/__init__.py +0 -0
  37. dataeval/_internal/models/tensorflow/autoencoder.py +317 -0
  38. dataeval/_internal/models/tensorflow/gmm.py +115 -0
  39. dataeval/_internal/models/tensorflow/losses.py +107 -0
  40. dataeval/_internal/models/tensorflow/pixelcnn.py +1106 -0
  41. dataeval/_internal/models/tensorflow/trainer.py +102 -0
  42. dataeval/_internal/models/tensorflow/utils.py +254 -0
  43. dataeval/_internal/workflows/sufficiency.py +555 -0
  44. dataeval/detectors/__init__.py +29 -0
  45. dataeval/flags/__init__.py +3 -0
  46. dataeval/metrics/__init__.py +7 -0
  47. dataeval/models/__init__.py +15 -0
  48. dataeval/models/tensorflow/__init__.py +6 -0
  49. dataeval/models/torch/__init__.py +8 -0
  50. dataeval/py.typed +0 -0
  51. dataeval/workflows/__init__.py +8 -0
  52. dataeval-0.61.0.dist-info/LICENSE.txt +21 -0
  53. dataeval-0.61.0.dist-info/METADATA +114 -0
  54. dataeval-0.61.0.dist-info/RECORD +55 -0
  55. dataeval-0.61.0.dist-info/WHEEL +4 -0
@@ -0,0 +1,114 @@
1
+ Metadata-Version: 2.1
2
+ Name: dataeval
3
+ Version: 0.61.0
4
+ Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
5
+ Home-page: https://dataeval.ai/
6
+ License: MIT
7
+ Author: Andrew Weng
8
+ Author-email: andrew.weng@ariacoustics.com
9
+ Maintainer: ARiA
10
+ Maintainer-email: dataeval@ariacoustics.com
11
+ Requires-Python: >=3.9,<3.12
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.9
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3 :: Only
21
+ Classifier: Topic :: Scientific/Engineering
22
+ Provides-Extra: all
23
+ Provides-Extra: tensorflow
24
+ Provides-Extra: torch
25
+ Requires-Dist: hdbscan (>=0.8.36)
26
+ Requires-Dist: maite
27
+ Requires-Dist: matplotlib ; extra == "torch" or extra == "all"
28
+ Requires-Dist: numpy (>1.24.3)
29
+ Requires-Dist: nvidia-cudnn-cu11 (>=8.6.0.163) ; extra == "tensorflow" or extra == "torch" or extra == "all"
30
+ Requires-Dist: pillow (>=10.3.0)
31
+ Requires-Dist: scikit-learn (>=1.5.0)
32
+ Requires-Dist: scipy (>=1.10)
33
+ Requires-Dist: tensorflow (>=2.14.1,<2.16) ; extra == "tensorflow" or extra == "all"
34
+ Requires-Dist: tensorflow-io-gcs-filesystem (>=0.35.0,<0.37) ; extra == "tensorflow" or extra == "all"
35
+ Requires-Dist: tensorflow_probability (>=0.22.1,<0.24) ; extra == "tensorflow" or extra == "all"
36
+ Requires-Dist: torch (>=2.0.1,!=2.2.0) ; extra == "torch" or extra == "all"
37
+ Requires-Dist: xxhash (>=3.3)
38
+ Project-URL: Documentation, https://dataeval.readthedocs.io/
39
+ Project-URL: Repository, https://github.com/aria-ml/dataeval/
40
+ Description-Content-Type: text/markdown
41
+
42
+ # DataEval
43
+
44
+ ## About DataEval
45
+
46
+ DataEval focuses on characterizing image data and its impact on model performance across classification and object-detection tasks.
47
+
48
+ <!-- start about -->
49
+
50
+ **Model-agnostic metrics that bound real-world performance**
51
+ - relevance/completeness/coverage
52
+ - metafeatures (data complexity)
53
+
54
+ **Model-specific metrics that guide model selection and training**
55
+ - dataset sufficiency
56
+ - data/model complexity mismatch
57
+
58
+ **Metrics for post-deployment monitoring of data with bounds on model performance to guide retraining**
59
+ - dataset-shift metrics
60
+ - model performance bounds under covariate shift
61
+ - guidance on sampling to assess model error and model retraining
62
+
63
+ <!-- end about -->
64
+
65
+ ## Getting Started
66
+
67
+ ### Requirements
68
+ - Python 3.9-3.11
69
+
70
+ ### Installing DataEval
71
+
72
+ You can install DataEval directly from pypi.org using the following command. The optional dependencies of DataEval are `torch`, `tensorflow` and `all`. Using `torch` enables Sufficiency metrics, and `tensorflow` enables OOD Detection.
73
+
74
+ ```
75
+ pip install dataeval[all]
76
+ ```
77
+
78
+ ### Installing DataEval from GitHub
79
+
80
+ To install DataEval from source locally on Ubuntu, you will need `git-lfs` to download larger, binary source files and `poetry` for project dependency management.
81
+
82
+ ```
83
+ sudo apt-get install git-lfs
84
+ pip install poetry
85
+ ```
86
+
87
+ Pull the source down and change to the DataEval project directory.
88
+ ```
89
+ git clone https://github.com/aria-ml/dataeval.git
90
+ cd dataeval
91
+ ```
92
+
93
+
94
+
95
+ Install DataEval with optional dependencies for development.
96
+ ```
97
+ poetry install --all-extras --with dev
98
+ ```
99
+
100
+ Now that DataEval is installed, you can run commands in the poetry virtual environment by prefixing shell commands with `poetry run`, or activate the virtual environment directly in the shell.
101
+ ```
102
+ poetry shell
103
+ ```
104
+
105
+ ### Documentation and Tutorials
106
+ For more ideas on getting started using DataEval in your workflow, additional information and tutorials are in our Sphinx documentation hosted on [Read the Docs](https://dataeval.readthedocs.io/).
107
+
108
+ ## Attribution
109
+ This project uses code from the [Alibi-Detect](https://github.com/SeldonIO/alibi-detect) python library developed by SeldonIO. Additional documentation from the developers are also available [here](https://docs.seldon.io/projects/alibi-detect/en/stable/).
110
+
111
+ ## POCs
112
+ - **POC**: Scott Swan @scott.swan
113
+ - **DPOC**: Andrew Weng @aweng
114
+
@@ -0,0 +1,55 @@
1
+ dataeval/__init__.py,sha256=hoMLXpAqvfNQ6BtdLD49hOXWMUp5_4zBg_IPeuGPyZA,408
2
+ dataeval/_internal/detectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
+ dataeval/_internal/detectors/clusterer.py,sha256=zuWW7qfbQWQ777TqBOsDp2_fEbFmuUzNqOvTWv8xijo,19193
4
+ dataeval/_internal/detectors/drift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
+ dataeval/_internal/detectors/drift/base.py,sha256=b6_kUHGPtfDnN6S9gNkldpoaGwdRKw6ohGI4wGega24,8981
6
+ dataeval/_internal/detectors/drift/cvm.py,sha256=3YePeStvRSKVFqHEWshngXhJTCh3cn9-9B-Ou7FQHgM,3897
7
+ dataeval/_internal/detectors/drift/ks.py,sha256=PW4qB4XbDUhVxpKJeA39-0GYVgeAxWggipBh2bhKDKg,3924
8
+ dataeval/_internal/detectors/drift/mmd.py,sha256=ZJu28o6Brs0-pt3PVY2ysMKVfLYXaEz754I-lSLn1hM,6965
9
+ dataeval/_internal/detectors/drift/torch.py,sha256=NsQYfDVRcCGmU8k6oBG_aVzmML1zre-xUKBVK1W680o,10872
10
+ dataeval/_internal/detectors/drift/uncertainty.py,sha256=WeTQWZDL00-cs50neoJzs_9xBIdo-xxcj6bx0DApCQY,5263
11
+ dataeval/_internal/detectors/duplicates.py,sha256=6MVsopmgA4NccWtriHHvz_3y3tgWWu_vzFcxpXgS5DI,1469
12
+ dataeval/_internal/detectors/linter.py,sha256=Y8XIAbzMUe1UuYQLsZ1UyF1yVle2ExuzUmDQD2TF0z0,2792
13
+ dataeval/_internal/detectors/ood/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
14
+ dataeval/_internal/detectors/ood/ae.py,sha256=nM-uI-7h548yEc25iKlPYXTdG6RqDPA7Mup5IqPW7cg,2576
15
+ dataeval/_internal/detectors/ood/aegmm.py,sha256=05yOyP1RDOqVWW30weW_YAgciwBYsUJy35nPSuKQD10,2340
16
+ dataeval/_internal/detectors/ood/base.py,sha256=IxDhU4T8vlURvKkqakrwgUnWhl5a_u9_kXvrKkzWrog,6771
17
+ dataeval/_internal/detectors/ood/llr.py,sha256=EGgj1LV0qq-fE9RoWwL_6WXPaSTfhbnXkq25VtTiB3s,10029
18
+ dataeval/_internal/detectors/ood/vae.py,sha256=uh0QJ3b1_SQllHhb7BSt7IV369dmHjlU5PBwrOMesQg,2892
19
+ dataeval/_internal/detectors/ood/vaegmm.py,sha256=Yr5dKWSEcfP0xqqHJIM98QhS62oS8aYMj6edl26bKHI,2766
20
+ dataeval/_internal/flags.py,sha256=dRApeFkdSXFbYHSmvzgUP78zH8jUGtfzKFfLQtX0Q18,883
21
+ dataeval/_internal/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
22
+ dataeval/_internal/metrics/base.py,sha256=hUxJO-k0L7_nfach6xMW96YuMuu--A8Ypz7plYKGQA8,2623
23
+ dataeval/_internal/metrics/ber.py,sha256=HI2Cn-YWkqPz-mVZ8JgwubgjKYFyquq7L3yuKo-s8vc,3970
24
+ dataeval/_internal/metrics/coverage.py,sha256=eSOiCyqTN6t9OnhgFcPOp6HIReQvNL3tw6PzenrJ8aw,2700
25
+ dataeval/_internal/metrics/divergence.py,sha256=aihP84c8SUQ7IMUmBebFKtJxbH9VDfV_R-QtWEdUkvA,2823
26
+ dataeval/_internal/metrics/hash.py,sha256=5hdxMU2mFA9GxjF1MfhE_ztjJh7Ku1XpP82B_N54dwc,2782
27
+ dataeval/_internal/metrics/parity.py,sha256=9JunlLgrdbB2EPDKEtUDQnkBrsNiFJZD58dg3lSiYgE,6823
28
+ dataeval/_internal/metrics/stats.py,sha256=WnuO0tNnpz7mYi5_rmxdk8wZoEZFczo1DrV828lgcVk,12118
29
+ dataeval/_internal/metrics/uap.py,sha256=7QmJb-wM2B6c9ORa9kAar657KhDFScjNOGhpbuTv8YA,1149
30
+ dataeval/_internal/metrics/utils.py,sha256=u1kkGtS0irnx9dZTo9MahA-_4_uIorPDttQkBe8iU7U,4120
31
+ dataeval/_internal/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
32
+ dataeval/_internal/models/pytorch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
33
+ dataeval/_internal/models/pytorch/autoencoder.py,sha256=iK3Z9claesU_pJkRaiFJIZ9zKZg-Qj8ugzVYTTokDbE,6123
34
+ dataeval/_internal/models/pytorch/blocks.py,sha256=pm2xwsDZjZJYXrhhiz8husvh2vHmrkFMSYEn-EDUD5Q,1354
35
+ dataeval/_internal/models/pytorch/utils.py,sha256=Qgwym1PxGuwxbXCKUT-8r6Iyrxqm7x94oj45Vf5_CjE,1675
36
+ dataeval/_internal/models/tensorflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
37
+ dataeval/_internal/models/tensorflow/autoencoder.py,sha256=rErnOfDFTd7e4brSGQ2Lr1x1kNjSEHdbOREOtUfIhIM,9975
38
+ dataeval/_internal/models/tensorflow/gmm.py,sha256=wnqQKm3fURuvBROUd2fitCqzKViDo-g0-Djr3TBHZ3U,3640
39
+ dataeval/_internal/models/tensorflow/losses.py,sha256=3y6tHm7PTQ7hmasJDwTXjdARjCUWycoXqSyXJ1uT2mM,3766
40
+ dataeval/_internal/models/tensorflow/pixelcnn.py,sha256=B5cwB2IGPw-7b8klt82j_60g_IvqSiDELxvbiBYJtAo,48068
41
+ dataeval/_internal/models/tensorflow/trainer.py,sha256=2KHtMRniVselCaDXeb8QEfX-wMRsPfT1xiG2gUQgelg,4090
42
+ dataeval/_internal/models/tensorflow/utils.py,sha256=uK_fQ1JXUSVi0kgnhd9eRArlr36OzXUEdL4inJZCs-8,8579
43
+ dataeval/_internal/workflows/sufficiency.py,sha256=QZQdhERVr3EmPA2sAFFudSDL4KLM0eAvYgV38jhjzaI,18374
44
+ dataeval/detectors/__init__.py,sha256=I2e7YWb55RRlKQll85Z6KdN5wdBa53smn-_fcZIsCwA,1507
45
+ dataeval/flags/__init__.py,sha256=1-HmwmtfPkHWwqXUjDwWko396qAKBeaSvqVsQZLrzD0,170
46
+ dataeval/metrics/__init__.py,sha256=uVz0GtRvCsh_r08qaBM-jc4bjcfAb1yzlIyx6zfPc9Y,358
47
+ dataeval/models/__init__.py,sha256=onevPb5wznCggowBnVT0OUa8uBJXZCbrkFuek1UFvOs,293
48
+ dataeval/models/tensorflow/__init__.py,sha256=A1XRxVGHefuvh_WpaKE1x95pRD1FecuFp66iuNPA_5U,424
49
+ dataeval/models/torch/__init__.py,sha256=su7P9DF9LChlVCNHWG6d7s_yeIfWQbhCYWIkzJe0Qig,190
50
+ dataeval/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
+ dataeval/workflows/__init__.py,sha256=ObgS1cVYFRzFZWbNzGs2OcU02IVkJkAMHNnlnSNTMCE,208
52
+ dataeval-0.61.0.dist-info/LICENSE.txt,sha256=Kpzcfobf1HlqafF-EX6dQLw9TlJiaJzfgvLQFukyXYw,1060
53
+ dataeval-0.61.0.dist-info/METADATA,sha256=-d5akTEAHSqWXZ1JeCTkIvxDc-HdECfNxcLWQlFqhhs,4238
54
+ dataeval-0.61.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
55
+ dataeval-0.61.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: poetry-core 1.9.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any