dataeval 0.61.0__tar.gz

dataeval-0.61.0/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 ARiA
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dataeval-0.61.0/PKG-INFO

@@ -0,0 +1,114 @@
+Metadata-Version: 2.1
+Name: dataeval
+Version: 0.61.0
+Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
+Home-page: https://dataeval.ai/
+License: MIT
+Author: Andrew Weng
+Author-email: andrew.weng@ariacoustics.com
+Maintainer: ARiA
+Maintainer-email: dataeval@ariacoustics.com
+Requires-Python: >=3.9,<3.12
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering
+Provides-Extra: all
+Provides-Extra: tensorflow
+Provides-Extra: torch
+Requires-Dist: hdbscan (>=0.8.36)
+Requires-Dist: maite
+Requires-Dist: matplotlib ; extra == "torch" or extra == "all"
+Requires-Dist: numpy (>1.24.3)
+Requires-Dist: nvidia-cudnn-cu11 (>=8.6.0.163) ; extra == "tensorflow" or extra == "torch" or extra == "all"
+Requires-Dist: pillow (>=10.3.0)
+Requires-Dist: scikit-learn (>=1.5.0)
+Requires-Dist: scipy (>=1.10)
+Requires-Dist: tensorflow (>=2.14.1,<2.16) ; extra == "tensorflow" or extra == "all"
+Requires-Dist: tensorflow-io-gcs-filesystem (>=0.35.0,<0.37) ; extra == "tensorflow" or extra == "all"
+Requires-Dist: tensorflow_probability (>=0.22.1,<0.24) ; extra == "tensorflow" or extra == "all"
+Requires-Dist: torch (>=2.0.1,!=2.2.0) ; extra == "torch" or extra == "all"
+Requires-Dist: xxhash (>=3.3)
+Project-URL: Documentation, https://dataeval.readthedocs.io/
+Project-URL: Repository, https://github.com/aria-ml/dataeval/
+Description-Content-Type: text/markdown
+
+# DataEval
+
+## About DataEval
+
+DataEval focuses on characterizing image data and its impact on model performance across classification and object-detection tasks.
+
+<!-- start about -->
+
+**Model-agnostic metrics that bound real-world performance**
+- relevance/completeness/coverage
+- metafeatures (data complexity)
+
+**Model-specific metrics that guide model selection and training**
+- dataset sufficiency
+- data/model complexity mismatch
+
+**Metrics for post-deployment monitoring of data with bounds on model performance to guide retraining**
+- dataset-shift metrics
+- model performance bounds under covariate shift
+- guidance on sampling to assess model error and model retraining
+
+<!-- end about -->
+
+## Getting Started
+
+### Requirements
+- Python 3.9-3.11
+
+### Installing DataEval
+
+You can install DataEval directly from pypi.org using the following command.  The optional dependencies of DataEval are `torch`, `tensorflow` and `all`.  Using `torch` enables Sufficiency metrics, and `tensorflow` enables OOD Detection.
+
+```
+pip install dataeval[all]
+```
+
+### Installing DataEval from GitHub
+
+To install DataEval from source locally on Ubuntu, you will need `git-lfs` to download larger, binary source files and `poetry` for project dependency management.
+
+```
+sudo apt-get install git-lfs
+pip install poetry
+```
+
+Pull the source down and change to the DataEval project directory.
+```
+git clone https://github.com/aria-ml/dataeval.git
+cd dataeval
+```
+
+
+
+Install DataEval with optional dependencies for development.
+```
+poetry install --all-extras --with dev
+```
+
+Now that DataEval is installed, you can run commands in the poetry virtual environment by prefixing shell commands with `poetry run`, or activate the virtual environment directly in the shell.
+```
+poetry shell
+```
+
+### Documentation and Tutorials
+For more ideas on getting started using DataEval in your workflow, additional information and tutorials are in our Sphinx documentation hosted on [Read the Docs](https://dataeval.readthedocs.io/).
+
+## Attribution
+This project uses code from the [Alibi-Detect](https://github.com/SeldonIO/alibi-detect) python library developed by SeldonIO.  Additional documentation from the developers are also available [here](https://docs.seldon.io/projects/alibi-detect/en/stable/).
+
+## POCs
+- **POC**: Scott Swan @scott.swan
+- **DPOC**: Andrew Weng @aweng
+

dataeval-0.61.0/README.md

@@ -0,0 +1,72 @@
+# DataEval
+
+## About DataEval
+
+DataEval focuses on characterizing image data and its impact on model performance across classification and object-detection tasks.
+
+<!-- start about -->
+
+**Model-agnostic metrics that bound real-world performance**
+- relevance/completeness/coverage
+- metafeatures (data complexity)
+
+**Model-specific metrics that guide model selection and training**
+- dataset sufficiency
+- data/model complexity mismatch
+
+**Metrics for post-deployment monitoring of data with bounds on model performance to guide retraining**
+- dataset-shift metrics
+- model performance bounds under covariate shift
+- guidance on sampling to assess model error and model retraining
+
+<!-- end about -->
+
+## Getting Started
+
+### Requirements
+- Python 3.9-3.11
+
+### Installing DataEval
+
+You can install DataEval directly from pypi.org using the following command.  The optional dependencies of DataEval are `torch`, `tensorflow` and `all`.  Using `torch` enables Sufficiency metrics, and `tensorflow` enables OOD Detection.
+
+```
+pip install dataeval[all]
+```
+
+### Installing DataEval from GitHub
+
+To install DataEval from source locally on Ubuntu, you will need `git-lfs` to download larger, binary source files and `poetry` for project dependency management.
+
+```
+sudo apt-get install git-lfs
+pip install poetry
+```
+
+Pull the source down and change to the DataEval project directory.
+```
+git clone https://github.com/aria-ml/dataeval.git
+cd dataeval
+```
+
+
+
+Install DataEval with optional dependencies for development.
+```
+poetry install --all-extras --with dev
+```
+
+Now that DataEval is installed, you can run commands in the poetry virtual environment by prefixing shell commands with `poetry run`, or activate the virtual environment directly in the shell.
+```
+poetry shell
+```
+
+### Documentation and Tutorials
+For more ideas on getting started using DataEval in your workflow, additional information and tutorials are in our Sphinx documentation hosted on [Read the Docs](https://dataeval.readthedocs.io/).
+
+## Attribution
+This project uses code from the [Alibi-Detect](https://github.com/SeldonIO/alibi-detect) python library developed by SeldonIO.  Additional documentation from the developers are also available [here](https://docs.seldon.io/projects/alibi-detect/en/stable/).
+
+## POCs
+- **POC**: Scott Swan @scott.swan
+- **DPOC**: Andrew Weng @aweng

dataeval-0.61.0/pyproject.toml

@@ -0,0 +1,177 @@
+[tool.poetry]
+name = "dataeval"
+version = "0.61.0" # dynamic
+description = "DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks"
+license = "MIT"
+readme = "README.md"
+homepage = "https://dataeval.ai/"
+repository = "https://github.com/aria-ml/dataeval/"
+documentation = "https://dataeval.readthedocs.io/"
+
+authors = [
+  "Andrew Weng <andrew.weng@ariacoustics.com>",
+  "James Gleeson <james.gleeson@ariacoustics.com>",
+  "Scott Swan <scott.swan@ariacoustics.com>",
+  "Shaun Jullens <shaun.jullens@ariacoustics.com>",
+  "Thayer Fisher <thayer.fisher@ariacoustics.com>",
+]
+maintainers = [
+  "ARiA <dataeval@ariacoustics.com>"
+]
+
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Operating System :: OS Independent",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Topic :: Scientific/Engineering",
+]
+
+packages = [
+  {include = "dataeval", from = "src"}
+]
+
+[tool.poetry.dependencies]
+# required
+python = ">=3.9,<3.12"
+hdbscan = {version = ">=0.8.36"}
+numpy = {version = ">1.24.3"}
+pillow = {version = ">=10.3.0"}
+scipy = {version = ">=1.10"}
+scikit-learn = {version = ">=1.5.0"}
+xxhash = {version = ">=3.3"}
+maite = {version = "*"}
+
+# optional
+matplotlib = {version = "*", optional = true}
+nvidia-cudnn-cu11 = {version = ">=8.6.0.163", optional = true}
+tensorflow = {version = ">=2.14.1, <2.16", optional = true}
+tensorflow-io-gcs-filesystem = {version = ">=0.35.0, <0.37", optional = true}
+tensorflow_probability = {version = ">=0.22.1, <0.24", optional = true}
+torch = {version = ">=2.0.1, !=2.2.0", source = "pytorch", optional = true}
+
+[tool.poetry.extras]
+tensorflow = ["tensorflow", "tensorflow-io-gcs-filesystem", "tensorflow_probability", "nvidia-cudnn-cu11"]
+torch = ["torch", "matplotlib", "nvidia-cudnn-cu11"]
+all = ["matplotlib", "nvidia-cudnn-cu11", "tensorflow", "tensorflow-io-gcs-filesystem", "tensorflow_probability", "torch"]
+
+[tool.poetry.group.dev]
+optional = true
+
+[tool.poetry.group.dev.dependencies]
+tox = {version = "*"}
+tox-uv = {version = "*"}
+uv = {version = "*"}
+poetry = {version = "*"}
+# lint
+ruff = {version = "*"}
+codespell = {version = "*", extras = ["toml"]}
+# unit
+pytest = {version = "*"}
+pytest-cov = {version = "*"}
+pytest-xdist = {version = "*"}
+coverage = {version = "*", extras = ["toml"]}
+torchmetrics = {version = ">=1.0.0", source = "pytorch"}
+# type
+pyright = {version = "*, !=1.1.340"}
+# docs
+certifi = {version = ">=2024.07.04", python = "~3.11"}
+enum_tools = {version = "0.12.0", extras = ["sphinx"], python = "~3.11"}
+ipykernel = {version = "6.26.0", python = "~3.11"}
+ipywidgets = {version = "8.1.1", python = "~3.11"}
+jupyter-client = {version = "8.6.0", python = "~3.11"}
+jupyter-cache = {version = "*", python = "~3.11"}
+myst-nb = {version = "1.0.0", python = "~3.11"} 
+protobuf = {version = "4.25.3", python = "~3.11"} 
+sphinx-rtd-size = {version = "0.2.0", python = "~3.11"}
+sphinx-rtd-theme = {version = "1.3.0", python = "~3.11"}
+sphinx-design = {version = "*", python = "~3.11"}
+sphinx-tabs = {version = "*", python = "~3.11"}
+Sphinx = {version = "7.2.6", python = "~3.11"}
+tensorflow-datasets = {version = "4.9.3", python = "~3.11"}
+torchvision = {version = ">=0.16.0", source = "pytorch", python = "~3.11"}
+
+[[tool.poetry.source]]
+name = "pytorch"
+url = "https://download.pytorch.org/whl/cu118"
+priority = "explicit"
+
+[tool.poetry-dynamic-versioning]
+enable = false
+vcs = "git"
+style = "semver"
+pattern = "v(?P<base>\\d+\\.\\d+\\.\\d+)$"
+
+[tool.poetry-dynamic-versioning.substitution]
+files = ["src/dataeval/__init__.py"]
+
+[tool.pyright]
+reportMissingImports = false
+
+[tool.pytest.ini_options]
+norecursedirs = ["prototype"]
+addopts = ["--pythonwarnings=ignore::DeprecationWarning", "--verbose", "--durations=20", "--durations-min=1.0"]
+
+[tool.coverage.run]
+source = ["dataeval"]
+branch = true
+
+[tool.coverage.report]
+exclude_also = [
+  "raise NotImplementedError"
+]
+omit = [
+  "*/_internal/models/tensorflow/pixelcnn.py",
+  "*/_prototype/*",
+  "/tmp/*",
+  "tests/*"
+]
+fail_under = 90
+
+# Ruff rules - https://docs.astral.sh/ruff/rules/
+[tool.ruff]
+exclude = [
+  ".devcontainer",
+  ".github",
+  ".vscode",
+  ".jupyter_cache",
+  "*env*",
+  "output",
+  "_build",
+  ".tox",
+  "prototype",
+]
+line-length = 120
+indent-width = 4
+target-version = "py38"
+extend-include = ["*.ipynb"]
+
+[tool.ruff.lint]
+select = ["A", "E", "F", "C4", "I", "UP", "NPY", "SIM", "RUF100"]
+ignore = ["NPY002"]
+fixable = ["ALL"]
+unfixable = []
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+per-file-ignores = { "*.ipynb" = ["E402"] }
+
+[tool.ruff.lint.isort]
+known-first-party = ["dataeval"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+docstring-code-format = true
+docstring-code-line-length = "dynamic"
+
+[tool.codespell]
+skip = './*env*,./prototype,./docs/.jupyter_cache,./.tox,CHANGELOG.md,poetry.lock,./output,*.html'
+
+[build-system]
+requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
+build-backend = "poetry_dynamic_versioning.backend"

dataeval-0.61.0/src/dataeval/__init__.py

@@ -0,0 +1,18 @@
+from importlib.util import find_spec
+
+from . import detectors, flags, metrics
+
+__version__ = "0.61.0"
+
+__all__ = ["detectors", "flags", "metrics"]
+
+if find_spec("torch") is not None:  # pragma: no cover
+    from . import models, workflows
+
+    __all__ += ["models", "workflows"]
+elif find_spec("tensorflow") is not None:  # pragma: no cover
+    from . import models
+
+    __all__ += ["models"]
+
+del find_spec