checkmaite 0.2.0__tar.gz → 0.2.2__tar.gz

{checkmaite-0.2.0 → checkmaite-0.2.2}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: checkmaite
-Version: 0.2.0
+Version: 0.2.2
 Summary: checkmaite - an implementation of T&E tools using maite
-License: TBD
+License: Apache-2.0
 License-File: LICENSE.md
 Keywords: jatic,artificial intelligence,testing and evaluation
 Author: OpenTeams
@@ -16,11 +16,13 @@ Provides-Extra: aws
 Provides-Extra: azure
 Provides-Extra: cloud
 Provides-Extra: gcs
+Provides-Extra: onnx
+Provides-Extra: onnx-cuda
 Provides-Extra: reporting
 Provides-Extra: ui
 Requires-Dist: adlfs (>=2023.1.0) ; extra == "azure"
 Requires-Dist: adlfs (>=2023.1.0) ; extra == "cloud"
-Requires-Dist: dataeval (==1.0.3)
+Requires-Dist: dataeval (==1.0.6)
 Requires-Dist: dataeval-plots (==0.0.6)
 Requires-Dist: fsspec (>=2023.1.0)
 Requires-Dist: gcsfs (>=2023.1.0) ; extra == "cloud"
@@ -33,8 +35,12 @@ Requires-Dist: maite (==0.9.2)
 Requires-Dist: markdown (>=3.5) ; extra == "reporting"
 Requires-Dist: matplotlib (>=3.8.4,<3.10.8)
 Requires-Dist: nbconvert (>=7.16.6) ; extra == "ui"
-Requires-Dist: nrtk[pybsm] (==0.27.1)
+Requires-Dist: nrtk[pybsm] (==1.0.3)
 Requires-Dist: numba (>=0.60.0)
+Requires-Dist: onnx (>=1.14.0) ; extra == "onnx"
+Requires-Dist: onnx (>=1.14.0) ; extra == "onnx-cuda"
+Requires-Dist: onnxruntime (>=1.14.0,<1.24.0) ; extra == "onnx"
+Requires-Dist: onnxruntime-gpu (>=1.14.0,<1.24.0) ; extra == "onnx-cuda"
 Requires-Dist: opencv-python-headless (>=4.9.0.80)
 Requires-Dist: pandas (>=2)
 Requires-Dist: panel (>=1.8.7) ; extra == "ui"
@@ -69,21 +75,21 @@ Checkmaite is an API and a UI application which makes testing and evaluation of
 
 It was built as an integration point for all [CDAO JATIC](https://cdao.pages.jatic.net/public/) tools and has since expanded focus. Our goal is to make T&E easier for analysts!
 
-To learn more please visit our [published documentation](https://jatic.pages.jatic.net/reference-implementation/reference-implementation/).
+To learn more please visit our [published documentation](https://openteams-ai.github.io/checkmaite/).
 
 ## Installation
 
-For detailed installation instructions please refer to [Setup Guide](https://jatic.pages.jatic.net/reference-implementation/reference-implementation/community/setup.html)
+For detailed installation instructions please refer to [Setup Guide](https://openteams-ai.github.io/checkmaite/get-started/install_setup.html)
 
 ## Usage
 
-To learn how to get started using `checkmaite`, please visit our [quick start guide](https://jatic.pages.jatic.net/reference-implementation/reference-implementation/index.html#who-are-the-target-users-of-the-reference-implementation).
+To learn how to get started using `checkmaite`, please visit our [quick start guide](https://openteams-ai.github.io/checkmaite/get-started/index.html).
 
-There are also [How-to Guides](https://jatic.pages.jatic.net/reference-implementation/reference-implementation/how-tos/index.html) & [Tutorials](https://jatic.pages.jatic.net/reference-implementation/reference-implementation/tutorials/index.html) that will help get new users up to speed on the available functionality.
+There are also [How-to Guides](https://openteams-ai.github.io/checkmaite/development/index.html) & [Tutorials](https://openteams-ai.github.io/checkmaite/tool-usage/index.html) that will help get new users up to speed on the available functionality.
 
 ## Contributing
 
-The `checkmaite` team welcomes contributions of all forms - questions, documentation, and code contributions. Please visit our [contribution guide](https://jatic.pages.jatic.net/reference-implementation/reference-implementation/community/contributing.html).
+The `checkmaite` team welcomes contributions of all forms - questions, documentation, and code contributions. Please visit our [contribution guide](https://openteams-ai.github.io/checkmaite/development/contributing.html).
 
 ## Authors and acknowledgment
 

{checkmaite-0.2.0 → checkmaite-0.2.2}/README.md

@@ -9,21 +9,21 @@ Checkmaite is an API and a UI application which makes testing and evaluation of
 
 It was built as an integration point for all [CDAO JATIC](https://cdao.pages.jatic.net/public/) tools and has since expanded focus. Our goal is to make T&E easier for analysts!
 
-To learn more please visit our [published documentation](https://jatic.pages.jatic.net/reference-implementation/reference-implementation/).
+To learn more please visit our [published documentation](https://openteams-ai.github.io/checkmaite/).
 
 ## Installation
 
-For detailed installation instructions please refer to [Setup Guide](https://jatic.pages.jatic.net/reference-implementation/reference-implementation/community/setup.html)
+For detailed installation instructions please refer to [Setup Guide](https://openteams-ai.github.io/checkmaite/get-started/install_setup.html)
 
 ## Usage
 
-To learn how to get started using `checkmaite`, please visit our [quick start guide](https://jatic.pages.jatic.net/reference-implementation/reference-implementation/index.html#who-are-the-target-users-of-the-reference-implementation).
+To learn how to get started using `checkmaite`, please visit our [quick start guide](https://openteams-ai.github.io/checkmaite/get-started/index.html).
 
-There are also [How-to Guides](https://jatic.pages.jatic.net/reference-implementation/reference-implementation/how-tos/index.html) & [Tutorials](https://jatic.pages.jatic.net/reference-implementation/reference-implementation/tutorials/index.html) that will help get new users up to speed on the available functionality.
+There are also [How-to Guides](https://openteams-ai.github.io/checkmaite/development/index.html) & [Tutorials](https://openteams-ai.github.io/checkmaite/tool-usage/index.html) that will help get new users up to speed on the available functionality.
 
 ## Contributing
 
-The `checkmaite` team welcomes contributions of all forms - questions, documentation, and code contributions. Please visit our [contribution guide](https://jatic.pages.jatic.net/reference-implementation/reference-implementation/community/contributing.html).
+The `checkmaite` team welcomes contributions of all forms - questions, documentation, and code contributions. Please visit our [contribution guide](https://openteams-ai.github.io/checkmaite/development/contributing.html).
 
 ## Authors and acknowledgment
 

{checkmaite-0.2.0 → checkmaite-0.2.2}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry_dynamic_versioning.backend"
 name = "checkmaite"
 description = "checkmaite - an implementation of T&E tools using maite"
 readme = "README.md"
-license = { text = "TBD" }
+license = { text = "Apache-2.0" }
 authors = [
     { name = "OpenTeams" },
 ]
@@ -24,7 +24,7 @@ classifiers = [
 ]
 requires-python = ">=3.10, <3.13"
 dynamic = ["dependencies"]
-version = "0.2.0"
+version = "0.2.2"
 
 [tool.conda-lock]
 channels = ['conda-forge', 'fastai']
@@ -56,10 +56,10 @@ pattern = "default-unprefixed"  # required because our tags don't start with v
 fix-shallow-repository = true
 
 [tool.poetry.dependencies]
-dataeval = "==1.0.3"
+dataeval = "==1.0.6"
 dataeval-plots = "==0.0.6"                                        # We specify pypi source as dataeval_plots wont be on conda-forge
 maite = "==0.9.2"
-nrtk = { version = "==0.27.1", extras = ["pybsm"] }
+nrtk = { version = "==1.0.3", extras = ["pybsm"] }
 xaitk-jatic = "==0.8.1"
 # external deps
 fsspec = ">=2023.1.0"                                              # arbitrary, required for cloud storage support
@@ -74,6 +74,8 @@ ipython = ">=8.13,<9.0"                                            # necessary,
 matplotlib = ">=3.8.4,< 3.10.8"                                    # arbitrary, required by panel, 3.10.8 limit can be removed when artifact is released on PyPI
 nbconvert = { version = ">=7.16.6", optional = true }              # arbitrary, required for serving panel notebooks via CLI
 numba = ">=0.60.0"                                                 # arbitrary, needed to run visdrone model
+onnx = { version = ">=1.14.0", optional = true }                       # arbitrary, required for ONNX model validation
+onnxruntime = { version = ">=1.14.0,<1.24.0", optional = true }       # necessary upper pin, 1.24.x lacks cp310 wheels
 opencv-python-headless = ">=4.9.0.80"                              # necessary lower pin, required by nrtk usage here
 pandas = ">=2"                                                     # arbitrary, required for data handling
 panel = { version = ">=1.8.7", optional = true }                   # necessary for FileDropper widget, and for progression of dashboards
@@ -139,6 +141,14 @@ reporting = [
     "markdown>=3.5",
     "xhtml2pdf>=0.2.16",
 ]
+onnx = [
+    "onnx>=1.14.0",
+    "onnxruntime>=1.14.0,<1.24.0", # 1.24.x lacks CPython 3.10 wheels
+]
+onnx-cuda = [
+    "onnx>=1.14.0",
+    "onnxruntime-gpu>=1.14.0,<1.24.0", # 1.24.x lacks CPython 3.10 wheels
+]
 aws = ["s3fs>=2023.1.0"]
 gcs = ["gcsfs>=2023.1.0"]
 azure = ["adlfs>=2023.1.0"]

{checkmaite-0.2.0 → checkmaite-0.2.2}/src/checkmaite/__init__.py

@@ -9,15 +9,6 @@ import os
 import sys
 import warnings
 
-# Filter Pydantic warnings about TypedDict ReadOnly qualifiers
-# These occur because ReadOnly is a static type annotation but Pydantic attempts runtime validation.  As long as we are
-# using MAITE objects in our Pydantics models, these warnings will continue to appear. They are harmless but noisy.
-warnings.filterwarnings(
-    "ignore",
-    message=r"Items? .* on TypedDict class .* (?:is|are) using the `ReadOnly` qualifier\. Pydantic will not protect.*",
-    category=UserWarning,
-)
-
 
 def _configure_torch_mps_fallback() -> None:
     mps_env_var = "PYTORCH_ENABLE_MPS_FALLBACK"
@@ -112,6 +103,11 @@ PosixPath('/home/user/my-cache')
 
 from checkmaite.core import cached_tasks, capability_core, image_classification, object_detection, report  # noqa: E402
 from checkmaite.core._cache import binary_de_serializer  # noqa: E402
+from checkmaite.core.analytics_store._provenance import (  # noqa: E402
+    configure_provenance,
+    get_provenance_defaults,
+    reset_provenance,
+)
 
 __all__ = [
     "__version__",
@@ -122,4 +118,7 @@ __all__ = [
     "capability_core",
     "binary_de_serializer",
     "report",
+    "configure_provenance",
+    "get_provenance_defaults",
+    "reset_provenance",
 ]

{checkmaite-0.2.0 → checkmaite-0.2.2}/src/checkmaite/core/_common/dataeval_bias_capability.py

@@ -3,6 +3,7 @@ from pathlib import Path
 from typing import Any, Literal, cast
 
 import dataeval_plots as dep
+import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 import polars as pl
@@ -106,7 +107,7 @@ class DataevalBiasCoverageOutputs(CapabilityOutputsBase):
     image: Image | None = None
 
 
-class DataevalBiasOutputs(pydantic.BaseModel):
+class DataevalBiasOutputs(CapabilityOutputsBase):
     balance: DataevalBiasBalanceOutputs | None = None
     diversity: DataevalBiasDiversityOutputs | None = None
     coverage: DataevalBiasCoverageOutputs
@@ -444,7 +445,9 @@ class DataevalBiasBase(Capability[DataevalBiasOutputs, TDataset, TModel, TMetric
         if len({np.asarray(dataset[i][0]).shape for i in idxs}) == 1:
             cov_dict["image"] = dep.plot(dataset, indices=idxs)
 
-        return DataevalBiasOutputs.model_validate({"balance": bal_dict, "diversity": div_dict, "coverage": cov_dict})
+        outputs = DataevalBiasOutputs.model_validate({"balance": bal_dict, "diversity": div_dict, "coverage": cov_dict})
+        plt.close("all")
+        return outputs
 
 
 def generate_table_of_contents(deck: str) -> dict[str, Any]:  # pragma: no cover

{checkmaite-0.2.0 → checkmaite-0.2.2}/src/checkmaite/core/_common/dataeval_shift_capability.py

@@ -76,7 +76,7 @@ class DataevalShiftOODOutputs(CapabilityOutputsBase):
     ood_knn: DataevalShiftOODKNNOutput
 
 
-class DataevalShiftOutputs(pydantic.BaseModel):
+class DataevalShiftOutputs(CapabilityOutputsBase):
     drift: DataevalShiftDriftOutputs
     ood: DataevalShiftOODOutputs
 

checkmaite-0.2.2/src/checkmaite/core/_common/dataset_utils.py

@@ -0,0 +1,7 @@
+from upath import UPath
+
+_IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".tif", ".tiff"}
+
+
+def _is_image_path(path: UPath) -> bool:
+    return path.suffix.lower() in _IMAGE_SUFFIXES

{checkmaite-0.2.0 → checkmaite-0.2.2}/src/checkmaite/core/_common/maite_evaluation_capability.py

@@ -3,7 +3,6 @@ from typing import Any
 
 import matplotlib.pyplot as plt
 import numpy as np
-import pydantic
 from matplotlib.figure import Figure
 
 from checkmaite.core._utils import (
@@ -16,6 +15,7 @@ from checkmaite.core.cached_tasks import evaluate
 from checkmaite.core.capability_core import (
     Capability,
     CapabilityConfigBase,
+    CapabilityOutputsBase,
     CapabilityRunBase,
     Number,
     TDataset,
@@ -61,7 +61,7 @@ class MaiteEvaluationConfig(CapabilityConfigBase):
     pass
 
 
-class MaiteEvaluationOutputs(pydantic.BaseModel):
+class MaiteEvaluationOutputs(CapabilityOutputsBase):
     overall_metric_name: str
     result: dict[str, float]
     class_metrics: dict[str, float | None] | None

{checkmaite-0.2.0 → checkmaite-0.2.2}/src/checkmaite/core/_common/nrtk_robustness_capability.py

@@ -3,7 +3,7 @@ import re
 from typing import Any
 
 import pandas as pd
-from nrtk.interfaces.perturb_image_factory import PerturbImageFactory
+from nrtk.interfaces import PerturbImageFactory
 from pydantic import Field
 from smqtk_core.configuration import from_config_dict