validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/vm_models/test/threshold_test_result.py

@@ -1,75 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-"""
-ThresholdTestResult models
-"""
-from dataclasses import dataclass
-from typing import List, Optional
-
-from .result_summary import ResultSummary
-
-
-@dataclass
-class ThresholdTestResult:
-    """
-    ThresholdTestResult model
-    """
-
-    values: dict
-    test_name: Optional[str] = None  # Optionally allow a name for an individual test
-    column: Optional[
-        str
-    ] = None  # Optionally track the results for an individual column
-    passed: Optional[bool] = None  # Optionally per-result pass/fail
-
-    def serialize(self):
-        """
-        Serializes the ThresholdTestResult to a dictionary so it can be sent to the API
-        """
-        test_result = {
-            "values": self.values,
-        }
-
-        if self.test_name is not None:
-            test_result["test_name"] = self.test_name
-
-        if self.column is not None:
-            test_result["column"] = self.column
-
-        if self.passed is not None:
-            test_result["passed"] = self.passed
-
-        return test_result
-
-
-@dataclass
-class ThresholdTestResults:
-    """
-    ThresholdTestResults model
-    """
-
-    test_name: str
-    ref_id: str
-    params: dict
-    passed: bool
-    results: List[ThresholdTestResult]
-    summary: Optional[ResultSummary]
-
-    def serialize(self):
-        """
-        Serializes the ThresholdTestResults to a dictionary so it can be sent to the API
-        """
-        test_results = {
-            "test_name": self.test_name,
-            "ref_id": self.ref_id,
-            "params": self.params,
-            "passed": self.passed,
-            "results": [result.serialize() for result in self.results],
-        }
-
-        if self.summary is not None:
-            test_results["summary"] = self.summary.serialize()
-
-        return test_results

validmind/vm_models/test_context.py

@@ -1,259 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-"""
-TestContext
-"""
-
-# TODO: lets think about refactoring this file since it deals with both the `TestContext` and the `TestInput`.
-# TestInput was previously used for both storing a global "context" for running tests to be able to
-# share data, as well as for storing the inputs to those tests.
-# We've since split this into two separate concepts.
-# https://app.shortcut.com/validmind/story/2468/allow-arbitrary-test-context
-# There is more changes to come around how we handle test inputs, so once we iron out that, we can refactor
-
-from dataclasses import dataclass
-from typing import ClassVar, List, Optional
-
-from validmind.input_registry import input_registry
-
-from ..errors import MissingRequiredTestInputError
-from ..logging import get_logger
-from .dataset.dataset import VMDataset
-from .model import VMModel
-
-# More human readable context names for error messages
-CONTEXT_NAMES = {
-    "dataset": "Dataset",
-    "model": "Model",
-    "models": "Models",
-    "model.train_ds": "Model Training Dataset",
-    "model.test_ds": "Model Testing Dataset",
-    "model.validation_ds": "Model Validation Dataset",
-    "train_ds": "Training Dataset",
-    "test_ds": "Testing Dataset",
-    "validation_ds": "Validation Dataset",
-}
-
-logger = get_logger(__name__)
-
-
-@dataclass
-class TestContext:
-    """
-    Holds context that can be used by tests to run.
-    Allows us to store data that needs to be reused
-    across different tests/metrics such as shared dataset metrics, etc.
-    """
-
-    # Custom context data that can be set by metrics or tests using this context
-    context_data: Optional[dict] = None
-
-    # TODO: here for backwards compatibility, remove this when appropriate
-    dataset: VMDataset = None
-    model: VMModel = None
-    models: List[VMModel] = None
-
-    def set_context_data(self, key, value):
-        if self.context_data is None:
-            self.context_data = {}
-
-        self.context_data[key] = value
-
-    def get_context_data(self, key):
-        if self.context_data is None:
-            return None
-
-        return self.context_data.get(key)
-
-
-class TestInput:
-    """Holds models, datasets and other arbitrary inputs for test(s)
-
-    Attributes:
-        dataset (VMDataset): Single dataset for dataset-only tests
-        model (VMModel): Single model for model-related tests
-        models (List[VMModel]): Multiple models for model comparison tests
-        ... (any): Any other arbitrary inputs that can be used by tests
-    """
-
-    def __init__(self, inputs):
-        """Initialize with either a dictionary of inputs"""
-        for key, value in inputs.items():
-            # 1) retrieve input object from input registry if an input_id string is provided
-            # 2) check the input_id type if a list of inputs (mix of strings and objects) is provided
-            # 3) if its a dict, it should contain the `input_id` key as well as other options
-            if isinstance(value, str):
-                value = input_registry.get(key=value)
-            elif isinstance(value, list) or isinstance(value, tuple):
-                value = [
-                    input_registry.get(key=v) if isinstance(v, str) else v
-                    for v in value
-                ]
-            elif isinstance(value, dict):
-                assert "input_id" in value, (
-                    "Input dictionary must contain an 'input_id' key "
-                    "to retrieve the input object from the input registry."
-                )
-                value = input_registry.get(key=value.get("input_id")).with_options(
-                    **{k: v for k, v in value.items() if k != "input_id"}
-                )
-
-            setattr(self, key, value)
-
-    def __getitem__(self, key):
-        """Allow accessing inputs via `self.inputs["input_name"]`"""
-        return getattr(self, key)
-
-    def __repr__(self):
-        """Human-readable string representation of the object."""
-        attrs = ",\n    ".join(
-            f"{key}={value!r}" for key, value in self.__dict__.items()
-        )
-        return f"{self.__class__.__name__}(\n    {attrs}\n)"
-
-
-class InputAccessTrackerProxy:
-    """Proxy object to track TestInput attribute access on a per-test basis"""
-
-    def __init__(self, inputs):
-        self._inputs = inputs
-        self._accessed = set()
-
-    def __getattr__(self, name):
-        # Track access only if the attribute actually exists in the inputs
-        if hasattr(self._inputs, name):
-            input = getattr(self._inputs, name)
-            # if the input is a list of inputs, track each input individually
-            if isinstance(input, list) or isinstance(input, tuple):
-                for i in input:
-                    self._accessed.add(i.input_id)
-            else:
-                self._accessed.add(input.input_id)
-            return getattr(self._inputs, name)
-
-        raise AttributeError(
-            f"'{type(self._inputs).__name__}' object has no attribute '{name}'"
-        )
-
-    def get_accessed(self):
-        # Provide the list of accessed input names
-        return list(self._accessed)
-
-
-@dataclass
-class TestUtils:
-    """Utility methods for classes that receive a TestContext"""
-
-    required_inputs: ClassVar[List[str]]
-
-    context: Optional[TestContext] = None
-    inputs: Optional[TestInput] = None  # gets overwritten to be a proxy when accessed
-
-    def __getattribute__(self, name):
-        # Intercept attribute access
-        if name == "inputs":
-            inputs = super().__getattribute__(name)
-
-            # when accessing inputs for the first time, wrap them in tracker proxy
-            if inputs is not None and not isinstance(inputs, InputAccessTrackerProxy):
-                inputs = InputAccessTrackerProxy(inputs)
-                super().__setattr__(name, inputs)  # overwrite to avoid re-wrapping
-
-            return inputs
-
-        return super().__getattribute__(name)
-
-    def get_accessed_inputs(self):
-        """Return a list of inputs that were accessed for this test"""
-        if isinstance(self.inputs, InputAccessTrackerProxy):
-            return self.inputs.get_accessed()
-
-        return []
-
-    def _get_input_dict(self):
-        """Return a dictionary of all inputs"""
-        if isinstance(self.inputs, InputAccessTrackerProxy):
-            return self.inputs._inputs.__dict__
-
-        return self.inputs.__dict__
-
-    def _get_legacy_input(self, key):
-        """Retrieve an input from the Test Input or, for backwards compatibility,
-        the Test Context
-
-        We should remove this once we all tests (including customer tests) are
-        using `self.inputs.<input_name>` instead of `self.<input_name>`.
-        """
-        try:
-            _input = getattr(self.inputs, key)
-        except AttributeError:
-            # in case any code is still manually creating a TestContext instead of
-            # a TestInput, we'll still support that for now by checking there
-            _input = getattr(self.context, key)
-
-        return _input
-
-    @property
-    def dataset(self):
-        """[DEPRECATED] Returns the input dataset for the test"""
-        logger.warning(
-            "Accesing the input dataset using `self.dataset` is deprecated. "
-            "Use `self.inputs.dataset` instead."
-        )
-        return self._get_legacy_input("dataset")
-
-    @property
-    def model(self):
-        """[DEPRECATED] Returns the input model for the test"""
-        logger.warning(
-            "Accesing the input model using `self.model` is deprecated. "
-            "Use `self.inputs.model` instead."
-        )
-        return self._get_legacy_input("model")
-
-    @property
-    def models(self):
-        """[DEPRECATED] Returns the input models for the test"""
-        logger.warning(
-            "Accesing the input models using `self.models` is deprecated. "
-            "Use `self.inputs.models` instead."
-        )
-        return self._get_legacy_input("models")
-
-    def validate_inputs(self):
-        """
-        Validates that the required inputs are present in the test input object.
-
-        Raises:
-            MissingRequiredTestInputError: If a required context element is missing.
-        """
-
-        def recursive_attr_check(obj, attr_chain):
-            """
-            Recursively checks if the given object has the specified attribute chain.
-
-            Args:
-                obj: The object to check.
-                attr_chain: A string representing the attribute chain, separated by dots.
-
-            Returns:
-                True if the object has the attribute chain, False otherwise.
-            """
-            attrs = attr_chain.split(".")
-            if not hasattr(obj, attrs[0]) or getattr(obj, attrs[0]) is None:
-                return False
-            return len(attrs) == 1 or recursive_attr_check(
-                getattr(obj, attrs[0]),
-                ".".join(attrs[1:]),
-            )
-
-        required_inputs = self.required_inputs or []
-        for element in required_inputs:
-            if not recursive_attr_check(self.inputs, element):
-                context_name = CONTEXT_NAMES.get(element, element)
-                raise MissingRequiredTestInputError(
-                    f"{context_name} '{element}' is a required input and must be "
-                    "passed as part of the test inputs dictionary."
-                )

validmind-2.5.24.dist-info/METADATA

@@ -1,118 +0,0 @@
-Metadata-Version: 2.1
-Name: validmind
-Version: 2.5.24
-Summary: ValidMind Library
-License: Commercial License
-Author: Andres Rodriguez
-Author-email: andres@validmind.ai
-Requires-Python: >=3.8.1,<3.12
-Classifier: License :: Other/Proprietary License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Provides-Extra: all
-Provides-Extra: huggingface
-Provides-Extra: llm
-Provides-Extra: pytorch
-Provides-Extra: r-support
-Requires-Dist: aiohttp[speedups]
-Requires-Dist: arch
-Requires-Dist: bert-score (>=0.3.13)
-Requires-Dist: catboost
-Requires-Dist: datasets (>=2.10.0,<3.0.0)
-Requires-Dist: evaluate
-Requires-Dist: ipywidgets
-Requires-Dist: kaleido (>=0.2.1,!=0.2.1.post1)
-Requires-Dist: langchain-openai (>=0.1.8) ; extra == "all" or extra == "llm"
-Requires-Dist: langdetect
-Requires-Dist: latex2mathml (>=3.77.0)
-Requires-Dist: llvmlite ; python_version >= "3.8" and python_full_version <= "3.11.0"
-Requires-Dist: matplotlib
-Requires-Dist: mistune (>=3.0.2,<4.0.0)
-Requires-Dist: nest-asyncio (>=1.6.0,<2.0.0)
-Requires-Dist: nltk (>=3.8.1,<4.0.0)
-Requires-Dist: numba (<0.59.0)
-Requires-Dist: numpy
-Requires-Dist: openai (>=1)
-Requires-Dist: pandas (>=1.1,<=2.0.3)
-Requires-Dist: plotly
-Requires-Dist: plotly-express
-Requires-Dist: polars
-Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"
-Requires-Dist: python-dotenv
-Requires-Dist: ragas (>=0.1.19) ; extra == "all" or extra == "llm"
-Requires-Dist: rouge (>=1)
-Requires-Dist: rpy2 (>=3.5.10,<4.0.0) ; extra == "all" or extra == "r-support"
-Requires-Dist: scikit-learn
-Requires-Dist: scipy
-Requires-Dist: scorecardpy (>=0.1.9.6,<0.2.0.0)
-Requires-Dist: seaborn
-Requires-Dist: sentencepiece (>=0.2.0,<0.3.0) ; extra == "all" or extra == "huggingface" or extra == "llm"
-Requires-Dist: sentry-sdk (>=1.24.0,<2.0.0)
-Requires-Dist: shap (==0.44.1)
-Requires-Dist: statsmodels
-Requires-Dist: tabulate (>=0.8.9,<0.9.0)
-Requires-Dist: textblob (>=0.18.0.post0,<0.19.0)
-Requires-Dist: torch (>=1.10.0) ; extra == "all" or extra == "llm" or extra == "pytorch"
-Requires-Dist: tqdm
-Requires-Dist: transformers (>=4.32.0,<5.0.0) ; extra == "all" or extra == "huggingface" or extra == "llm"
-Requires-Dist: xgboost (>=1.5.2,<3)
-Requires-Dist: ydata-profiling
-Description-Content-Type: text/markdown
-
-# ValidMind Library
-
-ValidMind’s Python Library contains a suite of developer tools and methods designed to automate
-the documentation and validation of your models.
-
-The Library is designed to be model agnostic. If your model is built in Python, ValidMind's
-Python library will provide all the standard functionality without requiring your developers to rewrite any functions.
-
-The Library provides a rich suite of documentation tools and test suites, from documenting
-descriptions of your dataset to testing your models for weak spots and overfit areas. The Developer
-Framework helps you automate the generation of model documentation by feeding the ValidMind platform with
-documentation artifacts and test results to the ValidMind platform.
-
-## Installation
-
-To install the ValidMind Library and all optional dependencies, run:
-
-```bash
-pip install validmind[all]
-```
-
-To install the Library without optional dependencies (core functionality only), run:
-
-```bash
-pip install validmind
-```
-
-### Extra dependencies
-
-The Library has optional dependencies that can be installed separately to support additional model types and tests.
-
-- **LLM Support**: To be able to run tests for Large Language Models (LLMs), install the `llm` extra:
-
-    ```bash
-    pip install validmind[llm]
-    ```
-
-- **PyTorch Models**: To use pytorch models with the Library, install the `torch` extra:
-
-    ```bash
-    pip install validmind[torch]
-    ```
-
-- **Hugging Face Transformers**: To use Hugging Face Transformers models with the Library, install the `transformers` extra:
-
-    ```bash
-    pip install validmind[transformers]
-    ```
-
-- **R Models**: To use R models with the Library, install the `r` extra:
-
-    ```bash
-    pip install validmind[r-support]
-    ```
-