validmind 2.5.18__py3-none-any.whl → 2.5.23__py3-none-any.whl

validmind/datasets/nlp/__init__.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-Example datasets that can be used with the developer framework.
+Example datasets that can be used with the library.
 """
 
 __all__ = [

validmind/errors.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-This module contains all the custom errors that are used in the developer framework.
+This module contains all the custom errors that are used in the library.
 
 The following base errors are defined for others:
 - BaseError
@@ -207,6 +207,23 @@ class MissingRequiredTestInputError(BaseError):
     pass
 
 
+class MissingDependencyError(BaseError):
+    """
+    When a required dependency is missing.
+    """
+
+    def __init__(self, message="", required_dependencies=None, extra=None):
+        """
+        Args:
+            message (str): The error message.
+            required_dependencies (list): A list of required dependencies.
+            extra (str): The particular validmind `extra` that will install the missing dependencies.
+        """
+        super().__init__(message)
+        self.required_dependencies = required_dependencies or []
+        self.extra = extra
+
+
 class MissingRExtrasError(BaseError):
     """
     When the R extras have not been installed.
@@ -219,14 +236,6 @@ class MissingRExtrasError(BaseError):
         )
 
 
-class MissingRunCUIDError(APIRequestError):
-    """
-    When data is being sent to the API but the run_cuid is missing.
-    """
-
-    pass
-
-
 class MissingTextContentIdError(APIRequestError):
     """
     When a Text object is sent to the API without a content_id.
@@ -243,30 +252,14 @@ class MissingTextContentsError(APIRequestError):
     pass
 
 
-class MissingProjectIdError(BaseError):
+class MissingModelIdError(BaseError):
     def description(self, *args, **kwargs):
         return (
             self.message
-            or "Project ID must be provided either as an environment variable or as an argument to init."
+            or "Model ID must be provided either as an environment variable or as an argument to init."
         )
 
 
-class StartTestRunFailedError(APIRequestError):
-    """
-    When the API was not able to start a test run.
-    """
-
-    pass
-
-
-class TestRunNotFoundError(APIRequestError):
-    """
-    When a test run is not found in the API.
-    """
-
-    pass
-
-
 class TestInputInvalidDatasetError(BaseError):
     """
     When an invalid dataset is used in a test context.
@@ -352,11 +345,8 @@ def raise_api_error(error_string):
         "missing_text": MissingTextContentsError,
         "invalid_text_object": InvalidTextObjectError,
         "invalid_content_id_prefix": InvalidContentIdPrefixError,
-        "missing_run_cuid": MissingRunCUIDError,
-        "test_run_not_found": TestRunNotFoundError,
         "invalid_metric_results": InvalidMetricResultsError,
         "invalid_test_results": InvalidTestResultsError,
-        "start_test_run_failed": StartTestRunFailedError,
     }
 
     error_class = error_map.get(api_code, APIRequestError)

validmind/tests/data_validation/ProtectedClassesCombination.py

@@ -7,19 +7,27 @@ import sys
 import pandas as pd
 import plotly.graph_objects as go
 import plotly.subplots as sp
-from fairlearn.metrics import (
-    MetricFrame,
-    count,
-    demographic_parity_ratio,
-    equalized_odds_ratio,
-    false_positive_rate,
-    selection_rate,
-    true_positive_rate,
-)
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 from validmind.logging import get_logger
 
+try:
+    from fairlearn.metrics import (
+        MetricFrame,
+        count,
+        demographic_parity_ratio,
+        equalized_odds_ratio,
+        false_positive_rate,
+        selection_rate,
+        true_positive_rate,
+    )
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `fairlearn` for ProtectedClassesCombination.",
+        required_dependencies=["fairlearn"],
+    ) from e
+
 logger = get_logger(__name__)
 
 

validmind/tests/data_validation/ProtectedClassesDisparity.py

@@ -5,15 +5,23 @@
 import io
 import sys
 
-import aequitas.plot as ap
 import pandas as pd
-from aequitas.bias import Bias
-from aequitas.group import Group
-from aequitas.plotting import Plot
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 from validmind.logging import get_logger
 
+try:
+    import aequitas.plot as ap
+    from aequitas.bias import Bias
+    from aequitas.group import Group
+    from aequitas.plotting import Plot
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `aequitas` for ProtectedClassesDisparity.",
+        required_dependencies=["aequitas"],
+    ) from e
+
 logger = get_logger(__name__)
 
 

validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.py

@@ -7,20 +7,28 @@ import sys
 
 import matplotlib.pyplot as plt
 import pandas as pd
-from fairlearn.metrics import (
-    MetricFrame,
-    count,
-    demographic_parity_ratio,
-    equalized_odds_ratio,
-    false_negative_rate,
-    false_positive_rate,
-    true_positive_rate,
-)
-from fairlearn.postprocessing import ThresholdOptimizer, plot_threshold_optimizer
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 from validmind.logging import get_logger
 
+try:
+    from fairlearn.metrics import (
+        MetricFrame,
+        count,
+        demographic_parity_ratio,
+        equalized_odds_ratio,
+        false_negative_rate,
+        false_positive_rate,
+        true_positive_rate,
+    )
+    from fairlearn.postprocessing import ThresholdOptimizer, plot_threshold_optimizer
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `fairlearn` for ProtectedClassesThresholdOptimizer.",
+        required_dependencies=["fairlearn"],
+    ) from e
+
 logger = get_logger(__name__)
 
 

validmind/tests/load.py

@@ -15,7 +15,7 @@ from uuid import uuid4
 import pandas as pd
 from ipywidgets import HTML, Accordion
 
-from ..errors import LoadTestError
+from ..errors import LoadTestError, MissingDependencyError
 from ..html_templates.content_blocks import test_content_block_html
 from ..logging import get_logger
 from ..unit_metrics.composite import load_composite_metric
@@ -88,10 +88,30 @@ def list_tests(
     Returns:
         list or pandas.DataFrame: A list of all tests or a formatted table.
     """
-    tests = {
-        test_id: load_test(test_id, reload=True)
-        for test_id in test_store.get_test_ids()
-    }
+    # tests = {
+    #     test_id: load_test(test_id, reload=True)
+    #     for test_id in test_store.get_test_ids()
+    # }
+    tests = {}
+    for test_id in test_store.get_test_ids():
+        try:
+            tests[test_id] = load_test(test_id, reload=True)
+        except MissingDependencyError as e:
+            # skip tests that have missing dependencies
+            logger.debug(str(e))
+
+            if e.extra:
+                logger.info(
+                    f"Skipping `{test_id}` as it requires extra dependencies: {e.required_dependencies}."
+                    f" Please run `pip install validmind[{e.extra}]` to view and run this test."
+                )
+            else:
+                logger.info(
+                    f"Skipping `{test_id}` as it requires missing dependencies: {e.required_dependencies}."
+                    " Please install the missing dependencies to view and run this test."
+                )
+
+            continue
 
     # first search by the filter string since it's the most general search
     if filter is not None:

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import answer_correctness
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for AnswerCorrectness. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -88,12 +100,6 @@ def AnswerCorrectness(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import answer_correctness
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/AnswerRelevance.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import answer_relevancy
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for AnswerRelevance. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -92,12 +104,6 @@ def AnswerRelevance(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import answer_relevancy
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/AnswerSimilarity.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import answer_similarity
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for AnswerSimilarity. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -78,12 +90,6 @@ def AnswerSimilarity(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import answer_similarity
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/AspectCritique.py

@@ -8,9 +8,28 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import AspectCritic
+    from ragas.metrics._aspect_critic import (
+        coherence,
+        conciseness,
+        correctness,
+        harmfulness,
+        maliciousness,
+    )
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for AspectCritique. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 LOWER_IS_BETTER_ASPECTS = ["harmfulness", "maliciousness"]
 
 
@@ -101,19 +120,6 @@ def AspectCritique(
     )
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import AspectCritic
-        from ragas.metrics._aspect_critic import (
-            coherence,
-            conciseness,
-            correctness,
-            harmfulness,
-            maliciousness,
-        )
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     built_in_aspects = {
         "coherence": coherence,
         "conciseness": conciseness,

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import context_entity_recall
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for ContextEntityRecall. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -84,12 +96,6 @@ def ContextEntityRecall(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import context_entity_recall
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import context_precision
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for ContextPrecision. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
@@ -79,12 +91,6 @@ def ContextPrecision(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import context_precision
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import context_recall
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for ContextRecall. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
@@ -79,12 +91,6 @@ def ContextRecall(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import context_recall
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/ContextUtilization.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import context_utilization
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for ContextUtilization. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
@@ -107,12 +119,6 @@ def ContextUtilization(
     - Requires proper context retrieval to be effective; irrelevant context chunks can skew the results.
     - Dependent on large sample sizes to provide stable and reliable estimates of utilization performance.
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import context_utilization
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import faithfulness
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for Faithfulness. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -78,12 +90,6 @@ def Faithfulness(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import faithfulness
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/NoiseSensitivity.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import noise_sensitivity_relevant
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for NoiseSensitivity. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -100,12 +112,6 @@ def NoiseSensitivity(
     - Primarily applicable to tasks like text QA, text generation, and text summarization where contextual relevance is
     critical.
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import noise_sensitivity_relevant
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/sklearn/ClassifierPerformance.py

@@ -67,6 +67,7 @@ class ClassifierPerformance(Metric):
         "multiclass_classification",
         "model_performance",
     ]
+    default_params = {"average": "macro"}
 
     def summary(self, metric_value: dict):
         """
@@ -134,11 +135,13 @@ class ClassifierPerformance(Metric):
         if len(np.unique(y_true)) > 2:
             y_pred = self.inputs.dataset.y_pred(self.inputs.model)
             y_true = y_true.astype(y_pred.dtype)
-            roc_auc = multiclass_roc_auc_score(y_true, y_pred)
+            roc_auc = multiclass_roc_auc_score(
+                y_true, y_pred, average=self.params["average"]
+            )
         else:
             y_prob = self.inputs.dataset.y_prob(self.inputs.model)
             y_true = y_true.astype(y_prob.dtype).flatten()
-            roc_auc = roc_auc_score(y_true, y_prob)
+            roc_auc = roc_auc_score(y_true, y_prob, average=self.params["average"])
 
         report["roc_auc"] = roc_auc