validmind 2.5.18__tar.gz → 2.5.19__tar.gz

{validmind-2.5.18 → validmind-2.5.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.5.18
+Version: 2.5.19
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez

{validmind-2.5.18 → validmind-2.5.19}/pyproject.toml

@@ -10,7 +10,7 @@ description = "ValidMind Developer Framework"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.5.18"
+version = "2.5.19"
 
 [tool.poetry.dependencies]
 aiohttp = {extras = ["speedups"], version = "*"}

validmind-2.5.19/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.5.19"

{validmind-2.5.18 → validmind-2.5.19}/validmind/errors.py

@@ -207,6 +207,23 @@ class MissingRequiredTestInputError(BaseError):
     pass
 
 
+class MissingDependencyError(BaseError):
+    """
+    When a required dependency is missing.
+    """
+
+    def __init__(self, message="", required_dependencies=None, extra=None):
+        """
+        Args:
+            message (str): The error message.
+            required_dependencies (list): A list of required dependencies.
+            extra (str): The particular validmind `extra` that will install the missing dependencies.
+        """
+        super().__init__(message)
+        self.required_dependencies = required_dependencies or []
+        self.extra = extra
+
+
 class MissingRExtrasError(BaseError):
     """
     When the R extras have not been installed.

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/data_validation/ProtectedClassesCombination.py

@@ -7,19 +7,27 @@ import sys
 import pandas as pd
 import plotly.graph_objects as go
 import plotly.subplots as sp
-from fairlearn.metrics import (
-    MetricFrame,
-    count,
-    demographic_parity_ratio,
-    equalized_odds_ratio,
-    false_positive_rate,
-    selection_rate,
-    true_positive_rate,
-)
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 from validmind.logging import get_logger
 
+try:
+    from fairlearn.metrics import (
+        MetricFrame,
+        count,
+        demographic_parity_ratio,
+        equalized_odds_ratio,
+        false_positive_rate,
+        selection_rate,
+        true_positive_rate,
+    )
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `fairlearn` for ProtectedClassesCombination.",
+        required_dependencies=["fairlearn"],
+    ) from e
+
 logger = get_logger(__name__)
 
 

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/data_validation/ProtectedClassesDisparity.py

@@ -5,15 +5,23 @@
 import io
 import sys
 
-import aequitas.plot as ap
 import pandas as pd
-from aequitas.bias import Bias
-from aequitas.group import Group
-from aequitas.plotting import Plot
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 from validmind.logging import get_logger
 
+try:
+    import aequitas.plot as ap
+    from aequitas.bias import Bias
+    from aequitas.group import Group
+    from aequitas.plotting import Plot
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `aequitas` for ProtectedClassesDisparity.",
+        required_dependencies=["aequitas"],
+    ) from e
+
 logger = get_logger(__name__)
 
 

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.py

@@ -7,20 +7,28 @@ import sys
 
 import matplotlib.pyplot as plt
 import pandas as pd
-from fairlearn.metrics import (
-    MetricFrame,
-    count,
-    demographic_parity_ratio,
-    equalized_odds_ratio,
-    false_negative_rate,
-    false_positive_rate,
-    true_positive_rate,
-)
-from fairlearn.postprocessing import ThresholdOptimizer, plot_threshold_optimizer
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 from validmind.logging import get_logger
 
+try:
+    from fairlearn.metrics import (
+        MetricFrame,
+        count,
+        demographic_parity_ratio,
+        equalized_odds_ratio,
+        false_negative_rate,
+        false_positive_rate,
+        true_positive_rate,
+    )
+    from fairlearn.postprocessing import ThresholdOptimizer, plot_threshold_optimizer
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `fairlearn` for ProtectedClassesThresholdOptimizer.",
+        required_dependencies=["fairlearn"],
+    ) from e
+
 logger = get_logger(__name__)
 
 

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/load.py

@@ -15,7 +15,7 @@ from uuid import uuid4
 import pandas as pd
 from ipywidgets import HTML, Accordion
 
-from ..errors import LoadTestError
+from ..errors import LoadTestError, MissingDependencyError
 from ..html_templates.content_blocks import test_content_block_html
 from ..logging import get_logger
 from ..unit_metrics.composite import load_composite_metric
@@ -88,10 +88,26 @@ def list_tests(
     Returns:
         list or pandas.DataFrame: A list of all tests or a formatted table.
     """
-    tests = {
-        test_id: load_test(test_id, reload=True)
-        for test_id in test_store.get_test_ids()
-    }
+    tests = {}
+    for test_id in test_store.get_test_ids():
+        try:
+            tests[test_id] = load_test(test_id, reload=True)
+        except MissingDependencyError as e:
+            # skip tests that have missing dependencies
+            logger.debug(str(e))
+
+            if e.extra:
+                logger.info(
+                    f"Skipping `{test_id}` as it requires extra dependencies: {e.required_dependencies}."
+                    f" Please run `pip install validmind[{e.extra}]` to view and run this test."
+                )
+            else:
+                logger.info(
+                    f"Skipping `{test_id}` as it requires missing dependencies: {e.required_dependencies}."
+                    " Please install the missing dependencies to view and run this test."
+                )
+
+            continue
 
     # first search by the filter string since it's the most general search
     if filter is not None:

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import answer_correctness
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for AnswerCorrectness. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -88,11 +100,6 @@ def AnswerCorrectness(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import answer_correctness
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
 
     warnings.filterwarnings(
         "ignore",

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/model_validation/ragas/AnswerRelevance.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import answer_relevancy
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for AnswerRelevance. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -92,12 +104,6 @@ def AnswerRelevance(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import answer_relevancy
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/model_validation/ragas/AnswerSimilarity.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import answer_similarity
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for AnswerSimilarity. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -78,12 +90,6 @@ def AnswerSimilarity(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import answer_similarity
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/model_validation/ragas/AspectCritique.py

@@ -8,9 +8,28 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import AspectCritic
+    from ragas.metrics._aspect_critic import (
+        coherence,
+        conciseness,
+        correctness,
+        harmfulness,
+        maliciousness,
+    )
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for AspectCritique. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 LOWER_IS_BETTER_ASPECTS = ["harmfulness", "maliciousness"]
 
 
@@ -101,19 +120,6 @@ def AspectCritique(
     )
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import AspectCritic
-        from ragas.metrics._aspect_critic import (
-            coherence,
-            conciseness,
-            correctness,
-            harmfulness,
-            maliciousness,
-        )
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     built_in_aspects = {
         "coherence": coherence,
         "conciseness": conciseness,

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import context_entity_recall
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for ContextEntityRecall. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -84,12 +96,6 @@ def ContextEntityRecall(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import context_entity_recall
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import context_precision
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for ContextPrecision. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
@@ -79,12 +91,6 @@ def ContextPrecision(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import context_precision
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/model_validation/ragas/ContextRecall.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import context_recall
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for ContextRecall. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
@@ -79,12 +91,6 @@ def ContextRecall(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import context_recall
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/model_validation/ragas/ContextUtilization.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import context_utilization
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for ContextUtilization. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
@@ -107,12 +119,6 @@ def ContextUtilization(
     - Requires proper context retrieval to be effective; irrelevant context chunks can skew the results.
     - Dependent on large sample sizes to provide stable and reliable estimates of utilization performance.
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import context_utilization
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/model_validation/ragas/Faithfulness.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import faithfulness
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for Faithfulness. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -78,12 +90,6 @@ def Faithfulness(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import faithfulness
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.5.18 → validmind-2.5.19}/validmind/tests/model_validation/ragas/NoiseSensitivity.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import noise_sensitivity_relevant
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for NoiseSensitivity. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -100,12 +112,6 @@ def NoiseSensitivity(
     - Primarily applicable to tasks like text QA, text generation, and text summarization where contextual relevance is
     critical.
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import noise_sensitivity_relevant
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind-2.5.18/validmind/__version__.py

@@ -1 +0,0 @@
-__version__ = "2.5.18"