validmind 2.3.5__tar.gz → 2.4.1__tar.gz

{validmind-2.3.5 → validmind-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.3.5
+Version: 2.4.1
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez

{validmind-2.3.5 → validmind-2.4.1}/pyproject.toml

@@ -10,7 +10,7 @@ description = "ValidMind Developer Framework"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.3.5"
+version = "2.4.1"
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<3.12"

validmind-2.4.1/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.4.1"

{validmind-2.3.5 → validmind-2.4.1}/validmind/ai/test_descriptions.py

@@ -218,6 +218,7 @@ def get_description_metadata(
     summary=None,
     figures=None,
     prefix="metric_description",
+    should_generate=True,
 ):
     """Get Metadata Dictionary for a Test or Metric Result
 
@@ -238,6 +239,7 @@ def get_description_metadata(
         summary (Any): The test summary or results to interpret
         figures (List[Figure]): The figures to attach to the test suite result
         prefix (str): The prefix to use for the content ID (Default: "metric_description")
+        should_generate (bool): Whether to generate the description or not (Default: True)
 
     Returns:
         dict: The metadata object to be logged with the test results
@@ -250,7 +252,12 @@ def get_description_metadata(
     # TODO: fix circular import
     from validmind.ai.utils import is_configured
 
-    if (summary or figures) and not env_disabled and is_configured():
+    if (
+        should_generate
+        and (summary or figures)
+        and not env_disabled
+        and is_configured()
+    ):
         revision_name = AI_REVISION_NAME
 
         # get description future and set it as the description in the metadata

{validmind-2.3.5 → validmind-2.4.1}/validmind/ai/utils.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import os
+from urllib.parse import urljoin
 
 from openai import AzureOpenAI, Client, OpenAI
 
@@ -63,7 +64,7 @@ def get_client_and_model():
                     # TODO: improve this to be a bit more dynamic
                     "http://localhost:4000/genai"
                     if "localhost" in get_api_host()
-                    else f"{get_api_host()}/genai"
+                    else urljoin(get_api_host(), "/genai")
                 ),
                 api_key=response["key"],
             )

{validmind-2.3.5 → validmind-2.4.1}/validmind/client.py

@@ -162,6 +162,7 @@ def init_dataset(
         raise UnsupportedDatasetError(
             "Only Pandas datasets and Tensor Datasets are supported at the moment."
         )
+
     if __log:
         log_input(
             name=input_id,

{validmind-2.3.5 → validmind-2.4.1}/validmind/template.py

@@ -22,6 +22,8 @@ CONTENT_TYPE_MAP = {
     "dynamic": "Dynamic Content",
     "text": "Text",
     "risk_assessment": "Risk Assessment",
+    "assessment_summary": "Assessment Summary",
+    "guideline": "Guideline Assessment",
 }
 
 

validmind-2.4.1/validmind/tests/__init__.py

@@ -0,0 +1,39 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""ValidMind Tests Module"""
+
+from ..errors import LoadTestError
+from ..logging import get_logger
+from .decorator import metric, tags, tasks, test
+from .load import describe_test, list_tests, load_test
+from .metadata import list_tags, list_tasks, list_tasks_and_tags
+from .run import run_test
+from .test_providers import LocalTestProvider, TestProvider, register_test_provider
+
+logger = get_logger(__name__)
+
+
+__all__ = [
+    "data_validation",
+    "model_validation",
+    "prompt_validation",
+    "list_tests",
+    "load_test",
+    "describe_test",
+    "run_test",
+    "register_test_provider",
+    "LoadTestError",
+    "LocalTestProvider",
+    "TestProvider",
+    # Metadata
+    "list_tags",
+    "list_tasks",
+    "list_tasks_and_tags",
+    # Decorators for functional metrics
+    "test",
+    "metric",  # DEPRECATED
+    "tags",
+    "tasks",
+]

validmind-2.4.1/validmind/tests/_store.py

@@ -0,0 +1,102 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Module for storing loaded tests and test providers"""
+
+
+def singleton(cls):
+    """Decorator to make a class a singleton"""
+    instances = {}
+
+    def get_instance(*args, **kwargs):
+        if cls not in instances:
+            instances[cls] = cls(*args, **kwargs)
+        return instances[cls]
+
+    return get_instance
+
+
+@singleton
+class TestProviderStore:
+    """Singleton class for storing test providers"""
+
+    def __init__(self):
+        self.test_providers = {}
+
+    def has_test_provider(self, namespace: str) -> bool:
+        """Check if a test provider exists by namespace
+
+        Args:
+            namespace (str): The namespace of the test provider
+
+        Returns:
+            bool: True if the test provider exists
+        """
+        return namespace in self.test_providers
+
+    def get_test_provider(self, namespace: str):
+        """Get a test provider by namespace
+
+        Args:
+            namespace (str): The namespace of the test provider
+
+        Returns:
+            TestProvider: The test provider
+        """
+        return self.test_providers.get(namespace)
+
+    def register_test_provider(self, namespace: str, test_provider) -> None:
+        """Register an external test provider
+
+        Args:
+            namespace (str): The namespace of the test provider
+            test_provider (TestProvider): The test provider
+        """
+        self.test_providers[namespace] = test_provider
+
+
+class TestStore:
+    """Singleton class for storing loaded tests"""
+
+    def __init__(self):
+        self.tests = {}
+        self.custom_tests = {}
+
+    def get_test(self, test_id: str):
+        """Get a test by test ID
+
+        Args:
+            test_id (str): The test ID
+
+        Returns:
+            object: The test class or function
+        """
+        return self.tests.get(test_id)
+
+    def get_custom_test(self, test_id: str):
+        """Get a custom test by test ID
+
+        Args:
+            test_id (str): The test ID
+
+        Returns:
+            object: The test class or function
+        """
+        return self.custom_tests.get(test_id)
+
+    def get_test_ids(self) -> list:
+        """Get all registered test IDs"""
+        return list(self.tests.keys())
+
+    def register_test(self, test_id: str, test: object = None):
+        """Register a test"""
+        self.tests[test_id] = test
+
+    def register_custom_test(self, test_id: str, test: object):
+        """Register a single one-off custom test"""
+        self.custom_tests[test_id] = test
+
+
+test_store = TestStore()
+test_provider_store = TestProviderStore()

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/ACFandPACFPlot.py

@@ -50,15 +50,13 @@ class ACFandPACFPlot(Metric):
 
     name = "acf_pacf_plot"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": [
-            "time_series_data",
-            "forecasting",
-            "statistical_test",
-            "visualization",
-        ],
-    }
+    tasks = ["regression"]
+    tags = [
+        "time_series_data",
+        "forecasting",
+        "statistical_test",
+        "visualization",
+    ]
 
     def run(self):
         # Check if index is datetime

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/ADF.py

@@ -47,16 +47,14 @@ class ADF(Metric):
 
     name = "adf"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": [
-            "time_series_data",
-            "statsmodels",
-            "forecasting",
-            "statistical_test",
-            "stationarity",
-        ],
-    }
+    tasks = ["regression"]
+    tags = [
+        "time_series_data",
+        "statsmodels",
+        "forecasting",
+        "statistical_test",
+        "stationarity",
+    ]
 
     def summary(self, metric_value: dict):
         table = pd.DataFrame.from_dict(metric_value, orient="index")

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/ANOVAOneWayTable.py

@@ -57,16 +57,14 @@ class ANOVAOneWayTable(Metric):
     name = "anova_one_way_table"
     required_inputs = ["dataset"]
     default_params = {"features": None, "p_threshold": 0.05}
-    metadata = {
-        "task_types": ["classification"],
-        "tags": [
-            "tabular_data",
-            "statistical_test",
-            "multiclass_classification",
-            "binary_classification",
-            "numerical_data",
-        ],
-    }
+    tasks = ["classification"]
+    tags = [
+        "tabular_data",
+        "statistical_test",
+        "multiclass_classification",
+        "binary_classification",
+        "numerical_data",
+    ]
 
     def run(self):
         features = self.params["features"]

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/AutoAR.py

@@ -61,10 +61,8 @@ class AutoAR(Metric):
     name = "auto_ar"
     required_inputs = ["dataset"]
     default_params = {"max_ar_order": 3}
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "statsmodels", "forecasting", "statistical_test"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data", "statsmodels", "forecasting", "statistical_test"]
 
     def run(self):
         if "max_ar_order" not in self.params:

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/AutoMA.py

@@ -57,10 +57,8 @@ class AutoMA(Metric):
     name = "auto_ma"
     required_inputs = ["dataset"]
     default_params = {"max_ma_order": 3}
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "statsmodels", "forecasting", "statistical_test"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data", "statsmodels", "forecasting", "statistical_test"]
 
     def run(self):
         if "max_ma_order" not in self.params:

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/AutoSeasonality.py

@@ -61,16 +61,14 @@ class AutoSeasonality(Metric):
     name = "auto_seasonality"
     required_inputs = ["dataset"]
     default_params = {"min_period": 1, "max_period": 4}
-    metadata = {
-        "task_types": ["regression"],
-        "tags": [
-            "time_series_data",
-            "forecasting",
-            "statistical_test",
-            "statsmodels",
-            "seasonality",
-        ],
-    }
+    tasks = ["regression"]
+    tags = [
+        "time_series_data",
+        "forecasting",
+        "statistical_test",
+        "statsmodels",
+        "seasonality",
+    ]
 
     def evaluate_seasonal_periods(self, series, min_period, max_period):
         seasonal_periods = []

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/AutoStationarity.py

@@ -54,16 +54,14 @@ class AutoStationarity(Metric):
     name = "auto_stationarity"
     required_inputs = ["dataset"]
     default_params = {"max_order": 5, "threshold": 0.05}
-    metadata = {
-        "task_types": ["regression"],
-        "tags": [
-            "time_series_data",
-            "statsmodels",
-            "forecasting",
-            "statistical_test",
-            "stationarity",
-        ],
-    }
+    tasks = ["regression"]
+    tags = [
+        "time_series_data",
+        "statsmodels",
+        "forecasting",
+        "statistical_test",
+        "stationarity",
+    ]
 
     def run(self):
         if "max_order" not in self.params:

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/BivariateFeaturesBarPlots.py

@@ -56,16 +56,14 @@ class BivariateFeaturesBarPlots(Metric):
     name = "bivariate_features_bar_plots"
     required_inputs = ["dataset"]
     default_params = {"features_pairs": None}
-    metadata = {
-        "task_types": ["classification"],
-        "tags": [
-            "tabular_data",
-            "categorical_data",
-            "binary_classification",
-            "multiclass_classification",
-            "visualization",
-        ],
-    }
+    tasks = ["classification"]
+    tags = [
+        "tabular_data",
+        "categorical_data",
+        "binary_classification",
+        "multiclass_classification",
+        "visualization",
+    ]
 
     def run(self):
         features_pairs = self.params["features_pairs"]

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/BivariateHistograms.py

@@ -55,16 +55,14 @@ class BivariateHistograms(Metric):
     name = "bivariate_histograms"
     required_inputs = ["dataset"]
     default_params = {"features_pairs": None, "target_filter": None}
-    metadata = {
-        "task_types": ["classification"],
-        "tags": [
-            "tabular_data",
-            "categorical_data",
-            "binary_classification",
-            "multiclass_classification",
-            "visualization",
-        ],
-    }
+    tasks = ["classification"]
+    tags = [
+        "tabular_data",
+        "categorical_data",
+        "binary_classification",
+        "multiclass_classification",
+        "visualization",
+    ]
 
     def plot_bivariate_histogram(self, features_pairs, target_filter):
         status_var = self.inputs.dataset.target_column

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/BivariateScatterPlots.py

@@ -54,16 +54,14 @@ class BivariateScatterPlots(Metric):
     name = "bivariate_scatter_plots"
     required_inputs = ["dataset"]
     default_params = {"selected_columns": None}
-    metadata = {
-        "task_types": ["classification"],
-        "tags": [
-            "tabular_data",
-            "categorical_data",
-            "binary_classification",
-            "multiclass_classification",
-            "visualization",
-        ],
-    }
+    tasks = ["classification"]
+    tags = [
+        "tabular_data",
+        "categorical_data",
+        "binary_classification",
+        "multiclass_classification",
+        "visualization",
+    ]
 
     def plot_bivariate_scatter(self, columns):
         figures = []

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/ChiSquaredFeaturesTable.py

@@ -54,16 +54,14 @@ class ChiSquaredFeaturesTable(Metric):
     name = "chi_squared_features_table"
     required_inputs = ["dataset"]
     default_params = {"cat_features": None, "p_threshold": 0.05}
-    metadata = {
-        "task_types": ["classification"],
-        "tags": [
-            "tabular_data",
-            "categorical_data",
-            "statistical_test",
-            "binary_classification",
-            "multiclass_classification",
-        ],
-    }
+    tasks = ["classification"]
+    tags = [
+        "tabular_data",
+        "categorical_data",
+        "statistical_test",
+        "binary_classification",
+        "multiclass_classification",
+    ]
 
     def run(self):
         target_column = self.inputs.dataset.target_column

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/ClassImbalance.py

@@ -73,10 +73,8 @@ class ClassImbalance(ThresholdTest):
     name = "class_imbalance"
     required_inputs = ["dataset"]
     default_params = {"min_percent_threshold": 10}
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["tabular_data", "binary_classification", "multiclass_classification"],
-    }
+    tasks = ["classification"]
+    tags = ["tabular_data", "binary_classification", "multiclass_classification"]
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         return ResultSummary(

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/DFGLSArch.py

@@ -53,10 +53,8 @@ class DFGLSArch(Metric):
 
     name = "dickey_fuller_gls"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "forecasting", "unit_root_test"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data", "forecasting", "unit_root_test"]
 
     def run(self):
         """

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/DatasetDescription.py

@@ -69,15 +69,13 @@ class DatasetDescription(Metric):
 
     name = "dataset_description"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": [
-            "classification",
-            "regression",
-            "text_classification",
-            "text_summarization",
-        ],
-        "tags": ["tabular_data", "time_series_data", "text_data"],
-    }
+    tasks = [
+        "classification",
+        "regression",
+        "text_classification",
+        "text_summarization",
+    ]
+    tags = ["tabular_data", "time_series_data", "text_data"]
 
     def summary(self, metric_value):
         """

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/DatasetSplit.py

@@ -48,15 +48,14 @@ class DatasetSplit(Metric):
 
     name = "dataset_split"
     required_inputs = ["datasets"]
-    metadata = {
-        "task_types": [
-            "classification",
-            "regression",
-            "text_classification",
-            "text_summarization",
-        ],
-        "tags": ["tabular_data", "time_series_data", "text_data"],
-    }
+    tasks = [
+        "classification",
+        "regression",
+        "text_classification",
+        "text_summarization",
+    ]
+
+    tags = ["tabular_data", "time_series_data", "text_data"]
 
     dataset_labels = {
         "train_ds": "Training",

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/DescriptiveStatistics.py

@@ -53,10 +53,8 @@ class DescriptiveStatistics(Metric):
 
     name = "descriptive_statistics"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": ["tabular_data", "time_series_data"],
-    }
+    tasks = ["classification", "regression"]
+    tags = ["tabular_data", "time_series_data"]
 
     def get_summary_statistics_numerical(self, df, numerical_fields):
         percentiles = [0.25, 0.5, 0.75, 0.90, 0.95]

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/Duplicates.py

@@ -55,10 +55,8 @@ class Duplicates(ThresholdTest):
     name = "duplicates"
     required_inputs = ["dataset"]
     default_params = {"min_threshold": 1}
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": ["tabular_data", "data_quality", "text_data"],
-    }
+    tasks = ["classification", "regression"]
+    tags = ["tabular_data", "data_quality", "text_data"]
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         """

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/EngleGrangerCoint.py

@@ -51,10 +51,8 @@ class EngleGrangerCoint(Metric):
     name = "engle_granger_coint"
     required_inputs = ["dataset"]
     default_params = {"threshold": 0.05}
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "statistical_test", "forecasting"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data", "statistical_test", "forecasting"]
 
     def run(self):
         threshold = self.params["threshold"]

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/FeatureTargetCorrelationPlot.py

@@ -48,10 +48,8 @@ class FeatureTargetCorrelationPlot(Metric):
     name = "feature_target_correlation_plot"
     required_inputs = ["dataset"]
     default_params = {"features": None, "fig_height": 600}
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": ["tabular_data", "visualization", "feature_importance", "correlation"],
-    }
+    tasks = ["classification", "regression"]
+    tags = ["tabular_data", "visualization", "feature_importance", "correlation"]
 
     def run(self):
         fig_height = self.params["fig_height"]

{validmind-2.3.5 → validmind-2.4.1}/validmind/tests/data_validation/HeatmapFeatureCorrelations.py

@@ -56,10 +56,8 @@ class HeatmapFeatureCorrelations(Metric):
     name = "heatmap_feature_correlations"
     required_inputs = ["dataset"]
     default_params = {"declutter": None, "fontsize": None, "num_features": None}
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": ["tabular_data", "visualization", "correlation"],
-    }
+    tasks = ["classification", "regression"]
+    tags = ["tabular_data", "visualization", "correlation"]
 
     def run(self):
         features = self.params.get("features")