validmind 2.5.6__py3-none-any.whl → 2.5.15__py3-none-any.whl

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.5.6"
+__version__ = "2.5.15"

validmind/ai/test_descriptions.py

@@ -4,9 +4,11 @@
 
 import os
 from concurrent.futures import ThreadPoolExecutor
+from typing import Union
 
 from validmind.utils import md_to_html
 
+from ..client_config import client_config
 from ..logging import get_logger
 
 __executor = ThreadPoolExecutor()
@@ -110,10 +112,11 @@ def generate_description(
     test_id: str,
     test_description: str,
     test_summary: str,
+    metric: Union[float, int] = None,
     figures: list = None,
 ):
     """Generate the description for the test results"""
-    if not test_summary and not figures:
+    if not test_summary and not figures and not metric:
         raise ValueError("No summary or figures provided - cannot generate description")
 
     # TODO: fix circular import
@@ -130,6 +133,13 @@ def generate_description(
         else test_description
     )
 
+    if metric:
+        metric_summary = f"**Metric Value**: {metric}"
+        if test_summary:
+            test_summary = metric_summary + "\n" + test_summary
+        else:
+            test_summary = metric_summary
+
     if test_summary:
         logger.debug(
             f"Generating description for test {test_name} with stringified summary"
@@ -198,11 +208,16 @@ def background_generate_description(
     test_description: str,
     test_summary: str,
     figures: list = None,
+    metric: Union[int, float] = None,
 ):
     def wrapped():
         try:
             return generate_description(
-                test_id, test_description, test_summary, figures
+                test_id=test_id,
+                test_description=test_description,
+                test_summary=test_summary,
+                figures=figures,
+                metric=metric,
             )
         except Exception as e:
             logger.error(f"Failed to generate description: {e}")
@@ -217,6 +232,7 @@ def get_description_metadata(
     default_description,
     summary=None,
     figures=None,
+    metric=None,
     prefix="metric_description",
     should_generate=True,
 ):
@@ -238,16 +254,18 @@ def get_description_metadata(
         default_description (str): The default description for the test
         summary (Any): The test summary or results to interpret
         figures (List[Figure]): The figures to attach to the test suite result
+        metric (Union[int, float]): Unit metrics attached to the test result
         prefix (str): The prefix to use for the content ID (Default: "metric_description")
         should_generate (bool): Whether to generate the description or not (Default: True)
 
     Returns:
         dict: The metadata object to be logged with the test results
     """
-    env_disabled = os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1") in [
-        "0",
-        "false",
-    ]
+    # Check the feature flag first, then the environment variable
+    llm_descriptions_enabled = (
+        client_config.can_generate_llm_test_descriptions()
+        and os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1") not in ["0", "false"]
+    )
 
     # TODO: fix circular import
     from validmind.ai.utils import is_configured
@@ -255,7 +273,7 @@ def get_description_metadata(
     if (
         should_generate
         and (summary or figures)
-        and not env_disabled
+        and llm_descriptions_enabled
         and is_configured()
     ):
         revision_name = AI_REVISION_NAME
@@ -267,6 +285,7 @@ def get_description_metadata(
             test_description=default_description,
             test_summary=summary,
             figures=figures,
+            metric=metric,
         )
 
     else:

validmind/api_client.py

@@ -186,12 +186,24 @@ def __ping() -> Dict[str, Any]:
     client_config.project = client_info["project"]
     client_config.documentation_template = client_info.get("documentation_template", {})
     client_config.feature_flags = client_info.get("feature_flags", {})
+    client_config.model = client_info.get("model", {})
+    client_config.document_type = client_info.get(
+        "document_type", "model_documentation"
+    )
 
     if ack_connected:
-        logger.info(
-            f"Connected to ValidMind... Current Model: {client_config.project['name']}"
-            f" ({client_config.project['cuid']})"
-        )
+        if client_config.model:
+            logger.info(
+                f"🎉 Connected to ValidMind!\n"
+                f"📊 Model: {client_config.model.get('name', 'N/A')} "
+                f"(ID: {client_config.model.get('cuid', 'N/A')})\n"
+                f"📁 Document Type: {client_config.document_type}"
+            )
+        else:
+            logger.info(
+                f"Connected to ValidMind... Current Model: {client_config.project['name']}"
+                f" ({client_config.project['cuid']})"
+            )
 
 
 def reload():
@@ -331,32 +343,6 @@ async def log_figures(figures: List[Figure]) -> Dict[str, Any]:
     Returns:
         dict: The response from the API
     """
-    # this actually slows things down - better to log them in parallel
-    # if client_config.can_log_figures():  # check if the backend supports batch logging
-    #     try:
-    #         data = {}
-    #         files = {}
-    #         for figure in figures:
-    #             data.update(
-    #                 {f"{k}-{figure.key}": v for k, v in figure.serialize().items()}
-    #             )
-    #             files.update(
-    #                 {
-    #                     f"{k}-{figure.key}": v
-    #                     for k, v in figure.serialize_files().items()
-    #                 }
-    #             )
-
-    #         return await _post(
-    #             "log_figures",
-    #             data=data,
-    #             files=files,
-    #         )
-    #     except Exception as e:
-    #         logger.error("Error logging figures to ValidMind API")
-    #         raise e
-
-    # else:
     return await asyncio.gather(*[log_figure(figure) for figure in figures])
 
 
@@ -416,11 +402,11 @@ async def log_metrics(
     Returns:
         dict: The response from the API
     """
-    params = {}
+    request_params = {}
     if section_id:
-        params["section_id"] = section_id
+        request_params["section_id"] = section_id
     if position is not None:
-        params["position"] = position
+        request_params["position"] = position
 
     data = []
 
@@ -430,7 +416,7 @@ async def log_metrics(
             "inputs": inputs,
         }
 
-        if output_template and client_config.can_log_output_template():
+        if output_template:
             metric_data["output_template"] = output_template
 
         data.append(metric_data)
@@ -438,7 +424,7 @@ async def log_metrics(
     try:
         return await _post(
             "log_metrics",
-            params=params,
+            params=request_params,
             data=json.dumps(data, cls=NumpyEncoder, allow_nan=False),
         )
     except Exception as e:
@@ -469,16 +455,16 @@ async def log_test_result(
     Returns:
         dict: The response from the API
     """
-    params = {}
+    request_params = {}
     if section_id:
-        params["section_id"] = section_id
+        request_params["section_id"] = section_id
     if position is not None:
-        params["position"] = position
+        request_params["position"] = position
 
     try:
         return await _post(
             "log_test_results",
-            params=params,
+            params=request_params,
             data=json.dumps(
                 {
                     **result.serialize(),
@@ -503,7 +489,7 @@ def log_test_results(
 
     Args:
         results (list): A list of ThresholdTestResults objects
-        inputs (list): A list of input keys (names) that were used to run the test
+        inputs (list): A list of input IDs that were used to run the test
 
     Raises:
         Exception: If the API call fails
@@ -522,11 +508,11 @@ def log_test_results(
     return responses
 
 
-def log_input(name: str, type: str, metadata: Dict[str, Any]) -> Dict[str, Any]:
+def log_input(input_id: str, type: str, metadata: Dict[str, Any]) -> Dict[str, Any]:
     """Logs input information - internal use for now (don't expose via public API)
 
     Args:
-        name (str): The name of the input
+        input_id (str): The input_id of the input
         type (str): The type of the input
         metadata (dict): The metadata of the input
 
@@ -542,7 +528,7 @@ def log_input(name: str, type: str, metadata: Dict[str, Any]) -> Dict[str, Any]:
             "log_input",
             data=json.dumps(
                 {
-                    "name": name,
+                    "name": input_id,
                     "type": type,
                     "metadata": metadata,
                 },
@@ -555,6 +541,66 @@ def log_input(name: str, type: str, metadata: Dict[str, Any]) -> Dict[str, Any]:
         raise e
 
 
+async def alog_metric(
+    key: str,
+    value: float,
+    inputs: Optional[List[str]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    recorded_at: Optional[str] = None,
+) -> None:
+    """See log_metric for details"""
+    if not key or not isinstance(key, str):
+        raise ValueError("`key` must be a non-empty string")
+
+    if not value or not isinstance(value, (int, float)):
+        raise ValueError("`value` must be a scalar (int or float)")
+
+    try:
+        return await _post(
+            "log_unit_metric",
+            data=json.dumps(
+                {
+                    "key": key,
+                    "value": value,
+                    "inputs": inputs or [],
+                    "params": params or {},
+                    "recorded_at": recorded_at,
+                },
+                cls=NumpyEncoder,
+                allow_nan=False,
+            ),
+        )
+    except Exception as e:
+        logger.error("Error logging metric to ValidMind API")
+        raise e
+
+
+def log_metric(
+    key: str,
+    value: float,
+    inputs: Optional[List[str]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    recorded_at: Optional[str] = None,
+) -> None:
+    """Logs a unit metric
+
+    Unit metrics are key-value pairs where the key is the metric name and the value is
+    a scalar (int or float). These key-value pairs are associated with the currently
+    selected model (inventory model in the ValidMind Platform) and keys can be logged
+    to over time to create a history of the metric. On the platform, these metrics
+    will be used to create plots/visualizations for documentation and dashboards etc.
+
+    Args:
+        key (str): The metric key
+        value (float): The metric value
+        inputs (list, optional): A list of input IDs that were used to compute the metric.
+        params (dict, optional): Dictionary of parameters used to compute the metric.
+        recorded_at (str, optional): The timestamp of the metric. Server will use
+            current time if not provided.
+    """
+    run_async(alog_metric, key, value, inputs, params, recorded_at)
+
+
 def start_run() -> str:
     """Starts a new test run
 

validmind/client.py

@@ -164,7 +164,7 @@ def init_dataset(
 
     if __log:
         log_input(
-            name=input_id,
+            input_id=input_id,
             type="dataset",
             metadata=get_dataset_info(vm_dataset),
         )
@@ -265,7 +265,7 @@ def init_model(
 
     if __log:
         log_input(
-            name=input_id,
+            input_id=input_id,
             type="model",
             metadata=metadata,
         )

validmind/client_config.py

@@ -18,7 +18,9 @@ class ClientConfig:
     """
 
     project: object
+    model: object
     feature_flags: dict
+    document_type: str
     documentation_template: object
     running_on_colab: bool = False
 
@@ -34,21 +36,16 @@ class ClientConfig:
         except ImportError:
             self.running_on_colab = False
 
-    def is_json_plots_enabled(self):
-        """
-        Returns True if the JSON plots feature flag is enabled on the backend
-        """
-        return self.feature_flags.get("generate_json_plots", False)
-
-    def can_log_figures(self):
-        """Returns True if the client can log figures to the API"""
-        return self.feature_flags.get("log_figures", False)
-
-    def can_log_output_template(self):
-        """Returns True if the client can log output templates to the API"""
-        return self.feature_flags.get("output_templates", False)
+    def can_generate_llm_test_descriptions(self):
+        """Returns True if the client can generate LLM based test descriptions"""
+        return self.feature_flags.get("llm_test_descriptions", True)
 
 
 client_config = ClientConfig(
-    project=None, feature_flags={}, documentation_template=None
+    project=None,
+    model=None,
+    feature_flags={},
+    document_type="model_documentation",
+    documentation_template=None,
+    running_on_colab=False,
 )

validmind/datasets/regression/fred_timeseries.py

@@ -68,65 +68,56 @@ def convert_to_levels(diff_df, original_df, target_column):
     return levels_df
 
 
-def get_demo_test_config(test_suite=None):
+def get_demo_test_config():
 
     default_config = {}
 
     default_config["validmind.data_validation.TimeSeriesDescription"] = {
-        "inputs": {
-            "dataset": "raw_ds",
+        "input_grid": {
+            "dataset": [
+                "raw_ds",
+                "train_diff_ds",
+                "test_diff_ds",
+                "train_ds",
+                "test_ds",
+            ],
         }
     }
     default_config["validmind.data_validation.TimeSeriesLinePlot"] = {
-        "inputs": {
-            "dataset": "raw_ds",
+        "input_grid": {
+            "dataset": ["raw_ds"],
         }
     }
     default_config["validmind.data_validation.TimeSeriesMissingValues"] = {
-        "inputs": {
-            "dataset": "raw_ds",
+        "input_grid": {
+            "dataset": [
+                "raw_ds",
+                "train_diff_ds",
+                "test_diff_ds",
+                "train_ds",
+                "test_ds",
+            ],
         }
     }
     default_config["validmind.data_validation.SeasonalDecompose"] = {
-        "inputs": {
-            "dataset": "raw_ds",
+        "input_grid": {
+            "dataset": ["raw_ds"],
         }
     }
-    default_config[
-        "validmind.data_validation.TimeSeriesDescriptiveStatistics:train_diff_data"
-    ] = {
-        "inputs": {
-            "dataset": "train_diff_ds",
+    default_config["validmind.data_validation.TimeSeriesDescriptiveStatistics"] = {
+        "input_grid": {
+            "dataset": ["train_diff_ds", "test_diff_ds"],
         }
     }
-    default_config[
-        "validmind.data_validation.TimeSeriesDescriptiveStatistics:test_diff_data"
-    ] = {
-        "inputs": {
-            "dataset": "test_diff_ds",
-        }
-    }
-    default_config["validmind.data_validation.TimeSeriesOutliers:train_diff_data"] = {
-        "inputs": {
-            "dataset": "train_diff_ds",
+    default_config["validmind.data_validation.TimeSeriesOutliers"] = {
+        "input_grid": {
+            "dataset": ["train_diff_ds", "test_diff_ds"],
         },
         "params": {"zscore_threshold": 4},
     }
-    default_config["validmind.data_validation.TimeSeriesOutliers:test_diff_data"] = {
-        "inputs": {
-            "dataset": "test_diff_ds",
-        },
-        "params": {"zscore_threshold": 4},
-    }
-    default_config["validmind.data_validation.TimeSeriesHistogram:train_diff_data"] = {
-        "inputs": {
-            "dataset": "train_diff_ds",
-        },
-        "params": {"nbins": 100},
-    }
-    default_config["validmind.data_validation.TimeSeriesHistogram:test_diff_data"] = {
-        "inputs": {
-            "dataset": "test_diff_ds",
+    default_config["validmind.data_validation.TimeSeriesHistogram"] = {
+        "input_grid": {
+            "dataset": ["train_diff_ds", "test_diff_ds"],
         },
         "params": {"nbins": 100},
     }
@@ -135,57 +126,37 @@ def get_demo_test_config(test_suite=None):
             "datasets": ["train_diff_ds", "test_diff_ds"],
         }
     }
-    default_config["validmind.model_validation.ModelMetadataComparison"] = {
-        "inputs": {
-            "models": ["random_forests_model", "gradient_boosting_model"],
-        }
-    }
-    default_config[
-        "validmind.model_validation.sklearn.RegressionErrorsComparison:train_data"
-    ] = {
-        "inputs": {
-            "datasets": ["train_ds", "train_ds"],
-            "models": ["random_forests_model", "gradient_boosting_model"],
+    default_config["validmind.model_validation.ModelMetadata"] = {
+        "input_grid": {
+            "model": ["random_forests_model", "gradient_boosting_model"],
         }
     }
-    default_config[
-        "validmind.model_validation.sklearn.RegressionErrorsComparison:test_data"
-    ] = {
-        "inputs": {
-            "datasets": ["test_ds", "test_ds"],
-            "models": ["random_forests_model", "gradient_boosting_model"],
+    default_config["validmind.model_validation.sklearn.RegressionErrors"] = {
+        "input_grid": {
+            "dataset": ["train_ds", "test_ds"],
+            "model": ["random_forests_model", "gradient_boosting_model"],
         }
     }
-    default_config[
-        "validmind.model_validation.sklearn.RegressionR2SquareComparison:train_data"
-    ] = {
-        "inputs": {
-            "datasets": ["train_ds", "train_ds"],
-            "models": ["random_forests_model", "gradient_boosting_model"],
-        }
-    }
-    default_config[
-        "validmind.model_validation.sklearn.RegressionR2SquareComparison:test_data"
-    ] = {
-        "inputs": {
-            "datasets": ["test_ds", "test_ds"],
-            "models": ["random_forests_model", "gradient_boosting_model"],
+    default_config["validmind.model_validation.sklearn.RegressionR2Square"] = {
+        "input_grid": {
+            "dataset": ["train_ds", "test_ds"],
+            "model": ["random_forests_model", "gradient_boosting_model"],
         }
     }
     default_config[
         "validmind.model_validation.TimeSeriesR2SquareBySegments:train_data"
     ] = {
-        "inputs": {
-            "datasets": ["train_ds", "train_ds"],
-            "models": ["random_forests_model", "gradient_boosting_model"],
+        "input_grid": {
+            "dataset": ["train_ds"],
+            "model": ["random_forests_model", "gradient_boosting_model"],
         }
     }
     default_config[
         "validmind.model_validation.TimeSeriesR2SquareBySegments:test_data"
     ] = {
-        "inputs": {
-            "datasets": ["test_ds", "test_ds"],
-            "models": ["random_forests_model", "gradient_boosting_model"],
+        "input_grid": {
+            "dataset": ["test_ds"],
+            "model": ["random_forests_model", "gradient_boosting_model"],
         },
         "params": {
             "segments": {
@@ -194,78 +165,36 @@ def get_demo_test_config(test_suite=None):
             }
         },
     }
-    default_config[
-        "validmind.model_validation.TimeSeriesPredictionsPlot:train_data"
-    ] = {
-        "inputs": {
-            "datasets": ["train_ds", "train_ds"],
-            "models": ["random_forests_model", "gradient_boosting_model"],
-        }
-    }
-    default_config["validmind.model_validation.TimeSeriesPredictionsPlot:test_data"] = {
-        "inputs": {
-            "datasets": ["test_ds", "test_ds"],
-            "models": ["random_forests_model", "gradient_boosting_model"],
-        }
-    }
-    default_config[
-        "validmind.model_validation.TimeSeriesPredictionWithCI:random_forests_model"
-    ] = {
-        "inputs": {
-            "dataset": "test_ds",
-            "model": "random_forests_model",
-        }
-    }
-    default_config[
-        "validmind.model_validation.TimeSeriesPredictionWithCI:gradient_boosting_model"
-    ] = {
-        "inputs": {
-            "dataset": "test_ds",
-            "model": "gradient_boosting_model",
-        }
-    }
-    default_config["validmind.model_validation.ModelPredictionResiduals:train_data"] = {
-        "inputs": {
-            "datasets": ["train_ds", "train_ds"],
-            "models": ["random_forests_model", "gradient_boosting_model"],
+    default_config["validmind.model_validation.TimeSeriesPredictionsPlot"] = {
+        "input_grid": {
+            "dataset": ["train_ds", "test_ds"],
+            "model": ["random_forests_model", "gradient_boosting_model"],
         }
     }
-    default_config["validmind.model_validation.ModelPredictionResiduals:test_data"] = {
-        "inputs": {
-            "datasets": ["test_ds", "test_ds"],
-            "models": ["random_forests_model", "gradient_boosting_model"],
-        }
-    }
-    default_config[
-        "validmind.model_validation.sklearn.FeatureImportanceComparison:train_data"
-    ] = {
-        "inputs": {
-            "datasets": ["train_ds", "train_ds"],
-            "models": ["random_forests_model", "gradient_boosting_model"],
+    default_config["validmind.model_validation.TimeSeriesPredictionWithCI"] = {
+        "input_grid": {
+            "dataset": ["train_ds", "test_ds"],
+            "model": ["random_forests_model", "gradient_boosting_model"],
         }
     }
-    default_config[
-        "validmind.model_validation.sklearn.FeatureImportanceComparison:test_data"
-    ] = {
-        "inputs": {
-            "datasets": ["test_ds", "test_ds"],
-            "models": ["random_forests_model", "gradient_boosting_model"],
+    default_config["validmind.model_validation.ModelPredictionResiduals"] = {
+        "input_grid": {
+            "dataset": ["train_ds", "test_ds"],
+            "model": ["random_forests_model", "gradient_boosting_model"],
         }
     }
-    default_config[
-        "validmind.model_validation.sklearn.PermutationFeatureImportance:random_forests_model"
-    ] = {
-        "inputs": {
-            "dataset": "test_ds",
-            "model": "random_forests_model",
+    default_config["validmind.model_validation.sklearn.FeatureImportance"] = {
+        "input_grid": {
+            "dataset": ["train_ds", "test_ds"],
+            "model": ["random_forests_model", "gradient_boosting_model"],
         }
     }
     default_config[
-        "validmind.model_validation.sklearn.PermutationFeatureImportance:gradient_boosting_model"
+        "validmind.model_validation.sklearn.PermutationFeatureImportance"
     ] = {
-        "inputs": {
-            "dataset": "test_ds",
-            "model": "gradient_boosting_model",
+        "input_grid": {
+            "dataset": ["train_ds", "test_ds"],
+            "model": ["random_forests_model", "gradient_boosting_model"],
         }
     }
 

validmind/template.py

@@ -18,6 +18,7 @@ logger = get_logger(__name__)
 CONTENT_TYPE_MAP = {
     "test": "Threshold Test",
     "metric": "Metric",
+    "unit_metric": "Unit Metric",
     "metadata_text": "Metadata Text",
     "dynamic": "Dynamic Content",
     "text": "Text",

validmind/test_suites/__init__.py

@@ -42,7 +42,6 @@ from .time_series import (
     TimeSeriesDataset,
     TimeSeriesModelValidation,
     TimeSeriesMultivariate,
-    TimeSeriesSensitivity,
     TimeSeriesUnivariate,
 )
 
@@ -78,7 +77,6 @@ core_test_suites = {
     TimeSeriesDataset.suite_id: TimeSeriesDataset,
     TimeSeriesModelValidation.suite_id: TimeSeriesModelValidation,
     TimeSeriesMultivariate.suite_id: TimeSeriesMultivariate,
-    TimeSeriesSensitivity.suite_id: TimeSeriesSensitivity,
     TimeSeriesUnivariate.suite_id: TimeSeriesUnivariate,
 }