validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/comparison.py

@@ -0,0 +1,362 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from itertools import product
+from typing import Any, Dict, List, Tuple, Union
+
+import pandas as pd
+
+from validmind.logging import get_logger
+from validmind.utils import test_id_to_name
+from validmind.vm_models.figure import (
+    is_matplotlib_figure,
+    is_plotly_figure,
+    is_png_image,
+)
+from validmind.vm_models.input import VMInput
+from validmind.vm_models.result import ResultTable, TestResult
+
+logger = get_logger(__name__)
+
+
+def _cartesian_product(grid: Dict[str, List[Any]]) -> List[Dict[str, Any]]:
+    """
+    Generate all possible combinations for a grid of inputs or parameters.
+
+    Args:
+        grid: A dictionary where each key corresponds to a parameter name and the associated list contains possible values.
+
+    Returns:
+        A list of dictionaries representing all combinations of the parameter values.
+
+    Example:
+        _cartesian_product({"a": [1, 2], "b": [3, 4]})
+        >>> [{'a': 1, 'b': 3}, {'a': 1, 'b': 4}, {'a': 2, 'b': 3}, {'a': 2, 'b': 4}]
+    """
+    if not grid:
+        return [{}]
+    return [dict(zip(grid.keys(), values)) for values in product(*grid.values())]
+
+
+def _get_input_key(input_obj_or_list: Union[VMInput, List[VMInput]]) -> str:
+    """Create a key for a given input or list of inputs"""
+    if isinstance(input_obj_or_list, list):
+        return ",".join(item.input_id for item in input_obj_or_list)
+
+    return input_obj_or_list.input_id
+
+
+def _get_unique_inputs(results: List[TestResult]) -> Dict[str, set]:
+    """Get only the inputs that are not the same across all results"""
+    unique_inputs = {}
+
+    for result in results:
+        if not result.inputs:
+            continue
+
+        for func_input_name, input_obj_or_list in result.inputs.items():
+            if isinstance(input_obj_or_list, list):
+                key = ",".join(_get_input_key(item) for item in input_obj_or_list)
+            else:
+                key = _get_input_key(input_obj_or_list)
+
+            unique_inputs.setdefault(func_input_name, set()).add(key)
+
+    return unique_inputs
+
+
+def _get_unique_params(results: List[TestResult]) -> Dict[str, List[Any]]:
+    """Get only the params that are not the same across all results"""
+    param_values = {}
+    for result in results:
+        if not result.params:
+            continue
+
+        for name, value in result.params.items():
+            param_values.setdefault(name, []).append(value)
+
+    unique_params = {}
+    for name, values in param_values.items():
+        unique_values = []
+        for value in values:
+            if not any(value == x for x in unique_values):
+                unique_values.append(value)
+
+        unique_params[name] = unique_values
+
+    return unique_params
+
+
+def _get_table_metadata(
+    result: TestResult, results: List[TestResult]
+) -> Dict[str, Any]:
+    """Create a metadata dict with unique inputs and params for a table row"""
+    metadata = {}
+    unique_inputs = _get_unique_inputs(results)
+    unique_params = _get_unique_params(results)
+
+    if result.inputs:
+        for input_name, input_obj in result.inputs.items():
+            if len(unique_inputs[input_name]) > 1:
+                metadata[input_name] = _get_input_key(input_obj)
+
+    if result.params:
+        for param_name, param_value in result.params.items():
+            if len(unique_params[param_name]) > 1:
+                metadata[param_name] = param_value
+
+    return metadata
+
+
+def _combine_single_table(results: List[TestResult], table_index: int) -> pd.DataFrame:
+    """
+    Combine a single table across multiple test results.
+
+    Args:
+        results: A list of TestResult objects.
+        table_index: The index of the table to combine.
+
+    Returns:
+        A pandas DataFrame combining the tables with added metadata columns.
+    """
+    combined_tables = []
+
+    for result in results:
+        metadata = _get_table_metadata(result, results)
+        table_data = result.tables[table_index].data
+
+        if metadata:
+            metadata_df = pd.DataFrame([metadata] * len(table_data))
+            table_data = pd.concat([metadata_df, table_data], axis=1)
+
+        combined_tables.append(table_data)
+
+    return pd.concat(combined_tables, ignore_index=True)
+
+
+def _combine_tables(results: List[TestResult]) -> List[pd.DataFrame]:
+    """Combine tables from multiple test results
+
+    # TODO: retain table titles
+    """
+    if not results[0].tables:
+        return []
+
+    return [_combine_single_table(results, i) for i in range(len(results[0].tables))]
+
+
+def _build_input_param_string(result: TestResult, results: List[TestResult]) -> str:
+    """Build a string repr of unique inputs + params for a figure title"""
+    parts = []
+    unique_inputs = _get_unique_inputs(results)
+
+    # if theres only one unique value for an input or param, don't show it
+    # however, if there is only one unique value for all inputs then show it
+    if result.inputs:
+        should_show = all(
+            len(unique_inputs[input_name]) == 1 for input_name in unique_inputs
+        )
+        for input_name, input_obj in result.inputs.items():
+            if should_show or len(unique_inputs[input_name]) > 1:
+                input_val = _get_input_key(input_obj)
+                parts.append(f"{input_name}={input_val}")
+
+    # TODO: revisit this when we can create a value/title to show for params
+    # unique_params = _get_unique_params(results)
+    # # if theres only one unique value for a param, don't show it
+    # # however, if there is only one unique value for all params then show it as
+    # # long as there is no existing inputs in the parts list
+    # if result.params:
+    #     should_show = (
+    #         all(len(unique_params[param_name]) == 1 for param_name in unique_params)
+    #         and not parts
+    #     )
+    #     for param_name, param_value in result.params.items():
+    #         if should_show or len(unique_params[param_name]) > 1:
+    #             parts.append(f"{param_name}={param_value}")
+
+    return ", ".join(parts)
+
+
+def _update_figure_title(figure: Any, input_param_str: str) -> None:
+    """
+    Update the title of a figure with input and parameter information.
+
+    Args:
+        figure: A figure object (matplotlib, plotly, or PNG image).
+        input_param_str: A string of input and parameter information.
+
+    Raises:
+        ValueError: If the figure type is unsupported.
+    """
+    if not input_param_str:
+        return
+
+    new_title = f"{{curr_title}} (for {input_param_str})"
+
+    if is_matplotlib_figure(figure):
+        curr_title = figure._suptitle.get_text() if figure._suptitle else ""
+        figure.suptitle(new_title.format(curr_title=curr_title))
+    elif is_plotly_figure(figure):
+        curr_title = figure.layout.title.text
+        figure.layout.title.text = new_title.format(curr_title=curr_title)
+    elif is_png_image(figure):
+        logger.warning("Unable to update title for PNG image figure.")
+    else:
+        raise ValueError(f"Unsupported figure type: {type(figure)}")
+
+
+def _combine_figures(results: List[TestResult]) -> List[Any]:
+    """Combine figures from multiple test results (gets raw figure objects, not vm Figures)"""
+    combined_figures = []
+
+    for result in results:
+        for figure in result.figures or []:
+            # update the figure object in-place with the new title
+            _update_figure_title(
+                figure=figure.figure,
+                input_param_str=_build_input_param_string(result, results),
+            )
+            combined_figures.append(figure)
+
+    return combined_figures
+
+
+def _handle_metrics(results: List[TestResult]) -> List[Any]:
+    """Combine metrics from multiple test results"""
+    # add a table with the metric value so it is combined into a single table
+    for result in results:
+        if result.metric:
+            result.add_table(
+                ResultTable(
+                    data=[
+                        {
+                            "Metric": test_id_to_name(result.result_id),
+                            "Value": result.metric,
+                        }
+                    ],
+                    title=None,
+                )
+            )
+
+
+def _combine_dict_values(items_dict: Dict[str, Any]) -> Dict[str, Any]:
+    """Combine values for each key in a dictionary, keeping only unique values"""
+    combined = {}
+
+    for name, value in items_dict.items():
+        values = value if isinstance(value, list) else [value]
+
+        unique_values = []
+        for v in values:
+            if not any(v == x for x in unique_values):
+                unique_values.append(v)
+
+        combined[name] = unique_values[0] if len(unique_values) == 1 else unique_values
+
+    return combined
+
+
+def get_comparison_test_configs(
+    input_grid: Union[Dict[str, List[Any]], List[Dict[str, Any]], None] = None,
+    param_grid: Union[Dict[str, List[Any]], List[Dict[str, Any]], None] = None,
+    inputs: Union[Dict[str, Union[VMInput, List[VMInput]]], None] = None,
+    params: Union[Dict[str, Any], None] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Generate test configurations based on input and parameter grids.
+
+    Function inputs should be validated before calling this.
+
+    Args:
+        input_grid: A dictionary or list defining the grid of inputs.
+        param_grid: A dictionary or list defining the grid of parameters.
+        inputs: A dictionary of inputs.
+        params: A dictionary of parameters.
+
+    Returns:
+        A list of test configurations.
+    """
+
+    # Convert list of dicts to dict of lists if necessary
+    def list_to_dict(grid_list):
+        return {k: [d[k] for d in grid_list] for k in grid_list[0].keys()}
+
+    if isinstance(input_grid, list):
+        input_grid = list_to_dict(input_grid)
+
+    if isinstance(param_grid, list):
+        param_grid = list_to_dict(param_grid)
+
+    test_configs = []
+
+    if input_grid and param_grid:
+        input_combinations = _cartesian_product(input_grid)
+        param_combinations = _cartesian_product(param_grid)
+        test_configs = [
+            {"inputs": i, "params": p}
+            for i, p in product(input_combinations, param_combinations)
+        ]
+    elif input_grid:
+        input_combinations = _cartesian_product(input_grid)
+        test_configs = [
+            {"inputs": i, "params": params or {}} for i in input_combinations
+        ]
+    elif param_grid:
+        param_combinations = _cartesian_product(param_grid)
+        test_configs = [
+            {"inputs": inputs or {}, "params": p} for p in param_combinations
+        ]
+
+    return test_configs
+
+
+def combine_results(
+    results: List[TestResult],
+) -> Tuple[List[Any], Dict[str, List[Any]], Dict[str, List[Any]]]:
+    """
+    Combine multiple test results into a single set of outputs.
+
+    Args:
+        results: A list of TestResult objects to combine.
+
+    Returns:
+        A tuple containing:
+            - A list of combined outputs (tables and figures).
+            - A dictionary of inputs with lists of all values.
+            - A dictionary of parameters with lists of all values.
+    """
+    # metrics are added as a table to each result so later they can be combined
+    _handle_metrics(results)
+
+    combined_outputs = []
+    # handle tables (if any)
+    combined_outputs.extend(_combine_tables(results))
+    # handle figures (if any)
+    combined_outputs.extend(_combine_figures(results))
+    # handle threshold tests (i.e. tests that have pass/fail bool status)
+    if results[0].passed is not None:
+        combined_outputs.append(all(result.passed for result in results))
+
+    # combine inputs and params
+    combined_inputs = {}
+    combined_params = {}
+
+    for result in results:
+        if result.inputs:
+            for input_name, input_obj_or_list in result.inputs.items():
+                combined_inputs.setdefault(input_name, []).extend(
+                    input_obj_or_list
+                    if isinstance(input_obj_or_list, list)
+                    else [input_obj_or_list]
+                )
+
+        if result.params:
+            for param_name, param_value in result.params.items():
+                combined_params.setdefault(param_name, []).append(param_value)
+
+    combined_inputs = _combine_dict_values(combined_inputs)
+    combined_params = _combine_dict_values(combined_params)
+
+    return combined_outputs, combined_inputs, combined_params

validmind/tests/data_validation/ACFandPACFPlot.py

@@ -6,10 +6,13 @@ import pandas as pd
 import plotly.graph_objects as go
 from statsmodels.tsa.stattools import acf, pacf
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset
 
 
-class ACFandPACFPlot(Metric):
+@tags("time_series_data", "forecasting", "statistical_test", "visualization")
+@tasks("regression")
+def ACFandPACFPlot(dataset: VMDataset):
     """
     Analyzes time series data using Autocorrelation Function (ACF) and Partial Autocorrelation Function (PACF) plots to
     reveal trends and correlations.
@@ -49,74 +52,42 @@ class ACFandPACFPlot(Metric):
     - The plots can only represent linear correlations and fail to capture any non-linear relationships within the data.
     - The plots might be difficult for non-experts to interpret and should not replace more advanced analyses.
     """
-
-    name = "acf_pacf_plot"
-    required_inputs = ["dataset"]
-    tasks = ["regression"]
-    tags = [
-        "time_series_data",
-        "forecasting",
-        "statistical_test",
-        "visualization",
-    ]
-
-    def run(self):
-        # Check if index is datetime
-        if not pd.api.types.is_datetime64_any_dtype(self.inputs.dataset.df.index):
-            raise ValueError("Index must be a datetime type")
-
-        columns = list(self.inputs.dataset.df.columns)
-
-        df = self.inputs.dataset.df.dropna()
-
-        if not set(columns).issubset(set(df.columns)):
-            raise ValueError("Provided 'columns' must exist in the dataset")
-
-        figures = []
-
-        for col in df.columns:
-            series = df[col]
-
-            # Calculate the maximum number of lags based on the size of the dataset
-            max_lags = min(40, len(series) // 2 - 1)
-
-            # Calculate ACF and PACF values
-            acf_values = acf(series, nlags=max_lags)
-            pacf_values = pacf(series, nlags=max_lags)
-
-            # Create ACF plot using Plotly
-            acf_fig = go.Figure()
-            acf_fig.add_trace(go.Bar(x=list(range(len(acf_values))), y=acf_values))
-            acf_fig.update_layout(
-                title=f"ACF for {col}",
-                xaxis_title="Lag",
-                yaxis_title="ACF",
-                font=dict(size=18),
-            )
-
-            # Create PACF plot using Plotly
-            pacf_fig = go.Figure()
-            pacf_fig.add_trace(go.Bar(x=list(range(len(pacf_values))), y=pacf_values))
-            pacf_fig.update_layout(
-                title=f"PACF for {col}",
-                xaxis_title="Lag",
-                yaxis_title="PACF",
-                font=dict(size=18),
-            )
-
-            figures.append(
-                Figure(
-                    for_object=self,
-                    key=f"{self.key}:{col}_acf",
-                    figure=acf_fig,
-                )
-            )
-            figures.append(
-                Figure(
-                    for_object=self,
-                    key=f"{self.key}:{col}_pacf",
-                    figure=pacf_fig,
-                )
-            )
-
-        return self.cache_results(figures=figures)
+    if not pd.api.types.is_datetime64_any_dtype(dataset.df.index):
+        raise ValueError("Index must be a datetime type")
+
+    columns = list(dataset.df.columns)
+    df = dataset.df.dropna()
+
+    if not set(columns).issubset(set(df.columns)):
+        raise ValueError("Provided 'columns' must exist in the dataset")
+
+    figures = []
+    for col in df.columns:
+        series = df[col]
+        max_lags = min(40, len(series) // 2 - 1)
+
+        # Create ACF plot using Plotly
+        acf_values = acf(series, nlags=max_lags)
+        acf_fig = go.Figure()
+        acf_fig.add_trace(go.Bar(x=list(range(len(acf_values))), y=acf_values))
+        acf_fig.update_layout(
+            title=f"ACF for {col}",
+            xaxis_title="Lag",
+            yaxis_title="ACF",
+            font=dict(size=18),
+        )
+        figures.append(acf_fig)
+
+        # Create PACF plot using Plotly
+        pacf_values = pacf(series, nlags=max_lags)
+        pacf_fig = go.Figure()
+        pacf_fig.add_trace(go.Bar(x=list(range(len(pacf_values))), y=pacf_values))
+        pacf_fig.update_layout(
+            title=f"PACF for {col}",
+            xaxis_title="Lag",
+            yaxis_title="PACF",
+            font=dict(size=18),
+        )
+        figures.append(pacf_fig)
+
+    return tuple(figures)

validmind/tests/data_validation/ADF.py

@@ -2,19 +2,21 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import pandas as pd
 from statsmodels.tsa.stattools import adfuller
 
+from validmind import tags, tasks
 from validmind.logging import get_logger
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
+from validmind.vm_models import VMDataset
 
 logger = get_logger(__name__)
 
 
-@dataclass
-class ADF(Metric):
+@tags(
+    "time_series_data", "statsmodels", "forecasting", "statistical_test", "stationarity"
+)
+@tasks("regression")
+def ADF(dataset: VMDataset):
     """
     Assesses the stationarity of a time series dataset using the Augmented Dickey-Fuller (ADF) test.
 
@@ -51,84 +53,48 @@ class ADF(Metric):
     - It assumes the data follows an autoregressive process, which might not always be the case.
     - The test struggles with time series data that have structural breaks.
     """
+    df = dataset.df.dropna()
 
-    name = "adf"
-    required_inputs = ["dataset"]
-    tasks = ["regression"]
-    tags = [
-        "time_series_data",
-        "statsmodels",
-        "forecasting",
-        "statistical_test",
-        "stationarity",
-    ]
-
-    def summary(self, metric_value: dict):
-        table = pd.DataFrame.from_dict(metric_value, orient="index")
-        table = table.reset_index()
-        table.columns = [
-            "Feature",
-            "ADF Statistic",
-            "P-Value",
-            "Used Lag",
-            "Number of Observations",
-            "Critical Values",
-            "IC Best",
-        ]
-        table = table.rename_axis("Index", axis=1)
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=table,
-                    metadata=ResultTableMetadata(
-                        title="ADF Test Results for Each Feature"
-                    ),
-                ),
-            ]
+    if not isinstance(df.index, (pd.DatetimeIndex, pd.PeriodIndex)):
+        raise ValueError(
+            "Dataset index must be a datetime or period index for time series analysis."
         )
 
-    def run(self):
-        """
-        Calculates ADF metric for each of the dataset features
-        """
-        dataset = self.inputs.dataset.df
-
-        # Check if the dataset is a time series
-        if not isinstance(dataset.index, (pd.DatetimeIndex, pd.PeriodIndex)):
-            raise ValueError(
-                "Dataset index must be a datetime or period index for time series analysis."
-            )
-
-        # Preprocessing: Drop rows with any NaN values
-        if dataset.isnull().values.any():
-            logger.warning(
-                "Dataset contains missing values. Rows with NaNs will be dropped."
-            )
-            dataset = dataset.dropna()
-
-        adf_values = {}
-        for col in dataset.columns:
-            try:
-                adf_result = adfuller(dataset[col].values)
-                adf_values[col] = {
-                    "ADF Statistic": adf_result[0],
-                    "P-Value": adf_result[1],
-                    "Used Lag": adf_result[2],
-                    "Number of Observations": adf_result[3],
-                    "Critical Values": adf_result[4],
-                    "IC Best": adf_result[5],
-                }
-            except Exception as e:
-                logger.error(f"Error processing column '{col}': {e}")
-                adf_values[col] = {
-                    "ADF Statistic": None,
-                    "P-Value": None,
-                    "Used Lag": None,
-                    "Number of Observations": None,
-                    "Critical Values": None,
-                    "IC Best": None,
-                    "Error": str(e),
-                }
-
-        return self.cache_results(adf_values)
+    adf_values = {}
+    for col in df.columns:
+        try:
+            adf_result = adfuller(df[col].values)
+            adf_values[col] = {
+                "ADF Statistic": adf_result[0],
+                "P-Value": adf_result[1],
+                "Used Lag": adf_result[2],
+                "Number of Observations": adf_result[3],
+                "Critical Values": adf_result[4],
+                "IC Best": adf_result[5],
+            }
+        except Exception as e:
+            logger.error(f"Error processing column '{col}': {e}")
+            adf_values[col] = {
+                "ADF Statistic": None,
+                "P-Value": None,
+                "Used Lag": None,
+                "Number of Observations": None,
+                "Critical Values": None,
+                "IC Best": None,
+                "Error": str(e),
+            }
+
+    table = pd.DataFrame.from_dict(adf_values, orient="index")
+    table = table.reset_index()
+    table.columns = [
+        "Feature",
+        "ADF Statistic",
+        "P-Value",
+        "Used Lag",
+        "Number of Observations",
+        "Critical Values",
+        "IC Best",
+    ]
+    table = table.rename_axis("Index", axis=1)
+
+    return {"ADF Test Results for Each Feature": table}