validmind 2.3.3__py3-none-any.whl → 2.3.5__py3-none-any.whl

validmind/tests/data_validation/TimeSeriesHistogram.py

@@ -2,14 +2,14 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import matplotlib.pyplot as plt
-import pandas as pd
-import seaborn as sns
+import plotly.express as px
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
 
 
-class TimeSeriesHistogram(Metric):
+@tags("data_validation", "visualization")
+@tasks("regression", "time_series_forecasting")
+def TimeSeriesHistogram(dataset, nbins=30):
     """
     Visualizes distribution of time-series data using histograms and Kernel Density Estimation (KDE) lines.
 
@@ -20,7 +20,7 @@ class TimeSeriesHistogram(Metric):
     (kurtosis) underlying the data.
 
     **Test Mechanism**: This test operates on a specific column within the dataset that is required to have a datetime
-    type index. It goes through each column in the given dataset, creating a histogram with Seaborn's histplot
+    type index. It goes through each column in the given dataset, creating a histogram with Plotly's histplot
     function. In cases where the dataset includes more than one time-series (i.e., more than one column with a datetime
     type index), a distinct histogram is plotted for each series. Additionally, a kernel density estimate (KDE) line is
     drawn for each histogram, providing a visualization of the data's underlying probability distribution. The x and
@@ -48,46 +48,30 @@ class TimeSeriesHistogram(Metric):
     - The histogram's shape may be sensitive to the number of bins used.
     """
 
-    name = "time_series_histogram"
-    required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "visualization"],
-    }
+    df = dataset.df
 
-    def run(self):
-        # Check if index is datetime
-        if not pd.api.types.is_datetime64_any_dtype(self.inputs.dataset.df.index):
-            raise ValueError("Index must be a datetime type")
+    columns = list(dataset.df.columns)
 
-        columns = list(self.inputs.dataset.df.columns)
+    if not set(columns).issubset(set(df.columns)):
+        raise ValueError("Provided 'columns' must exist in the dataset")
 
-        df = self.inputs.dataset.df
-
-        if not set(columns).issubset(set(df.columns)):
-            raise ValueError("Provided 'columns' must exist in the dataset")
-
-        figures = []
-        for col in columns:
-            plt.figure()
-            fig, _ = plt.subplots()
-            ax = sns.histplot(data=df, x=col, kde=True)
-            plt.title(f"Histogram for {col}", weight="bold", fontsize=20)
-
-            plt.xticks(fontsize=18)
-            plt.yticks(fontsize=18)
-            ax.set_xlabel("")
-            ax.set_ylabel("")
-            figures.append(
-                Figure(
-                    for_object=self,
-                    key=f"{self.key}:{col}",
-                    figure=fig,
-                )
-            )
-
-        plt.close("all")
-
-        return self.cache_results(
-            figures=figures,
+    figures = []
+    for col in columns:
+        fig = px.histogram(
+            df, x=col, marginal="violin", nbins=nbins, title=f"Histogram for {col}"
+        )
+        fig.update_layout(
+            title={
+                "text": f"Histogram for {col}",
+                "y": 0.9,
+                "x": 0.5,
+                "xanchor": "center",
+                "yanchor": "top",
+            },
+            xaxis_title="",
+            yaxis_title="",
+            font=dict(size=18),
         )
+        figures.append(fig)
+
+    return tuple(figures)

validmind/tests/data_validation/TimeSeriesOutliers.py

@@ -4,11 +4,8 @@
 
 from dataclasses import dataclass
 
-import matplotlib.pyplot as plt
 import pandas as pd
-import seaborn as sns
-from ydata_profiling.config import Settings
-from ydata_profiling.model.typeset import ProfilingTypeSet
+import plotly.graph_objects as go
 
 from validmind.vm_models import (
     Figure,
@@ -93,7 +90,8 @@ class TimeSeriesOutliers(ThresholdTest):
         zScores = first_result.values["z-score"]
         dates = first_result.values["Date"]
         passFail = [
-            "Pass" if z < self.params["zscore_threshold"] else "Fail" for z in zScores
+            "Pass" if abs(z) < self.params["zscore_threshold"] else "Fail"
+            for z in zScores
         ]
 
         return ResultSummary(
@@ -116,25 +114,26 @@ class TimeSeriesOutliers(ThresholdTest):
         )
 
     def run(self):
+        # Initialize the test_results list
+        test_results = []
+
         # Check if the index of dataframe is datetime
         is_datetime = pd.api.types.is_datetime64_any_dtype(self.inputs.dataset.df.index)
         if not is_datetime:
             raise ValueError("Dataset must be provided with datetime index")
 
-        # Validate threshold paremeter
+        # Validate threshold parameter
         if "zscore_threshold" not in self.params:
             raise ValueError("zscore_threshold must be provided in params")
         zscore_threshold = self.params["zscore_threshold"]
 
         temp_df = self.inputs.dataset.df.copy()
         # temp_df = temp_df.dropna()
-        typeset = ProfilingTypeSet(Settings())
-        dataset_types = typeset.infer_type(temp_df)
-        test_results = []
-        test_figures = []
-        num_features_columns = [
-            k for k, v in dataset_types.items() if str(v) == "Numeric"
-        ]
+
+        # Infer numeric columns
+        num_features_columns = temp_df.select_dtypes(
+            include=["number"]
+        ).columns.tolist()
 
         outliers_table = self.identify_outliers(
             temp_df[num_features_columns], zscore_threshold
@@ -196,49 +195,39 @@ class TimeSeriesOutliers(ThresholdTest):
             df (pandas.DataFrame): Input data with time series.
             outliers_table (pandas.DataFrame): DataFrame with identified outliers.
         Returns:
-            matplotlib.figure.Figure: A matplotlib figure object with subplots for each variable.
+            list: A list of Figure objects with subplots for each variable.
         """
-        sns.set(style="darkgrid")
-        columns = list(self.inputs.dataset.df.columns)
         figures = []
 
-        for col in columns:
-            plt.figure()
-            fig, _ = plt.subplots()
-            column_index_name = df.index.name
-            ax = sns.lineplot(data=df.reset_index(), x=column_index_name, y=col)
+        for col in df.columns:
+            fig = go.Figure()
+
+            fig.add_trace(go.Scatter(x=df.index, y=df[col], mode="lines", name=col))
 
             if not outliers_table.empty:
                 variable_outliers = outliers_table[outliers_table["Variable"] == col]
-                for idx, row in variable_outliers.iterrows():
-                    date = row["Date"]
-                    outlier_value = df.loc[date, col]
-                    ax.scatter(
-                        date,
-                        outlier_value,
-                        marker="o",
-                        s=100,
-                        c="red",
-                        label="Outlier" if idx == 0 else "",
+                fig.add_trace(
+                    go.Scatter(
+                        x=variable_outliers["Date"],
+                        y=df.loc[variable_outliers["Date"], col],
+                        mode="markers",
+                        marker=dict(color="red", size=10),
+                        name="Outlier",
                     )
+                )
 
-            plt.xticks(fontsize=18)
-            plt.yticks(fontsize=18)
-            ax.set_xlabel("")
-            ax.set_ylabel("")
-            ax.set_title(
-                f"Time Series with Outliers for {col}", weight="bold", fontsize=20
+            fig.update_layout(
+                title=f"Time Series with Outliers for {col}",
+                xaxis_title="Date",
+                yaxis_title=col,
             )
 
-            ax.legend()
             figures.append(
                 Figure(
                     for_object=self,
-                    key=f"{self.name}:{col}",
+                    key=f"{self.name}:{col}_{self.inputs.dataset.input_id}",
                     figure=fig,
                 )
             )
 
-        # Do this if you want to prevent the figure from being displayed
-        plt.close("all")
         return figures

validmind/tests/model_validation/ModelMetadataComparison.py

@@ -0,0 +1,59 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+
+from validmind import tags, tasks
+from validmind.utils import get_model_info
+
+
+@tags("model_training", "metadata")
+@tasks("regression", "time_series_forecasting")
+def ModelMetadataComparison(models):
+    """
+    Compare metadata of different models and generate a summary table with the results.
+
+    **Purpose**: The purpose of this function is to compare the metadata of different models, including information about their architecture, framework, framework version, and programming language.
+
+    **Test Mechanism**: The function retrieves the metadata for each model using `get_model_info`, renames columns according to a predefined set of labels, and compiles this information into a summary table.
+
+    **Signs of High Risk**:
+    - Inconsistent or missing metadata across models can indicate potential issues in model documentation or management.
+    - Significant differences in framework versions or programming languages might pose challenges in model integration and deployment.
+
+    **Strengths**:
+    - Provides a clear comparison of essential model metadata.
+    - Standardizes metadata labels for easier interpretation and comparison.
+    - Helps identify potential compatibility or consistency issues across models.
+
+    **Limitations**:
+    - Assumes that the `get_model_info` function returns all necessary metadata fields.
+    - Relies on the correctness and completeness of the metadata provided by each model.
+    - Does not include detailed parameter information, focusing instead on high-level metadata.
+    """
+    column_labels = {
+        "architecture": "Modeling Technique",
+        "framework": "Modeling Framework",
+        "framework_version": "Framework Version",
+        "language": "Programming Language",
+    }
+
+    description = []
+
+    for model in models:
+        model_info = get_model_info(model)
+
+        # Rename columns based on provided labels
+        model_info_renamed = {
+            column_labels.get(k, k): v for k, v in model_info.items() if k != "params"
+        }
+
+        # Add model name or identifier if available
+        model_info_renamed = {"Model Name": model.input_id, **model_info_renamed}
+
+        description.append(model_info_renamed)
+
+    description_df = pd.DataFrame(description)
+
+    return description_df

validmind/tests/model_validation/ModelPredictionResiduals.py

@@ -0,0 +1,103 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+import plotly.graph_objects as go
+from scipy.stats import kstest
+
+from validmind import tags, tasks
+
+
+@tags("regression")
+@tasks("residual_analysis", "visualization")
+def ModelPredictionResiduals(
+    datasets, models, nbins=100, p_value_threshold=0.05, start_date=None, end_date=None
+):
+    """
+    Plot the residuals and histograms for each model, and generate a summary table
+    with the Kolmogorov-Smirnov normality test results.
+
+    **Purpose**: The purpose of this function is to visualize the residuals of model predictions and
+    assess the normality of residuals using the Kolmogorov-Smirnov test.
+
+    **Test Mechanism**: The function iterates through each dataset-model pair, calculates residuals, and generates
+    two figures for each model: one for the time series of residuals and one for the histogram of residuals.
+    It also calculates the KS test for normality and summarizes the results in a table.
+
+    **Signs of High Risk**:
+    - If the residuals are not normally distributed, it could indicate issues with model assumptions.
+    - High skewness or kurtosis in the residuals may indicate model misspecification.
+
+    **Strengths**:
+    - Provides a clear visualization of residuals over time and their distribution.
+    - Includes statistical tests to assess the normality of residuals.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access
+      the pandas DataFrame.
+    - Only generates plots for datasets with a datetime index, and will raise an error for other types of indices.
+    """
+
+    figures = []
+    summary = []
+
+    for dataset, model in zip(datasets, models):
+        df = dataset.df.copy()
+
+        # Filter DataFrame by date range if specified
+        if start_date:
+            df = df[df.index >= pd.to_datetime(start_date)]
+        if end_date:
+            df = df[df.index <= pd.to_datetime(end_date)]
+
+        y_true = dataset.y
+        y_pred = dataset.y_pred(model)
+        residuals = y_true - y_pred
+
+        # Plot residuals
+        residuals_fig = go.Figure()
+        residuals_fig.add_trace(
+            go.Scatter(x=df.index, y=residuals, mode="lines", name="Residuals")
+        )
+        residuals_fig.update_layout(
+            title=f"Residuals for {model.input_id}",
+            xaxis_title="Date",
+            yaxis_title="Residuals",
+            font=dict(size=16),
+            showlegend=False,
+        )
+        figures.append(residuals_fig)
+
+        # Plot histogram of residuals
+        hist_fig = go.Figure()
+        hist_fig.add_trace(go.Histogram(x=residuals, nbinsx=nbins, name="Residuals"))
+        hist_fig.update_layout(
+            title=f"Histogram of Residuals for {model.input_id}",
+            xaxis_title="Residuals",
+            yaxis_title="Frequency",
+            font=dict(size=16),
+            showlegend=False,
+        )
+        figures.append(hist_fig)
+
+        # Perform KS normality test
+        ks_stat, p_value = kstest(
+            residuals, "norm", args=(residuals.mean(), residuals.std())
+        )
+        ks_normality = "Normal" if p_value > p_value_threshold else "Not Normal"
+
+        summary.append(
+            {
+                "Model": model.input_id,
+                "KS Statistic": ks_stat,
+                "p-value": p_value,
+                "KS Normality": ks_normality,
+                "p-value Threshold": p_value_threshold,
+            }
+        )
+
+    # Create a summary DataFrame for the KS normality test results
+    summary_df = pd.DataFrame(summary)
+
+    return (summary_df, *figures)

validmind/tests/model_validation/TimeSeriesPredictionWithCI.py

@@ -0,0 +1,131 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from scipy.stats import norm
+
+from validmind import tags, tasks
+
+
+@tags("model_predictions", "visualization")
+@tasks("regression", "time_series_forecasting")
+def TimeSeriesPredictionWithCI(dataset, model, confidence=0.95):
+    """
+    Plot actual vs predicted values for a time series with confidence intervals and compute breaches.
+
+    **Purpose**: The purpose of this function is to visualize the actual versus predicted values for time series data, including confidence intervals, and to compute and report the number of breaches beyond these intervals.
+
+    **Test Mechanism**: The function calculates the standard deviation of prediction errors, determines the confidence intervals, and counts the number of actual values that fall outside these intervals (breaches). It then generates a plot with the actual values, predicted values, and confidence intervals, and returns a DataFrame summarizing the breach information.
+
+    **Signs of High Risk**:
+    - A high number of breaches indicates that the model's predictions are not reliable within the specified confidence level.
+    - Significant deviations between actual and predicted values may highlight model inadequacies or issues with data quality.
+
+    **Strengths**:
+    - Provides a visual representation of prediction accuracy and the uncertainty around predictions.
+    - Includes a statistical measure of prediction reliability through confidence intervals.
+    - Computes and reports breaches, offering a quantitative assessment of prediction performance.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with a datetime index.
+    - Requires that `dataset.y_pred(model)` returns the predicted values for the model.
+    - The calculation of confidence intervals assumes normally distributed errors, which may not hold for all datasets.
+    """
+    dataset_name = dataset.input_id
+    model_name = model.input_id
+    time_index = dataset.df.index  # Assuming the index of the dataset is datetime
+
+    # Get actual and predicted values
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Compute the standard deviation of the errors
+    errors = y_true - y_pred
+    std_error = np.std(errors)
+
+    # Compute z-score for the given confidence level
+    z_score = norm.ppf(1 - (1 - confidence) / 2)
+
+    # Compute confidence intervals
+    lower_conf = y_pred - z_score * std_error
+    upper_conf = y_pred + z_score * std_error
+
+    # Calculate breaches
+    upper_breaches = (y_true > upper_conf).sum()
+    lower_breaches = (y_true < lower_conf).sum()
+    total_breaches = upper_breaches + lower_breaches
+
+    # Create DataFrame
+    breaches_df = pd.DataFrame(
+        {
+            "Confidence Level": [confidence],
+            "Total Breaches": [total_breaches],
+            "Upper Breaches": [upper_breaches],
+            "Lower Breaches": [lower_breaches],
+        }
+    )
+
+    # Plotting
+    fig = go.Figure()
+
+    # Plot actual values
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=y_true,
+            mode="lines",
+            name="Actual Values",
+            line=dict(color="blue"),
+        )
+    )
+
+    # Plot predicted values
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=y_pred,
+            mode="lines",
+            name=f"Predicted by {model_name}",
+            line=dict(color="red"),
+        )
+    )
+
+    # Add confidence interval lower bound as an invisible line
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=lower_conf,
+            mode="lines",
+            line=dict(width=0),
+            showlegend=False,
+            name="CI Lower",
+        )
+    )
+
+    # Add confidence interval upper bound and fill area
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=upper_conf,
+            mode="lines",
+            fill="tonexty",
+            fillcolor="rgba(200, 200, 200, 0.5)",
+            line=dict(width=0),
+            showlegend=True,
+            name="Confidence Interval",
+        )
+    )
+
+    # Update layout
+    fig.update_layout(
+        title=f"Time Series Actual vs Predicted Values for {dataset_name} and {model_name}",
+        xaxis_title="Time",
+        yaxis_title="Values",
+        legend_title="Legend",
+        template="plotly_white",
+    )
+
+    return fig, breaches_df

validmind/tests/model_validation/TimeSeriesPredictionsPlot.py

@@ -0,0 +1,76 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import plotly.express as px
+import plotly.graph_objects as go
+
+from validmind import tags, tasks
+
+
+@tags("model_predictions", "visualization")
+@tasks("regression", "time_series_forecasting")
+def TimeSeriesPredictionsPlot(datasets, models):
+    """
+    Plot actual vs predicted values for time series data and generate a visual comparison for each model.
+
+    **Purpose**: The purpose of this function is to visualize the actual versus predicted values for time series data across different models.
+
+    **Test Mechanism**: The function iterates through each dataset-model pair, plots the actual values from the dataset, and overlays the predicted values from each model using Plotly for interactive visualization.
+
+    **Signs of High Risk**:
+    - Large discrepancies between actual and predicted values indicate poor model performance.
+    - Systematic deviations in predicted values can highlight model bias or issues with data patterns.
+
+    **Strengths**:
+    - Provides a clear visual comparison of model predictions against actual values.
+    - Uses Plotly for interactive and visually appealing plots.
+    - Can handle multiple models and datasets, displaying them with distinct colors.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with a datetime index.
+    - Requires that `dataset.y_pred(model)` returns the predicted values for the model.
+    - Visualization might become cluttered with a large number of models or datasets.
+    """
+    fig = go.Figure()
+
+    # Use Plotly's color sequence for different model predictions
+    colors = px.colors.qualitative.Plotly
+
+    # Plot actual values from the first dataset
+    dataset = datasets[0]
+    time_index = dataset.df.index  # Assuming the index of the dataset is datetime
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=dataset.y,
+            mode="lines",
+            name="Actual Values",
+            line=dict(color="blue"),
+        )
+    )
+
+    # Plot predicted values for each dataset-model pair
+    for idx, (dataset, model) in enumerate(zip(datasets, models)):
+        model_name = model.input_id
+        y_pred = dataset.y_pred(model)
+        fig.add_trace(
+            go.Scatter(
+                x=time_index,
+                y=y_pred,
+                mode="lines",
+                name=f"Predicted by {model_name}",
+                line=dict(color=colors[idx % len(colors)]),
+            )
+        )
+
+    # Update layout
+    fig.update_layout(
+        title="Time Series Actual vs Predicted Values",
+        xaxis_title="Time",
+        yaxis_title="Values",
+        legend_title="Legend",
+        template="plotly_white",
+    )
+
+    return fig