validmind 2.3.3__py3-none-any.whl → 2.4.0__py3-none-any.whl

validmind/tests/model_validation/ClusterSizeDistribution.py

@@ -51,13 +51,11 @@ class ClusterSizeDistribution(Metric):
 
     name = "cluster_size_distribution"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def run(self):
         y_true_train = self.inputs.dataset.y

validmind/tests/model_validation/FeaturesAUC.py

@@ -45,14 +45,12 @@ class FeaturesAUC(Metric):
         "fontsize": 12,
         "figure_height": 500,
     }
-    metadata = {
-        "task_types": ["classification"],
-        "tags": [
-            "feature_importance",
-            "AUC",
-            "visualization",
-        ],
-    }
+    tasks = ["classification"]
+    tags = [
+        "feature_importance",
+        "AUC",
+        "visualization",
+    ]
 
     def run(self):
         dataset = self.inputs.dataset

validmind/tests/model_validation/ModelMetadata.py

@@ -53,15 +53,14 @@ class ModelMetadata(Metric):
 
     name = "model_metadata"
     required_inputs = ["model"]
-    metadata = {
-        "task_types": [
-            "classification",
-            "regression",
-            "text_classification",
-            "text_summarization",
-        ],
-        "tags": ["model_metadata"],
-    }
+    tasks = [
+        "classification",
+        "regression",
+        "text_classification",
+        "text_summarization",
+    ]
+
+    tags = ["model_metadata"]
 
     column_labels = {
         "architecture": "Modeling Technique",

validmind/tests/model_validation/ModelMetadataComparison.py

@@ -0,0 +1,59 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+
+from validmind import tags, tasks
+from validmind.utils import get_model_info
+
+
+@tags("model_training", "metadata")
+@tasks("regression", "time_series_forecasting")
+def ModelMetadataComparison(models):
+    """
+    Compare metadata of different models and generate a summary table with the results.
+
+    **Purpose**: The purpose of this function is to compare the metadata of different models, including information about their architecture, framework, framework version, and programming language.
+
+    **Test Mechanism**: The function retrieves the metadata for each model using `get_model_info`, renames columns according to a predefined set of labels, and compiles this information into a summary table.
+
+    **Signs of High Risk**:
+    - Inconsistent or missing metadata across models can indicate potential issues in model documentation or management.
+    - Significant differences in framework versions or programming languages might pose challenges in model integration and deployment.
+
+    **Strengths**:
+    - Provides a clear comparison of essential model metadata.
+    - Standardizes metadata labels for easier interpretation and comparison.
+    - Helps identify potential compatibility or consistency issues across models.
+
+    **Limitations**:
+    - Assumes that the `get_model_info` function returns all necessary metadata fields.
+    - Relies on the correctness and completeness of the metadata provided by each model.
+    - Does not include detailed parameter information, focusing instead on high-level metadata.
+    """
+    column_labels = {
+        "architecture": "Modeling Technique",
+        "framework": "Modeling Framework",
+        "framework_version": "Framework Version",
+        "language": "Programming Language",
+    }
+
+    description = []
+
+    for model in models:
+        model_info = get_model_info(model)
+
+        # Rename columns based on provided labels
+        model_info_renamed = {
+            column_labels.get(k, k): v for k, v in model_info.items() if k != "params"
+        }
+
+        # Add model name or identifier if available
+        model_info_renamed = {"Model Name": model.input_id, **model_info_renamed}
+
+        description.append(model_info_renamed)
+
+    description_df = pd.DataFrame(description)
+
+    return description_df

validmind/tests/model_validation/ModelPredictionResiduals.py

@@ -0,0 +1,103 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+import plotly.graph_objects as go
+from scipy.stats import kstest
+
+from validmind import tags, tasks
+
+
+@tags("regression")
+@tasks("residual_analysis", "visualization")
+def ModelPredictionResiduals(
+    datasets, models, nbins=100, p_value_threshold=0.05, start_date=None, end_date=None
+):
+    """
+    Plot the residuals and histograms for each model, and generate a summary table
+    with the Kolmogorov-Smirnov normality test results.
+
+    **Purpose**: The purpose of this function is to visualize the residuals of model predictions and
+    assess the normality of residuals using the Kolmogorov-Smirnov test.
+
+    **Test Mechanism**: The function iterates through each dataset-model pair, calculates residuals, and generates
+    two figures for each model: one for the time series of residuals and one for the histogram of residuals.
+    It also calculates the KS test for normality and summarizes the results in a table.
+
+    **Signs of High Risk**:
+    - If the residuals are not normally distributed, it could indicate issues with model assumptions.
+    - High skewness or kurtosis in the residuals may indicate model misspecification.
+
+    **Strengths**:
+    - Provides a clear visualization of residuals over time and their distribution.
+    - Includes statistical tests to assess the normality of residuals.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access
+      the pandas DataFrame.
+    - Only generates plots for datasets with a datetime index, and will raise an error for other types of indices.
+    """
+
+    figures = []
+    summary = []
+
+    for dataset, model in zip(datasets, models):
+        df = dataset.df.copy()
+
+        # Filter DataFrame by date range if specified
+        if start_date:
+            df = df[df.index >= pd.to_datetime(start_date)]
+        if end_date:
+            df = df[df.index <= pd.to_datetime(end_date)]
+
+        y_true = dataset.y
+        y_pred = dataset.y_pred(model)
+        residuals = y_true - y_pred
+
+        # Plot residuals
+        residuals_fig = go.Figure()
+        residuals_fig.add_trace(
+            go.Scatter(x=df.index, y=residuals, mode="lines", name="Residuals")
+        )
+        residuals_fig.update_layout(
+            title=f"Residuals for {model.input_id}",
+            xaxis_title="Date",
+            yaxis_title="Residuals",
+            font=dict(size=16),
+            showlegend=False,
+        )
+        figures.append(residuals_fig)
+
+        # Plot histogram of residuals
+        hist_fig = go.Figure()
+        hist_fig.add_trace(go.Histogram(x=residuals, nbinsx=nbins, name="Residuals"))
+        hist_fig.update_layout(
+            title=f"Histogram of Residuals for {model.input_id}",
+            xaxis_title="Residuals",
+            yaxis_title="Frequency",
+            font=dict(size=16),
+            showlegend=False,
+        )
+        figures.append(hist_fig)
+
+        # Perform KS normality test
+        ks_stat, p_value = kstest(
+            residuals, "norm", args=(residuals.mean(), residuals.std())
+        )
+        ks_normality = "Normal" if p_value > p_value_threshold else "Not Normal"
+
+        summary.append(
+            {
+                "Model": model.input_id,
+                "KS Statistic": ks_stat,
+                "p-value": p_value,
+                "KS Normality": ks_normality,
+                "p-value Threshold": p_value_threshold,
+            }
+        )
+
+    # Create a summary DataFrame for the KS normality test results
+    summary_df = pd.DataFrame(summary)
+
+    return (summary_df, *figures)

validmind/tests/model_validation/RegressionResidualsPlot.py

@@ -52,12 +52,8 @@ class RegressionResidualsPlot(Metric):
 
     name = "regression_residuals_plot"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": [
-            "model_performance",
-        ],
-    }
+    tasks = ["regression"]
+    tags = ["model_performance"]
     default_params = {"bin_size": 0.1}
 
     def run(self):

validmind/tests/model_validation/TimeSeriesPredictionWithCI.py

@@ -0,0 +1,131 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from scipy.stats import norm
+
+from validmind import tags, tasks
+
+
+@tags("model_predictions", "visualization")
+@tasks("regression", "time_series_forecasting")
+def TimeSeriesPredictionWithCI(dataset, model, confidence=0.95):
+    """
+    Plot actual vs predicted values for a time series with confidence intervals and compute breaches.
+
+    **Purpose**: The purpose of this function is to visualize the actual versus predicted values for time series data, including confidence intervals, and to compute and report the number of breaches beyond these intervals.
+
+    **Test Mechanism**: The function calculates the standard deviation of prediction errors, determines the confidence intervals, and counts the number of actual values that fall outside these intervals (breaches). It then generates a plot with the actual values, predicted values, and confidence intervals, and returns a DataFrame summarizing the breach information.
+
+    **Signs of High Risk**:
+    - A high number of breaches indicates that the model's predictions are not reliable within the specified confidence level.
+    - Significant deviations between actual and predicted values may highlight model inadequacies or issues with data quality.
+
+    **Strengths**:
+    - Provides a visual representation of prediction accuracy and the uncertainty around predictions.
+    - Includes a statistical measure of prediction reliability through confidence intervals.
+    - Computes and reports breaches, offering a quantitative assessment of prediction performance.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with a datetime index.
+    - Requires that `dataset.y_pred(model)` returns the predicted values for the model.
+    - The calculation of confidence intervals assumes normally distributed errors, which may not hold for all datasets.
+    """
+    dataset_name = dataset.input_id
+    model_name = model.input_id
+    time_index = dataset.df.index  # Assuming the index of the dataset is datetime
+
+    # Get actual and predicted values
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Compute the standard deviation of the errors
+    errors = y_true - y_pred
+    std_error = np.std(errors)
+
+    # Compute z-score for the given confidence level
+    z_score = norm.ppf(1 - (1 - confidence) / 2)
+
+    # Compute confidence intervals
+    lower_conf = y_pred - z_score * std_error
+    upper_conf = y_pred + z_score * std_error
+
+    # Calculate breaches
+    upper_breaches = (y_true > upper_conf).sum()
+    lower_breaches = (y_true < lower_conf).sum()
+    total_breaches = upper_breaches + lower_breaches
+
+    # Create DataFrame
+    breaches_df = pd.DataFrame(
+        {
+            "Confidence Level": [confidence],
+            "Total Breaches": [total_breaches],
+            "Upper Breaches": [upper_breaches],
+            "Lower Breaches": [lower_breaches],
+        }
+    )
+
+    # Plotting
+    fig = go.Figure()
+
+    # Plot actual values
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=y_true,
+            mode="lines",
+            name="Actual Values",
+            line=dict(color="blue"),
+        )
+    )
+
+    # Plot predicted values
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=y_pred,
+            mode="lines",
+            name=f"Predicted by {model_name}",
+            line=dict(color="red"),
+        )
+    )
+
+    # Add confidence interval lower bound as an invisible line
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=lower_conf,
+            mode="lines",
+            line=dict(width=0),
+            showlegend=False,
+            name="CI Lower",
+        )
+    )
+
+    # Add confidence interval upper bound and fill area
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=upper_conf,
+            mode="lines",
+            fill="tonexty",
+            fillcolor="rgba(200, 200, 200, 0.5)",
+            line=dict(width=0),
+            showlegend=True,
+            name="Confidence Interval",
+        )
+    )
+
+    # Update layout
+    fig.update_layout(
+        title=f"Time Series Actual vs Predicted Values for {dataset_name} and {model_name}",
+        xaxis_title="Time",
+        yaxis_title="Values",
+        legend_title="Legend",
+        template="plotly_white",
+    )
+
+    return fig, breaches_df

validmind/tests/model_validation/TimeSeriesPredictionsPlot.py

@@ -0,0 +1,76 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import plotly.express as px
+import plotly.graph_objects as go
+
+from validmind import tags, tasks
+
+
+@tags("model_predictions", "visualization")
+@tasks("regression", "time_series_forecasting")
+def TimeSeriesPredictionsPlot(datasets, models):
+    """
+    Plot actual vs predicted values for time series data and generate a visual comparison for each model.
+
+    **Purpose**: The purpose of this function is to visualize the actual versus predicted values for time series data across different models.
+
+    **Test Mechanism**: The function iterates through each dataset-model pair, plots the actual values from the dataset, and overlays the predicted values from each model using Plotly for interactive visualization.
+
+    **Signs of High Risk**:
+    - Large discrepancies between actual and predicted values indicate poor model performance.
+    - Systematic deviations in predicted values can highlight model bias or issues with data patterns.
+
+    **Strengths**:
+    - Provides a clear visual comparison of model predictions against actual values.
+    - Uses Plotly for interactive and visually appealing plots.
+    - Can handle multiple models and datasets, displaying them with distinct colors.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with a datetime index.
+    - Requires that `dataset.y_pred(model)` returns the predicted values for the model.
+    - Visualization might become cluttered with a large number of models or datasets.
+    """
+    fig = go.Figure()
+
+    # Use Plotly's color sequence for different model predictions
+    colors = px.colors.qualitative.Plotly
+
+    # Plot actual values from the first dataset
+    dataset = datasets[0]
+    time_index = dataset.df.index  # Assuming the index of the dataset is datetime
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=dataset.y,
+            mode="lines",
+            name="Actual Values",
+            line=dict(color="blue"),
+        )
+    )
+
+    # Plot predicted values for each dataset-model pair
+    for idx, (dataset, model) in enumerate(zip(datasets, models)):
+        model_name = model.input_id
+        y_pred = dataset.y_pred(model)
+        fig.add_trace(
+            go.Scatter(
+                x=time_index,
+                y=y_pred,
+                mode="lines",
+                name=f"Predicted by {model_name}",
+                line=dict(color=colors[idx % len(colors)]),
+            )
+        )
+
+    # Update layout
+    fig.update_layout(
+        title="Time Series Actual vs Predicted Values",
+        xaxis_title="Time",
+        yaxis_title="Values",
+        legend_title="Legend",
+        template="plotly_white",
+    )
+
+    return fig

validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py

@@ -0,0 +1,103 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+
+import pandas as pd
+import plotly.express as px
+from sklearn import metrics
+
+from validmind import tags, tasks
+
+
+@tags("model_performance", "sklearn")
+@tasks("regression", "time_series_forecasting")
+def TimeSeriesR2SquareBySegments(datasets, models, segments=None):
+    """
+    Plot R-Squared values for each model over specified time segments and generate a bar chart
+    with the results.
+
+    **Purpose**: The purpose of this function is to plot the R-Squared values for different models applied to various segments of the time series data.
+
+    **Parameters**:
+    - datasets: List of datasets to evaluate.
+    - models: List of models to evaluate.
+    - segments: Dictionary with 'start_date' and 'end_date' keys containing lists of start and end dates for each segments. If None, the time series will be segmented into two halves.
+
+    **Test Mechanism**: The function iterates through each dataset-model pair, calculates the R-Squared values for specified time segments, and generates a bar chart with these results.
+
+    **Signs of High Risk**:
+    - If the R-Squared values are significantly low for certain segments, it could indicate that the model is not explaining much of the variability in the dataset for those segments.
+
+    **Strengths**:
+    - Provides a visual representation of model performance across different time segments.
+    - Allows for identification of segments where models perform poorly.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with `y`, `y_pred`, and `feature_columns` attributes.
+    - Requires that `dataset.y_pred(model)` returns the predicted values for the model.
+    - Assumes that `y_true` and `y_pred` are pandas Series with datetime indices.
+    """
+    results_list = []
+
+    for dataset, model in zip(datasets, models):
+        dataset_name = dataset.input_id
+        model_name = model.input_id
+
+        y_true = dataset.y
+        y_pred = dataset.y_pred(model)
+
+        # Ensure y_true and y_pred are pandas Series with the same index
+        if not isinstance(y_true, pd.Series):
+            y_true = pd.Series(y_true, index=dataset.df.index)
+        if not isinstance(y_pred, pd.Series):
+            y_pred = pd.Series(y_pred, index=dataset.df.index)
+
+        index = dataset.df.index
+
+        if segments is None:
+            mid_point = len(index) // 2
+            segments = {
+                "start_date": [index.min(), index[mid_point]],
+                "end_date": [index[mid_point - 1], index.max()],
+            }
+
+        for segment_index, (start_date, end_date) in enumerate(
+            zip(segments["start_date"], segments["end_date"])
+        ):
+            mask = (index >= start_date) & (index <= end_date)
+            y_true_segment = y_true.loc[mask]
+            y_pred_segment = y_pred.loc[mask]
+
+            if len(y_true_segment) > 0 and len(y_pred_segment) > 0:
+                r2s = metrics.r2_score(y_true_segment, y_pred_segment)
+                results_list.append(
+                    {
+                        "Model": model_name,
+                        "Dataset": dataset_name,
+                        "Segments": f"Segment {segment_index + 1}",
+                        "Start Date": start_date,
+                        "End Date": end_date,
+                        "R-Squared": r2s,
+                    }
+                )
+
+    # Convert results list to a DataFrame
+    results_df = pd.DataFrame(results_list)
+
+    # Plotting
+    fig = px.bar(
+        results_df,
+        x="Segments",
+        y="R-Squared",
+        color="Model",
+        barmode="group",
+        title="R-Squared Comparison by Segment and Model",
+        labels={
+            "R-Squared": "R-Squared Value",
+            "Segment": "Time Segment",
+            "Model": "Model",
+        },
+    )
+
+    return fig, results_df

validmind/tests/model_validation/embeddings/ClusterDistribution.py

@@ -51,10 +51,8 @@ class ClusterDistribution(Metric):
     default_params = {
         "num_clusters": 5,
     }
-    metadata = {
-        "task_types": ["feature_extraction"],
-        "tags": ["llm", "text_data", "text_embeddings", "visualization"],
-    }
+    tasks = ["feature_extraction"]
+    tags = ["llm", "text_data", "text_embeddings", "visualization"]
 
     def run(self):
         # run kmeans clustering on embeddings

validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py

@@ -50,10 +50,8 @@ class CosineSimilarityDistribution(Metric):
 
     name = "Text Embeddings Cosine Similarity Distribution"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["feature_extraction"],
-        "tags": ["llm", "text_data", "text_embeddings", "visualization"],
-    }
+    tasks = ["feature_extraction"]
+    tags = ["llm", "text_data", "text_embeddings", "visualization"]
 
     def run(self):
         # Compute cosine similarity

validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py

@@ -53,10 +53,8 @@ class DescriptiveAnalytics(Metric):
 
     name = "Descriptive Analytics for Text Embeddings Models"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["feature_extraction"],
-        "tags": ["llm", "text_data", "text_embeddings", "visualization"],
-    }
+    tasks = ["feature_extraction"]
+    tags = ["llm", "text_data", "text_embeddings", "visualization"]
 
     def run(self):
         # Assuming y_pred returns a 2D array of embeddings [samples, features]

validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py

@@ -53,10 +53,8 @@ class EmbeddingsVisualization2D(Metric):
         "cluster_column": None,
         "perplexity": 30,
     }
-    metadata = {
-        "task_types": ["feature_extraction"],
-        "tags": ["llm", "text_data", "text_embeddings", "visualization"],
-    }
+    tasks = ["feature_extraction"]
+    tags = ["llm", "text_data", "text_embeddings", "visualization"]
 
     def run(self):
         cluster_column = self.params.get("cluster_column")

validmind/tests/model_validation/embeddings/StabilityAnalysis.py

@@ -29,10 +29,8 @@ class StabilityAnalysis(ThresholdTest):
     default_params = {
         "mean_similarity_threshold": 0.7,
     }
-    metadata = {
-        "task_types": ["feature_extraction"],
-        "tags": ["llm", "text_data", "text_embeddings", "visualization"],
-    }
+    tasks = ["feature_extraction"]
+    tags = ["llm", "text_data", "text_embeddings", "visualization"]
 
     @abstractmethod
     def perturb_data(self, data: str) -> str:

validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py

@@ -48,13 +48,11 @@ class AdjustedMutualInformation(ClusterPerformance):
 
     name = "adjusted_mutual_information"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def metric_info(self):
         return {"Adjusted Mutual Information": metrics.adjusted_mutual_info_score}

validmind/tests/model_validation/sklearn/AdjustedRandIndex.py

@@ -47,13 +47,11 @@ class AdjustedRandIndex(ClusterPerformance):
 
     name = "adjusted_rand_index"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def metric_info(self):
         return {"Adjusted Rand Index": metrics.adjusted_rand_score}