validmind 2.3.3__py3-none-any.whl → 2.3.5__py3-none-any.whl

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.3.3"
+__version__ = "2.3.5"

validmind/datasets/regression/fred_timeseries.py

@@ -0,0 +1,272 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import os
+
+import pandas as pd
+
+current_path = os.path.dirname(os.path.abspath(__file__))
+mortgage30us_path = os.path.join(current_path, "datasets", "fred", "MORTGAGE30US.csv")
+fedfunds_path = os.path.join(current_path, "datasets", "fred", "FEDFUNDS.csv")
+gs10_path = os.path.join(current_path, "datasets", "fred", "GS10.csv")
+unrate_path = os.path.join(current_path, "datasets", "fred", "UNRATE.csv")
+
+target_column = "MORTGAGE30US"
+feature_columns = ["FEDFUNDS", "GS10", "UNRATE"]
+
+
+def get_common_date_range(dfs):
+    start_dates = [df.index.min() for df in dfs]
+    end_dates = [df.index.max() for df in dfs]
+
+    common_start_date = max(start_dates)
+    common_end_date = min(end_dates)
+
+    return common_start_date, common_end_date
+
+
+def align_date_range(dfs, start_date, end_date):
+    return [df.loc[start_date:end_date] for df in dfs]
+
+
+def load_data():
+    mortgage30us = pd.read_csv(
+        mortgage30us_path, parse_dates=["DATE"], index_col="DATE"
+    )
+    fedfunds = pd.read_csv(fedfunds_path, parse_dates=["DATE"], index_col="DATE")
+    gs10 = pd.read_csv(gs10_path, parse_dates=["DATE"], index_col="DATE")
+    unrate = pd.read_csv(unrate_path, parse_dates=["DATE"], index_col="DATE")
+
+    # Resample mortgage30us to monthly frequency
+    mortgage30us = mortgage30us.resample("MS").last()
+
+    # Get the common date range
+    common_start_date, common_end_date = get_common_date_range(
+        [mortgage30us, fedfunds, gs10, unrate]
+    )
+
+    # Align the date range for all dataframes
+    mortgage30us, fedfunds, gs10, unrate = align_date_range(
+        [mortgage30us, fedfunds, gs10, unrate], common_start_date, common_end_date
+    )
+
+    # Combine into a single DataFrame
+    df = pd.concat([mortgage30us, fedfunds, gs10, unrate], axis=1, join="inner")
+    df.columns = [target_column] + feature_columns
+
+    return df
+
+
+# Convert data back to levels
+def convert_to_levels(diff_df, original_df, target_column):
+    """
+    Convert differenced data back to original levels.
+    """
+    previous_values = original_df[target_column].shift(1).dropna()
+    levels_df = diff_df.add(previous_values, axis=0)
+    return levels_df
+
+
+def get_demo_test_config(test_suite=None):
+
+    default_config = {}
+
+    default_config["validmind.data_validation.TimeSeriesDescription"] = {
+        "inputs": {
+            "dataset": "raw_ds",
+        }
+    }
+    default_config["validmind.data_validation.TimeSeriesLinePlot"] = {
+        "inputs": {
+            "dataset": "raw_ds",
+        }
+    }
+    default_config["validmind.data_validation.TimeSeriesMissingValues"] = {
+        "inputs": {
+            "dataset": "raw_ds",
+        }
+    }
+    default_config["validmind.data_validation.SeasonalDecompose"] = {
+        "inputs": {
+            "dataset": "raw_ds",
+        }
+    }
+    default_config[
+        "validmind.data_validation.TimeSeriesDescriptiveStatistics:train_diff_data"
+    ] = {
+        "inputs": {
+            "dataset": "train_diff_ds",
+        }
+    }
+    default_config[
+        "validmind.data_validation.TimeSeriesDescriptiveStatistics:test_diff_data"
+    ] = {
+        "inputs": {
+            "dataset": "test_diff_ds",
+        }
+    }
+    default_config["validmind.data_validation.TimeSeriesOutliers:train_diff_data"] = {
+        "inputs": {
+            "dataset": "train_diff_ds",
+        },
+        "params": {"zscore_threshold": 4},
+    }
+    default_config["validmind.data_validation.TimeSeriesOutliers:test_diff_data"] = {
+        "inputs": {
+            "dataset": "test_diff_ds",
+        },
+        "params": {"zscore_threshold": 4},
+    }
+    default_config["validmind.data_validation.TimeSeriesHistogram:train_diff_data"] = {
+        "inputs": {
+            "dataset": "train_diff_ds",
+        },
+        "params": {"nbins": 100},
+    }
+    default_config["validmind.data_validation.TimeSeriesHistogram:test_diff_data"] = {
+        "inputs": {
+            "dataset": "test_diff_ds",
+        },
+        "params": {"nbins": 100},
+    }
+    default_config["validmind.data_validation.DatasetSplit"] = {
+        "inputs": {
+            "datasets": ["train_diff_ds", "test_diff_ds"],
+        }
+    }
+    default_config["validmind.model_validation.ModelMetadataComparison"] = {
+        "inputs": {
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.RegressionErrorsComparison:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.RegressionErrorsComparison:test_data"
+    ] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.RegressionR2SquareComparison:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.RegressionR2SquareComparison:test_data"
+    ] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesR2SquareBySegments:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesR2SquareBySegments:test_data"
+    ] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        },
+        "params": {
+            "segments": {
+                "start_date": ["2012-11-01", "2018-02-01"],
+                "end_date": ["2018-01-01", "2023-03-01"],
+            }
+        },
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesPredictionsPlot:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config["validmind.model_validation.TimeSeriesPredictionsPlot:test_data"] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesPredictionWithCI:random_forests_model"
+    ] = {
+        "inputs": {
+            "dataset": "test_ds",
+            "model": "random_forests_model",
+        }
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesPredictionWithCI:gradient_boosting_model"
+    ] = {
+        "inputs": {
+            "dataset": "test_ds",
+            "model": "gradient_boosting_model",
+        }
+    }
+    default_config["validmind.model_validation.ModelPredictionResiduals:train_data"] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config["validmind.model_validation.ModelPredictionResiduals:test_data"] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.FeatureImportanceComparison:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.FeatureImportanceComparison:test_data"
+    ] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.PermutationFeatureImportance:random_forests_model"
+    ] = {
+        "inputs": {
+            "dataset": "test_ds",
+            "model": "random_forests_model",
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.PermutationFeatureImportance:gradient_boosting_model"
+    ] = {
+        "inputs": {
+            "dataset": "test_ds",
+            "model": "gradient_boosting_model",
+        }
+    }
+
+    return default_config

validmind/tests/__types__.py

@@ -18,9 +18,12 @@ TestID = Literal[
     "validmind.prompt_validation.NegativeInstruction",
     "validmind.prompt_validation.Conciseness",
     "validmind.prompt_validation.Delimitation",
+    "validmind.model_validation.ModelPredictionResiduals",
     "validmind.model_validation.BertScore",
+    "validmind.model_validation.TimeSeriesPredictionsPlot",
     "validmind.model_validation.RegardScore",
     "validmind.model_validation.BleuScore",
+    "validmind.model_validation.TimeSeriesPredictionWithCI",
     "validmind.model_validation.RegressionResidualsPlot",
     "validmind.model_validation.FeaturesAUC",
     "validmind.model_validation.ContextualRecall",
@@ -30,6 +33,8 @@ TestID = Literal[
     "validmind.model_validation.ClusterSizeDistribution",
     "validmind.model_validation.TokenDisparity",
     "validmind.model_validation.ToxicityScore",
+    "validmind.model_validation.ModelMetadataComparison",
+    "validmind.model_validation.TimeSeriesR2SquareBySegments",
     "validmind.model_validation.embeddings.CosineSimilarityComparison",
     "validmind.model_validation.embeddings.EmbeddingsVisualization2D",
     "validmind.model_validation.embeddings.StabilityAnalysisRandomNoise",
@@ -77,11 +82,14 @@ TestID = Literal[
     "validmind.model_validation.sklearn.RegressionR2Square",
     "validmind.model_validation.sklearn.RegressionErrors",
     "validmind.model_validation.sklearn.ClusterPerformance",
+    "validmind.model_validation.sklearn.FeatureImportanceComparison",
     "validmind.model_validation.sklearn.TrainingTestDegradation",
+    "validmind.model_validation.sklearn.RegressionErrorsComparison",
     "validmind.model_validation.sklearn.HyperParametersTuning",
     "validmind.model_validation.sklearn.KMeansClustersOptimization",
     "validmind.model_validation.sklearn.ModelsPerformanceComparison",
     "validmind.model_validation.sklearn.WeakspotsDiagnosis",
+    "validmind.model_validation.sklearn.RegressionR2SquareComparison",
     "validmind.model_validation.sklearn.PopulationStabilityIndex",
     "validmind.model_validation.sklearn.MinimumAccuracy",
     "validmind.model_validation.statsmodels.RegressionModelsCoeffs",
@@ -118,6 +126,7 @@ TestID = Literal[
     "validmind.data_validation.TabularCategoricalBarPlots",
     "validmind.data_validation.AutoStationarity",
     "validmind.data_validation.DescriptiveStatistics",
+    "validmind.data_validation.TimeSeriesDescription",
     "validmind.data_validation.ANOVAOneWayTable",
     "validmind.data_validation.TargetRateBarPlots",
     "validmind.data_validation.PearsonCorrelationMatrix",
@@ -154,6 +163,7 @@ TestID = Literal[
     "validmind.data_validation.ClassImbalance",
     "validmind.data_validation.IQROutliersBarPlot",
     "validmind.data_validation.DFGLSArch",
+    "validmind.data_validation.TimeSeriesDescriptiveStatistics",
     "validmind.data_validation.AutoAR",
     "validmind.data_validation.TabularDateTimeHistograms",
     "validmind.data_validation.ADF",

validmind/tests/data_validation/SeasonalDecompose.py

@@ -4,10 +4,10 @@
 
 import warnings
 
-import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-import seaborn as sns
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
 from scipy import stats
 from statsmodels.tsa.seasonal import seasonal_decompose
 
@@ -132,7 +132,6 @@ class SeasonalDecompose(Metric):
                 inferred_freq = pd.infer_freq(series.index)
 
                 if inferred_freq is not None:
-                    logger.info(f"Frequency of {col}: {inferred_freq}")
 
                     # Only take finite values to seasonal_decompose
                     sd = seasonal_decompose(
@@ -142,58 +141,87 @@ class SeasonalDecompose(Metric):
 
                     results[col] = self.serialize_seasonal_decompose(sd)
 
-                    # Create subplots
-                    fig, axes = plt.subplots(3, 2)
-                    width, _ = fig.get_size_inches()
-                    fig.set_size_inches(width, 15)
-                    fig.subplots_adjust(hspace=0.3)
-                    fig.suptitle(
-                        f"Seasonal Decomposition for {col}",
-                        fontsize=20,
-                        weight="bold",
-                        y=0.95,
+                    # Create subplots using Plotly
+                    fig = make_subplots(
+                        rows=3,
+                        cols=2,
+                        subplot_titles=(
+                            "Observed",
+                            "Trend",
+                            "Seasonal",
+                            "Residuals",
+                            "Histogram and KDE of Residuals",
+                            "Normal Q-Q Plot of Residuals",
+                        ),
+                        vertical_spacing=0.1,
                     )
 
-                    # Original seasonal decomposition plots
                     # Observed
-                    sd.observed.plot(ax=axes[0, 0])
-                    axes[0, 0].set_title("Observed", fontsize=18)
-                    axes[0, 0].set_xlabel("")
-                    axes[0, 0].tick_params(axis="both", labelsize=18)
+                    fig.add_trace(
+                        go.Scatter(x=sd.observed.index, y=sd.observed, name="Observed"),
+                        row=1,
+                        col=1,
+                    )
 
                     # Trend
-                    sd.trend.plot(ax=axes[0, 1])
-                    axes[0, 1].set_title("Trend", fontsize=18)
-                    axes[0, 1].set_xlabel("")
-                    axes[0, 1].tick_params(axis="both", labelsize=18)
+                    fig.add_trace(
+                        go.Scatter(x=sd.trend.index, y=sd.trend, name="Trend"),
+                        row=1,
+                        col=2,
+                    )
 
                     # Seasonal
-                    sd.seasonal.plot(ax=axes[1, 0])
-                    axes[1, 0].set_title("Seasonal", fontsize=18)
-                    axes[1, 0].set_xlabel("")
-                    axes[1, 0].tick_params(axis="both", labelsize=18)
+                    fig.add_trace(
+                        go.Scatter(x=sd.seasonal.index, y=sd.seasonal, name="Seasonal"),
+                        row=2,
+                        col=1,
+                    )
 
                     # Residuals
-                    sd.resid.plot(ax=axes[1, 1])
-                    axes[1, 1].set_title("Residuals", fontsize=18)
-                    axes[1, 1].set_xlabel("")
-                    axes[1, 1].tick_params(axis="both", labelsize=18)
+                    fig.add_trace(
+                        go.Scatter(x=sd.resid.index, y=sd.resid, name="Residuals"),
+                        row=2,
+                        col=2,
+                    )
 
                     # Histogram with KDE
                     residuals = sd.resid.dropna()
-                    sns.histplot(residuals, kde=True, ax=axes[2, 0])
-                    axes[2, 0].set_title("Histogram and KDE of Residuals", fontsize=18)
-                    axes[2, 0].set_xlabel("")
-                    axes[2, 0].tick_params(axis="both", labelsize=18)
+                    fig.add_trace(
+                        go.Histogram(x=residuals, nbinsx=100, name="Residuals"),
+                        row=3,
+                        col=1,
+                    )
 
                     # Normal Q-Q plot
-                    stats.probplot(residuals, plot=axes[2, 1])
-                    axes[2, 1].set_title("Normal Q-Q Plot of Residuals", fontsize=18)
-                    axes[2, 1].set_xlabel("")
-                    axes[2, 1].tick_params(axis="both", labelsize=18)
+                    qq = stats.probplot(residuals, plot=None)
+                    qq_line_slope, qq_line_intercept = stats.linregress(
+                        qq[0][0], qq[0][1]
+                    )[:2]
+                    qq_line = qq_line_slope * np.array(qq[0][0]) + qq_line_intercept
+
+                    fig.add_trace(
+                        go.Scatter(
+                            x=qq[0][0], y=qq[0][1], mode="markers", name="QQ plot"
+                        ),
+                        row=3,
+                        col=2,
+                    )
+                    fig.add_trace(
+                        go.Scatter(
+                            x=qq[0][0],
+                            y=qq_line,
+                            mode="lines",
+                            name="QQ line",
+                        ),
+                        row=3,
+                        col=2,
+                    )
 
-                    # Do this if you want to prevent the figure from being displayed
-                    plt.close("all")
+                    fig.update_layout(
+                        height=1000,
+                        title_text=f"Seasonal Decomposition for {col}",
+                        showlegend=False,
+                    )
 
                     figures.append(
                         Figure(

validmind/tests/data_validation/TimeSeriesDescription.py

@@ -0,0 +1,74 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+
+from validmind import tags, tasks
+
+
+@tags("time_series_data", "analysis")
+@tasks("regression")
+def TimeSeriesDescription(dataset):
+    """
+    Generates a detailed analysis for the provided time series dataset.
+
+    **Purpose**: The purpose of the TimeSeriesDescription function is to analyze an individual time series
+    by providing a summary of key statistics. This helps in understanding trends, patterns, and data quality issues
+    within the time series.
+
+    **Test Mechanism**: The function extracts the time series data and provides a summary of key statistics.
+    The dataset is expected to have a datetime index. The function checks this and raises an error if the index is
+    not in datetime format. For each variable (column) in the dataset, appropriate statistics including start date,
+    end date, frequency, number of missing values, count, min, and max values are calculated.
+
+    **Signs of High Risk**:
+    - If the index of the dataset is not in datetime format, it could lead to errors in time-series analysis.
+    - Inconsistent or missing data within the dataset might affect the analysis of trends and patterns.
+
+    **Strengths**:
+    - This function provides a comprehensive summary of key statistics for each variable, helping to identify data quality
+      issues such as missing values.
+    - The function helps in understanding the distribution and range of the data by including min and max values.
+
+    **Limitations**:
+    - This function assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access
+      the pandas DataFrame.
+    - It only analyzes datasets with a datetime index and will raise an error for other types of indices.
+    - The function does not handle large datasets efficiently, and performance may degrade with very large datasets.
+    """
+
+    summary = []
+
+    df = (
+        dataset.df
+    )  # Assuming DataFrameDataset objects have a .df attribute to get the pandas DataFrame
+
+    if not pd.api.types.is_datetime64_any_dtype(df.index):
+        raise ValueError(f"Dataset {dataset.input_id} must have a datetime index")
+
+    for column in df.columns:
+        start_date = df.index.min().strftime("%Y-%m-%d")
+        end_date = df.index.max().strftime("%Y-%m-%d")
+        frequency = pd.infer_freq(df.index)
+        num_missing_values = df[column].isna().sum()
+        count = df[column].count()
+        min_value = df[column].min()
+        max_value = df[column].max()
+
+        summary.append(
+            {
+                "Variable": column,
+                "Start Date": start_date,
+                "End Date": end_date,
+                "Frequency": frequency,
+                "Num of Missing Values": num_missing_values,
+                "Count": count,
+                "Min Value": min_value,
+                "Max Value": max_value,
+            }
+        )
+
+    result_df = pd.DataFrame(summary)
+
+    return result_df

validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.py

@@ -0,0 +1,76 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+from scipy.stats import kurtosis, skew
+
+from validmind import tags, tasks
+
+
+@tags("time_series_data", "analysis")
+@tasks("regression")
+def TimeSeriesDescriptiveStatistics(dataset):
+    """
+    Generates a detailed table of descriptive statistics for the provided time series dataset.
+
+    **Purpose**: The purpose of the TimeSeriesDescriptiveStatistics function is to analyze an individual time series
+    by providing a summary of key descriptive statistics. This helps in understanding trends, patterns, and data quality issues
+    within the time series.
+
+    **Test Mechanism**: The function extracts the time series data and provides a summary of key descriptive statistics.
+    The dataset is expected to have a datetime index. The function checks this and raises an error if the index is
+    not in datetime format. For each variable (column) in the dataset, appropriate statistics including start date,
+    end date, min, mean, max, skewness, kurtosis, and count are calculated.
+
+    **Signs of High Risk**:
+    - If the index of the dataset is not in datetime format, it could lead to errors in time-series analysis.
+    - Inconsistent or missing data within the dataset might affect the analysis of trends and patterns.
+
+    **Strengths**:
+    - This function provides a comprehensive summary of key descriptive statistics for each variable, helping to identify data quality
+      issues and understand the distribution of the data.
+
+    **Limitations**:
+    - This function assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access
+      the pandas DataFrame.
+    - It only analyzes datasets with a datetime index and will raise an error for other types of indices.
+    - The function does not handle large datasets efficiently, and performance may degrade with very large datasets.
+    """
+
+    summary = []
+
+    df = (
+        dataset.df
+    )  # Assuming DataFrameDataset objects have a .df attribute to get the pandas DataFrame
+
+    if not pd.api.types.is_datetime64_any_dtype(df.index):
+        raise ValueError(f"Dataset {dataset.input_id} must have a datetime index")
+
+    for column in df.columns:
+        start_date = df.index.min().strftime("%Y-%m-%d")
+        end_date = df.index.max().strftime("%Y-%m-%d")
+        count = df[column].count()
+        min_value = df[column].min()
+        mean_value = df[column].mean()
+        max_value = df[column].max()
+        skewness_value = skew(df[column].dropna())
+        kurtosis_value = kurtosis(df[column].dropna())
+
+        summary.append(
+            {
+                "Variable": column,
+                "Start Date": start_date,
+                "End Date": end_date,
+                "Min": min_value,
+                "Mean": mean_value,
+                "Max": max_value,
+                "Skewness": skewness_value,
+                "Kurtosis": kurtosis_value,
+                "Count": count,
+            }
+        )
+
+    result_df = pd.DataFrame(summary)
+
+    return result_df