validmind 2.3.1__py3-none-any.whl → 2.3.5__py3-none-any.whl

validmind/tests/data_validation/SeasonalDecompose.py

@@ -4,10 +4,10 @@
 
 import warnings
 
-import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-import seaborn as sns
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
 from scipy import stats
 from statsmodels.tsa.seasonal import seasonal_decompose
 
@@ -132,7 +132,6 @@ class SeasonalDecompose(Metric):
                 inferred_freq = pd.infer_freq(series.index)
 
                 if inferred_freq is not None:
-                    logger.info(f"Frequency of {col}: {inferred_freq}")
 
                     # Only take finite values to seasonal_decompose
                     sd = seasonal_decompose(
@@ -142,58 +141,87 @@ class SeasonalDecompose(Metric):
 
                     results[col] = self.serialize_seasonal_decompose(sd)
 
-                    # Create subplots
-                    fig, axes = plt.subplots(3, 2)
-                    width, _ = fig.get_size_inches()
-                    fig.set_size_inches(width, 15)
-                    fig.subplots_adjust(hspace=0.3)
-                    fig.suptitle(
-                        f"Seasonal Decomposition for {col}",
-                        fontsize=20,
-                        weight="bold",
-                        y=0.95,
+                    # Create subplots using Plotly
+                    fig = make_subplots(
+                        rows=3,
+                        cols=2,
+                        subplot_titles=(
+                            "Observed",
+                            "Trend",
+                            "Seasonal",
+                            "Residuals",
+                            "Histogram and KDE of Residuals",
+                            "Normal Q-Q Plot of Residuals",
+                        ),
+                        vertical_spacing=0.1,
                     )
 
-                    # Original seasonal decomposition plots
                     # Observed
-                    sd.observed.plot(ax=axes[0, 0])
-                    axes[0, 0].set_title("Observed", fontsize=18)
-                    axes[0, 0].set_xlabel("")
-                    axes[0, 0].tick_params(axis="both", labelsize=18)
+                    fig.add_trace(
+                        go.Scatter(x=sd.observed.index, y=sd.observed, name="Observed"),
+                        row=1,
+                        col=1,
+                    )
 
                     # Trend
-                    sd.trend.plot(ax=axes[0, 1])
-                    axes[0, 1].set_title("Trend", fontsize=18)
-                    axes[0, 1].set_xlabel("")
-                    axes[0, 1].tick_params(axis="both", labelsize=18)
+                    fig.add_trace(
+                        go.Scatter(x=sd.trend.index, y=sd.trend, name="Trend"),
+                        row=1,
+                        col=2,
+                    )
 
                     # Seasonal
-                    sd.seasonal.plot(ax=axes[1, 0])
-                    axes[1, 0].set_title("Seasonal", fontsize=18)
-                    axes[1, 0].set_xlabel("")
-                    axes[1, 0].tick_params(axis="both", labelsize=18)
+                    fig.add_trace(
+                        go.Scatter(x=sd.seasonal.index, y=sd.seasonal, name="Seasonal"),
+                        row=2,
+                        col=1,
+                    )
 
                     # Residuals
-                    sd.resid.plot(ax=axes[1, 1])
-                    axes[1, 1].set_title("Residuals", fontsize=18)
-                    axes[1, 1].set_xlabel("")
-                    axes[1, 1].tick_params(axis="both", labelsize=18)
+                    fig.add_trace(
+                        go.Scatter(x=sd.resid.index, y=sd.resid, name="Residuals"),
+                        row=2,
+                        col=2,
+                    )
 
                     # Histogram with KDE
                     residuals = sd.resid.dropna()
-                    sns.histplot(residuals, kde=True, ax=axes[2, 0])
-                    axes[2, 0].set_title("Histogram and KDE of Residuals", fontsize=18)
-                    axes[2, 0].set_xlabel("")
-                    axes[2, 0].tick_params(axis="both", labelsize=18)
+                    fig.add_trace(
+                        go.Histogram(x=residuals, nbinsx=100, name="Residuals"),
+                        row=3,
+                        col=1,
+                    )
 
                     # Normal Q-Q plot
-                    stats.probplot(residuals, plot=axes[2, 1])
-                    axes[2, 1].set_title("Normal Q-Q Plot of Residuals", fontsize=18)
-                    axes[2, 1].set_xlabel("")
-                    axes[2, 1].tick_params(axis="both", labelsize=18)
+                    qq = stats.probplot(residuals, plot=None)
+                    qq_line_slope, qq_line_intercept = stats.linregress(
+                        qq[0][0], qq[0][1]
+                    )[:2]
+                    qq_line = qq_line_slope * np.array(qq[0][0]) + qq_line_intercept
+
+                    fig.add_trace(
+                        go.Scatter(
+                            x=qq[0][0], y=qq[0][1], mode="markers", name="QQ plot"
+                        ),
+                        row=3,
+                        col=2,
+                    )
+                    fig.add_trace(
+                        go.Scatter(
+                            x=qq[0][0],
+                            y=qq_line,
+                            mode="lines",
+                            name="QQ line",
+                        ),
+                        row=3,
+                        col=2,
+                    )
 
-                    # Do this if you want to prevent the figure from being displayed
-                    plt.close("all")
+                    fig.update_layout(
+                        height=1000,
+                        title_text=f"Seasonal Decomposition for {col}",
+                        showlegend=False,
+                    )
 
                     figures.append(
                         Figure(

validmind/tests/data_validation/TimeSeriesDescription.py

@@ -0,0 +1,74 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+
+from validmind import tags, tasks
+
+
+@tags("time_series_data", "analysis")
+@tasks("regression")
+def TimeSeriesDescription(dataset):
+    """
+    Generates a detailed analysis for the provided time series dataset.
+
+    **Purpose**: The purpose of the TimeSeriesDescription function is to analyze an individual time series
+    by providing a summary of key statistics. This helps in understanding trends, patterns, and data quality issues
+    within the time series.
+
+    **Test Mechanism**: The function extracts the time series data and provides a summary of key statistics.
+    The dataset is expected to have a datetime index. The function checks this and raises an error if the index is
+    not in datetime format. For each variable (column) in the dataset, appropriate statistics including start date,
+    end date, frequency, number of missing values, count, min, and max values are calculated.
+
+    **Signs of High Risk**:
+    - If the index of the dataset is not in datetime format, it could lead to errors in time-series analysis.
+    - Inconsistent or missing data within the dataset might affect the analysis of trends and patterns.
+
+    **Strengths**:
+    - This function provides a comprehensive summary of key statistics for each variable, helping to identify data quality
+      issues such as missing values.
+    - The function helps in understanding the distribution and range of the data by including min and max values.
+
+    **Limitations**:
+    - This function assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access
+      the pandas DataFrame.
+    - It only analyzes datasets with a datetime index and will raise an error for other types of indices.
+    - The function does not handle large datasets efficiently, and performance may degrade with very large datasets.
+    """
+
+    summary = []
+
+    df = (
+        dataset.df
+    )  # Assuming DataFrameDataset objects have a .df attribute to get the pandas DataFrame
+
+    if not pd.api.types.is_datetime64_any_dtype(df.index):
+        raise ValueError(f"Dataset {dataset.input_id} must have a datetime index")
+
+    for column in df.columns:
+        start_date = df.index.min().strftime("%Y-%m-%d")
+        end_date = df.index.max().strftime("%Y-%m-%d")
+        frequency = pd.infer_freq(df.index)
+        num_missing_values = df[column].isna().sum()
+        count = df[column].count()
+        min_value = df[column].min()
+        max_value = df[column].max()
+
+        summary.append(
+            {
+                "Variable": column,
+                "Start Date": start_date,
+                "End Date": end_date,
+                "Frequency": frequency,
+                "Num of Missing Values": num_missing_values,
+                "Count": count,
+                "Min Value": min_value,
+                "Max Value": max_value,
+            }
+        )
+
+    result_df = pd.DataFrame(summary)
+
+    return result_df

validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.py

@@ -0,0 +1,76 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+from scipy.stats import kurtosis, skew
+
+from validmind import tags, tasks
+
+
+@tags("time_series_data", "analysis")
+@tasks("regression")
+def TimeSeriesDescriptiveStatistics(dataset):
+    """
+    Generates a detailed table of descriptive statistics for the provided time series dataset.
+
+    **Purpose**: The purpose of the TimeSeriesDescriptiveStatistics function is to analyze an individual time series
+    by providing a summary of key descriptive statistics. This helps in understanding trends, patterns, and data quality issues
+    within the time series.
+
+    **Test Mechanism**: The function extracts the time series data and provides a summary of key descriptive statistics.
+    The dataset is expected to have a datetime index. The function checks this and raises an error if the index is
+    not in datetime format. For each variable (column) in the dataset, appropriate statistics including start date,
+    end date, min, mean, max, skewness, kurtosis, and count are calculated.
+
+    **Signs of High Risk**:
+    - If the index of the dataset is not in datetime format, it could lead to errors in time-series analysis.
+    - Inconsistent or missing data within the dataset might affect the analysis of trends and patterns.
+
+    **Strengths**:
+    - This function provides a comprehensive summary of key descriptive statistics for each variable, helping to identify data quality
+      issues and understand the distribution of the data.
+
+    **Limitations**:
+    - This function assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access
+      the pandas DataFrame.
+    - It only analyzes datasets with a datetime index and will raise an error for other types of indices.
+    - The function does not handle large datasets efficiently, and performance may degrade with very large datasets.
+    """
+
+    summary = []
+
+    df = (
+        dataset.df
+    )  # Assuming DataFrameDataset objects have a .df attribute to get the pandas DataFrame
+
+    if not pd.api.types.is_datetime64_any_dtype(df.index):
+        raise ValueError(f"Dataset {dataset.input_id} must have a datetime index")
+
+    for column in df.columns:
+        start_date = df.index.min().strftime("%Y-%m-%d")
+        end_date = df.index.max().strftime("%Y-%m-%d")
+        count = df[column].count()
+        min_value = df[column].min()
+        mean_value = df[column].mean()
+        max_value = df[column].max()
+        skewness_value = skew(df[column].dropna())
+        kurtosis_value = kurtosis(df[column].dropna())
+
+        summary.append(
+            {
+                "Variable": column,
+                "Start Date": start_date,
+                "End Date": end_date,
+                "Min": min_value,
+                "Mean": mean_value,
+                "Max": max_value,
+                "Skewness": skewness_value,
+                "Kurtosis": kurtosis_value,
+                "Count": count,
+            }
+        )
+
+    result_df = pd.DataFrame(summary)
+
+    return result_df

validmind/tests/data_validation/TimeSeriesHistogram.py

@@ -2,14 +2,14 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import matplotlib.pyplot as plt
-import pandas as pd
-import seaborn as sns
+import plotly.express as px
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
 
 
-class TimeSeriesHistogram(Metric):
+@tags("data_validation", "visualization")
+@tasks("regression", "time_series_forecasting")
+def TimeSeriesHistogram(dataset, nbins=30):
     """
     Visualizes distribution of time-series data using histograms and Kernel Density Estimation (KDE) lines.
 
@@ -20,7 +20,7 @@ class TimeSeriesHistogram(Metric):
     (kurtosis) underlying the data.
 
     **Test Mechanism**: This test operates on a specific column within the dataset that is required to have a datetime
-    type index. It goes through each column in the given dataset, creating a histogram with Seaborn's histplot
+    type index. It goes through each column in the given dataset, creating a histogram with Plotly's histplot
     function. In cases where the dataset includes more than one time-series (i.e., more than one column with a datetime
     type index), a distinct histogram is plotted for each series. Additionally, a kernel density estimate (KDE) line is
     drawn for each histogram, providing a visualization of the data's underlying probability distribution. The x and
@@ -48,46 +48,30 @@ class TimeSeriesHistogram(Metric):
     - The histogram's shape may be sensitive to the number of bins used.
     """
 
-    name = "time_series_histogram"
-    required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "visualization"],
-    }
+    df = dataset.df
 
-    def run(self):
-        # Check if index is datetime
-        if not pd.api.types.is_datetime64_any_dtype(self.inputs.dataset.df.index):
-            raise ValueError("Index must be a datetime type")
+    columns = list(dataset.df.columns)
 
-        columns = list(self.inputs.dataset.df.columns)
+    if not set(columns).issubset(set(df.columns)):
+        raise ValueError("Provided 'columns' must exist in the dataset")
 
-        df = self.inputs.dataset.df
-
-        if not set(columns).issubset(set(df.columns)):
-            raise ValueError("Provided 'columns' must exist in the dataset")
-
-        figures = []
-        for col in columns:
-            plt.figure()
-            fig, _ = plt.subplots()
-            ax = sns.histplot(data=df, x=col, kde=True)
-            plt.title(f"Histogram for {col}", weight="bold", fontsize=20)
-
-            plt.xticks(fontsize=18)
-            plt.yticks(fontsize=18)
-            ax.set_xlabel("")
-            ax.set_ylabel("")
-            figures.append(
-                Figure(
-                    for_object=self,
-                    key=f"{self.key}:{col}",
-                    figure=fig,
-                )
-            )
-
-        plt.close("all")
-
-        return self.cache_results(
-            figures=figures,
+    figures = []
+    for col in columns:
+        fig = px.histogram(
+            df, x=col, marginal="violin", nbins=nbins, title=f"Histogram for {col}"
+        )
+        fig.update_layout(
+            title={
+                "text": f"Histogram for {col}",
+                "y": 0.9,
+                "x": 0.5,
+                "xanchor": "center",
+                "yanchor": "top",
+            },
+            xaxis_title="",
+            yaxis_title="",
+            font=dict(size=18),
         )
+        figures.append(fig)
+
+    return tuple(figures)

validmind/tests/data_validation/TimeSeriesOutliers.py

@@ -4,11 +4,8 @@
 
 from dataclasses import dataclass
 
-import matplotlib.pyplot as plt
 import pandas as pd
-import seaborn as sns
-from ydata_profiling.config import Settings
-from ydata_profiling.model.typeset import ProfilingTypeSet
+import plotly.graph_objects as go
 
 from validmind.vm_models import (
     Figure,
@@ -93,7 +90,8 @@ class TimeSeriesOutliers(ThresholdTest):
         zScores = first_result.values["z-score"]
         dates = first_result.values["Date"]
         passFail = [
-            "Pass" if z < self.params["zscore_threshold"] else "Fail" for z in zScores
+            "Pass" if abs(z) < self.params["zscore_threshold"] else "Fail"
+            for z in zScores
         ]
 
         return ResultSummary(
@@ -116,25 +114,26 @@ class TimeSeriesOutliers(ThresholdTest):
         )
 
     def run(self):
+        # Initialize the test_results list
+        test_results = []
+
         # Check if the index of dataframe is datetime
         is_datetime = pd.api.types.is_datetime64_any_dtype(self.inputs.dataset.df.index)
         if not is_datetime:
             raise ValueError("Dataset must be provided with datetime index")
 
-        # Validate threshold paremeter
+        # Validate threshold parameter
         if "zscore_threshold" not in self.params:
             raise ValueError("zscore_threshold must be provided in params")
         zscore_threshold = self.params["zscore_threshold"]
 
         temp_df = self.inputs.dataset.df.copy()
         # temp_df = temp_df.dropna()
-        typeset = ProfilingTypeSet(Settings())
-        dataset_types = typeset.infer_type(temp_df)
-        test_results = []
-        test_figures = []
-        num_features_columns = [
-            k for k, v in dataset_types.items() if str(v) == "Numeric"
-        ]
+
+        # Infer numeric columns
+        num_features_columns = temp_df.select_dtypes(
+            include=["number"]
+        ).columns.tolist()
 
         outliers_table = self.identify_outliers(
             temp_df[num_features_columns], zscore_threshold
@@ -196,49 +195,39 @@ class TimeSeriesOutliers(ThresholdTest):
             df (pandas.DataFrame): Input data with time series.
             outliers_table (pandas.DataFrame): DataFrame with identified outliers.
         Returns:
-            matplotlib.figure.Figure: A matplotlib figure object with subplots for each variable.
+            list: A list of Figure objects with subplots for each variable.
         """
-        sns.set(style="darkgrid")
-        columns = list(self.inputs.dataset.df.columns)
         figures = []
 
-        for col in columns:
-            plt.figure()
-            fig, _ = plt.subplots()
-            column_index_name = df.index.name
-            ax = sns.lineplot(data=df.reset_index(), x=column_index_name, y=col)
+        for col in df.columns:
+            fig = go.Figure()
+
+            fig.add_trace(go.Scatter(x=df.index, y=df[col], mode="lines", name=col))
 
             if not outliers_table.empty:
                 variable_outliers = outliers_table[outliers_table["Variable"] == col]
-                for idx, row in variable_outliers.iterrows():
-                    date = row["Date"]
-                    outlier_value = df.loc[date, col]
-                    ax.scatter(
-                        date,
-                        outlier_value,
-                        marker="o",
-                        s=100,
-                        c="red",
-                        label="Outlier" if idx == 0 else "",
+                fig.add_trace(
+                    go.Scatter(
+                        x=variable_outliers["Date"],
+                        y=df.loc[variable_outliers["Date"], col],
+                        mode="markers",
+                        marker=dict(color="red", size=10),
+                        name="Outlier",
                     )
+                )
 
-            plt.xticks(fontsize=18)
-            plt.yticks(fontsize=18)
-            ax.set_xlabel("")
-            ax.set_ylabel("")
-            ax.set_title(
-                f"Time Series with Outliers for {col}", weight="bold", fontsize=20
+            fig.update_layout(
+                title=f"Time Series with Outliers for {col}",
+                xaxis_title="Date",
+                yaxis_title=col,
             )
 
-            ax.legend()
             figures.append(
                 Figure(
                     for_object=self,
-                    key=f"{self.name}:{col}",
+                    key=f"{self.name}:{col}_{self.inputs.dataset.input_id}",
                     figure=fig,
                 )
             )
 
-        # Do this if you want to prevent the figure from being displayed
-        plt.close("all")
         return figures

validmind/tests/decorator.py

@@ -226,6 +226,18 @@ def _get_save_func(func, test_id):
 
 
 def metric(func_or_id):
+    """
+    DEPRECATED, use @vm.test instead
+    """
+    # print a deprecation notice and call the test() function instead
+    logger.warning(
+        "The @vm.metric decorator is deprecated and will be removed in a future release. "
+        "Please use @vm.test instead."
+    )
+    return test(func_or_id)
+
+
+def test(func_or_id):
     """Decorator for creating and registering metrics with the ValidMind framework.
 
     Creates a metric object and registers it with ValidMind under the provided ID. If

validmind/tests/model_validation/ModelMetadataComparison.py

@@ -0,0 +1,59 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+
+from validmind import tags, tasks
+from validmind.utils import get_model_info
+
+
+@tags("model_training", "metadata")
+@tasks("regression", "time_series_forecasting")
+def ModelMetadataComparison(models):
+    """
+    Compare metadata of different models and generate a summary table with the results.
+
+    **Purpose**: The purpose of this function is to compare the metadata of different models, including information about their architecture, framework, framework version, and programming language.
+
+    **Test Mechanism**: The function retrieves the metadata for each model using `get_model_info`, renames columns according to a predefined set of labels, and compiles this information into a summary table.
+
+    **Signs of High Risk**:
+    - Inconsistent or missing metadata across models can indicate potential issues in model documentation or management.
+    - Significant differences in framework versions or programming languages might pose challenges in model integration and deployment.
+
+    **Strengths**:
+    - Provides a clear comparison of essential model metadata.
+    - Standardizes metadata labels for easier interpretation and comparison.
+    - Helps identify potential compatibility or consistency issues across models.
+
+    **Limitations**:
+    - Assumes that the `get_model_info` function returns all necessary metadata fields.
+    - Relies on the correctness and completeness of the metadata provided by each model.
+    - Does not include detailed parameter information, focusing instead on high-level metadata.
+    """
+    column_labels = {
+        "architecture": "Modeling Technique",
+        "framework": "Modeling Framework",
+        "framework_version": "Framework Version",
+        "language": "Programming Language",
+    }
+
+    description = []
+
+    for model in models:
+        model_info = get_model_info(model)
+
+        # Rename columns based on provided labels
+        model_info_renamed = {
+            column_labels.get(k, k): v for k, v in model_info.items() if k != "params"
+        }
+
+        # Add model name or identifier if available
+        model_info_renamed = {"Model Name": model.input_id, **model_info_renamed}
+
+        description.append(model_info_renamed)
+
+    description_df = pd.DataFrame(description)
+
+    return description_df