validmind 2.5.15__py3-none-any.whl → 2.5.19__py3-none-any.whl

validmind/errors.py

@@ -207,6 +207,23 @@ class MissingRequiredTestInputError(BaseError):
     pass
 
 
+class MissingDependencyError(BaseError):
+    """
+    When a required dependency is missing.
+    """
+
+    def __init__(self, message="", required_dependencies=None, extra=None):
+        """
+        Args:
+            message (str): The error message.
+            required_dependencies (list): A list of required dependencies.
+            extra (str): The particular validmind `extra` that will install the missing dependencies.
+        """
+        super().__init__(message)
+        self.required_dependencies = required_dependencies or []
+        self.extra = extra
+
+
 class MissingRExtrasError(BaseError):
     """
     When the R extras have not been installed.

validmind/tests/__types__.py

@@ -33,7 +33,6 @@ TestID = Literal[
     "validmind.model_validation.ClusterSizeDistribution",
     "validmind.model_validation.TokenDisparity",
     "validmind.model_validation.ToxicityScore",
-    "validmind.model_validation.ModelMetadata",
     "validmind.model_validation.TimeSeriesR2SquareBySegments",
     "validmind.model_validation.embeddings.CosineSimilarityComparison",
     "validmind.model_validation.embeddings.EmbeddingsVisualization2D",
@@ -53,12 +52,13 @@ TestID = Literal[
     "validmind.model_validation.ragas.ContextEntityRecall",
     "validmind.model_validation.ragas.Faithfulness",
     "validmind.model_validation.ragas.AspectCritique",
+    "validmind.model_validation.ragas.NoiseSensitivity",
     "validmind.model_validation.ragas.AnswerSimilarity",
     "validmind.model_validation.ragas.AnswerCorrectness",
     "validmind.model_validation.ragas.ContextRecall",
     "validmind.model_validation.ragas.ContextPrecision",
     "validmind.model_validation.ragas.AnswerRelevance",
-    "validmind.model_validation.sklearn.RegressionModelsPerformanceComparison",
+    "validmind.model_validation.ragas.ContextUtilization",
     "validmind.model_validation.sklearn.AdjustedMutualInformation",
     "validmind.model_validation.sklearn.SilhouettePlot",
     "validmind.model_validation.sklearn.RobustnessDiagnosis",
@@ -77,35 +77,35 @@ TestID = Literal[
     "validmind.model_validation.sklearn.ClassifierPerformance",
     "validmind.model_validation.sklearn.VMeasure",
     "validmind.model_validation.sklearn.MinimumF1Score",
+    "validmind.model_validation.sklearn.RegressionPerformance",
     "validmind.model_validation.sklearn.ROCCurve",
     "validmind.model_validation.sklearn.RegressionR2Square",
     "validmind.model_validation.sklearn.RegressionErrors",
     "validmind.model_validation.sklearn.ClusterPerformance",
-    "validmind.model_validation.sklearn.FeatureImportance",
     "validmind.model_validation.sklearn.TrainingTestDegradation",
+    "validmind.model_validation.sklearn.RegressionErrorsComparison",
+    "validmind.model_validation.sklearn.FeatureImportance",
     "validmind.model_validation.sklearn.HyperParametersTuning",
     "validmind.model_validation.sklearn.KMeansClustersOptimization",
     "validmind.model_validation.sklearn.ModelsPerformanceComparison",
     "validmind.model_validation.sklearn.WeakspotsDiagnosis",
+    "validmind.model_validation.sklearn.RegressionR2SquareComparison",
     "validmind.model_validation.sklearn.PopulationStabilityIndex",
     "validmind.model_validation.sklearn.MinimumAccuracy",
-    "validmind.model_validation.statsmodels.RegressionModelCoeffs",
-    "validmind.model_validation.statsmodels.BoxPierce",
-    "validmind.model_validation.statsmodels.RegressionCoeffsPlot",
+    "validmind.model_validation.statsmodels.RegressionModelSensitivityPlot",
+    "validmind.model_validation.statsmodels.RegressionModelForecastPlotLevels",
     "validmind.model_validation.statsmodels.ScorecardHistogram",
-    "validmind.model_validation.statsmodels.LJungBox",
-    "validmind.model_validation.statsmodels.JarqueBera",
     "validmind.model_validation.statsmodels.KolmogorovSmirnov",
-    "validmind.model_validation.statsmodels.ShapiroWilk",
     "validmind.model_validation.statsmodels.CumulativePredictionProbabilities",
     "validmind.model_validation.statsmodels.RegressionFeatureSignificance",
     "validmind.model_validation.statsmodels.RegressionModelSummary",
+    "validmind.model_validation.statsmodels.RegressionCoeffs",
     "validmind.model_validation.statsmodels.Lilliefors",
-    "validmind.model_validation.statsmodels.RunsTest",
     "validmind.model_validation.statsmodels.RegressionPermutationFeatureImportance",
     "validmind.model_validation.statsmodels.PredictionProbabilitiesHistogram",
     "validmind.model_validation.statsmodels.AutoARIMA",
     "validmind.model_validation.statsmodels.GINITable",
+    "validmind.model_validation.statsmodels.RegressionModelForecastPlot",
     "validmind.model_validation.statsmodels.DurbinWatsonTest",
     "validmind.ongoing_monitoring.PredictionCorrelation",
     "validmind.ongoing_monitoring.PredictionAcrossEachFeature",
@@ -113,9 +113,11 @@ TestID = Literal[
     "validmind.ongoing_monitoring.TargetPredictionDistributionPlot",
     "validmind.data_validation.IQROutliersTable",
     "validmind.data_validation.Skewness",
+    "validmind.data_validation.BoxPierce",
     "validmind.data_validation.Duplicates",
     "validmind.data_validation.MissingValuesBarPlot",
     "validmind.data_validation.DatasetDescription",
+    "validmind.data_validation.ProtectedClassesCombination",
     "validmind.data_validation.ZivotAndrewsArch",
     "validmind.data_validation.ScatterPlot",
     "validmind.data_validation.TimeSeriesOutliers",
@@ -123,7 +125,9 @@ TestID = Literal[
     "validmind.data_validation.AutoStationarity",
     "validmind.data_validation.DescriptiveStatistics",
     "validmind.data_validation.TimeSeriesDescription",
+    "validmind.data_validation.LJungBox",
     "validmind.data_validation.TargetRateBarPlots",
+    "validmind.data_validation.JarqueBera",
     "validmind.data_validation.PearsonCorrelationMatrix",
     "validmind.data_validation.FeatureTargetCorrelationPlot",
     "validmind.data_validation.TabularNumericalHistograms",
@@ -133,9 +137,11 @@ TestID = Literal[
     "validmind.data_validation.MissingValues",
     "validmind.data_validation.PhillipsPerronArch",
     "validmind.data_validation.RollingStatsPlot",
+    "validmind.data_validation.ProtectedClassesDisparity",
     "validmind.data_validation.TabularDescriptionTables",
     "validmind.data_validation.AutoMA",
     "validmind.data_validation.UniqueRows",
+    "validmind.data_validation.ShapiroWilk",
     "validmind.data_validation.TooManyZeroValues",
     "validmind.data_validation.HighPearsonCorrelation",
     "validmind.data_validation.ACFandPACFPlot",
@@ -146,10 +152,12 @@ TestID = Literal[
     "validmind.data_validation.TimeSeriesLinePlot",
     "validmind.data_validation.KPSS",
     "validmind.data_validation.AutoSeasonality",
+    "validmind.data_validation.ProtectedClassesDescription",
     "validmind.data_validation.BivariateScatterPlots",
     "validmind.data_validation.EngleGrangerCoint",
     "validmind.data_validation.TimeSeriesMissingValues",
     "validmind.data_validation.TimeSeriesHistogram",
+    "validmind.data_validation.RunsTest",
     "validmind.data_validation.LaggedCorrelationHeatmap",
     "validmind.data_validation.SeasonalDecompose",
     "validmind.data_validation.WOEBinPlots",
@@ -159,6 +167,7 @@ TestID = Literal[
     "validmind.data_validation.TimeSeriesDescriptiveStatistics",
     "validmind.data_validation.AutoAR",
     "validmind.data_validation.TabularDateTimeHistograms",
+    "validmind.data_validation.ProtectedClassesThresholdOptimizer",
     "validmind.data_validation.ADF",
     "validmind.data_validation.nlp.Toxicity",
     "validmind.data_validation.nlp.PolarityAndSubjectivity",

validmind/tests/{model_validation/statsmodels → data_validation}/BoxPierce.py

@@ -2,12 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import pandas as pd
 from statsmodels.stats.diagnostic import acorr_ljungbox
 
-from validmind.vm_models import Metric
+from validmind import tags, tasks
 
 
-class BoxPierce(Metric):
+@tasks("regression")
+@tags("time_series_data", "forecasting", "statistical_test", "statsmodels")
+def BoxPierce(dataset):
     """
     Detects autocorrelation in time-series data through the Box-Pierce test to validate model performance.
 
@@ -51,25 +54,18 @@ class BoxPierce(Metric):
     - Applicability is limited to time-series data, which limits its overall utility.
     """
 
-    name = "box_pierce"
-    required_inputs = ["dataset"]
-    tasks = ["regression"]
-    tags = ["time_series_data", "forecasting", "statistical_test", "statsmodels"]
-
-    def run(self):
-        """
-        Calculates Box-Pierce test for each of the dataset features
-        """
-        x_train = self.inputs.dataset.df
-
-        box_pierce_values = {}
-        for col in x_train.columns:
-            bp_results = acorr_ljungbox(
-                x_train[col].values, boxpierce=True, return_df=True
-            )
-            box_pierce_values[col] = {
-                "stat": bp_results.iloc[0]["lb_stat"],
-                "pvalue": bp_results.iloc[0]["lb_pvalue"],
-            }
-
-        return self.cache_results(box_pierce_values)
+    df = dataset.df
+
+    box_pierce_values = {}
+    for col in df.columns:
+        bp_results = acorr_ljungbox(df[col].values, boxpierce=True, return_df=True)
+        box_pierce_values[col] = {
+            "stat": bp_results.iloc[0]["lb_stat"],
+            "pvalue": bp_results.iloc[0]["lb_pvalue"],
+        }
+
+    box_pierce_df = pd.DataFrame.from_dict(box_pierce_values, orient="index")
+    box_pierce_df.reset_index(inplace=True)
+    box_pierce_df.columns = ["column", "stat", "pvalue"]
+
+    return box_pierce_df

validmind/tests/data_validation/ChiSquaredFeaturesTable.py

@@ -7,6 +7,7 @@ import pandas as pd
 from scipy.stats import chi2_contingency
 
 from validmind import tags, tasks
+from validmind.errors import SkipTestError
 
 
 @tags("tabular_data", "categorical_data", "statistical_test")
@@ -55,9 +56,11 @@ def ChiSquaredFeaturesTable(dataset, p_threshold=0.05):
     """
 
     target_column = dataset.target_column
-
     features = dataset.feature_columns_categorical
 
+    if not features:
+        raise SkipTestError("No categorical features found in dataset")
+
     results_df = _chi_squared_categorical_feature_selection(
         dataset.df, features, target_column, p_threshold
     )

validmind/tests/{model_validation/statsmodels → data_validation}/JarqueBera.py

@@ -2,12 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import pandas as pd
 from statsmodels.stats.stattools import jarque_bera
 
-from validmind.vm_models import Metric
+from validmind import tags, tasks
 
 
-class JarqueBera(Metric):
+@tasks("classification", "regression")
+@tags("tabular_data", "data_distribution", "statistical_test", "statsmodels")
+def JarqueBera(dataset):
     """
     Assesses normality of dataset features in an ML model using the Jarque-Bera test.
 
@@ -48,31 +51,20 @@ class JarqueBera(Metric):
     even for minor deviations in larger datasets.
     """
 
-    name = "jarque_bera"
-    required_inputs = ["dataset"]
-    tasks = ["classification", "regression"]
-    tags = [
-        "tabular_data",
-        "data_distribution",
-        "statistical_test",
-        "statsmodels",
-    ]
-
-    def run(self):
-        """
-        Calculates JB for each of the dataset features
-        """
-        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
-
-        jb_values = {}
-        for col in x_train.columns:
-            jb_stat, jb_pvalue, jb_skew, jb_kurtosis = jarque_bera(x_train[col].values)
-
-            jb_values[col] = {
-                "stat": jb_stat,
-                "pvalue": jb_pvalue,
-                "skew": jb_skew,
-                "kurtosis": jb_kurtosis,
-            }
-
-        return self.cache_results(jb_values)
+    df = dataset.df[dataset.feature_columns_numeric]
+
+    jb_values = {}
+    for col in df.columns:
+        jb_stat, jb_pvalue, jb_skew, jb_kurtosis = jarque_bera(df[col].values)
+        jb_values[col] = {
+            "stat": jb_stat,
+            "pvalue": jb_pvalue,
+            "skew": jb_skew,
+            "kurtosis": jb_kurtosis,
+        }
+
+    jb_df = pd.DataFrame.from_dict(jb_values, orient="index")
+    jb_df.reset_index(inplace=True)
+    jb_df.columns = ["column", "stat", "pvalue", "skew", "kurtosis"]
+
+    return jb_df

validmind/tests/{model_validation/statsmodels → data_validation}/LJungBox.py

@@ -2,12 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import pandas as pd
 from statsmodels.stats.diagnostic import acorr_ljungbox
 
-from validmind.vm_models import Metric
+from validmind import tags, tasks
 
 
-class LJungBox(Metric):
+@tasks("regression")
+@tags("time_series_data", "forecasting", "statistical_test", "statsmodels")
+def LJungBox(dataset):
     """
     Assesses autocorrelations in dataset features by performing a Ljung-Box test on each feature.
 
@@ -20,11 +23,11 @@ class LJungBox(Metric):
 
     ### Test Mechanism
 
-    The test operates by iterating over each feature within the training dataset and applying the `acorr_ljungbox`
+    The test operates by iterating over each feature within the dataset and applying the `acorr_ljungbox`
     function from the `statsmodels.stats.diagnostic` library. This function calculates the Ljung-Box statistic and
-    p-value for each feature. These results are then stored in a dictionary where the keys are the feature names and
-    the values are dictionaries containing the statistic and p-value respectively. Generally, a lower p-value indicates
-    a higher likelihood of significant autocorrelations within the feature.
+    p-value for each feature. These results are then stored in a pandas DataFrame where the columns are the feature names,
+    statistic, and p-value respectively. Generally, a lower p-value indicates a higher likelihood of significant
+    autocorrelations within the feature.
 
     ### Signs of High Risk
 
@@ -41,30 +44,23 @@ class LJungBox(Metric):
     ### Limitations
 
     - Cannot detect all types of non-linearity or complex interrelationships among variables.
-    - Testing individual features may not fully encapsulate the dynamics of the data if features interact with each
-    other.
+    - Testing individual features may not fully encapsulate the dynamics of the data if features interact with each other.
     - Designed more for traditional statistical models and may not be fully compatible with certain types of complex
-    machine learning models.
+      machine learning models.
     """
 
-    name = "ljung_box"
-    required_inputs = ["dataset"]
-    tasks = ["regression"]
-    tags = ["time_series_data", "forecasting", "statistical_test", "statsmodels"]
+    df = dataset.df
 
-    def run(self):
-        """
-        Calculates Ljung-Box test for each of the dataset features
-        """
-        x_train = self.inputs.dataset.df
+    ljung_box_values = {}
+    for col in df.columns:
+        lb_results = acorr_ljungbox(df[col].values, return_df=True)
+        ljung_box_values[col] = {
+            "stat": lb_results.iloc[0]["lb_stat"],
+            "pvalue": lb_results.iloc[0]["lb_pvalue"],
+        }
 
-        ljung_box_values = {}
-        for col in x_train.columns:
-            lb_results = acorr_ljungbox(x_train[col].values, return_df=True)
+    ljung_box_df = pd.DataFrame.from_dict(ljung_box_values, orient="index")
+    ljung_box_df.reset_index(inplace=True)
+    ljung_box_df.columns = ["column", "stat", "pvalue"]
 
-            ljung_box_values[col] = {
-                "stat": lb_results["lb_stat"].values[0],
-                "pvalue": lb_results["lb_pvalue"].values[0],
-            }
-
-        return self.cache_results(ljung_box_values)
+    return ljung_box_df

validmind/tests/data_validation/ProtectedClassesCombination.py

@@ -0,0 +1,205 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import sys
+
+import pandas as pd
+import plotly.graph_objects as go
+import plotly.subplots as sp
+
+from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
+from validmind.logging import get_logger
+
+try:
+    from fairlearn.metrics import (
+        MetricFrame,
+        count,
+        demographic_parity_ratio,
+        equalized_odds_ratio,
+        false_positive_rate,
+        selection_rate,
+        true_positive_rate,
+    )
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `fairlearn` for ProtectedClassesCombination.",
+        required_dependencies=["fairlearn"],
+    ) from e
+
+logger = get_logger(__name__)
+
+
+@tags("bias_and_fairness")
+@tasks("classification", "regression")
+def ProtectedClassesCombination(dataset, model, protected_classes=None):
+    """
+    Visualizes combinations of protected classes and their corresponding error metric differences.
+
+    ### Purpose
+
+    This test aims to provide insights into how different combinations of protected classes affect various error metrics,
+    particularly the false negative rate (FNR) and false positive rate (FPR). By visualizing these combinations,
+    it helps identify potential biases or disparities in model performance across different intersectional groups.
+
+    ### Test Mechanism
+
+    The test performs the following steps:
+    1. Combines the specified protected class columns to create a single multi-class category.
+    2. Calculates error metrics (FNR, FPR, etc.) for each combination of protected classes.
+    3. Generates visualizations showing the distribution of these metrics across all class combinations.
+
+    ### Signs of High Risk
+
+    - Large disparities in FNR or FPR across different protected class combinations.
+    - Consistent patterns of higher error rates for specific combinations of protected attributes.
+    - Unexpected or unexplainable variations in error metrics between similar group combinations.
+
+    ### Strengths
+
+    - Provides a comprehensive view of intersectional fairness across multiple protected attributes.
+    - Allows for easy identification of potentially problematic combinations of protected classes.
+    - Visualizations make it easier to spot patterns or outliers in model performance across groups.
+
+    ### Limitations
+
+    - May become complex and difficult to interpret with a large number of protected classes or combinations.
+    - Does not provide statistical significance of observed differences.
+    - Visualization alone may not capture all nuances of intersectional fairness.
+    """
+
+    if sys.version_info < (3, 9):
+        raise RuntimeError("This test requires Python 3.9 or higher.")
+
+    if protected_classes is None:
+        logger.warning(
+            "No protected classes provided. Please pass the 'protected_classes' parameter to run this test."
+        )
+        return pd.DataFrame()
+
+    # Construct a function dictionary for figures
+    my_metrics = {
+        "fpr": false_positive_rate,
+        "tpr": true_positive_rate,
+        "selection rate": selection_rate,
+        "count": count,
+    }
+
+    # Construct a MetricFrame for figures
+    mf = MetricFrame(
+        metrics=my_metrics,
+        y_true=dataset.y,
+        y_pred=dataset.y_pred(model),
+        sensitive_features=dataset._df[protected_classes],
+    )
+
+    # Combine protected class columns to create a single multi-class category for the x-axis
+    metrics_by_group = mf.by_group.reset_index()
+    metrics_by_group["class_combination"] = metrics_by_group[protected_classes].apply(
+        lambda row: ", ".join(row.values.astype(str)), axis=1
+    )
+
+    # Create the subplots for the bar plots
+    fig = sp.make_subplots(
+        rows=2,
+        cols=2,
+        subplot_titles=[
+            "False Positive Rate",
+            "True Positive Rate",
+            "Selection Rate",
+            "Count",
+        ],
+    )
+
+    # Add bar plots for each metric
+    fig.add_trace(
+        go.Bar(
+            x=metrics_by_group["class_combination"],
+            y=metrics_by_group["fpr"],
+            name="FPR",
+        ),
+        row=1,
+        col=1,
+    )
+    fig.add_trace(
+        go.Bar(
+            x=metrics_by_group["class_combination"],
+            y=metrics_by_group["tpr"],
+            name="TPR",
+        ),
+        row=1,
+        col=2,
+    )
+    fig.add_trace(
+        go.Bar(
+            x=metrics_by_group["class_combination"],
+            y=metrics_by_group["selection rate"],
+            name="Selection Rate",
+        ),
+        row=2,
+        col=1,
+    )
+    fig.add_trace(
+        go.Bar(
+            x=metrics_by_group["class_combination"],
+            y=metrics_by_group["count"],
+            name="Count",
+        ),
+        row=2,
+        col=2,
+    )
+
+    # Update layout of the figure to match the original style
+    fig.update_layout(
+        title="Show all metrics",
+        height=800,
+        width=900,
+        barmode="group",
+        legend=dict(orientation="h", yanchor="bottom", y=-0.3, xanchor="center", x=0.5),
+        margin=dict(t=50),
+        font=dict(size=12),
+    )
+
+    # Rotate x-axis labels for better readability
+    fig.update_xaxes(tickangle=45, row=1, col=1)
+    fig.update_xaxes(tickangle=45, row=1, col=2)
+    fig.update_xaxes(tickangle=45, row=2, col=1)
+    fig.update_xaxes(tickangle=45, row=2, col=2)
+
+    # Extract demographic parity ratio and equalized odds ratio
+    m_dpr = []
+    m_eqo = []
+    for protected_class in protected_classes:
+        m_dpr.append(
+            demographic_parity_ratio(
+                y_true=dataset.y,
+                y_pred=dataset.y_pred(model),
+                sensitive_features=dataset._df[[protected_class]],
+            )
+        )
+        m_eqo.append(
+            equalized_odds_ratio(
+                y_true=dataset.y,
+                y_pred=dataset.y_pred(model),
+                sensitive_features=dataset._df[[protected_class]],
+            )
+        )
+
+    # Create a DataFrame for the demographic parity and equalized odds ratio
+    dpr_eor_df = pd.DataFrame(
+        columns=protected_classes,
+        index=["demographic parity ratio", "equal odds ratio"],
+    )
+
+    for i in range(len(m_dpr)):
+        dpr_eor_df[protected_classes[i]]["demographic parity ratio"] = round(
+            m_dpr[i], 2
+        )
+        dpr_eor_df[protected_classes[i]]["equal odds ratio"] = round(m_eqo[i], 2)
+
+    return (
+        {"Class Combination Table": metrics_by_group},
+        {"DPR and EOR table": dpr_eor_df},
+        fig,
+    )