validmind 2.5.8__py3-none-any.whl → 2.5.18__py3-none-any.whl

validmind/tests/data_validation/PhillipsPerronArch.py

@@ -17,36 +17,43 @@ logger = get_logger(__name__)
 @dataclass
 class PhillipsPerronArch(Metric):
     """
-    Executes Phillips-Perron test to assess the stationarity of time series data in each ML model feature.
-
-    **Purpose**: The Phillips-Perron (PP) test is used to establish the order of integration in time series data,
-    testing a null hypothesis that a time series is unit-root non-stationary. This is vital in forecasting and
-    understanding the stochastic behavior of data within machine learning models. Essentially, the PP test aids in
-    confirming the robustness of results and generating valid predictions from regression analysis models.
-
-    **Test Mechanism**: The PP test is conducted for each feature in the dataset. A data frame is created from the
-    dataset, and for each column in this frame, the PhillipsPerron method calculates the statistic value, p-value, used
-    lags, and number of observations. This process computes the PP metric for each feature and stores the results for
-    future reference.
-
-    **Signs of High Risk**:
-    - A high P-value could imply that the series has a unit root and is therefore non-stationary.
-    - Test statistic values that surpass critical values indicate additional evidence of non-stationarity.
-    - A high 'usedlag' value for a series could point towards autocorrelation issues which could further impede the
-    model's performance.
-
-    **Strengths**:
-    - Resilience against heteroskedasticity in the error term is a significant strength of the PP test.
-    - Its capacity to handle long time series data.
-    - Its ability to determine whether the time series is stationary or not, influencing the selection of suitable
-    models for forecasting.
-
-    **Limitations**:
-    - The PP test can only be employed within a univariate time series framework.
-    - The test relies on asymptotic theory, which means the test's power can significantly diminish for small sample
-    sizes.
-    - The need to convert non-stationary time series into stationary series through differencing might lead to loss of
-    vital data points.
+    Assesses the stationarity of time series data in each feature of the ML model using the Phillips-Perron test.
+
+    ### Purpose
+
+    The Phillips-Perron (PP) test is used to determine the stationarity of time series data for each feature in a
+    dataset, which is crucial for forecasting tasks. It tests the null hypothesis that a time series is unit-root
+    non-stationary. This is vital for understanding the stochastic behavior of the data and ensuring the robustness and
+    validity of predictions generated by regression analysis models.
+
+    ### Test Mechanism
+
+    The PP test is conducted for each feature in the dataset as follows:
+    - A data frame is created from the dataset.
+    - For each column, the Phillips-Perron method calculates the test statistic, p-value, lags used, and number of
+    observations.
+    - The results are then stored for each feature, providing a metric that indicates the stationarity of the time
+    series data.
+
+    ### Signs of High Risk
+
+    - A high p-value, indicating that the series has a unit root and is non-stationary.
+    - Test statistic values exceeding critical values, suggesting non-stationarity.
+    - High 'usedlag' value, pointing towards autocorrelation issues that may degrade model performance.
+
+    ### Strengths
+
+    - Resilience against heteroskedasticity in the error term.
+    - Effective for long time series data.
+    - Helps in determining whether the time series is stationary, aiding in the selection of suitable forecasting
+    models.
+
+    ### Limitations
+
+    - Applicable only within a univariate time series framework.
+    - Relies on asymptotic theory, which may reduce the test’s power for small sample sizes.
+    - Non-stationary time series must be converted to stationary series through differencing, potentially leading to
+    loss of important data points.
     """
 
     name = "phillips_perron"

validmind/tests/data_validation/ProtectedClassesCombination.py

@@ -0,0 +1,197 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import sys
+
+import pandas as pd
+import plotly.graph_objects as go
+import plotly.subplots as sp
+from fairlearn.metrics import (
+    MetricFrame,
+    count,
+    demographic_parity_ratio,
+    equalized_odds_ratio,
+    false_positive_rate,
+    selection_rate,
+    true_positive_rate,
+)
+
+from validmind import tags, tasks
+from validmind.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@tags("bias_and_fairness")
+@tasks("classification", "regression")
+def ProtectedClassesCombination(dataset, model, protected_classes=None):
+    """
+    Visualizes combinations of protected classes and their corresponding error metric differences.
+
+    ### Purpose
+
+    This test aims to provide insights into how different combinations of protected classes affect various error metrics,
+    particularly the false negative rate (FNR) and false positive rate (FPR). By visualizing these combinations,
+    it helps identify potential biases or disparities in model performance across different intersectional groups.
+
+    ### Test Mechanism
+
+    The test performs the following steps:
+    1. Combines the specified protected class columns to create a single multi-class category.
+    2. Calculates error metrics (FNR, FPR, etc.) for each combination of protected classes.
+    3. Generates visualizations showing the distribution of these metrics across all class combinations.
+
+    ### Signs of High Risk
+
+    - Large disparities in FNR or FPR across different protected class combinations.
+    - Consistent patterns of higher error rates for specific combinations of protected attributes.
+    - Unexpected or unexplainable variations in error metrics between similar group combinations.
+
+    ### Strengths
+
+    - Provides a comprehensive view of intersectional fairness across multiple protected attributes.
+    - Allows for easy identification of potentially problematic combinations of protected classes.
+    - Visualizations make it easier to spot patterns or outliers in model performance across groups.
+
+    ### Limitations
+
+    - May become complex and difficult to interpret with a large number of protected classes or combinations.
+    - Does not provide statistical significance of observed differences.
+    - Visualization alone may not capture all nuances of intersectional fairness.
+    """
+
+    if sys.version_info < (3, 9):
+        raise RuntimeError("This test requires Python 3.9 or higher.")
+
+    if protected_classes is None:
+        logger.warning(
+            "No protected classes provided. Please pass the 'protected_classes' parameter to run this test."
+        )
+        return pd.DataFrame()
+
+    # Construct a function dictionary for figures
+    my_metrics = {
+        "fpr": false_positive_rate,
+        "tpr": true_positive_rate,
+        "selection rate": selection_rate,
+        "count": count,
+    }
+
+    # Construct a MetricFrame for figures
+    mf = MetricFrame(
+        metrics=my_metrics,
+        y_true=dataset.y,
+        y_pred=dataset.y_pred(model),
+        sensitive_features=dataset._df[protected_classes],
+    )
+
+    # Combine protected class columns to create a single multi-class category for the x-axis
+    metrics_by_group = mf.by_group.reset_index()
+    metrics_by_group["class_combination"] = metrics_by_group[protected_classes].apply(
+        lambda row: ", ".join(row.values.astype(str)), axis=1
+    )
+
+    # Create the subplots for the bar plots
+    fig = sp.make_subplots(
+        rows=2,
+        cols=2,
+        subplot_titles=[
+            "False Positive Rate",
+            "True Positive Rate",
+            "Selection Rate",
+            "Count",
+        ],
+    )
+
+    # Add bar plots for each metric
+    fig.add_trace(
+        go.Bar(
+            x=metrics_by_group["class_combination"],
+            y=metrics_by_group["fpr"],
+            name="FPR",
+        ),
+        row=1,
+        col=1,
+    )
+    fig.add_trace(
+        go.Bar(
+            x=metrics_by_group["class_combination"],
+            y=metrics_by_group["tpr"],
+            name="TPR",
+        ),
+        row=1,
+        col=2,
+    )
+    fig.add_trace(
+        go.Bar(
+            x=metrics_by_group["class_combination"],
+            y=metrics_by_group["selection rate"],
+            name="Selection Rate",
+        ),
+        row=2,
+        col=1,
+    )
+    fig.add_trace(
+        go.Bar(
+            x=metrics_by_group["class_combination"],
+            y=metrics_by_group["count"],
+            name="Count",
+        ),
+        row=2,
+        col=2,
+    )
+
+    # Update layout of the figure to match the original style
+    fig.update_layout(
+        title="Show all metrics",
+        height=800,
+        width=900,
+        barmode="group",
+        legend=dict(orientation="h", yanchor="bottom", y=-0.3, xanchor="center", x=0.5),
+        margin=dict(t=50),
+        font=dict(size=12),
+    )
+
+    # Rotate x-axis labels for better readability
+    fig.update_xaxes(tickangle=45, row=1, col=1)
+    fig.update_xaxes(tickangle=45, row=1, col=2)
+    fig.update_xaxes(tickangle=45, row=2, col=1)
+    fig.update_xaxes(tickangle=45, row=2, col=2)
+
+    # Extract demographic parity ratio and equalized odds ratio
+    m_dpr = []
+    m_eqo = []
+    for protected_class in protected_classes:
+        m_dpr.append(
+            demographic_parity_ratio(
+                y_true=dataset.y,
+                y_pred=dataset.y_pred(model),
+                sensitive_features=dataset._df[[protected_class]],
+            )
+        )
+        m_eqo.append(
+            equalized_odds_ratio(
+                y_true=dataset.y,
+                y_pred=dataset.y_pred(model),
+                sensitive_features=dataset._df[[protected_class]],
+            )
+        )
+
+    # Create a DataFrame for the demographic parity and equalized odds ratio
+    dpr_eor_df = pd.DataFrame(
+        columns=protected_classes,
+        index=["demographic parity ratio", "equal odds ratio"],
+    )
+
+    for i in range(len(m_dpr)):
+        dpr_eor_df[protected_classes[i]]["demographic parity ratio"] = round(
+            m_dpr[i], 2
+        )
+        dpr_eor_df[protected_classes[i]]["equal odds ratio"] = round(m_eqo[i], 2)
+
+    return (
+        {"Class Combination Table": metrics_by_group},
+        {"DPR and EOR table": dpr_eor_df},
+        fig,
+    )

validmind/tests/data_validation/ProtectedClassesDescription.py

@@ -0,0 +1,130 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+
+import pandas as pd
+import plotly.graph_objects as go
+
+from validmind import tags, tasks
+from validmind.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@tags("bias_and_fairness", "descriptive_statistics")
+@tasks("classification", "regression")
+def ProtectedClassesDescription(dataset, protected_classes=None):
+    """
+    Visualizes the distribution of protected classes in the dataset relative to the target variable
+    and provides descriptive statistics.
+
+    ### Purpose
+
+    The ProtectedClassesDescription test aims to identify potential biases or significant differences in the
+    distribution of target outcomes across different protected classes. This visualization and statistical summary
+    help in understanding the relationship between protected attributes and the target variable, which is crucial
+    for assessing fairness in machine learning models.
+
+    ### Test Mechanism
+
+    The function creates interactive stacked bar charts for each specified protected class using Plotly.
+    Additionally, it generates a single table of descriptive statistics for all protected classes, including:
+    - Protected class and category
+    - Count and percentage of each category within the protected class
+    - Mean, median, and mode of the target variable for each category
+    - Standard deviation of the target variable for each category
+    - Minimum and maximum values of the target variable for each category
+
+    ### Signs of High Risk
+
+    - Significant imbalances in the distribution of target outcomes across different categories of a protected class.
+    - Large disparities in mean, median, or mode of the target variable across categories.
+    - Underrepresentation or overrepresentation of certain groups within protected classes.
+    - High standard deviations in certain categories, indicating potential volatility or outliers.
+
+    ### Strengths
+
+    - Provides both visual and statistical representation of potential biases in the dataset.
+    - Allows for easy identification of imbalances in target variable distribution across protected classes.
+    - Interactive plots enable detailed exploration of the data.
+    - Consolidated statistical summary provides quantitative measures to complement visual analysis.
+    - Applicable to both classification and regression tasks.
+
+    ### Limitations
+
+    - Does not provide advanced statistical measures of bias or fairness.
+    - May become cluttered if there are many categories within a protected class or many unique target values.
+    - Interpretation may require domain expertise to understand the implications of observed disparities.
+    - Does not account for intersectionality or complex interactions between multiple protected attributes.
+    """
+
+    if protected_classes is None:
+        logger.warning(
+            "No protected classes provided. Please pass the 'protected_classes' parameter to run this test."
+        )
+        return pd.DataFrame()
+
+    figures = []
+    all_stats = []
+
+    df = dataset._df
+    target = dataset.target_column
+
+    for protected_class in protected_classes:
+        # Create the stacked bar chart
+        counts = df.groupby([protected_class, target]).size().unstack(fill_value=0)
+        fig = go.Figure()
+        for col in counts.columns:
+            fig.add_trace(
+                go.Bar(
+                    x=counts.index,
+                    y=counts[col],
+                    name=str(col),
+                    text=counts[col],
+                    textposition="auto",
+                )
+            )
+
+        fig.update_layout(
+            title=f"Distribution of {protected_class} by {target}",
+            xaxis_title=protected_class,
+            yaxis_title="Count",
+            barmode="stack",
+            showlegend=True,
+            legend_title=target,
+        )
+
+        figures.append(fig)
+
+        # Get unique values in the target column
+        target_labels = df[target].unique()
+
+        for category in df[protected_class].unique():
+            category_data = df[df[protected_class] == category]
+            stats = {
+                "Protected Class": protected_class,
+                "Category": category,
+                "Count": len(category_data),
+                "Percentage": len(category_data) / len(df) * 100,
+            }
+
+            # Add mean for each target label
+            for label in target_labels:
+                label_data = category_data[category_data[target] == label]
+                stats[f"Rate {target}: {label}"] = (
+                    len(label_data) / len(category_data) * 100
+                )
+
+            all_stats.append(stats)
+
+    # Create a single DataFrame with all statistics
+    stats_df = pd.DataFrame(all_stats)
+    stats_df = stats_df.round(2)  # Round to 2 decimal places for readability
+
+    # Sort the DataFrame by Protected Class and Count (descending)
+    stats_df = stats_df.sort_values(
+        ["Protected Class", "Count"], ascending=[True, False]
+    )
+
+    return (stats_df, *tuple(figures))

validmind/tests/data_validation/ProtectedClassesDisparity.py

@@ -0,0 +1,133 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import io
+import sys
+
+import aequitas.plot as ap
+import pandas as pd
+from aequitas.bias import Bias
+from aequitas.group import Group
+from aequitas.plotting import Plot
+
+from validmind import tags, tasks
+from validmind.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@tags("bias_and_fairness")
+@tasks("classification", "regression")
+def ProtectedClassesDisparity(
+    dataset,
+    model,
+    protected_classes=None,
+    disparity_tolerance=1.25,
+    metrics=["fnr", "fpr", "tpr"],
+):
+    """
+    Investigates disparities in model performance across different protected class segments.
+
+    ### Purpose
+
+    This test aims to identify and quantify potential biases in model outcomes by comparing various performance metrics
+    across different segments of protected classes. It helps in assessing whether the model produces discriminatory
+    outcomes for certain groups, which is crucial for ensuring fairness in machine learning models.
+
+    ### Test Mechanism
+
+    The test performs the following steps:
+    1. Calculates performance metrics (e.g., false negative rate, false positive rate, true positive rate) for each segment
+       of the specified protected classes.
+    2. Computes disparity ratios by comparing these metrics between different segments and a reference group.
+    3. Generates visualizations showing the disparities and their relation to a user-defined disparity tolerance threshold.
+    4. Produces a comprehensive table with various disparity metrics for detailed analysis.
+
+    ### Signs of High Risk
+
+    - Disparity ratios exceeding the specified disparity tolerance threshold.
+    - Consistent patterns of higher error rates or lower performance for specific protected class segments.
+    - Statistically significant differences in performance metrics across segments.
+
+    ### Strengths
+
+    - Provides a comprehensive view of model fairness across multiple protected attributes and metrics.
+    - Allows for easy identification of problematic disparities through visual and tabular representations.
+    - Customizable disparity tolerance threshold to align with specific use-case requirements.
+    - Applicable to various performance metrics, offering a multi-faceted analysis of model fairness.
+
+    ### Limitations
+
+    - Relies on a predefined reference group for each protected class, which may not always be the most appropriate choice.
+    - Does not account for intersectionality between different protected attributes.
+    - The interpretation of results may require domain expertise to understand the implications of observed disparities.
+    """
+
+    if protected_classes is None:
+        logger.warning(
+            "No protected classes provided. Please pass the 'protected_classes' parameter to run this test."
+        )
+        return pd.DataFrame()
+
+    if sys.version_info < (3, 9):
+        raise RuntimeError("This test requires Python 3.9 or higher.")
+
+    df = dataset._df
+
+    for protected_class in protected_classes:
+        # make the dataset compatible for the python package of interest
+        df[protected_class] = pd.Categorical(df[protected_class]).astype("object")
+
+    df["score"] = dataset.y_pred(model).astype(int)
+    df["label_value"] = df[dataset.target_column].astype(int)
+
+    # let map the attributes for each protected class
+    # default use reference that is most observable for dictionary
+    attributes_and_reference_groups = {}
+    for protected_class in protected_classes:
+        attributes_and_reference_groups.update(
+            {protected_class: df[protected_class].value_counts().idxmax()}
+        )
+
+    attributes_to_audit = list(attributes_and_reference_groups.keys())
+
+    # Initialize Aequitas
+    g = Group()
+    b = Bias()
+    aqp = Plot()
+
+    columns_to_include = (
+        protected_classes + [dataset.target_column] + ["score", "label_value"]
+    )
+
+    # get_crosstabs returns a dataframe of the group counts and group value bias metrics.
+    xtab, _ = g.get_crosstabs(df[columns_to_include], attr_cols=attributes_to_audit)
+    bdf = b.get_disparity_predefined_groups(
+        xtab,
+        original_df=df[columns_to_include],
+        ref_groups_dict=attributes_and_reference_groups,
+        alpha=0.05,
+        mask_significance=True,
+    )
+
+    plots = []
+    for protected_class in protected_classes:
+        plot = ap.disparity(
+            bdf, metrics, protected_class, fairness_threshold=disparity_tolerance
+        )
+
+        buf = io.BytesIO()  # create a bytes array to save the image into in memory
+        plot.save(
+            buf, format="png"
+        )  # as long as the above library is installed, this will work
+        plots.append(buf.getvalue())
+
+    string = "_disparity"
+    metrics_adj = [x + string for x in metrics]
+
+    table = bdf[["attribute_name", "attribute_value"] + b.list_disparities(bdf)]
+    plots.append(aqp.plot_disparity_all(bdf, metrics=metrics_adj))
+    plots_return = tuple(plots)
+
+    return (table, *plots_return)