validmind 2.5.15__py3-none-any.whl → 2.5.19__py3-none-any.whl

validmind/tests/data_validation/ProtectedClassesDescription.py

@@ -0,0 +1,130 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+
+import pandas as pd
+import plotly.graph_objects as go
+
+from validmind import tags, tasks
+from validmind.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@tags("bias_and_fairness", "descriptive_statistics")
+@tasks("classification", "regression")
+def ProtectedClassesDescription(dataset, protected_classes=None):
+    """
+    Visualizes the distribution of protected classes in the dataset relative to the target variable
+    and provides descriptive statistics.
+
+    ### Purpose
+
+    The ProtectedClassesDescription test aims to identify potential biases or significant differences in the
+    distribution of target outcomes across different protected classes. This visualization and statistical summary
+    help in understanding the relationship between protected attributes and the target variable, which is crucial
+    for assessing fairness in machine learning models.
+
+    ### Test Mechanism
+
+    The function creates interactive stacked bar charts for each specified protected class using Plotly.
+    Additionally, it generates a single table of descriptive statistics for all protected classes, including:
+    - Protected class and category
+    - Count and percentage of each category within the protected class
+    - Mean, median, and mode of the target variable for each category
+    - Standard deviation of the target variable for each category
+    - Minimum and maximum values of the target variable for each category
+
+    ### Signs of High Risk
+
+    - Significant imbalances in the distribution of target outcomes across different categories of a protected class.
+    - Large disparities in mean, median, or mode of the target variable across categories.
+    - Underrepresentation or overrepresentation of certain groups within protected classes.
+    - High standard deviations in certain categories, indicating potential volatility or outliers.
+
+    ### Strengths
+
+    - Provides both visual and statistical representation of potential biases in the dataset.
+    - Allows for easy identification of imbalances in target variable distribution across protected classes.
+    - Interactive plots enable detailed exploration of the data.
+    - Consolidated statistical summary provides quantitative measures to complement visual analysis.
+    - Applicable to both classification and regression tasks.
+
+    ### Limitations
+
+    - Does not provide advanced statistical measures of bias or fairness.
+    - May become cluttered if there are many categories within a protected class or many unique target values.
+    - Interpretation may require domain expertise to understand the implications of observed disparities.
+    - Does not account for intersectionality or complex interactions between multiple protected attributes.
+    """
+
+    if protected_classes is None:
+        logger.warning(
+            "No protected classes provided. Please pass the 'protected_classes' parameter to run this test."
+        )
+        return pd.DataFrame()
+
+    figures = []
+    all_stats = []
+
+    df = dataset._df
+    target = dataset.target_column
+
+    for protected_class in protected_classes:
+        # Create the stacked bar chart
+        counts = df.groupby([protected_class, target]).size().unstack(fill_value=0)
+        fig = go.Figure()
+        for col in counts.columns:
+            fig.add_trace(
+                go.Bar(
+                    x=counts.index,
+                    y=counts[col],
+                    name=str(col),
+                    text=counts[col],
+                    textposition="auto",
+                )
+            )
+
+        fig.update_layout(
+            title=f"Distribution of {protected_class} by {target}",
+            xaxis_title=protected_class,
+            yaxis_title="Count",
+            barmode="stack",
+            showlegend=True,
+            legend_title=target,
+        )
+
+        figures.append(fig)
+
+        # Get unique values in the target column
+        target_labels = df[target].unique()
+
+        for category in df[protected_class].unique():
+            category_data = df[df[protected_class] == category]
+            stats = {
+                "Protected Class": protected_class,
+                "Category": category,
+                "Count": len(category_data),
+                "Percentage": len(category_data) / len(df) * 100,
+            }
+
+            # Add mean for each target label
+            for label in target_labels:
+                label_data = category_data[category_data[target] == label]
+                stats[f"Rate {target}: {label}"] = (
+                    len(label_data) / len(category_data) * 100
+                )
+
+            all_stats.append(stats)
+
+    # Create a single DataFrame with all statistics
+    stats_df = pd.DataFrame(all_stats)
+    stats_df = stats_df.round(2)  # Round to 2 decimal places for readability
+
+    # Sort the DataFrame by Protected Class and Count (descending)
+    stats_df = stats_df.sort_values(
+        ["Protected Class", "Count"], ascending=[True, False]
+    )
+
+    return (stats_df, *tuple(figures))

validmind/tests/data_validation/ProtectedClassesDisparity.py

@@ -0,0 +1,141 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import io
+import sys
+
+import pandas as pd
+
+from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
+from validmind.logging import get_logger
+
+try:
+    import aequitas.plot as ap
+    from aequitas.bias import Bias
+    from aequitas.group import Group
+    from aequitas.plotting import Plot
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `aequitas` for ProtectedClassesDisparity.",
+        required_dependencies=["aequitas"],
+    ) from e
+
+logger = get_logger(__name__)
+
+
+@tags("bias_and_fairness")
+@tasks("classification", "regression")
+def ProtectedClassesDisparity(
+    dataset,
+    model,
+    protected_classes=None,
+    disparity_tolerance=1.25,
+    metrics=["fnr", "fpr", "tpr"],
+):
+    """
+    Investigates disparities in model performance across different protected class segments.
+
+    ### Purpose
+
+    This test aims to identify and quantify potential biases in model outcomes by comparing various performance metrics
+    across different segments of protected classes. It helps in assessing whether the model produces discriminatory
+    outcomes for certain groups, which is crucial for ensuring fairness in machine learning models.
+
+    ### Test Mechanism
+
+    The test performs the following steps:
+    1. Calculates performance metrics (e.g., false negative rate, false positive rate, true positive rate) for each segment
+       of the specified protected classes.
+    2. Computes disparity ratios by comparing these metrics between different segments and a reference group.
+    3. Generates visualizations showing the disparities and their relation to a user-defined disparity tolerance threshold.
+    4. Produces a comprehensive table with various disparity metrics for detailed analysis.
+
+    ### Signs of High Risk
+
+    - Disparity ratios exceeding the specified disparity tolerance threshold.
+    - Consistent patterns of higher error rates or lower performance for specific protected class segments.
+    - Statistically significant differences in performance metrics across segments.
+
+    ### Strengths
+
+    - Provides a comprehensive view of model fairness across multiple protected attributes and metrics.
+    - Allows for easy identification of problematic disparities through visual and tabular representations.
+    - Customizable disparity tolerance threshold to align with specific use-case requirements.
+    - Applicable to various performance metrics, offering a multi-faceted analysis of model fairness.
+
+    ### Limitations
+
+    - Relies on a predefined reference group for each protected class, which may not always be the most appropriate choice.
+    - Does not account for intersectionality between different protected attributes.
+    - The interpretation of results may require domain expertise to understand the implications of observed disparities.
+    """
+
+    if protected_classes is None:
+        logger.warning(
+            "No protected classes provided. Please pass the 'protected_classes' parameter to run this test."
+        )
+        return pd.DataFrame()
+
+    if sys.version_info < (3, 9):
+        raise RuntimeError("This test requires Python 3.9 or higher.")
+
+    df = dataset._df
+
+    for protected_class in protected_classes:
+        # make the dataset compatible for the python package of interest
+        df[protected_class] = pd.Categorical(df[protected_class]).astype("object")
+
+    df["score"] = dataset.y_pred(model).astype(int)
+    df["label_value"] = df[dataset.target_column].astype(int)
+
+    # let map the attributes for each protected class
+    # default use reference that is most observable for dictionary
+    attributes_and_reference_groups = {}
+    for protected_class in protected_classes:
+        attributes_and_reference_groups.update(
+            {protected_class: df[protected_class].value_counts().idxmax()}
+        )
+
+    attributes_to_audit = list(attributes_and_reference_groups.keys())
+
+    # Initialize Aequitas
+    g = Group()
+    b = Bias()
+    aqp = Plot()
+
+    columns_to_include = (
+        protected_classes + [dataset.target_column] + ["score", "label_value"]
+    )
+
+    # get_crosstabs returns a dataframe of the group counts and group value bias metrics.
+    xtab, _ = g.get_crosstabs(df[columns_to_include], attr_cols=attributes_to_audit)
+    bdf = b.get_disparity_predefined_groups(
+        xtab,
+        original_df=df[columns_to_include],
+        ref_groups_dict=attributes_and_reference_groups,
+        alpha=0.05,
+        mask_significance=True,
+    )
+
+    plots = []
+    for protected_class in protected_classes:
+        plot = ap.disparity(
+            bdf, metrics, protected_class, fairness_threshold=disparity_tolerance
+        )
+
+        buf = io.BytesIO()  # create a bytes array to save the image into in memory
+        plot.save(
+            buf, format="png"
+        )  # as long as the above library is installed, this will work
+        plots.append(buf.getvalue())
+
+    string = "_disparity"
+    metrics_adj = [x + string for x in metrics]
+
+    table = bdf[["attribute_name", "attribute_value"] + b.list_disparities(bdf)]
+    plots.append(aqp.plot_disparity_all(bdf, metrics=metrics_adj))
+    plots_return = tuple(plots)
+
+    return (table, *plots_return)

validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.py

@@ -0,0 +1,180 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import json
+import sys
+
+import matplotlib.pyplot as plt
+import pandas as pd
+
+from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
+from validmind.logging import get_logger
+
+try:
+    from fairlearn.metrics import (
+        MetricFrame,
+        count,
+        demographic_parity_ratio,
+        equalized_odds_ratio,
+        false_negative_rate,
+        false_positive_rate,
+        true_positive_rate,
+    )
+    from fairlearn.postprocessing import ThresholdOptimizer, plot_threshold_optimizer
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `fairlearn` for ProtectedClassesThresholdOptimizer.",
+        required_dependencies=["fairlearn"],
+    ) from e
+
+logger = get_logger(__name__)
+
+
+@tags("bias_and_fairness")
+@tasks("classification", "regression")
+def ProtectedClassesThresholdOptimizer(
+    dataset, pipeline=None, protected_classes=None, X_train=None, y_train=None
+):
+    """
+    Obtains a classifier by applying group-specific thresholds to the provided estimator.
+
+    ### Purpose
+
+    This test aims to optimize the fairness of a machine learning model by applying different
+    classification thresholds for different protected groups. It helps in mitigating bias and
+    achieving more equitable outcomes across different demographic groups.
+
+    ### Test Mechanism
+
+    The test uses Fairlearn's ThresholdOptimizer to:
+    1. Fit an optimizer on the training data, considering protected classes.
+    2. Apply optimized thresholds to make predictions on the test data.
+    3. Calculate and report various fairness metrics.
+    4. Visualize the optimized thresholds.
+
+    ### Signs of High Risk
+
+    - Large disparities in fairness metrics (e.g., Demographic Parity Ratio, Equalized Odds Ratio)
+      across different protected groups.
+    - Significant differences in False Positive Rates (FPR) or True Positive Rates (TPR) between groups.
+    - Thresholds that vary widely across different protected groups.
+
+    ### Strengths
+
+    - Provides a post-processing method to improve model fairness without modifying the original model.
+    - Allows for balancing multiple fairness criteria simultaneously.
+    - Offers visual insights into the threshold optimization process.
+
+    ### Limitations
+
+    - May lead to a decrease in overall model performance while improving fairness.
+    - Requires access to protected attribute information at prediction time.
+    - The effectiveness can vary depending on the chosen fairness constraint and objective.
+    """
+
+    if sys.version_info < (3, 9):
+        raise RuntimeError("This test requires Python 3.9 or higher.")
+
+    if (
+        pipeline is None
+        or protected_classes is None
+        or X_train is None
+        or y_train is None
+    ):
+        logger.warning(
+            "Missing required parameters. Please provide pipeline, protected_classes, X_train, and y_train."
+        )
+        return pd.DataFrame()
+
+    test_df = dataset.df
+
+    threshold_optimizer = initialize_and_fit_optimizer(
+        pipeline, X_train, y_train, X_train[protected_classes]
+    )
+
+    fig = plot_thresholds(threshold_optimizer)
+
+    target = dataset.target_column
+    y_pred_opt = make_predictions(threshold_optimizer, test_df, protected_classes)
+
+    fairness_metrics = calculate_fairness_metrics(
+        test_df, target, y_pred_opt, protected_classes
+    )
+
+    return (
+        {"DPR and EOR Table": fairness_metrics.reset_index()},
+        fig,
+    )
+
+
+def initialize_and_fit_optimizer(pipeline, X_train, y_train, protected_classes_df):
+    threshold_optimizer = ThresholdOptimizer(
+        estimator=pipeline,
+        objective="balanced_accuracy_score",
+        constraints="demographic_parity",
+        predict_method="predict_proba",
+        prefit=False,
+    )
+    threshold_optimizer.fit(X_train, y_train, sensitive_features=protected_classes_df)
+    return threshold_optimizer
+
+
+def plot_thresholds(threshold_optimizer):
+    fig = plt.figure()
+    plot_threshold_optimizer(threshold_optimizer, show_plot=False)
+    return fig
+
+
+def make_predictions(threshold_optimizer, test_df, protected_classes):
+    y_pred_opt = threshold_optimizer.predict(
+        test_df, sensitive_features=test_df[protected_classes]
+    )
+    return y_pred_opt
+
+
+def calculate_fairness_metrics(test_df, target, y_pred_opt, protected_classes):
+    fairness_metrics = pd.DataFrame(
+        columns=protected_classes,
+        index=["demographic parity ratio", "equal odds ratio"],
+    )
+
+    for feature in protected_classes:
+        dpr = demographic_parity_ratio(
+            y_true=test_df[target],
+            y_pred=y_pred_opt,
+            sensitive_features=test_df[[feature]],
+        )
+        eor = equalized_odds_ratio(
+            y_true=test_df[target],
+            y_pred=y_pred_opt,
+            sensitive_features=test_df[[feature]],
+        )
+        fairness_metrics[feature] = [round(dpr, 2), round(eor, 2)]
+
+    return fairness_metrics
+
+
+def calculate_group_metrics(test_df, target, y_pred_opt, protected_classes):
+    metrics = {
+        "fpr": false_positive_rate,
+        "tpr": true_positive_rate,
+        "fnr": false_negative_rate,
+        "count": count,
+    }
+    mf = MetricFrame(
+        metrics=metrics,
+        y_true=test_df[target],
+        y_pred=y_pred_opt,
+        sensitive_features=test_df[protected_classes],
+    )
+    group_metrics = mf.by_group
+    return group_metrics
+
+
+def get_thresholds_by_group(threshold_optimizer):
+    threshold_rules = threshold_optimizer.interpolated_thresholder_.interpolation_dict
+    thresholds = json.dumps(threshold_rules, default=str, indent=4)
+    thresholds_df = pd.DataFrame.from_records(json.loads(thresholds))
+    return thresholds_df

validmind/tests/{model_validation/statsmodels → data_validation}/RunsTest.py

@@ -2,12 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import pandas as pd
 from statsmodels.sandbox.stats.runs import runstest_1samp
 
-from validmind.vm_models import Metric
+from validmind import tags, tasks
 
 
-class RunsTest(Metric):
+@tasks("classification", "regression")
+@tags("tabular_data", "statistical_test", "statsmodels")
+def RunsTest(dataset):
     """
     Executes Runs Test on ML model to detect non-random patterns in output data sequence.
 
@@ -52,24 +55,18 @@ class RunsTest(Metric):
     - Does not provide model performance evaluation; it is used to detect patterns in the sequence of outputs only.
     """
 
-    name = "runs_test"
-    required_inputs = ["dataset"]
-    tasks = ["classification", "regression"]
-    tags = ["tabular_data", "statistical_test", "statsmodels"]
+    df = dataset.df[dataset.feature_columns_numeric]
 
-    def run(self):
-        """
-        Calculates the run test for each of the dataset features
-        """
-        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
+    runs_test_values = {}
+    for col in df.columns:
+        runs_stat, runs_p_value = runstest_1samp(df[col].values)
+        runs_test_values[col] = {
+            "stat": runs_stat,
+            "pvalue": runs_p_value,
+        }
 
-        runs_test_values = {}
-        for col in x_train.columns:
-            runs_stat, runs_p_value = runstest_1samp(x_train[col].values)
+    runs_test_df = pd.DataFrame.from_dict(runs_test_values, orient="index")
+    runs_test_df.reset_index(inplace=True)
+    runs_test_df.columns = ["feature", "stat", "pvalue"]
 
-            runs_test_values[col] = {
-                "stat": runs_stat,
-                "pvalue": runs_p_value,
-            }
-
-        return self.cache_results(runs_test_values)
+    return runs_test_df

validmind/tests/{model_validation/statsmodels → data_validation}/ShapiroWilk.py

@@ -2,12 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import pandas as pd
 from scipy import stats
 
-from validmind.vm_models import Metric
+from validmind import tags, tasks
 
 
-class ShapiroWilk(Metric):
+@tasks("classification", "regression")
+@tags("tabular_data", "data_distribution", "statistical_test")
+def ShapiroWilk(dataset):
     """
     Evaluates feature-wise normality of training data using the Shapiro-Wilk test.
 
@@ -49,23 +52,18 @@ class ShapiroWilk(Metric):
     - Lastly, the Shapiro-Wilk test is not optimally suited for processing data with pronounced skewness or kurtosis.
     """
 
-    name = "shapiro_wilk"
-    required_inputs = ["dataset"]
-    tasks = ["classification", "regression"]
-    tags = ["tabular_data", "data_distribution", "statistical_test"]
-
-    def run(self):
-        """
-        Calculates Shapiro-Wilk test for each of the dataset features.
-        """
-        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
-        sw_values = {}
-        for col in x_train.columns:
-            sw_stat, sw_pvalue = stats.shapiro(x_train[col].values)
-
-            sw_values[col] = {
-                "stat": sw_stat,
-                "pvalue": sw_pvalue,
-            }
-
-        return self.cache_results(sw_values)
+    df = dataset.df[dataset.feature_columns_numeric]
+
+    sw_values = {}
+    for col in df.columns:
+        sw_stat, sw_pvalue = stats.shapiro(df[col].values)
+        sw_values[col] = {
+            "stat": sw_stat,
+            "pvalue": sw_pvalue,
+        }
+
+    sw_df = pd.DataFrame.from_dict(sw_values, orient="index")
+    sw_df.reset_index(inplace=True)
+    sw_df.columns = ["column", "stat", "pvalue"]
+
+    return sw_df

validmind/tests/data_validation/nlp/Hashtags.py

@@ -9,8 +9,7 @@ Threshold based tests
 import re
 from dataclasses import dataclass
 
-import matplotlib.pyplot as plt
-import seaborn as sns
+import plotly.graph_objects as go
 
 from validmind.vm_models import Figure, ThresholdTest, VMDataset
 
@@ -74,25 +73,23 @@ class Hashtags(ThresholdTest):
         text_column = self.inputs.dataset.text_column
 
         def find_hash(text):
-            line = re.findall(r"(?<=#)\w+", text)
-            return " ".join(line)
-
-        temp = (
-            self.inputs.dataset.df[text_column]
-            .apply(lambda x: find_hash(x))
-            .value_counts()[:][1 : self.params["top_hashtags"]]
-        )
-        temp = (
-            temp.to_frame()
-            .reset_index()
-            .rename(columns={"index": "Hashtag", text_column: "count"})
-        )
+            return re.findall(r"(?<=#)\w+", str(text))
+
+        # Extract hashtags from the text column and count occurrences
+        hashtags = self.inputs.dataset.df[text_column].apply(find_hash).explode()
+        temp = hashtags.value_counts().head(self.params["top_hashtags"])
+
+        print(f"temp: {temp}")
 
         figures = []
         if not temp.empty:
-            fig = plt.figure()
-            sns.barplot(x="Hashtag", y="count", data=temp)
-            plt.xticks(rotation=90)
+            fig = go.Figure(data=[go.Bar(x=temp.index, y=temp.values)])
+            fig.update_layout(
+                title="Top Hashtags",
+                xaxis_title="Hashtag",
+                yaxis_title="Count",
+                xaxis_tickangle=-45,
+            )
             figures.append(
                 Figure(
                     for_object=self,
@@ -100,7 +97,5 @@ class Hashtags(ThresholdTest):
                     figure=fig,
                 )
             )
-            # Do this if you want to prevent the figure from being displayed
-            plt.close("all")
 
         return self.cache_results([], passed=True, figures=figures)

validmind/tests/data_validation/nlp/TextDescription.py

@@ -84,7 +84,6 @@ class TextDescription(Metric):
     tags = ["nlp", "text_data", "visualization"]
 
     def general_text_metrics(self, df, text_column):
-        nltk.download("punkt", quiet=True)
         results = []
 
         for text in df[text_column]:
@@ -175,6 +174,9 @@ class TextDescription(Metric):
         if not isinstance(self.inputs.dataset, VMDataset):
             raise ValueError("TextDescription requires a validmind Dataset object")
 
+        # download nltk data
+        nltk.download("punkt_tab", quiet=True)
+
         df_text_description = self.text_description_table(
             self.inputs.dataset.df, self.params
         )

validmind/tests/load.py

@@ -15,7 +15,7 @@ from uuid import uuid4
 import pandas as pd
 from ipywidgets import HTML, Accordion
 
-from ..errors import LoadTestError
+from ..errors import LoadTestError, MissingDependencyError
 from ..html_templates.content_blocks import test_content_block_html
 from ..logging import get_logger
 from ..unit_metrics.composite import load_composite_metric
@@ -88,10 +88,26 @@ def list_tests(
     Returns:
         list or pandas.DataFrame: A list of all tests or a formatted table.
     """
-    tests = {
-        test_id: load_test(test_id, reload=True)
-        for test_id in test_store.get_test_ids()
-    }
+    tests = {}
+    for test_id in test_store.get_test_ids():
+        try:
+            tests[test_id] = load_test(test_id, reload=True)
+        except MissingDependencyError as e:
+            # skip tests that have missing dependencies
+            logger.debug(str(e))
+
+            if e.extra:
+                logger.info(
+                    f"Skipping `{test_id}` as it requires extra dependencies: {e.required_dependencies}."
+                    f" Please run `pip install validmind[{e.extra}]` to view and run this test."
+                )
+            else:
+                logger.info(
+                    f"Skipping `{test_id}` as it requires missing dependencies: {e.required_dependencies}."
+                    " Please install the missing dependencies to view and run this test."
+                )
+
+            continue
 
     # first search by the filter string since it's the most general search
     if filter is not None: