validmind 2.5.15__py3-none-any.whl → 2.5.18__py3-none-any.whl

validmind/tests/data_validation/ProtectedClassesDisparity.py

@@ -0,0 +1,133 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import io
+import sys
+
+import aequitas.plot as ap
+import pandas as pd
+from aequitas.bias import Bias
+from aequitas.group import Group
+from aequitas.plotting import Plot
+
+from validmind import tags, tasks
+from validmind.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@tags("bias_and_fairness")
+@tasks("classification", "regression")
+def ProtectedClassesDisparity(
+    dataset,
+    model,
+    protected_classes=None,
+    disparity_tolerance=1.25,
+    metrics=["fnr", "fpr", "tpr"],
+):
+    """
+    Investigates disparities in model performance across different protected class segments.
+
+    ### Purpose
+
+    This test aims to identify and quantify potential biases in model outcomes by comparing various performance metrics
+    across different segments of protected classes. It helps in assessing whether the model produces discriminatory
+    outcomes for certain groups, which is crucial for ensuring fairness in machine learning models.
+
+    ### Test Mechanism
+
+    The test performs the following steps:
+    1. Calculates performance metrics (e.g., false negative rate, false positive rate, true positive rate) for each segment
+       of the specified protected classes.
+    2. Computes disparity ratios by comparing these metrics between different segments and a reference group.
+    3. Generates visualizations showing the disparities and their relation to a user-defined disparity tolerance threshold.
+    4. Produces a comprehensive table with various disparity metrics for detailed analysis.
+
+    ### Signs of High Risk
+
+    - Disparity ratios exceeding the specified disparity tolerance threshold.
+    - Consistent patterns of higher error rates or lower performance for specific protected class segments.
+    - Statistically significant differences in performance metrics across segments.
+
+    ### Strengths
+
+    - Provides a comprehensive view of model fairness across multiple protected attributes and metrics.
+    - Allows for easy identification of problematic disparities through visual and tabular representations.
+    - Customizable disparity tolerance threshold to align with specific use-case requirements.
+    - Applicable to various performance metrics, offering a multi-faceted analysis of model fairness.
+
+    ### Limitations
+
+    - Relies on a predefined reference group for each protected class, which may not always be the most appropriate choice.
+    - Does not account for intersectionality between different protected attributes.
+    - The interpretation of results may require domain expertise to understand the implications of observed disparities.
+    """
+
+    if protected_classes is None:
+        logger.warning(
+            "No protected classes provided. Please pass the 'protected_classes' parameter to run this test."
+        )
+        return pd.DataFrame()
+
+    if sys.version_info < (3, 9):
+        raise RuntimeError("This test requires Python 3.9 or higher.")
+
+    df = dataset._df
+
+    for protected_class in protected_classes:
+        # make the dataset compatible for the python package of interest
+        df[protected_class] = pd.Categorical(df[protected_class]).astype("object")
+
+    df["score"] = dataset.y_pred(model).astype(int)
+    df["label_value"] = df[dataset.target_column].astype(int)
+
+    # let map the attributes for each protected class
+    # default use reference that is most observable for dictionary
+    attributes_and_reference_groups = {}
+    for protected_class in protected_classes:
+        attributes_and_reference_groups.update(
+            {protected_class: df[protected_class].value_counts().idxmax()}
+        )
+
+    attributes_to_audit = list(attributes_and_reference_groups.keys())
+
+    # Initialize Aequitas
+    g = Group()
+    b = Bias()
+    aqp = Plot()
+
+    columns_to_include = (
+        protected_classes + [dataset.target_column] + ["score", "label_value"]
+    )
+
+    # get_crosstabs returns a dataframe of the group counts and group value bias metrics.
+    xtab, _ = g.get_crosstabs(df[columns_to_include], attr_cols=attributes_to_audit)
+    bdf = b.get_disparity_predefined_groups(
+        xtab,
+        original_df=df[columns_to_include],
+        ref_groups_dict=attributes_and_reference_groups,
+        alpha=0.05,
+        mask_significance=True,
+    )
+
+    plots = []
+    for protected_class in protected_classes:
+        plot = ap.disparity(
+            bdf, metrics, protected_class, fairness_threshold=disparity_tolerance
+        )
+
+        buf = io.BytesIO()  # create a bytes array to save the image into in memory
+        plot.save(
+            buf, format="png"
+        )  # as long as the above library is installed, this will work
+        plots.append(buf.getvalue())
+
+    string = "_disparity"
+    metrics_adj = [x + string for x in metrics]
+
+    table = bdf[["attribute_name", "attribute_value"] + b.list_disparities(bdf)]
+    plots.append(aqp.plot_disparity_all(bdf, metrics=metrics_adj))
+    plots_return = tuple(plots)
+
+    return (table, *plots_return)

validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.py

@@ -0,0 +1,172 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import json
+import sys
+
+import matplotlib.pyplot as plt
+import pandas as pd
+from fairlearn.metrics import (
+    MetricFrame,
+    count,
+    demographic_parity_ratio,
+    equalized_odds_ratio,
+    false_negative_rate,
+    false_positive_rate,
+    true_positive_rate,
+)
+from fairlearn.postprocessing import ThresholdOptimizer, plot_threshold_optimizer
+
+from validmind import tags, tasks
+from validmind.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@tags("bias_and_fairness")
+@tasks("classification", "regression")
+def ProtectedClassesThresholdOptimizer(
+    dataset, pipeline=None, protected_classes=None, X_train=None, y_train=None
+):
+    """
+    Obtains a classifier by applying group-specific thresholds to the provided estimator.
+
+    ### Purpose
+
+    This test aims to optimize the fairness of a machine learning model by applying different
+    classification thresholds for different protected groups. It helps in mitigating bias and
+    achieving more equitable outcomes across different demographic groups.
+
+    ### Test Mechanism
+
+    The test uses Fairlearn's ThresholdOptimizer to:
+    1. Fit an optimizer on the training data, considering protected classes.
+    2. Apply optimized thresholds to make predictions on the test data.
+    3. Calculate and report various fairness metrics.
+    4. Visualize the optimized thresholds.
+
+    ### Signs of High Risk
+
+    - Large disparities in fairness metrics (e.g., Demographic Parity Ratio, Equalized Odds Ratio)
+      across different protected groups.
+    - Significant differences in False Positive Rates (FPR) or True Positive Rates (TPR) between groups.
+    - Thresholds that vary widely across different protected groups.
+
+    ### Strengths
+
+    - Provides a post-processing method to improve model fairness without modifying the original model.
+    - Allows for balancing multiple fairness criteria simultaneously.
+    - Offers visual insights into the threshold optimization process.
+
+    ### Limitations
+
+    - May lead to a decrease in overall model performance while improving fairness.
+    - Requires access to protected attribute information at prediction time.
+    - The effectiveness can vary depending on the chosen fairness constraint and objective.
+    """
+
+    if sys.version_info < (3, 9):
+        raise RuntimeError("This test requires Python 3.9 or higher.")
+
+    if (
+        pipeline is None
+        or protected_classes is None
+        or X_train is None
+        or y_train is None
+    ):
+        logger.warning(
+            "Missing required parameters. Please provide pipeline, protected_classes, X_train, and y_train."
+        )
+        return pd.DataFrame()
+
+    test_df = dataset.df
+
+    threshold_optimizer = initialize_and_fit_optimizer(
+        pipeline, X_train, y_train, X_train[protected_classes]
+    )
+
+    fig = plot_thresholds(threshold_optimizer)
+
+    target = dataset.target_column
+    y_pred_opt = make_predictions(threshold_optimizer, test_df, protected_classes)
+
+    fairness_metrics = calculate_fairness_metrics(
+        test_df, target, y_pred_opt, protected_classes
+    )
+
+    return (
+        {"DPR and EOR Table": fairness_metrics.reset_index()},
+        fig,
+    )
+
+
+def initialize_and_fit_optimizer(pipeline, X_train, y_train, protected_classes_df):
+    threshold_optimizer = ThresholdOptimizer(
+        estimator=pipeline,
+        objective="balanced_accuracy_score",
+        constraints="demographic_parity",
+        predict_method="predict_proba",
+        prefit=False,
+    )
+    threshold_optimizer.fit(X_train, y_train, sensitive_features=protected_classes_df)
+    return threshold_optimizer
+
+
+def plot_thresholds(threshold_optimizer):
+    fig = plt.figure()
+    plot_threshold_optimizer(threshold_optimizer, show_plot=False)
+    return fig
+
+
+def make_predictions(threshold_optimizer, test_df, protected_classes):
+    y_pred_opt = threshold_optimizer.predict(
+        test_df, sensitive_features=test_df[protected_classes]
+    )
+    return y_pred_opt
+
+
+def calculate_fairness_metrics(test_df, target, y_pred_opt, protected_classes):
+    fairness_metrics = pd.DataFrame(
+        columns=protected_classes,
+        index=["demographic parity ratio", "equal odds ratio"],
+    )
+
+    for feature in protected_classes:
+        dpr = demographic_parity_ratio(
+            y_true=test_df[target],
+            y_pred=y_pred_opt,
+            sensitive_features=test_df[[feature]],
+        )
+        eor = equalized_odds_ratio(
+            y_true=test_df[target],
+            y_pred=y_pred_opt,
+            sensitive_features=test_df[[feature]],
+        )
+        fairness_metrics[feature] = [round(dpr, 2), round(eor, 2)]
+
+    return fairness_metrics
+
+
+def calculate_group_metrics(test_df, target, y_pred_opt, protected_classes):
+    metrics = {
+        "fpr": false_positive_rate,
+        "tpr": true_positive_rate,
+        "fnr": false_negative_rate,
+        "count": count,
+    }
+    mf = MetricFrame(
+        metrics=metrics,
+        y_true=test_df[target],
+        y_pred=y_pred_opt,
+        sensitive_features=test_df[protected_classes],
+    )
+    group_metrics = mf.by_group
+    return group_metrics
+
+
+def get_thresholds_by_group(threshold_optimizer):
+    threshold_rules = threshold_optimizer.interpolated_thresholder_.interpolation_dict
+    thresholds = json.dumps(threshold_rules, default=str, indent=4)
+    thresholds_df = pd.DataFrame.from_records(json.loads(thresholds))
+    return thresholds_df

validmind/tests/{model_validation/statsmodels → data_validation}/RunsTest.py

@@ -2,12 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import pandas as pd
 from statsmodels.sandbox.stats.runs import runstest_1samp
 
-from validmind.vm_models import Metric
+from validmind import tags, tasks
 
 
-class RunsTest(Metric):
+@tasks("classification", "regression")
+@tags("tabular_data", "statistical_test", "statsmodels")
+def RunsTest(dataset):
     """
     Executes Runs Test on ML model to detect non-random patterns in output data sequence.
 
@@ -52,24 +55,18 @@ class RunsTest(Metric):
     - Does not provide model performance evaluation; it is used to detect patterns in the sequence of outputs only.
     """
 
-    name = "runs_test"
-    required_inputs = ["dataset"]
-    tasks = ["classification", "regression"]
-    tags = ["tabular_data", "statistical_test", "statsmodels"]
+    df = dataset.df[dataset.feature_columns_numeric]
 
-    def run(self):
-        """
-        Calculates the run test for each of the dataset features
-        """
-        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
+    runs_test_values = {}
+    for col in df.columns:
+        runs_stat, runs_p_value = runstest_1samp(df[col].values)
+        runs_test_values[col] = {
+            "stat": runs_stat,
+            "pvalue": runs_p_value,
+        }
 
-        runs_test_values = {}
-        for col in x_train.columns:
-            runs_stat, runs_p_value = runstest_1samp(x_train[col].values)
+    runs_test_df = pd.DataFrame.from_dict(runs_test_values, orient="index")
+    runs_test_df.reset_index(inplace=True)
+    runs_test_df.columns = ["feature", "stat", "pvalue"]
 
-            runs_test_values[col] = {
-                "stat": runs_stat,
-                "pvalue": runs_p_value,
-            }
-
-        return self.cache_results(runs_test_values)
+    return runs_test_df

validmind/tests/{model_validation/statsmodels → data_validation}/ShapiroWilk.py

@@ -2,12 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import pandas as pd
 from scipy import stats
 
-from validmind.vm_models import Metric
+from validmind import tags, tasks
 
 
-class ShapiroWilk(Metric):
+@tasks("classification", "regression")
+@tags("tabular_data", "data_distribution", "statistical_test")
+def ShapiroWilk(dataset):
     """
     Evaluates feature-wise normality of training data using the Shapiro-Wilk test.
 
@@ -49,23 +52,18 @@ class ShapiroWilk(Metric):
     - Lastly, the Shapiro-Wilk test is not optimally suited for processing data with pronounced skewness or kurtosis.
     """
 
-    name = "shapiro_wilk"
-    required_inputs = ["dataset"]
-    tasks = ["classification", "regression"]
-    tags = ["tabular_data", "data_distribution", "statistical_test"]
-
-    def run(self):
-        """
-        Calculates Shapiro-Wilk test for each of the dataset features.
-        """
-        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
-        sw_values = {}
-        for col in x_train.columns:
-            sw_stat, sw_pvalue = stats.shapiro(x_train[col].values)
-
-            sw_values[col] = {
-                "stat": sw_stat,
-                "pvalue": sw_pvalue,
-            }
-
-        return self.cache_results(sw_values)
+    df = dataset.df[dataset.feature_columns_numeric]
+
+    sw_values = {}
+    for col in df.columns:
+        sw_stat, sw_pvalue = stats.shapiro(df[col].values)
+        sw_values[col] = {
+            "stat": sw_stat,
+            "pvalue": sw_pvalue,
+        }
+
+    sw_df = pd.DataFrame.from_dict(sw_values, orient="index")
+    sw_df.reset_index(inplace=True)
+    sw_df.columns = ["column", "stat", "pvalue"]
+
+    return sw_df

validmind/tests/data_validation/nlp/Hashtags.py

@@ -9,8 +9,7 @@ Threshold based tests
 import re
 from dataclasses import dataclass
 
-import matplotlib.pyplot as plt
-import seaborn as sns
+import plotly.graph_objects as go
 
 from validmind.vm_models import Figure, ThresholdTest, VMDataset
 
@@ -74,25 +73,23 @@ class Hashtags(ThresholdTest):
         text_column = self.inputs.dataset.text_column
 
         def find_hash(text):
-            line = re.findall(r"(?<=#)\w+", text)
-            return " ".join(line)
-
-        temp = (
-            self.inputs.dataset.df[text_column]
-            .apply(lambda x: find_hash(x))
-            .value_counts()[:][1 : self.params["top_hashtags"]]
-        )
-        temp = (
-            temp.to_frame()
-            .reset_index()
-            .rename(columns={"index": "Hashtag", text_column: "count"})
-        )
+            return re.findall(r"(?<=#)\w+", str(text))
+
+        # Extract hashtags from the text column and count occurrences
+        hashtags = self.inputs.dataset.df[text_column].apply(find_hash).explode()
+        temp = hashtags.value_counts().head(self.params["top_hashtags"])
+
+        print(f"temp: {temp}")
 
         figures = []
         if not temp.empty:
-            fig = plt.figure()
-            sns.barplot(x="Hashtag", y="count", data=temp)
-            plt.xticks(rotation=90)
+            fig = go.Figure(data=[go.Bar(x=temp.index, y=temp.values)])
+            fig.update_layout(
+                title="Top Hashtags",
+                xaxis_title="Hashtag",
+                yaxis_title="Count",
+                xaxis_tickangle=-45,
+            )
             figures.append(
                 Figure(
                     for_object=self,
@@ -100,7 +97,5 @@ class Hashtags(ThresholdTest):
                     figure=fig,
                 )
             )
-            # Do this if you want to prevent the figure from being displayed
-            plt.close("all")
 
         return self.cache_results([], passed=True, figures=figures)

validmind/tests/data_validation/nlp/TextDescription.py

@@ -84,7 +84,6 @@ class TextDescription(Metric):
     tags = ["nlp", "text_data", "visualization"]
 
     def general_text_metrics(self, df, text_column):
-        nltk.download("punkt", quiet=True)
         results = []
 
         for text in df[text_column]:
@@ -175,6 +174,9 @@ class TextDescription(Metric):
         if not isinstance(self.inputs.dataset, VMDataset):
             raise ValueError("TextDescription requires a validmind Dataset object")
 
+        # download nltk data
+        nltk.download("punkt_tab", quiet=True)
+
         df_text_description = self.text_description_table(
             self.inputs.dataset.df, self.params
         )

validmind/tests/model_validation/ContextualRecall.py

@@ -58,6 +58,9 @@ def ContextualRecall(dataset, model):
     - Models that effectively use infrequent words might be undervalued, as these words might not overlap as often.
     """
 
+    # download nltk data
+    nltk.download("punkt_tab", quiet=True)
+
     y_true = dataset.y
     y_pred = dataset.y_pred(model)
 

validmind/tests/model_validation/ragas/AspectCritique.py

@@ -103,8 +103,8 @@ def AspectCritique(
     """
     try:
         from ragas import evaluate
-        from ragas.metrics.critique import AspectCritique as _AspectCritique
-        from ragas.metrics.critique import (
+        from ragas.metrics import AspectCritic
+        from ragas.metrics._aspect_critic import (
             coherence,
             conciseness,
             correctness,
@@ -114,7 +114,7 @@ def AspectCritique(
     except ImportError:
         raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
 
-    aspect_map = {
+    built_in_aspects = {
         "coherence": coherence,
         "conciseness": conciseness,
         "correctness": correctness,
@@ -136,16 +136,15 @@ def AspectCritique(
 
     df = get_renamed_columns(dataset._df, required_columns)
 
-    built_in_aspects = [aspect_map[aspect] for aspect in aspects]
     custom_aspects = (
         [
-            _AspectCritique(name=name, definition=description)
+            AspectCritic(name=name, definition=description)
             for name, description in additional_aspects
         ]
         if additional_aspects
         else []
     )
-    all_aspects = [*built_in_aspects, *custom_aspects]
+    all_aspects = [built_in_aspects[aspect] for aspect in aspects] + custom_aspects
 
     result_df = evaluate(
         Dataset.from_pandas(df), metrics=all_aspects, **get_ragas_config()