validmind 2.7.2__py3-none-any.whl → 2.7.5__py3-none-any.whl

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.7.2"
+__version__ = "2.7.5"

validmind/ai/test_descriptions.py

@@ -65,6 +65,23 @@ def prompt_to_message(role, prompt):
     return {"role": role, "content": content}
 
 
+def _get_llm_global_context():
+
+    # Get the context from the environment variable
+    context = os.getenv("VALIDMIND_LLM_DESCRIPTIONS_CONTEXT", "")
+
+    # Check if context should be used (similar to descriptions enabled pattern)
+    context_enabled = os.getenv(
+        "VALIDMIND_LLM_DESCRIPTIONS_CONTEXT_ENABLED", "1"
+    ) not in [
+        "0",
+        "false",
+    ]
+
+    # Only use context if it's enabled and not empty
+    return context if context_enabled and context else None
+
+
 def generate_description(
     test_id: str,
     test_description: str,
@@ -79,15 +96,11 @@ def generate_description(
             "No tables, unit metric or figures provided - cannot generate description"
         )
 
-    # # TODO: fix circular import
-    # from validmind.ai.utils import get_client_and_model
-
     client, model = get_client_and_model()
 
     # get last part of test id
     test_name = title or test_id.split(".")[-1]
 
-    # TODO: fully support metrics
     if metric is not None:
         tables = [] if not tables else tables
         tables.append(
@@ -108,12 +121,15 @@ def generate_description(
     else:
         summary = None
 
+    context = _get_llm_global_context()
+
     input_data = {
         "test_name": test_name,
         "test_description": test_description,
         "title": title,
         "summary": summary,
         "figures": [figure._get_b64_url() for figure in ([] if tables else figures)],
+        "context": context,
     }
     system, user = _load_prompt()
 

validmind/ai/test_result_description/user.jinja

@@ -8,6 +8,11 @@
 
 Generate a description of the following result of the test using the instructions given in your system prompt.
 
+{%- if context %}
+**Context**:
+{{ context }}
+{%- endif %}
+
 {%- if summary %}
 **Test Result Tables** *(Raw Data)*:
 {{ summary }}

validmind/datasets/credit_risk/lending_club.py

@@ -355,33 +355,76 @@ def _woebin(df):
         return bins_df
 
 
-def split(df, add_constant=False):
+def split(df, validation_size=None, test_size=0.2, add_constant=False):
+    """
+    Split dataset into train, validation (optional), and test sets.
+
+    Args:
+        df: Input DataFrame
+        validation_split: If None, returns train/test split. If float, returns train/val/test split
+        test_size: Proportion of data for test set (default: 0.2)
+        add_constant: Whether to add constant column for statsmodels (default: False)
+
+    Returns:
+        If validation_size is None:
+            train_df, test_df
+        If validation_size is float:
+            train_df, validation_df, test_df
+    """
     df = df.copy()
 
-    # Splitting the dataset into training and test sets
-    train_df, test_df = train_test_split(df, test_size=0.2, random_state=42)
+    # First split off the test set
+    train_val_df, test_df = train_test_split(df, test_size=test_size, random_state=42)
 
     if add_constant:
-        # Add a constant to the model for both training and testing datasets
-        train_df = sm.add_constant(train_df)
         test_df = sm.add_constant(test_df)
 
-    # Calculate and print details for the training dataset
-    print("After splitting the dataset into training and test sets:")
-    print(
-        f"Training Dataset:\nRows: {train_df.shape[0]}\nColumns: {train_df.shape[1]}\nMissing values: {train_df.isnull().sum().sum()}\n"
+    if validation_size is None:
+        if add_constant:
+            train_val_df = sm.add_constant(train_val_df)
+
+        # Print details for two-way split
+        print("After splitting the dataset into training and test sets:")
+        print(
+            f"Training Dataset:\nRows: {train_val_df.shape[0]}\nColumns: {train_val_df.shape[1]}\n"
+            f"Missing values: {train_val_df.isnull().sum().sum()}\n"
+        )
+        print(
+            f"Test Dataset:\nRows: {test_df.shape[0]}\nColumns: {test_df.shape[1]}\n"
+            f"Missing values: {test_df.isnull().sum().sum()}\n"
+        )
+
+        return train_val_df, test_df
+
+    # Calculate validation size as proportion of remaining data
+    val_size = validation_size / (1 - test_size)
+    train_df, validation_df = train_test_split(
+        train_val_df, test_size=val_size, random_state=42
     )
 
-    # Calculate and print details for the test dataset
+    if add_constant:
+        train_df = sm.add_constant(train_df)
+        validation_df = sm.add_constant(validation_df)
+
+    # Print details for three-way split
+    print("After splitting the dataset into training, validation, and test sets:")
     print(
-        f"Test Dataset:\nRows: {test_df.shape[0]}\nColumns: {test_df.shape[1]}\nMissing values: {test_df.isnull().sum().sum()}\n"
+        f"Training Dataset:\nRows: {train_df.shape[0]}\nColumns: {train_df.shape[1]}\n"
+        f"Missing values: {train_df.isnull().sum().sum()}\n"
+    )
+    print(
+        f"Validation Dataset:\nRows: {validation_df.shape[0]}\nColumns: {validation_df.shape[1]}\n"
+        f"Missing values: {validation_df.isnull().sum().sum()}\n"
+    )
+    print(
+        f"Test Dataset:\nRows: {test_df.shape[0]}\nColumns: {test_df.shape[1]}\n"
+        f"Missing values: {test_df.isnull().sum().sum()}\n"
     )
 
-    return train_df, test_df
+    return train_df, validation_df, test_df
 
 
 def compute_scores(probabilities):
-
     target_score = score_params["target_score"]
     target_odds = score_params["target_odds"]
     pdo = score_params["pdo"]
@@ -389,6 +432,393 @@ def compute_scores(probabilities):
     factor = pdo / np.log(2)
     offset = target_score - (factor * np.log(target_odds))
 
-    scores = offset + factor * np.log(probabilities / (1 - probabilities))
+    # Add negative sign to reverse the relationship
+    scores = offset - factor * np.log(probabilities / (1 - probabilities))
 
     return scores
+
+
+def get_demo_test_config(x_test=None, y_test=None):
+    """Get demo test configuration.
+
+    Args:
+        x_test: Test features DataFrame
+        y_test: Test target Series
+
+    Returns:
+        dict: Test configuration dictionary
+    """
+    default_config = {}
+
+    # RAW DATA TESTS
+    default_config["validmind.data_validation.DatasetDescription:raw_data"] = {
+        "inputs": {
+            "dataset": "raw_dataset",
+        }
+    }
+    default_config["validmind.data_validation.DescriptiveStatistics:raw_data"] = {
+        "inputs": {
+            "dataset": "raw_dataset",
+        }
+    }
+    default_config["validmind.data_validation.MissingValues:raw_data"] = {
+        "inputs": {
+            "dataset": "raw_dataset",
+        },
+        "params": {"min_threshold": 1},
+    }
+    default_config["validmind.data_validation.ClassImbalance:raw_data"] = {
+        "inputs": {
+            "dataset": "raw_dataset",
+        },
+        "params": {"min_percent_threshold": 10},
+    }
+    default_config["validmind.data_validation.Duplicates:raw_data"] = {
+        "inputs": {
+            "dataset": "raw_dataset",
+        },
+        "params": {"min_threshold": 1},
+    }
+    default_config["validmind.data_validation.HighCardinality:raw_data"] = {
+        "inputs": {
+            "dataset": "raw_dataset",
+        },
+        "params": {
+            "num_threshold": 100,
+            "percent_threshold": 0.1,
+            "threshold_type": "percent",
+        },
+    }
+    default_config["validmind.data_validation.Skewness:raw_data"] = {
+        "inputs": {
+            "dataset": "raw_dataset",
+        },
+        "params": {"max_threshold": 1},
+    }
+    default_config["validmind.data_validation.UniqueRows:raw_data"] = {
+        "inputs": {
+            "dataset": "raw_dataset",
+        },
+        "params": {"min_percent_threshold": 1},
+    }
+    default_config["validmind.data_validation.TooManyZeroValues:raw_data"] = {
+        "inputs": {
+            "dataset": "raw_dataset",
+        },
+        "params": {"max_percent_threshold": 0.03},
+    }
+    default_config["validmind.data_validation.IQROutliersTable:raw_data"] = {
+        "inputs": {
+            "dataset": "raw_dataset",
+        },
+        "params": {"threshold": 5},
+    }
+
+    # PREPROCESSED DATA TESTS
+    default_config[
+        "validmind.data_validation.DescriptiveStatistics:preprocessed_data"
+    ] = {
+        "inputs": {
+            "dataset": "preprocess_dataset",
+        }
+    }
+    default_config[
+        "validmind.data_validation.TabularDescriptionTables:preprocessed_data"
+    ] = {
+        "inputs": {
+            "dataset": "preprocess_dataset",
+        }
+    }
+    default_config["validmind.data_validation.MissingValues:preprocessed_data"] = {
+        "inputs": {
+            "dataset": "preprocess_dataset",
+        },
+        "params": {"min_threshold": 1},
+    }
+    default_config[
+        "validmind.data_validation.TabularNumericalHistograms:preprocessed_data"
+    ] = {
+        "inputs": {
+            "dataset": "preprocess_dataset",
+        }
+    }
+    default_config[
+        "validmind.data_validation.TabularCategoricalBarPlots:preprocessed_data"
+    ] = {
+        "inputs": {
+            "dataset": "preprocess_dataset",
+        }
+    }
+    default_config["validmind.data_validation.TargetRateBarPlots:preprocessed_data"] = {
+        "inputs": {
+            "dataset": "preprocess_dataset",
+        },
+        "params": {"default_column": "loan_status"},
+    }
+
+    # DEVELOPMENT DATA TESTS
+    default_config[
+        "validmind.data_validation.DescriptiveStatistics:development_data"
+    ] = {"input_grid": {"dataset": ["train_dataset", "test_dataset"]}}
+
+    default_config[
+        "validmind.data_validation.TabularDescriptionTables:development_data"
+    ] = {"input_grid": {"dataset": ["train_dataset", "test_dataset"]}}
+
+    default_config["validmind.data_validation.ClassImbalance:development_data"] = {
+        "input_grid": {"dataset": ["train_dataset", "test_dataset"]},
+        "params": {"min_percent_threshold": 10},
+    }
+
+    default_config["validmind.data_validation.UniqueRows:development_data"] = {
+        "input_grid": {"dataset": ["train_dataset", "test_dataset"]},
+        "params": {"min_percent_threshold": 1},
+    }
+
+    default_config[
+        "validmind.data_validation.TabularNumericalHistograms:development_data"
+    ] = {"input_grid": {"dataset": ["train_dataset", "test_dataset"]}}
+
+    # FEATURE SELECTION TESTS
+    default_config["validmind.data_validation.MutualInformation:development_data"] = {
+        "input_grid": {"dataset": ["train_dataset", "test_dataset"]},
+        "params": {"min_threshold": 0.01},
+    }
+
+    default_config[
+        "validmind.data_validation.PearsonCorrelationMatrix:development_data"
+    ] = {"input_grid": {"dataset": ["train_dataset", "test_dataset"]}}
+
+    default_config[
+        "validmind.data_validation.HighPearsonCorrelation:development_data"
+    ] = {
+        "input_grid": {"dataset": ["train_dataset", "test_dataset"]},
+        "params": {"max_threshold": 0.3, "top_n_correlations": 10},
+    }
+
+    default_config["validmind.data_validation.WOEBinTable"] = {
+        "input_grid": {"dataset": ["preprocess_dataset"]},
+        "params": {"breaks_adj": breaks_adj},
+    }
+
+    default_config["validmind.data_validation.WOEBinPlots"] = {
+        "input_grid": {"dataset": ["preprocess_dataset"]},
+        "params": {"breaks_adj": breaks_adj},
+    }
+
+    # MODEL TRAINING TESTS
+    default_config["validmind.data_validation.DatasetSplit"] = {
+        "inputs": {"datasets": ["train_dataset", "test_dataset"]}
+    }
+
+    default_config["validmind.model_validation.ModelMetadata"] = {
+        "input_grid": {"model": ["xgb_model", "rf_model"]}
+    }
+
+    default_config["validmind.model_validation.sklearn.ModelParameters"] = {
+        "input_grid": {"model": ["xgb_model", "rf_model"]}
+    }
+
+    # MODEL SELECTION TESTS
+    default_config["validmind.model_validation.statsmodels.GINITable"] = {
+        "input_grid": {
+            "dataset": ["train_dataset", "test_dataset"],
+            "model": ["xgb_model", "rf_model"],
+        }
+    }
+
+    default_config["validmind.model_validation.sklearn.ClassifierPerformance"] = {
+        "input_grid": {
+            "dataset": ["train_dataset", "test_dataset"],
+            "model": ["xgb_model", "rf_model"],
+        }
+    }
+
+    default_config[
+        "validmind.model_validation.sklearn.TrainingTestDegradation:XGBoost"
+    ] = {
+        "inputs": {"datasets": ["train_dataset", "test_dataset"], "model": "xgb_model"},
+        "params": {"max_threshold": 0.1},
+    }
+
+    default_config[
+        "validmind.model_validation.sklearn.TrainingTestDegradation:RandomForest"
+    ] = {
+        "inputs": {"datasets": ["train_dataset", "test_dataset"], "model": "rf_model"},
+        "params": {"max_threshold": 0.1},
+    }
+
+    default_config["validmind.model_validation.sklearn.HyperParametersTuning"] = {
+        "inputs": {"model": "xgb_model", "dataset": "train_dataset"},
+        "params": {
+            "param_grid": {"n_estimators": [50, 100]},
+            "scoring": ["roc_auc", "recall"],
+            "fit_params": {
+                "eval_set": [(x_test, y_test)],
+                "verbose": False,
+            },
+            "thresholds": [0.3, 0.5],
+        },
+    }
+
+    # MODEL PERFORMANCE - DISCRIMINATION TESTS
+    default_config["validmind.model_validation.sklearn.ROCCurve"] = {
+        "input_grid": {
+            "dataset": ["train_dataset", "test_dataset"],
+            "model": ["xgb_model"],
+        }
+    }
+
+    default_config["validmind.model_validation.sklearn.MinimumROCAUCScore"] = {
+        "input_grid": {
+            "dataset": ["train_dataset", "test_dataset"],
+            "model": ["xgb_model"],
+        },
+        "params": {"min_threshold": 0.5},
+    }
+
+    default_config[
+        "validmind.model_validation.statsmodels.PredictionProbabilitiesHistogram"
+    ] = {
+        "input_grid": {
+            "dataset": ["train_dataset", "test_dataset"],
+            "model": ["xgb_model"],
+        }
+    }
+
+    default_config[
+        "validmind.model_validation.statsmodels.CumulativePredictionProbabilities"
+    ] = {
+        "input_grid": {
+            "model": ["xgb_model"],
+            "dataset": ["train_dataset", "test_dataset"],
+        }
+    }
+
+    default_config["validmind.model_validation.sklearn.PopulationStabilityIndex"] = {
+        "inputs": {"datasets": ["train_dataset", "test_dataset"], "model": "xgb_model"},
+        "params": {"num_bins": 10, "mode": "fixed"},
+    }
+
+    # MODEL PERFORMANCE - ACCURACY TESTS
+    default_config["validmind.model_validation.sklearn.ConfusionMatrix"] = {
+        "input_grid": {
+            "dataset": ["train_dataset", "test_dataset"],
+            "model": ["xgb_model"],
+        }
+    }
+
+    default_config["validmind.model_validation.sklearn.MinimumAccuracy"] = {
+        "input_grid": {
+            "dataset": ["train_dataset", "test_dataset"],
+            "model": ["xgb_model"],
+        },
+        "params": {"min_threshold": 0.7},
+    }
+
+    default_config["validmind.model_validation.sklearn.MinimumF1Score"] = {
+        "input_grid": {
+            "dataset": ["train_dataset", "test_dataset"],
+            "model": ["xgb_model"],
+        },
+        "params": {"min_threshold": 0.5},
+    }
+
+    default_config["validmind.model_validation.sklearn.PrecisionRecallCurve"] = {
+        "input_grid": {
+            "dataset": ["train_dataset", "test_dataset"],
+            "model": ["xgb_model"],
+        }
+    }
+
+    default_config["validmind.model_validation.sklearn.CalibrationCurve"] = {
+        "input_grid": {
+            "dataset": ["train_dataset", "test_dataset"],
+            "model": ["xgb_model"],
+        }
+    }
+
+    default_config[
+        "validmind.model_validation.sklearn.ClassifierThresholdOptimization"
+    ] = {
+        "inputs": {"dataset": "train_dataset", "model": "xgb_model"},
+        "params": {
+            "target_recall": 0.8  # Find a threshold that achieves a recall of 80%
+        },
+    }
+
+    # MODEL PERFORMANCE - SCORING TESTS
+    default_config["validmind.model_validation.statsmodels.ScorecardHistogram"] = {
+        "input_grid": {"dataset": ["train_dataset", "test_dataset"]},
+        "params": {"score_column": "xgb_scores"},
+    }
+
+    default_config["validmind.data_validation.ScoreBandDefaultRates"] = {
+        "input_grid": {"dataset": ["train_dataset"], "model": ["xgb_model"]},
+        "params": {
+            "score_column": "xgb_scores",
+            "score_bands": [504, 537, 570],  # Creates four score bands
+        },
+    }
+
+    default_config["validmind.model_validation.sklearn.ScoreProbabilityAlignment"] = {
+        "input_grid": {"dataset": ["train_dataset"], "model": ["xgb_model"]},
+        "params": {"score_column": "xgb_scores"},
+    }
+
+    # MODEL DIAGNOSIS TESTS
+    default_config["validmind.model_validation.sklearn.WeakspotsDiagnosis"] = {
+        "inputs": {
+            "datasets": ["train_dataset", "test_dataset"],
+            "model": "xgb_model",
+        },
+    }
+
+    default_config["validmind.model_validation.sklearn.OverfitDiagnosis"] = {
+        "inputs": {
+            "model": "xgb_model",
+            "datasets": ["train_dataset", "test_dataset"],
+        },
+        "params": {"cut_off_threshold": 0.04},
+    }
+
+    default_config["validmind.model_validation.sklearn.RobustnessDiagnosis"] = {
+        "inputs": {
+            "datasets": ["train_dataset", "test_dataset"],
+            "model": "xgb_model",
+        },
+        "params": {
+            "scaling_factor_std_dev_list": [0.1, 0.2, 0.3, 0.4, 0.5],
+            "performance_decay_threshold": 0.05,
+        },
+    }
+
+    # EXPLAINABILITY TESTS
+    default_config[
+        "validmind.model_validation.sklearn.PermutationFeatureImportance"
+    ] = {
+        "input_grid": {
+            "dataset": ["train_dataset", "test_dataset"],
+            "model": ["xgb_model"],
+        }
+    }
+
+    default_config["validmind.model_validation.FeaturesAUC"] = {
+        "input_grid": {
+            "model": ["xgb_model"],
+            "dataset": ["train_dataset", "test_dataset"],
+        },
+    }
+
+    default_config["validmind.model_validation.sklearn.SHAPGlobalImportance"] = {
+        "input_grid": {
+            "model": ["xgb_model"],
+            "dataset": ["train_dataset", "test_dataset"],
+        },
+        "params": {
+            "kernel_explainer_samples": 10,
+            "tree_or_linear_explainer_samples": 200,
+        },
+    }
+
+    return default_config

validmind/tests/data_validation/MutualInformation.py

@@ -0,0 +1,129 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import plotly.graph_objects as go
+from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset
+from validmind.vm_models.result import RawData
+
+
+@tags("feature_selection", "data_analysis")
+@tasks("classification", "regression")
+def MutualInformation(
+    dataset: VMDataset, min_threshold: float = 0.01, task: str = "classification"
+):
+    """
+    Calculates mutual information scores between features and target variable to evaluate feature relevance.
+
+    ### Purpose
+
+    The Mutual Information test quantifies the predictive power of each feature by measuring its statistical
+    dependency with the target variable. This helps identify relevant features for model training and
+    detect potential redundant or irrelevant variables, supporting feature selection decisions and model
+    interpretability.
+
+    ### Test Mechanism
+
+    The test employs sklearn's mutual_info_classif/mutual_info_regression functions to compute mutual
+    information between each feature and the target. It produces a normalized score (0 to 1) for each
+    feature, where higher scores indicate stronger relationships. Results are presented in both tabular
+    format and visualized through a bar plot with a configurable threshold line.
+
+    ### Signs of High Risk
+
+    - Many features showing very low mutual information scores
+    - Key business features exhibiting unexpectedly low scores
+    - All features showing similar, low information content
+    - Large discrepancy between business importance and MI scores
+    - Highly skewed distribution of MI scores
+    - Critical features below the minimum threshold
+    - Unexpected zero or near-zero scores for known important features
+    - Inconsistent scores across different data samples
+
+    ### Strengths
+
+    - Captures non-linear relationships between features and target
+    - Scale-invariant measurement of feature relevance
+    - Works for both classification and regression tasks
+    - Provides interpretable scores (0 to 1 scale)
+    - Supports automated feature selection
+    - No assumptions about data distribution
+    - Handles numerical and categorical features
+    - Computationally efficient for most datasets
+
+    ### Limitations
+
+    - Requires sufficient data for reliable estimates
+    - May be computationally intensive for very large datasets
+    - Cannot detect redundant features (pairwise relationships)
+    - Sensitive to feature discretization for continuous variables
+    - Does not account for feature interactions
+    - May underestimate importance of rare but crucial events
+    - Cannot handle missing values directly
+    - May be affected by extreme class imbalance
+    """
+    if task not in ["classification", "regression"]:
+        raise ValueError("task must be either 'classification' or 'regression'")
+
+    X = dataset.x
+    y = dataset.y
+
+    # Select appropriate MI function based on task type
+    if task == "classification":
+        mi_scores = mutual_info_classif(X, y)
+    else:
+        mi_scores = mutual_info_regression(X, y)
+
+    # Create DataFrame for raw data
+    raw_data = RawData(
+        feature=dataset.feature_columns,
+        mutual_information_score=mi_scores.tolist(),
+        pass_fail=["Pass" if score >= min_threshold else "Fail" for score in mi_scores],
+    )
+
+    # Create Plotly figure
+    fig = go.Figure()
+
+    # Sort data for better visualization
+    sorted_indices = sorted(
+        range(len(mi_scores)), key=lambda k: mi_scores[k], reverse=True
+    )
+    sorted_features = [dataset.feature_columns[i] for i in sorted_indices]
+    sorted_scores = [mi_scores[i] for i in sorted_indices]
+
+    # Add bar plot
+    fig.add_trace(
+        go.Bar(
+            x=sorted_features,
+            y=sorted_scores,
+            marker_color=[
+                "blue" if score >= min_threshold else "red" for score in sorted_scores
+            ],
+            name="Mutual Information Score",
+        )
+    )
+
+    # Add threshold line
+    fig.add_hline(
+        y=min_threshold,
+        line_dash="dash",
+        line_color="gray",
+        annotation_text=f"Threshold ({min_threshold})",
+        annotation_position="right",
+    )
+
+    # Update layout
+    fig.update_layout(
+        title="Mutual Information Scores by Feature",
+        xaxis_title="Features",
+        yaxis_title="Mutual Information Score",
+        xaxis_tickangle=-45,
+        showlegend=False,
+        width=1000,
+        height=600,
+        template="plotly_white",
+    )
+
+    return raw_data, fig