validmind 2.0.7__py3-none-any.whl → 2.1.1__py3-none-any.whl

validmind/__init__.py

@@ -60,8 +60,7 @@ from .client import (  # noqa: E402
     run_documentation_tests,
     run_test_suite,
 )
-from .tests.decorator import metric
-from .unit_metrics import run_metric
+from .tests.decorator import metric, tags, tasks
 from .utils import run_async  # noqa: E402
 
 
@@ -111,6 +110,8 @@ __all__ = [  # noqa
     "reload",
     "run_documentation_tests",
     "run_test_suite",
+    "tags",
+    "tasks",
     "tests",
     "test_suites",
     "vm_models",
@@ -119,5 +120,4 @@ __all__ = [  # noqa
     "log_figure",
     "log_metrics",
     "log_test_results",
-    "run_metric",
 ]

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.0.7"
+__version__ = "2.1.1"

validmind/ai.py

@@ -7,8 +7,6 @@ import os
 
 from openai import AzureOpenAI, OpenAI
 
-from .utils import clean_docstring
-
 SYSTEM_PROMPT = """
 You are an expert data scientist and MRM specialist tasked with providing concise and'
 objective insights based on the results of quantitative model or dataset analysis.
@@ -23,22 +21,20 @@ Your analysis will act as the description of the result in the model documentati
 
 Avoid long sentences and complex vocabulary.
 Structure the response clearly and logically.
-Use Markdown syntax to format the response.
+Use valid Markdown syntax to format the response (tables are supported).
 Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
-Use the following format for the response:
+Use the following format for the response (feel free to modify slightly if necessary):
 ```
 **<Test Name>** <continue to explain what it does in detail>...
 
 The results of this test <detailed explanation of the results>...
 
-In summary the following key insights can be gained from this <Test Type>
+In summary the following key insights can be gained:
 
 - **<key insight 1 - title>**: <explanation of key insight 1>
 - ...<continue with any other key insights using the same format>
 ```
 It is very important that the text is nicely formatted and contains enough information to be useful to the user as documentation.
-
-- use valid markdown syntax: make sure to have two newlines between paragraphs and before bullet points etc.
 """.strip()
 USER_PROMPT = """
 Test ID: {test_name}
@@ -71,7 +67,7 @@ def __get_client_and_model():
 
     if "OPENAI_API_KEY" in os.environ:
         __client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
-        __model = os.environ.get("VM_OPENAI_MODEL", "gpt-4-turbo-preview")
+        __model = os.environ.get("VM_OPENAI_MODEL", "gpt-4-turbo")
 
     elif "AZURE_OPENAI_KEY" in os.environ:
         if "AZURE_OPENAI_ENDPOINT" not in os.environ:
@@ -111,7 +107,7 @@ class DescriptionFuture:
 
     def get_description(self):
         # This will block until the future is completed
-        return clean_docstring(self._future.result())
+        return self._future.result()
 
 
 def generate_description_async(
@@ -132,7 +128,7 @@ def generate_description_async(
             raise ValueError("No results, summary or figures provided")
 
         response = client.chat.completions.create(
-            model="gpt-4-1106-vision-preview",
+            model="gpt-4-turbo",
             messages=[
                 {"role": "system", "content": SYSTEM_PROMPT},
                 {
@@ -160,7 +156,7 @@ def generate_description_async(
         )
     else:
         response = client.chat.completions.create(
-            model="gpt-4-turbo-preview",
+            model="gpt-4-turbo",
             messages=[
                 {"role": "system", "content": SYSTEM_PROMPT},
                 {

validmind/api_client.py

@@ -16,6 +16,7 @@ from io import BytesIO
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import aiohttp
+import mistune
 import requests
 from aiohttp import FormData
 
@@ -294,32 +295,33 @@ async def log_figures(figures: List[Figure]) -> Dict[str, Any]:
     Returns:
         dict: The response from the API
     """
-    if client_config.can_log_figures():  # check if the backend supports batch logging
-        try:
-            data = {}
-            files = {}
-            for figure in figures:
-                data.update(
-                    {f"{k}-{figure.key}": v for k, v in figure.serialize().items()}
-                )
-                files.update(
-                    {
-                        f"{k}-{figure.key}": v
-                        for k, v in figure.serialize_files().items()
-                    }
-                )
-
-            return await _post(
-                "log_figures",
-                data=data,
-                files=files,
-            )
-        except Exception as e:
-            logger.error("Error logging figures to ValidMind API")
-            raise e
-
-    else:
-        return await asyncio.gather(*[log_figure(figure) for figure in figures])
+    # this actually slows things down - better to log them in parallel
+    # if client_config.can_log_figures():  # check if the backend supports batch logging
+    #     try:
+    #         data = {}
+    #         files = {}
+    #         for figure in figures:
+    #             data.update(
+    #                 {f"{k}-{figure.key}": v for k, v in figure.serialize().items()}
+    #             )
+    #             files.update(
+    #                 {
+    #                     f"{k}-{figure.key}": v
+    #                     for k, v in figure.serialize_files().items()
+    #                 }
+    #             )
+
+    #         return await _post(
+    #             "log_figures",
+    #             data=data,
+    #             files=files,
+    #         )
+    #     except Exception as e:
+    #         logger.error("Error logging figures to ValidMind API")
+    #         raise e
+
+    # else:
+    return await asyncio.gather(*[log_figure(figure) for figure in figures])
 
 
 async def log_metadata(
@@ -342,7 +344,7 @@ async def log_metadata(
     """
     metadata_dict = {"content_id": content_id}
     if text is not None:
-        metadata_dict["text"] = text
+        metadata_dict["text"] = mistune.html(text)
     if _json is not None:
         metadata_dict["json"] = _json
 

validmind/client.py

@@ -61,8 +61,13 @@ def init_dataset(
     """
     Initializes a VM Dataset, which can then be passed to other functions
     that can perform additional analysis and tests on the data. This function
-    also ensures we are reading a valid dataset type. We only support Pandas
-    DataFrames at the moment.
+    also ensures we are reading a valid dataset type.
+
+    The following dataset types are supported:
+    - Pandas DataFrame
+    - Polars DataFrame
+    - Numpy ndarray
+    - Torch TensorDataset
 
     Args:
         dataset : dataset from various python libraries
@@ -380,7 +385,7 @@ def preview_template():
 
 
 def run_documentation_tests(
-    section=None, send=True, fail_fast=False, inputs=None, **kwargs
+    section=None, send=True, fail_fast=False, inputs=None, config=None, **kwargs
 ):
     """Collect and run all the tests associated with a template
 
@@ -393,6 +398,7 @@ def run_documentation_tests(
         send (bool, optional): Whether to send the results to the ValidMind API. Defaults to True.
         fail_fast (bool, optional): Whether to stop running tests after the first failure. Defaults to False.
         inputs (dict, optional): A dictionary of test inputs to pass to the TestSuite
+        config: A dictionary of test parameters to override the defaults
         **kwargs: backwards compatibility for passing in test inputs using keyword arguments
 
     Returns:
@@ -421,6 +427,7 @@ def run_documentation_tests(
             send=send,
             fail_fast=fail_fast,
             inputs=inputs,
+            config=config,
             **kwargs,
         )
         test_suites[_section] = test_suite

validmind/datasets/credit_risk/__init__.py

@@ -0,0 +1,11 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""
+Entrypoint for credit risk datasets.
+"""
+
+__all__ = [
+    "lending_club",
+]

validmind/datasets/credit_risk/lending_club.py

@@ -0,0 +1,394 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import os
+
+import numpy as np
+import pandas as pd
+import scorecardpy as sc
+import statsmodels.api as sm
+from sklearn.model_selection import train_test_split
+
+current_path = os.path.dirname(os.path.abspath(__file__))
+dataset_path = os.path.join(current_path, "datasets")
+
+# URLs or file paths for online and offline data
+online_data_file = "https://vmai.s3.us-west-1.amazonaws.com/datasets/lending_club_loan_data_2007_2014.csv"
+offline_data_file = os.path.join(
+    dataset_path, "lending_club_loan_data_2007_2014_clean.csv.gz"
+)
+
+target_column = "loan_status"
+
+drop_columns = [
+    "Unnamed: 0",
+    "id",
+    "member_id",
+    "funded_amnt",
+    "emp_title",
+    "url",
+    "desc",
+    "application_type",
+    "title",
+    "zip_code",
+    "delinq_2yrs",
+    "mths_since_last_delinq",
+    "mths_since_last_record",
+    "mths_since_last_major_derog",
+    "revol_bal",
+    "total_rec_prncp",
+    "total_rec_late_fee",
+    "recoveries",
+    "out_prncp_inv",
+    "out_prncp",
+    "collection_recovery_fee",
+    "next_pymnt_d",
+    "initial_list_status",
+    "pub_rec",
+    "collections_12_mths_ex_med",
+    "policy_code",
+    "acc_now_delinq",
+    "pymnt_plan",
+    "tot_coll_amt",
+    "tot_cur_bal",
+    "total_rev_hi_lim",
+    "last_pymnt_d",
+    "last_credit_pull_d",
+    "earliest_cr_line",
+    "issue_d",
+    "addr_state",
+    "dti",
+    "revol_util",
+    "total_pymnt_inv",
+    "inq_last_6mths",
+    "total_rec_int",
+    "last_pymnt_amnt",
+]
+
+drop_features = [
+    "loan_amnt",
+    "funded_amnt_inv",
+    "total_pymnt",
+]
+
+categorical_variables = [
+    "term",
+    "grade",
+    "sub_grade",
+    "emp_length",
+    "home_ownership",
+    "verification_status",
+    "purpose",
+]
+
+breaks_adj = {
+    "loan_amnt": [5000, 10000, 15000, 20000, 25000],
+    "int_rate": [10, 15, 20],
+    "annual_inc": [50000, 100000, 150000],
+}
+
+score_params = {
+    "target_score": 600,
+    "target_odds": 50,
+    "pdo": 20,
+}
+
+
+def load_data(source="online"):
+    """
+    Load data from either an online source or offline files, automatically dropping specified columns for offline data.
+
+    :param source: 'online' for online data, 'offline' for offline files. Defaults to 'online'.
+    :return: DataFrame containing the loaded data.
+    """
+
+    if source == "online":
+        print(f"Loading data from an online source: {online_data_file}")
+        df = pd.read_csv(online_data_file)
+        df = _clean_data(df)
+
+    elif source == "offline":
+        print(f"Loading data from an offline .gz file: {offline_data_file}")
+        # Since we know the offline_data_file path ends with '.zip', we replace it with '.csv.gz'
+        gzip_file_path = offline_data_file.replace(".zip", ".csv.gz")
+        print(f"Attempting to read from .gz file: {gzip_file_path}")
+        # Read the CSV file directly from the .gz archive
+        df = pd.read_csv(gzip_file_path, compression="gzip")
+        print("Data loaded successfully.")
+    else:
+        raise ValueError("Invalid source specified. Choose 'online' or 'offline'.")
+
+    print(
+        f"Rows: {df.shape[0]}, Columns: {df.shape[1]}, Missing values: {df.isnull().sum().sum()}"
+    )
+    return df
+
+
+def _clean_data(df):
+    df = df.copy()
+
+    # Drop columns not relevant for application scorecards
+    df = df.drop(columns=drop_columns)
+
+    # Drop rows with missing target values
+    df.dropna(subset=[target_column], inplace=True)
+    print("Dropping rows with missing target values:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Drop columns with more than N percent missing values
+    missing_values = df.isnull().mean()
+    df = df.loc[:, missing_values < 0.7]
+    print("Dropping columns with more than 70% missing values:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Drop columns with only one unique value
+    unique_values = df.nunique()
+    df = df.loc[:, unique_values > 1]
+    print("Dropping columns with only one unique value:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Define the target variable for the model, representing loan default status.
+    df[target_column] = df[target_column].map({"Fully Paid": 0, "Charged Off": 1})
+
+    # Drop rows with NaN in target_column after mapping
+    df.dropna(subset=[target_column], inplace=True)
+    print("Dropping rows with missing target values:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    return df
+
+
+def preprocess(df):
+    df = df.copy()
+
+    # Convert the target variable to integer type for modeling.
+    df[target_column] = df[target_column].astype(int)
+
+    # Keep rows where purpose is 'debt_consolidation' or 'credit_card'
+    df = df[df["purpose"].isin(["debt_consolidation", "credit_card"])]
+    print("Filtering 'purpose' to 'debt_consolidation' and 'credit_card':")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Remove rows where grade is 'F' or 'G'
+    df = df[~df["grade"].isin(["F", "G"])]
+    print("Filtering out 'grade' F and G:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Remove rows where sub_grade starts with 'F' or 'G'
+    df = df[~df["sub_grade"].str.startswith(("F", "G"))]
+    print("Filtering out 'sub_grade' F and G:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Remove rows where home_ownership is 'OTHER', 'NONE', or 'ANY'
+    df = df[~df["home_ownership"].isin(["OTHER", "NONE", "ANY"])]
+    print("Filtering out 'home_ownership' OTHER, NONE, ANY:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Drop features that are not useful for modeling
+    df.drop(drop_features, axis=1, inplace=True)
+    print("Dropping specified features:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Drop rows with missing values
+    df.dropna(inplace=True)
+    print("Dropping rows with any missing values:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Preprocess emp_length column
+    df = _preprocess_emp_length(df)
+
+    # Preprocess term column
+    df = _preprocess_term(df)
+
+    return df
+
+
+def _preprocess_term(df):
+    df = df.copy()
+
+    # Remove ' months' and convert to integer
+    df["term"] = df["term"].str.replace(" months", "").astype(object)
+
+    return df
+
+
+def _preprocess_emp_length(df):
+    df = df.copy()
+
+    # Mapping string values to numbers
+    emp_length_map = {
+        "10+ years": 10,
+        "< 1 year": 0,
+        "1 year": 1,
+        "2 years": 2,
+        "3 years": 3,
+        "4 years": 4,
+        "5 years": 5,
+        "6 years": 6,
+        "7 years": 7,
+        "8 years": 8,
+        "9 years": 9,
+    }
+
+    # Apply the mapping to the emp_length column
+    df["emp_length"] = df["emp_length"].map(emp_length_map).astype(object)
+
+    # Drop rows where emp_length is NaN after mapping
+    # df.dropna(subset=["emp_length"], inplace=True)
+
+    return df
+
+
+def feature_engineering(df):
+    df = df.copy()
+
+    # WoE encoding of numerical and categorical features
+    df = woe_encoding(df)
+
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    return df
+
+
+def woe_encoding(df):
+    df = df.copy()
+
+    woe = _woebin(df)
+    bins = _woe_to_bins(woe)
+
+    # Make sure we don't transform the target column
+    if target_column in bins:
+        del bins[target_column]
+        print(f"Excluded {target_column} from WoE transformation.")
+
+    # Apply the WoE transformation
+    df = sc.woebin_ply(df, bins=bins)
+
+    print("Successfully converted features to WoE values.")
+
+    return df
+
+
+def _woe_to_bins(woe):
+    # Select and rename columns
+    transformed_df = woe[
+        [
+            "variable",
+            "bin",
+            "count",
+            "count_distr",
+            "good",
+            "bad",
+            "badprob",
+            "woe",
+            "bin_iv",
+            "total_iv",
+        ]
+    ].copy()
+    transformed_df.rename(columns={"bin_iv": "total_iv"}, inplace=True)
+
+    # Create 'is_special_values' column (assuming there are no special values)
+    transformed_df["is_special_values"] = False
+
+    # Transform 'bin' column into interval format and store it in 'breaks' column
+    transformed_df["breaks"] = transformed_df["bin"].apply(
+        lambda x: "[-inf, %s)" % x if isinstance(x, float) else "[%s, inf)" % x
+    )
+
+    # Group by 'variable' to create bins dictionary
+    bins = {}
+    for variable, group in transformed_df.groupby("variable"):
+        bins[variable] = group
+
+    return bins
+
+
+def _woebin(df):
+    """
+    This function performs automatic binning using WoE.
+    df: A pandas dataframe
+    target_column: The target variable in quotes, e.g. 'loan_status'
+    """
+
+    non_numeric_cols = df.select_dtypes(exclude=["int64", "float64"]).columns
+    df[non_numeric_cols] = df[non_numeric_cols].astype(str)
+
+    try:
+        print(
+            f"Performing binning with breaks_adj: {breaks_adj}"
+        )  # print the breaks_adj being used
+        bins = sc.woebin(df, target_column, breaks_list=breaks_adj)
+    except Exception as e:
+        print("Error during binning: ")
+        print(e)
+    else:
+        bins_df = pd.concat(bins.values(), keys=bins.keys())
+        bins_df.reset_index(inplace=True)
+        bins_df.drop(columns=["variable"], inplace=True)
+        bins_df.rename(columns={"level_0": "variable"}, inplace=True)
+
+        bins_df["bin_number"] = bins_df.groupby("variable").cumcount()
+
+        return bins_df
+
+
+def split(df, add_constant=False):
+    df = df.copy()
+
+    # Splitting the dataset into training and test sets
+    train_df, test_df = train_test_split(df, test_size=0.2, random_state=42)
+
+    if add_constant:
+        # Add a constant to the model for both training and testing datasets
+        train_df = sm.add_constant(train_df)
+        test_df = sm.add_constant(test_df)
+
+    # Calculate and print details for the training dataset
+    print("After splitting the dataset into training and test sets:")
+    print(
+        f"Training Dataset:\nRows: {train_df.shape[0]}\nColumns: {train_df.shape[1]}\nMissing values: {train_df.isnull().sum().sum()}\n"
+    )
+
+    # Calculate and print details for the test dataset
+    print(
+        f"Test Dataset:\nRows: {test_df.shape[0]}\nColumns: {test_df.shape[1]}\nMissing values: {test_df.isnull().sum().sum()}\n"
+    )
+
+    return train_df, test_df
+
+
+def compute_scores(probabilities):
+
+    target_score = score_params["target_score"]
+    target_odds = score_params["target_odds"]
+    pdo = score_params["pdo"]
+
+    factor = pdo / np.log(2)
+    offset = target_score - (factor * np.log(target_odds))
+
+    scores = offset + factor * np.log(probabilities / (1 - probabilities))
+
+    return scores

validmind/logging.py

@@ -68,10 +68,17 @@ def get_logger(name="validmind", log_level=None):
     logger = logging.getLogger(name)
     logger.setLevel(log_level or _get_log_level())
 
-    # Check if the handler is already added
-    if not any(isinstance(h, type(handler)) for h in logger.handlers):
+    # Clear existing handlers if any (or refine the existing logic as necessary)
+    # TODO: lets add some better handler management
+    if not any(
+        isinstance(h, type(handler)) and h.formatter._fmt == formatter._fmt
+        for h in logger.handlers
+    ):
         logger.addHandler(handler)
 
+    # Prevent logger from propagating to root logger
+    logger.propagate = False
+
     return logger
 
 

validmind/template.py

@@ -4,9 +4,9 @@
 
 from pprint import pformat
 
+import mistune
 from IPython.display import display
 from ipywidgets import HTML, Accordion, VBox
-from markdown import markdown
 
 from .html_templates.content_blocks import (
     failed_content_block_html,
@@ -75,7 +75,7 @@ def _create_content_widget(content):
             HTML(
                 test_content_block_html.format(
                     title=test_deets["Name"],
-                    description=markdown(test_deets["Description"]),
+                    description=mistune.html(test_deets["Description"]),
                     required_inputs=", ".join(
                         test_deets["Required Inputs"] or ["None"]
                     ),

validmind/test_suites/__init__.py

@@ -5,6 +5,8 @@
 """
 Entrypoint for test suites.
 """
+from inspect import getdoc
+
 import pandas as pd
 
 from ..logging import get_logger
@@ -139,7 +141,7 @@ def list_suites(pretty: bool = True):
             {
                 "ID": suite_id,
                 "Name": test_suite.__name__,
-                "Description": test_suite.__doc__.strip(),
+                "Description": getdoc(test_suite).strip(),
                 "Tests": ", ".join(_get_test_suite_test_ids(test_suite)),
             }
         )
@@ -167,7 +169,7 @@ def describe_suite(test_suite_id: str, verbose=False):
                     {
                         "ID": test_suite_id,
                         "Name": test_suite.__name__,
-                        "Description": test_suite.__doc__.strip(),
+                        "Description": getdoc(test_suite).strip(),
                         "Tests": ", ".join(_get_test_suite_test_ids(test_suite)),
                     }
                 ]