validmind 1.7.0__py3-none-any.whl → 1.8.1__py3-none-any.whl

validmind/__init__.py

@@ -30,7 +30,13 @@ from .api_client import (  # noqa: E402
     log_figure,
 )
 
-from .client import init_dataset, init_model, init_r_model, run_test_plan  # noqa: E402
+from .client import (  # noqa: E402
+    init_dataset,
+    init_model,
+    init_r_model,
+    run_test_plan,
+    run_test_suite,
+)
 
 # TODO: need to fix this import * situation
 from .data_validation import *  # noqa
@@ -45,6 +51,7 @@ __all__ = [  # noqa
     "init_r_model",
     "test_plans",
     "run_test_plan",
+    "run_test_suite",
     # Framework Logging API
     "log_dataset",
     "log_figure",

validmind/{client.pyx → client.py}

@@ -8,7 +8,8 @@ import xgboost as xgb
 from sklearn.linear_model import LinearRegression, LogisticRegression
 
 # from .model_validation import evaluate_model as mod_evaluate_model
-from .test_plans import get_by_name
+from .test_plans import get_by_name as get_test_plan_by_name
+from .test_suites import get_by_name as get_test_suite_by_name
 from .vm_models import (
     Dataset,
     DatasetTargets,
@@ -16,6 +17,7 @@ from .vm_models import (
     ModelAttributes,
     R_MODEL_TYPES,
     TestPlan,
+    TestSuite,
 )
 
 
@@ -65,10 +67,10 @@ def init_dataset(
 
 
 def init_model(
-        model: object,
-        train_ds: Dataset = None,
-        test_ds: Dataset = None,
-        validation_ds: Dataset = None
+    model: object,
+    train_ds: Dataset = None,
+    test_ds: Dataset = None,
+    validation_ds: Dataset = None,
 ) -> Model:
     """
     Initializes a VM Model, which can then be passed to other functions
@@ -92,7 +94,9 @@ def init_model(
             )
         )
 
-    return Model.init_vm_model(model, train_ds, test_ds, validation_ds, attributes=ModelAttributes())
+    return Model.init_vm_model(
+        model, train_ds, test_ds, validation_ds, attributes=ModelAttributes()
+    )
 
 
 def init_r_model(model_path: str, model_type: str) -> Model:
@@ -193,7 +197,7 @@ def run_test_plan(test_plan_name, send=True, **kwargs):
         dict: A dictionary of test results
     """
     try:
-        Plan: TestPlan = get_by_name(test_plan_name)
+        Plan: TestPlan = get_test_plan_by_name(test_plan_name)
     except ValueError as exc:
         raise ValueError(
             "Error retrieving test plan {}. {}".format(test_plan_name, str(exc))
@@ -211,37 +215,40 @@ def run_test_plan(test_plan_name, send=True, **kwargs):
     return plan
 
 
-# def evaluate_model(model, train_set, val_set, test_set, eval_opts=None, send=True):
-#     """
-#     Evaluates a model and logs results to the ValidMind API. This function will log information
-#     about the trained model (parameters, etc.), training metrics, test metrics, and run model
-#     evaluation tests.
-
-#     :param model: The model to evaluate. Only scikit-learn and XGBoost models are supported at the moment
-#     :param (pd.DataFrame, pd.DataFrame) train_set: (x_train, y_train) tuple
-#     :param (pd.DataFrame, pd.DataFrame) val_set: (x_val, y_val) tuple
-#     :param (pd.DataFrame, pd.DataFrame) test_set: (x_test, y_test) tuple
-#     :param dict eval_opts: A dictionary of options for the model evaluation
-#     :param bool send: Whether to post the test results to the API. send=False is useful for testing
-#     """
-#     print("Logging model metadata and parameters...")
-#     log_model(model)
-
-#     print("Extracting training/validation set metrics from trained model...")
-#     x_train, y_train = train_set
-#     x_val, y_val = val_set
-
-#     log_training_metrics(
-#         model, x_train.copy(), y_train.copy(), x_val.copy(), y_val.copy()
-#     )
-
-#     print("Running model evaluation tests...")
-#     eval_results = mod_evaluate_model(
-#         model,
-#         test_set=test_set,
-#         train_set=train_set,
-#         eval_opts=eval_opts,
-#         send=send,
-#     )
-
-#     return eval_results
+def run_test_suite(test_suite_name, send=True, **kwargs):
+    """High Level function for running a test suite
+
+    This function provides a high level interface for running a test suite. A test suite is
+    a collection of test plans. This function will automatically find the correct test suite
+    class based on the test_suite_name, initialize each of the test plans, and run them.
+
+    Args:
+        test_suite_name (str): The test suite name (e.g. 'binary_classifier_full_suite')
+        send (bool, optional): Whether to post the test results to the API. send=False is useful for testing. Defaults to True.
+        **kwargs: Additional keyword arguments to pass to the test suite. These will provide
+            the TestSuite instance with the necessary context to run the tests. e.g. dataset, model etc.
+            See the documentation for the specific test plan, metric or threshold test for more details.
+
+    Raises:
+        ValueError: If the test suite name is not found or if there is an error initializing the test suite
+
+    Returns:
+        TestSuite: the TestSuite instance
+    """
+    try:
+        Suite: TestSuite = get_test_suite_by_name(test_suite_name)
+    except ValueError as exc:
+        raise ValueError(
+            "Error retrieving test suite {}. {}".format(test_suite_name, str(exc))
+        )
+
+    try:
+        suite = Suite(**kwargs)
+    except ValueError as exc:
+        raise ValueError(
+            "Error initializing test suite {}. {}".format(test_suite_name, str(exc))
+        )
+
+    suite.run(send=send)
+
+    return suite

validmind/data_validation/{threshold_tests.pyx → threshold_tests.py}

@@ -389,7 +389,7 @@ class TimeSeriesOutliers(ThresholdTest):
             raise ValueError("zscore_threshold must be provided in params")
         zscore_threshold = self.params["zscore_threshold"]
 
-        temp_df = self.df.copy(deep=True)
+        temp_df = self.df.copy()
         temp_df = temp_df.dropna()
         typeset = ProfilingTypeSet(Settings())
         dataset_types = typeset.infer_type(temp_df)
@@ -462,7 +462,6 @@ class TimeSeriesOutliers(ThresholdTest):
         Returns:
             matplotlib.figure.Figure: A matplotlib figure object with subplots for each variable.
         """
-        print(outliers_table)
         sns.set(style="darkgrid")
         n_variables = len(df.columns)
         fig, axes = plt.subplots(n_variables, 1, sharex=True)

validmind/datasets/classification/{customer_churn.pyx → customer_churn.py}

@@ -11,7 +11,7 @@ from . import (
 )
 
 current_path = os.path.dirname(os.path.abspath(__file__))
-dataset_path = os.path.join(current_path, "..", "..", "..", "notebooks", "datasets")
+dataset_path = os.path.join(current_path, "datasets")
 
 drop_columns = ["RowNumber", "CustomerId", "Surname", "CreditScore"]
 boolean_columns = ["Gender"]