validmind 1.8.0__py3-none-any.whl → 1.8.1__py3-none-any.whl

validmind/__init__.py

@@ -30,7 +30,13 @@ from .api_client import (  # noqa: E402
     log_figure,
 )
 
-from .client import init_dataset, init_model, init_r_model, run_test_plan  # noqa: E402
+from .client import (  # noqa: E402
+    init_dataset,
+    init_model,
+    init_r_model,
+    run_test_plan,
+    run_test_suite,
+)
 
 # TODO: need to fix this import * situation
 from .data_validation import *  # noqa
@@ -45,6 +51,7 @@ __all__ = [  # noqa
     "init_r_model",
     "test_plans",
     "run_test_plan",
+    "run_test_suite",
     # Framework Logging API
     "log_dataset",
     "log_figure",

validmind/client.py

@@ -8,7 +8,8 @@ import xgboost as xgb
 from sklearn.linear_model import LinearRegression, LogisticRegression
 
 # from .model_validation import evaluate_model as mod_evaluate_model
-from .test_plans import get_by_name
+from .test_plans import get_by_name as get_test_plan_by_name
+from .test_suites import get_by_name as get_test_suite_by_name
 from .vm_models import (
     Dataset,
     DatasetTargets,
@@ -16,6 +17,7 @@ from .vm_models import (
     ModelAttributes,
     R_MODEL_TYPES,
     TestPlan,
+    TestSuite,
 )
 
 
@@ -65,10 +67,10 @@ def init_dataset(
 
 
 def init_model(
-        model: object,
-        train_ds: Dataset = None,
-        test_ds: Dataset = None,
-        validation_ds: Dataset = None
+    model: object,
+    train_ds: Dataset = None,
+    test_ds: Dataset = None,
+    validation_ds: Dataset = None,
 ) -> Model:
     """
     Initializes a VM Model, which can then be passed to other functions
@@ -92,7 +94,9 @@ def init_model(
             )
         )
 
-    return Model.init_vm_model(model, train_ds, test_ds, validation_ds, attributes=ModelAttributes())
+    return Model.init_vm_model(
+        model, train_ds, test_ds, validation_ds, attributes=ModelAttributes()
+    )
 
 
 def init_r_model(model_path: str, model_type: str) -> Model:
@@ -193,7 +197,7 @@ def run_test_plan(test_plan_name, send=True, **kwargs):
         dict: A dictionary of test results
     """
     try:
-        Plan: TestPlan = get_by_name(test_plan_name)
+        Plan: TestPlan = get_test_plan_by_name(test_plan_name)
     except ValueError as exc:
         raise ValueError(
             "Error retrieving test plan {}. {}".format(test_plan_name, str(exc))
@@ -211,37 +215,40 @@ def run_test_plan(test_plan_name, send=True, **kwargs):
     return plan
 
 
-# def evaluate_model(model, train_set, val_set, test_set, eval_opts=None, send=True):
-#     """
-#     Evaluates a model and logs results to the ValidMind API. This function will log information
-#     about the trained model (parameters, etc.), training metrics, test metrics, and run model
-#     evaluation tests.
-
-#     :param model: The model to evaluate. Only scikit-learn and XGBoost models are supported at the moment
-#     :param (pd.DataFrame, pd.DataFrame) train_set: (x_train, y_train) tuple
-#     :param (pd.DataFrame, pd.DataFrame) val_set: (x_val, y_val) tuple
-#     :param (pd.DataFrame, pd.DataFrame) test_set: (x_test, y_test) tuple
-#     :param dict eval_opts: A dictionary of options for the model evaluation
-#     :param bool send: Whether to post the test results to the API. send=False is useful for testing
-#     """
-#     print("Logging model metadata and parameters...")
-#     log_model(model)
-
-#     print("Extracting training/validation set metrics from trained model...")
-#     x_train, y_train = train_set
-#     x_val, y_val = val_set
-
-#     log_training_metrics(
-#         model, x_train.copy(), y_train.copy(), x_val.copy(), y_val.copy()
-#     )
-
-#     print("Running model evaluation tests...")
-#     eval_results = mod_evaluate_model(
-#         model,
-#         test_set=test_set,
-#         train_set=train_set,
-#         eval_opts=eval_opts,
-#         send=send,
-#     )
-
-#     return eval_results
+def run_test_suite(test_suite_name, send=True, **kwargs):
+    """High Level function for running a test suite
+
+    This function provides a high level interface for running a test suite. A test suite is
+    a collection of test plans. This function will automatically find the correct test suite
+    class based on the test_suite_name, initialize each of the test plans, and run them.
+
+    Args:
+        test_suite_name (str): The test suite name (e.g. 'binary_classifier_full_suite')
+        send (bool, optional): Whether to post the test results to the API. send=False is useful for testing. Defaults to True.
+        **kwargs: Additional keyword arguments to pass to the test suite. These will provide
+            the TestSuite instance with the necessary context to run the tests. e.g. dataset, model etc.
+            See the documentation for the specific test plan, metric or threshold test for more details.
+
+    Raises:
+        ValueError: If the test suite name is not found or if there is an error initializing the test suite
+
+    Returns:
+        TestSuite: the TestSuite instance
+    """
+    try:
+        Suite: TestSuite = get_test_suite_by_name(test_suite_name)
+    except ValueError as exc:
+        raise ValueError(
+            "Error retrieving test suite {}. {}".format(test_suite_name, str(exc))
+        )
+
+    try:
+        suite = Suite(**kwargs)
+    except ValueError as exc:
+        raise ValueError(
+            "Error initializing test suite {}. {}".format(test_suite_name, str(exc))
+        )
+
+    suite.run(send=send)
+
+    return suite

validmind/test_plans/__init__.py

@@ -8,12 +8,10 @@ import tabulate
 from ..vm_models import TestPlan
 from .binary_classifier import (
     BinaryClassifierMetrics,
-    BinaryClassifier,
     BinaryClassifierPerformance,
     BinaryClassifierDiagnosis,
 )
 from .tabular_datasets import (
-    TabularDataset,
     TabularDataQuality,
     TabularDatasetDescription,
     TimeSeriesDataQuality,
@@ -39,8 +37,6 @@ core_test_plans = {
     "binary_classifier_metrics": BinaryClassifierMetrics,
     "binary_classifier_validation": BinaryClassifierPerformance,
     "binary_classifier_model_diagnosis": BinaryClassifierDiagnosis,
-    "binary_classifier": BinaryClassifier,
-    "tabular_dataset": TabularDataset,
     "tabular_dataset_description": TabularDatasetDescription,
     "tabular_data_quality": TabularDataQuality,
     "normality_test_plan": NormalityTestPlan,

validmind/test_plans/binary_classifier.py

@@ -81,18 +81,3 @@ class BinaryClassifierDiagnosis(TestPlan):
     name = "binary_classifier_model_diagnosis"
     required_context = ["model"]
     tests = [OverfitDiagnosis, WeakspotsDiagnosis, RobustnessDiagnosis]
-
-
-class BinaryClassifier(TestPlan):
-    """
-    Test plan for sklearn classifier models that includes
-    both metrics and validation tests
-    """
-
-    name = "binary_classifier"
-    required_context = ["model"]
-    test_plans = [
-        BinaryClassifierMetrics,
-        BinaryClassifierPerformance,
-        BinaryClassifierDiagnosis,
-    ]

validmind/test_plans/tabular_datasets.py

@@ -72,19 +72,6 @@ class TimeSeriesDataQuality(TestPlan):
     tests = [TimeSeriesOutliers, TimeSeriesMissingValues, TimeSeriesFrequency]
 
 
-class TabularDataset(TestPlan):
-    """
-    Test plan for generic tabular datasets
-    """
-
-    name = "tabular_dataset"
-    required_context = ["dataset"]
-    test_plans = [
-        TabularDatasetDescription,
-        TabularDataQuality,
-    ]
-
-
 class TimeSeriesDataset(TestPlan):
     """
     Test plan for time series  datasets

validmind/test_suites/__init__.py

@@ -0,0 +1,73 @@
+"""
+Entrypoint for test suites.
+"""
+import tabulate
+
+from .test_suites import (
+    BinaryClassifierFullSuite,
+    BinaryClassifierModelValidation,
+    TabularDataset,
+)
+from ..vm_models import TestSuite
+
+core_test_suites = {
+    "binary_classifier_full_suite": BinaryClassifierFullSuite,
+    "binary_classifier_model_validation": BinaryClassifierModelValidation,
+    "tabular_dataset": TabularDataset,
+}
+
+# These test suites can be added by the user
+custom_test_suites = {}
+
+
+def _get_all_test_suites():
+    """
+    Returns a dictionary of all test suites.
+
+    Merge the core and custom test suites, with the custom suites
+    taking precedence, i.e. allowing overriding of core test suites
+    """
+    return {**core_test_suites, **custom_test_suites}
+
+
+def get_by_name(name: str):
+    """
+    Returns the test suite by name
+    """
+    all_test_suites = _get_all_test_suites()
+    if name in all_test_suites:
+        return all_test_suites[name]
+
+    raise ValueError(f"Test suite with name: '{name}' not found")
+
+
+def list_suites(pretty: bool = True):
+    """
+    Returns a list of all available test suites
+    """
+
+    all_test_suites = _get_all_test_suites()
+
+    if not pretty:
+        return list(all_test_suites.keys())
+
+    table = []
+    for name, test_suite in all_test_suites.items():
+        table.append(
+            {
+                "ID": name,
+                "Name": test_suite.__name__,
+                "Description": test_suite.__doc__.strip(),
+                "Test Plans": ", ".join(test_suite.test_plans),
+            }
+        )
+
+    return tabulate.tabulate(table, headers="keys", tablefmt="html")
+
+
+def register_test_suite(suite_id: str, suite: TestSuite):
+    """
+    Registers a custom test suite
+    """
+    custom_test_suites[suite_id] = suite
+    print(f"Registered test suite: {suite_id}")

validmind/test_suites/test_suites.py

@@ -0,0 +1,48 @@
+"""
+Default test suites provided by the developer framework.
+"""
+
+from ..vm_models import TestSuite
+
+
+class TabularDataset(TestSuite):
+    """
+    Test suite for tabular datasets.
+    """
+
+    required_context = ["dataset"]
+
+    test_plans = [
+        "tabular_dataset_description",
+        "tabular_data_quality",
+    ]
+
+
+class BinaryClassifierModelValidation(TestSuite):
+    """
+    Test suite for binary classification models.
+    """
+
+    required_context = ["model"]
+
+    test_plans = [
+        "binary_classifier_metrics",
+        "binary_classifier_validation",
+        "binary_classifier_model_diagnosis",
+    ]
+
+
+class BinaryClassifierFullSuite(TestSuite):
+    """
+    Full test suite for binary classification models.
+    """
+
+    required_context = ["dataset", "model"]
+
+    test_plans = [
+        "tabular_dataset_description",
+        "tabular_data_quality",
+        "binary_classifier_metrics",
+        "binary_classifier_validation",
+        "binary_classifier_model_diagnosis",
+    ]

validmind/vm_models/__init__.py

@@ -17,6 +17,7 @@ from .test_plan_result import (
     TestPlanTestResult,
 )
 from .test_result import TestResult, TestResults
+from .test_suite import TestSuite
 from .threshold_test import ThresholdTest
 
 __all__ = [
@@ -40,5 +41,6 @@ __all__ = [
     "TestPlanTestResult",
     "TestResult",
     "TestResults",
+    "TestSuite",
     "ThresholdTest",
 ]

validmind/vm_models/test_suite.py

@@ -0,0 +1,57 @@
+"""
+A TestSuite is a collection of TestPlans. It is a helpful way to organize
+TestPlans that are related to each other. For example, a TestSuite could be
+created for a specific use case or model methodology, to run a colllection
+of plans for data validation and model validation with a single function call.
+"""
+
+from dataclasses import dataclass
+from typing import ClassVar, List
+
+from .test_context import TestContext
+from .test_plan import TestPlan
+
+
+@dataclass
+class TestSuite(TestPlan):
+    """
+    Base class for test suites. Test suites are used to define any
+    arbitrary grouping of test plans that will be run on a dataset and/or model.
+    """
+
+    test_plans: ClassVar[List[str]] = []
+    # Stores a reference to the child test plan instances
+    # so we can access their results after running the test suite
+    _test_plan_instances: List[object] = None
+
+    def run(self, send=True):
+        """
+        Runs the test suite.
+        """
+        # Avoid circular import
+        from ..test_plans import get_by_name
+
+        self._test_plan_instances = []
+
+        if self.test_context is None:
+            self.test_context = TestContext(
+                dataset=self.dataset,
+                model=self.model,
+                models=self.models,
+            )
+
+        for test_plan_id in self.test_plans:
+            test_plan = get_by_name(test_plan_id)
+            test_plan_instance = test_plan(
+                config=self.config,
+                test_context=self.test_context,
+            )
+            test_plan_instance.run(send=send)
+            self._test_plan_instances.append(test_plan_instance)
+
+    @property
+    def results(self):
+        """
+        Returns the results of the test suite.
+        """
+        return [test_plan.results for test_plan in self._test_plan_instances]

{validmind-1.8.0.dist-info → validmind-1.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 1.8.0
+Version: 1.8.1
 Summary: ValidMind Developer Framework
 Author: Andres Rodriguez
 Author-email: andres@validmind.ai

{validmind-1.8.0.dist-info → validmind-1.8.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
-validmind/__init__.py,sha256=_-1gzCQo2v2TEjQPyRD2SLJ7UCOiaPVFWnTkxMf7gfY,1380
+validmind/__init__.py,sha256=ND6a4RZrm1QFdYvDnGqhfwoXFsC762CJrUzdPdcVtp0,1443
 validmind/api_client.py,sha256=fqEWQii5H5uS4gbjm2X05mKKfPGeQiV6VkuMf_fC2Gw,11353
-validmind/client.py,sha256=P5UC4BVrts1S83xfPu37QZvtETuVcIbEOT7eEN_oPhw,8945
+validmind/client.py,sha256=cwk8swBmclmT4POTUsRdGXAKIg5IxzfOvYPXrRVvKQM,9190
 validmind/data_validation/__init__.py,sha256=xytRpsfQ86fDnIZRoAO7GMVVU_TwWVMXxSCwm0mb45I,590
 validmind/data_validation/metrics.py,sha256=I-q_fQ4daFneSAS8-hdVw7SMwrUnSv3m_RdcOsxDB10,35109
 validmind/data_validation/threshold_tests.py,sha256=azKHJ31HTxffq8_rVCpsfXIumJ9i02HwpgPXsAKm5Ck,24265
@@ -35,13 +35,15 @@ validmind/model_validation/statsmodels/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
 validmind/model_validation/statsmodels/metrics.py,sha256=wGEDlqHYVxpzUCbheYRy5Y8OiUs7wwHuxFwdnrhSctA,28009
 validmind/model_validation/statsmodels/threshold_tests.py,sha256=SyEv7oaNIgiQ1ML3dQrGN54pSP88j9htOy4V2l4Ipwk,1317
 validmind/model_validation/utils.py,sha256=lOq-C_mucS5JZ-7V43PnSqYkGc5ybdGDbzhclWp4V3A,1657
-validmind/test_plans/__init__.py,sha256=HUlON7orGplvoGmszwhm1q1Keu9Lbw8nv5vWfsAROTE,5375
-validmind/test_plans/binary_classifier.py,sha256=ct_ZWwxNM1K245DA8wyoNtu2eDzJ92r-eDgKES_KKEE,2328
+validmind/test_plans/__init__.py,sha256=elptQlHZ1YK55Rm2nQd391gpGjbXakzC9xDk2Txj7dw,5251
+validmind/test_plans/binary_classifier.py,sha256=x5vUXPD3Xfqxv6041lGfyzo3v8a0OzRcsaspajEqmyg,1985
 validmind/test_plans/statsmodels_timeseries.py,sha256=bEHwL_RJY5SoiUQnbM0SrGI6dcW2jJmasoH3CAYvBYQ,2836
-validmind/test_plans/tabular_datasets.py,sha256=5uyGOGnt5ngTsSJ-0DdonAlbErhprq9dZdZ2c0EXKik,2068
+validmind/test_plans/tabular_datasets.py,sha256=Eoz_Rbf2JKsD61XI6ZAti10uBaTa22Y8pkyJ7RUMGUc,1822
 validmind/test_plans/time_series.py,sha256=ecBVWUh4fAS8g6ZAUQwbmuaVo7q7tyMVrCyp1TV8RD8,6821
+validmind/test_suites/__init__.py,sha256=rojE4xXj0LPq4BUKWkUe6N_Z3RDvjztmY3f4g28kMOw,1825
+validmind/test_suites/test_suites.py,sha256=XLhRLNKagrlBhVrGwIp965O7KqTpPUVVHdYGFGvc1A0,1000
 validmind/utils.py,sha256=6KnbRTUJt6dRXbNtMzLySkJ3hnVC5f_e4qP-MEIDUbk,6465
-validmind/vm_models/__init__.py,sha256=9KA42gT5a9ocSfum5b3UaXHxuaI_NWiKAXalOWg9G3E,1065
+validmind/vm_models/__init__.py,sha256=6Rnzxn5aaHYFkq1llchPNmdjSVJXh8mHSWEqW1gQs1M,1116
 validmind/vm_models/dataset.py,sha256=CE2mJwT27HMKCJAE9ssRb0gh8skdrSSyYiOk3v4QCSw,10425
 validmind/vm_models/dataset_utils.py,sha256=xWq6dbxe9fCUdpKCiEFvbxhy9t7cOKh0boS-zw6ynZo,8287
 validmind/vm_models/figure.py,sha256=EarfMm49J2BWf6UhsMFGr3VSdGu8v2ZewzArKKp_LfU,592
@@ -54,7 +56,8 @@ validmind/vm_models/test_context.py,sha256=nWiTIORPNEl7mKkYme8n2QaJyAotp034A2NK3
 validmind/vm_models/test_plan.py,sha256=qGCftt01Yvzz9RbFGDo4uTWT9XyDb4doTxq3HxC65Mk,9846
 validmind/vm_models/test_plan_result.py,sha256=STdD-l0d68ogJet7T5j06YhCTtrGkGP248aiaHfxq6k,19074
 validmind/vm_models/test_result.py,sha256=jiX8yb1NptBXZQ_pOyHloc8I6yS6zamYm0j9OWUqrHs,1698
+validmind/vm_models/test_suite.py,sha256=cxyVF8cY1_ycpRRzvDHcIw3kTOUWhSYk6ObtyyB4xdg,1811
 validmind/vm_models/threshold_test.py,sha256=uqTqHDf443_Xm2J4G5KBnMH0GLRVVIpv6V-A3f-tSvE,3621
-validmind-1.8.0.dist-info/METADATA,sha256=fJqqUj7v2oppsz8HCRoFhxcToRvI4Fl3nfI5QhLuw7s,1385
-validmind-1.8.0.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-validmind-1.8.0.dist-info/RECORD,,
+validmind-1.8.1.dist-info/METADATA,sha256=BIfuQCYGrkDXHtwjIMFTJ8vgjNR862_hMASgxqGaiCk,1385
+validmind-1.8.1.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+validmind-1.8.1.dist-info/RECORD,,