validmind 1.7.0__tar.gz → 1.8.1__tar.gz

{validmind-1.7.0 → validmind-1.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 1.7.0
+Version: 1.8.1
 Summary: ValidMind Developer Framework
 Author: Andres Rodriguez
 Author-email: andres@validmind.ai
@@ -12,7 +12,6 @@ Classifier: Programming Language :: Python :: 3.10
 Provides-Extra: r-support
 Requires-Dist: arch (>=5.4.0,<6.0.0)
 Requires-Dist: click (>=8.0.4,<9.0.0)
-Requires-Dist: cython (>=0.29.34,<0.30.0)
 Requires-Dist: dython (>=0.7.1,<0.8.0)
 Requires-Dist: ipython (>=8.11.0,<9.0.0)
 Requires-Dist: myst-parser (>=1.0.0,<2.0.0)
@@ -33,5 +32,4 @@ Requires-Dist: sphinx-rtd-theme (>=1.2.0,<2.0.0)
 Requires-Dist: statsmodels (>=0.13.5,<0.14.0)
 Requires-Dist: tabulate (>=0.8.9,<0.9.0)
 Requires-Dist: tqdm (>=4.64.0,<5.0.0)
-Requires-Dist: twine (>=4.0.2,<5.0.0)
 Requires-Dist: xgboost (>=1.5.2,<2.0.0)

{validmind-1.7.0 → validmind-1.8.1}/pyproject.toml

@@ -6,14 +6,18 @@ authors = [
   "Luis Pallares <luis@validmind.ai>",
 ]
 description = "ValidMind Developer Framework"
-include = ["validmind/*.c", "validmind/*.py", "validmind/*.pyx", "validmind/*.pyd", "validmind/*.so"]
+# include = [
+#   "validmind/**/*.c",
+#   "validmind/**/*.pyx",
+#   "validmind/**/*.pyd",
+#   "validmind/**/*.so",
+# ]
 name = "validmind"
-version = "1.7.0"
+version = "1.8.1"
 
 [tool.poetry.dependencies]
 arch = "^5.4.0"
 click = "^8.0.4"
-cython = "^0.29.34"
 dython = "^0.7.1"
 ipython = "^8.11.0"
 myst-parser = "^1.0.0"
@@ -35,17 +39,18 @@ sphinx-rtd-theme = "^1.2.0"
 statsmodels = "^0.13.5"
 tabulate = "^0.8.9"
 tqdm = "^4.64.0"
-twine = "^4.0.2"
 xgboost = "^1.5.2"
 
 [tool.poetry.group.dev.dependencies]
 black = "^22.1.0"
 click = "^8.1.3"
+cython = "^0.29.34"
 flake8 = "^4.0.1"
 ipykernel = "^6.22.0"
 jupyter = "^1.0.0"
 jupyterlab = "^3.6.3"
 pytest = "^5.2"
+twine = "^4.0.2"
 
 [tool.poetry.extras]
 r-support = ["rpy2"]

{validmind-1.7.0 → validmind-1.8.1}/validmind/__init__.py

@@ -30,7 +30,13 @@ from .api_client import (  # noqa: E402
     log_figure,
 )
 
-from .client import init_dataset, init_model, init_r_model, run_test_plan  # noqa: E402
+from .client import (  # noqa: E402
+    init_dataset,
+    init_model,
+    init_r_model,
+    run_test_plan,
+    run_test_suite,
+)
 
 # TODO: need to fix this import * situation
 from .data_validation import *  # noqa
@@ -45,6 +51,7 @@ __all__ = [  # noqa
     "init_r_model",
     "test_plans",
     "run_test_plan",
+    "run_test_suite",
     # Framework Logging API
     "log_dataset",
     "log_figure",

validmind-1.7.0/validmind/client.pyx → validmind-1.8.1/validmind/client.py

@@ -8,7 +8,8 @@ import xgboost as xgb
 from sklearn.linear_model import LinearRegression, LogisticRegression
 
 # from .model_validation import evaluate_model as mod_evaluate_model
-from .test_plans import get_by_name
+from .test_plans import get_by_name as get_test_plan_by_name
+from .test_suites import get_by_name as get_test_suite_by_name
 from .vm_models import (
     Dataset,
     DatasetTargets,
@@ -16,6 +17,7 @@ from .vm_models import (
     ModelAttributes,
     R_MODEL_TYPES,
     TestPlan,
+    TestSuite,
 )
 
 
@@ -65,10 +67,10 @@ def init_dataset(
 
 
 def init_model(
-        model: object,
-        train_ds: Dataset = None,
-        test_ds: Dataset = None,
-        validation_ds: Dataset = None
+    model: object,
+    train_ds: Dataset = None,
+    test_ds: Dataset = None,
+    validation_ds: Dataset = None,
 ) -> Model:
     """
     Initializes a VM Model, which can then be passed to other functions
@@ -92,7 +94,9 @@ def init_model(
             )
         )
 
-    return Model.init_vm_model(model, train_ds, test_ds, validation_ds, attributes=ModelAttributes())
+    return Model.init_vm_model(
+        model, train_ds, test_ds, validation_ds, attributes=ModelAttributes()
+    )
 
 
 def init_r_model(model_path: str, model_type: str) -> Model:
@@ -193,7 +197,7 @@ def run_test_plan(test_plan_name, send=True, **kwargs):
         dict: A dictionary of test results
     """
     try:
-        Plan: TestPlan = get_by_name(test_plan_name)
+        Plan: TestPlan = get_test_plan_by_name(test_plan_name)
     except ValueError as exc:
         raise ValueError(
             "Error retrieving test plan {}. {}".format(test_plan_name, str(exc))
@@ -211,37 +215,40 @@ def run_test_plan(test_plan_name, send=True, **kwargs):
     return plan
 
 
-# def evaluate_model(model, train_set, val_set, test_set, eval_opts=None, send=True):
-#     """
-#     Evaluates a model and logs results to the ValidMind API. This function will log information
-#     about the trained model (parameters, etc.), training metrics, test metrics, and run model
-#     evaluation tests.
-
-#     :param model: The model to evaluate. Only scikit-learn and XGBoost models are supported at the moment
-#     :param (pd.DataFrame, pd.DataFrame) train_set: (x_train, y_train) tuple
-#     :param (pd.DataFrame, pd.DataFrame) val_set: (x_val, y_val) tuple
-#     :param (pd.DataFrame, pd.DataFrame) test_set: (x_test, y_test) tuple
-#     :param dict eval_opts: A dictionary of options for the model evaluation
-#     :param bool send: Whether to post the test results to the API. send=False is useful for testing
-#     """
-#     print("Logging model metadata and parameters...")
-#     log_model(model)
-
-#     print("Extracting training/validation set metrics from trained model...")
-#     x_train, y_train = train_set
-#     x_val, y_val = val_set
-
-#     log_training_metrics(
-#         model, x_train.copy(), y_train.copy(), x_val.copy(), y_val.copy()
-#     )
-
-#     print("Running model evaluation tests...")
-#     eval_results = mod_evaluate_model(
-#         model,
-#         test_set=test_set,
-#         train_set=train_set,
-#         eval_opts=eval_opts,
-#         send=send,
-#     )
-
-#     return eval_results
+def run_test_suite(test_suite_name, send=True, **kwargs):
+    """High Level function for running a test suite
+
+    This function provides a high level interface for running a test suite. A test suite is
+    a collection of test plans. This function will automatically find the correct test suite
+    class based on the test_suite_name, initialize each of the test plans, and run them.
+
+    Args:
+        test_suite_name (str): The test suite name (e.g. 'binary_classifier_full_suite')
+        send (bool, optional): Whether to post the test results to the API. send=False is useful for testing. Defaults to True.
+        **kwargs: Additional keyword arguments to pass to the test suite. These will provide
+            the TestSuite instance with the necessary context to run the tests. e.g. dataset, model etc.
+            See the documentation for the specific test plan, metric or threshold test for more details.
+
+    Raises:
+        ValueError: If the test suite name is not found or if there is an error initializing the test suite
+
+    Returns:
+        TestSuite: the TestSuite instance
+    """
+    try:
+        Suite: TestSuite = get_test_suite_by_name(test_suite_name)
+    except ValueError as exc:
+        raise ValueError(
+            "Error retrieving test suite {}. {}".format(test_suite_name, str(exc))
+        )
+
+    try:
+        suite = Suite(**kwargs)
+    except ValueError as exc:
+        raise ValueError(
+            "Error initializing test suite {}. {}".format(test_suite_name, str(exc))
+        )
+
+    suite.run(send=send)
+
+    return suite

validmind-1.7.0/validmind/data_validation/threshold_tests.pyx → validmind-1.8.1/validmind/data_validation/threshold_tests.py

@@ -389,7 +389,7 @@ class TimeSeriesOutliers(ThresholdTest):
             raise ValueError("zscore_threshold must be provided in params")
         zscore_threshold = self.params["zscore_threshold"]
 
-        temp_df = self.df.copy(deep=True)
+        temp_df = self.df.copy()
         temp_df = temp_df.dropna()
         typeset = ProfilingTypeSet(Settings())
         dataset_types = typeset.infer_type(temp_df)
@@ -462,7 +462,6 @@ class TimeSeriesOutliers(ThresholdTest):
         Returns:
             matplotlib.figure.Figure: A matplotlib figure object with subplots for each variable.
         """
-        print(outliers_table)
         sns.set(style="darkgrid")
         n_variables = len(df.columns)
         fig, axes = plt.subplots(n_variables, 1, sharex=True)

validmind-1.7.0/validmind/datasets/classification/customer_churn.pyx → validmind-1.8.1/validmind/datasets/classification/customer_churn.py

@@ -11,7 +11,7 @@ from . import (
 )
 
 current_path = os.path.dirname(os.path.abspath(__file__))
-dataset_path = os.path.join(current_path, "..", "..", "..", "notebooks", "datasets")
+dataset_path = os.path.join(current_path, "datasets")
 
 drop_columns = ["RowNumber", "CustomerId", "Surname", "CreditScore"]
 boolean_columns = ["Gender"]