validmind 2.4.10__tar.gz → 2.4.13__tar.gz

{validmind-2.4.10 → validmind-2.4.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.4.10
+Version: 2.4.13
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez

{validmind-2.4.10 → validmind-2.4.13}/pyproject.toml

@@ -10,7 +10,7 @@ description = "ValidMind Developer Framework"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.4.10"
+version = "2.4.13"
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<3.12"

validmind-2.4.13/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.4.13"

{validmind-2.4.10 → validmind-2.4.13}/validmind/api_client.py

@@ -152,6 +152,7 @@ def _get_session() -> aiohttp.ClientSession:
                 "X-API-KEY": _api_key,
                 "X-API-SECRET": _api_secret,
                 "X-PROJECT-CUID": _model_cuid,
+                "X-MONITORING": str(_monitoring),
             }
         )
 

{validmind-2.4.10 → validmind-2.4.13}/validmind/client.py

@@ -48,7 +48,6 @@ def init_dataset(
     index_name: str = None,
     date_time_index: bool = False,
     columns: list = None,
-    options: dict = None,
     text_column: str = None,
     target_column: str = None,
     feature_columns: list = None,
@@ -72,7 +71,6 @@ def init_dataset(
     Args:
         dataset : dataset from various python libraries
         model (VMModel): ValidMind model object
-        options (dict): A dictionary of options for the dataset
         targets (vm.vm.DatasetTargets): A list of target variables
         target_column (str): The name of the target column in the dataset
         feature_columns (list): A list of names of feature columns in the dataset

{validmind-2.4.10 → validmind-2.4.13}/validmind/input_registry.py

@@ -6,6 +6,8 @@
 Central class to register inputs
 """
 
+from validmind.vm_models.input import VMInput
+
 from .errors import InvalidInputError
 
 
@@ -14,6 +16,12 @@ class InputRegistry:
         self.registry = {}
 
     def add(self, key, obj):
+        if not isinstance(obj, VMInput):
+            raise InvalidInputError(
+                f"Input object must be an instance of VMInput. "
+                f"Got {type(obj)} instead."
+            )
+
         self.registry[key] = obj
 
     def get(self, key):

{validmind-2.4.10 → validmind-2.4.13}/validmind/tests/data_validation/DatasetDescription.py

@@ -134,6 +134,7 @@ class DatasetDescription(Metric):
                     )
             else:
                 vm_dataset_variables[column] = {"id": column, "type": str(type)}
+
         return list(vm_dataset_variables.values())
 
     def describe_dataset_field(self, df, field):

{validmind-2.4.10 → validmind-2.4.13}/validmind/tests/run.py

@@ -83,32 +83,47 @@ def _combine_summaries(summaries: List[Dict[str, Any]]):
     )
 
 
-def _update_plotly_titles(figures, input_groups, title_template):
-    current_title = figures[0].figure.layout.title.text
+def _get_input_id(v):
+    if isinstance(v, str):
+        return v  # If v is a string, return it as is.
+    elif isinstance(v, list) and all(hasattr(item, "input_id") for item in v):
+        # If v is a list and all items have an input_id attribute, join their input_id values.
+        return ", ".join(item.input_id for item in v)
+    elif hasattr(v, "input_id"):
+        return v.input_id  # If v has an input_id attribute, return it.
+    return str(v)  # Otherwise, return the string representation of v.
+
+
+def _update_plotly_titles(figures, input_group, title_template):
+    for figure in figures:
+
+        current_title = figure.figure.layout.title.text
+
+        input_description = " and ".join(
+            f"{key}: {_get_input_id(value)}" for key, value in input_group.items()
+        )
 
-    for i, figure in enumerate(figures):
         figure.figure.layout.title.text = title_template.format(
             current_title=f"{current_title} " if current_title else "",
-            input_description=" and ".join(
-                f"{k}: {v if isinstance(v, str) else ', '.join(item.input_id for item in v) if isinstance(v, list) and all(hasattr(item, 'input_id') for item in v) else v.input_id}"
-                for k, v in input_groups[i].items()
-            ),
+            input_description=input_description,
         )
 
 
-def _update_matplotlib_titles(figures, input_groups, title_template):
-    current_title = (
-        figures[0].figure._suptitle.get_text() if figures[0].figure._suptitle else ""
-    )
+def _update_matplotlib_titles(figures, input_group, title_template):
+    for figure in figures:
+
+        current_title = (
+            figure.figure._suptitle.get_text() if figure.figure._suptitle else ""
+        )
+
+        input_description = " and ".join(
+            f"{key}: {_get_input_id(value)}" for key, value in input_group.items()
+        )
 
-    for i, figure in enumerate(figures):
         figure.figure.suptitle(
             title_template.format(
                 current_title=f"{current_title} " if current_title else "",
-                input_description=" and ".join(
-                    f"{k}: {v if isinstance(v, str) else ', '.join(item.input_id for item in v) if isinstance(v, list) and all(hasattr(item, 'input_id') for item in v) else v.input_id}"
-                    for k, v in input_groups[i].items()
-                ),
+                input_description=input_description,
             )
         )
 
@@ -120,11 +135,12 @@ def _combine_figures(figure_lists: List[List[Any]], input_groups: List[Dict[str,
 
     title_template = "{current_title}({input_description})"
 
-    for figures in list(zip(*figure_lists)):
+    for idx, figures in enumerate(figure_lists):
+        input_group = input_groups[idx]
         if is_plotly_figure(figures[0].figure):
-            _update_plotly_titles(figures, input_groups, title_template)
+            _update_plotly_titles(figures, input_group, title_template)
         elif is_matplotlib_figure(figures[0].figure):
-            _update_matplotlib_titles(figures, input_groups, title_template)
+            _update_matplotlib_titles(figures, input_group, title_template)
         else:
             logger.warning("Cannot properly annotate png figures")
 

{validmind-2.4.10 → validmind-2.4.13}/validmind/unit_metrics/__init__.py

@@ -80,7 +80,7 @@ def _serialize_dataset(dataset, model):
         and pre-computed prediction columns, addressing potential hash collisions.
     """
     return _fast_hash(
-        dataset.df[
+        dataset._df[
             [
                 *dataset.feature_columns,
                 dataset.target_column,

{validmind-2.4.10 → validmind-2.4.13}/validmind/vm_models/__init__.py

@@ -8,6 +8,7 @@ Models entrypoint
 
 from .dataset.dataset import VMDataset
 from .figure import Figure
+from .input import VMInput
 from .model import R_MODEL_TYPES, ModelAttributes, VMModel
 from .test.metric import Metric
 from .test.metric_result import MetricResult
@@ -20,6 +21,7 @@ from .test_suite.runner import TestSuiteRunner
 from .test_suite.test_suite import TestSuite
 
 __all__ = [
+    "VMInput",
     "VMDataset",
     "VMModel",
     "Figure",

{validmind-2.4.10 → validmind-2.4.13}/validmind/vm_models/dataset/dataset.py

@@ -7,6 +7,7 @@ Dataset class wrapper
 """
 
 import warnings
+from copy import deepcopy
 
 import numpy as np
 import pandas as pd
@@ -14,6 +15,7 @@ import polars as pl
 
 from validmind.logging import get_logger
 from validmind.models import FunctionModel, PipelineModel
+from validmind.vm_models.input import VMInput
 from validmind.vm_models.model import VMModel
 
 from .utils import ExtraColumns, as_df, compute_predictions, convert_index_to_datetime
@@ -21,7 +23,7 @@ from .utils import ExtraColumns, as_df, compute_predictions, convert_index_to_da
 logger = get_logger(__name__)
 
 
-class VMDataset:
+class VMDataset(VMInput):
     """Base class for VM datasets
 
     Child classes should be used to support new dataset types (tensor, polars etc)
@@ -60,7 +62,6 @@ class VMDataset:
         text_column: str = None,
         extra_columns: dict = None,
         target_class_labels: dict = None,
-        options: dict = None,
     ):
         """
         Initializes a VMDataset instance.
@@ -77,7 +78,6 @@ class VMDataset:
             feature_columns (str, optional): The feature column names of the dataset. Defaults to None.
             text_column (str, optional): The text column name of the dataset for nlp tasks. Defaults to None.
             target_class_labels (Dict, optional): The class labels for the target columns. Defaults to None.
-            options (Dict, optional): Additional options for the dataset. Defaults to None.
         """
         # initialize input_id
         self.input_id = input_id
@@ -101,8 +101,6 @@ class VMDataset:
         if date_time_index:
             self._df = convert_index_to_datetime(self._df)
 
-        self.options = options
-
         self.columns = columns or []
         self.column_aliases = {}
         self.target_column = target_column
@@ -199,6 +197,56 @@ class VMDataset:
                 "Cannot use precomputed probabilities without precomputed predictions"
             )
 
+    def with_options(self, **kwargs) -> "VMDataset":
+        """Support options provided when passing an input to run_test or run_test_suite
+
+        Example:
+        ```python
+        # to only use a certain subset of columns in the dataset:
+        run_test(
+            "validmind.SomeTestID",
+            inputs={
+                "dataset": {
+                    "input_id": "my_dataset_id",
+                    "columns": ["col1", "col2"],
+                }
+            }
+        )
+
+        # behind the scenes, this retrieves the dataset object (VMDataset) from the registry
+        # and then calls the `with_options()` method and passes `{"columns": ...}`
+        ```
+
+        Args:
+            **kwargs: Options:
+                - columns: Filter columns in the dataset
+
+        Returns:
+            VMDataset: A new instance of the dataset with only the specified columns
+        """
+        if "columns" in kwargs:
+            # filter columns (create a temp copy of self with only specified columns)
+            # TODO: need a more robust mechanism for this as we expand on this feature
+            columns = kwargs.pop("columns")
+
+            new = deepcopy(self)
+
+            new._set_feature_columns(
+                [col for col in new.feature_columns if col in columns]
+            )
+            new.text_column = new.text_column if new.text_column in columns else None
+            new.target_column = (
+                new.target_column if new.target_column in columns else None
+            )
+            new.extra_columns.extras = new.extra_columns.extras.intersection(columns)
+
+            return new
+
+        if kwargs:
+            raise NotImplementedError(
+                f"Options {kwargs} are not supported for this input"
+            )
+
     def assign_predictions(
         self,
         model: VMModel,
@@ -444,7 +492,6 @@ class DataFrameDataset(VMDataset):
         feature_columns: list = None,
         text_column: str = None,
         target_class_labels: dict = None,
-        options: dict = None,
         date_time_index: bool = False,
     ):
         """
@@ -459,7 +506,6 @@ class DataFrameDataset(VMDataset):
             feature_columns (list, optional): The feature columns of the dataset. Defaults to None.
             text_column (str, optional): The text column name of the dataset for NLP tasks. Defaults to None.
             target_class_labels (dict, optional): The class labels for the target columns. Defaults to None.
-            options (dict, optional): Additional options for the dataset. Defaults to None.
             date_time_index (bool, optional): Whether to use date-time index. Defaults to False.
         """
         index = None
@@ -478,7 +524,6 @@ class DataFrameDataset(VMDataset):
             feature_columns=feature_columns,
             text_column=text_column,
             target_class_labels=target_class_labels,
-            options=options,
             date_time_index=date_time_index,
         )
 
@@ -498,7 +543,6 @@ class PolarsDataset(VMDataset):
         feature_columns: list = None,
         text_column: str = None,
         target_class_labels: dict = None,
-        options: dict = None,
         date_time_index: bool = False,
     ):
         """
@@ -513,7 +557,6 @@ class PolarsDataset(VMDataset):
             feature_columns (list, optional): The feature columns of the dataset. Defaults to None.
             text_column (str, optional): The text column name of the dataset for NLP tasks. Defaults to None.
             target_class_labels (dict, optional): The class labels for the target columns. Defaults to None.
-            options (dict, optional): Additional options for the dataset. Defaults to None.
             date_time_index (bool, optional): Whether to use date-time index. Defaults to False.
         """
         super().__init__(
@@ -528,7 +571,6 @@ class PolarsDataset(VMDataset):
             feature_columns=feature_columns,
             text_column=text_column,
             target_class_labels=target_class_labels,
-            options=options,
             date_time_index=date_time_index,
         )
 
@@ -551,7 +593,6 @@ class TorchDataset(VMDataset):
         feature_columns: list = None,
         text_column: str = None,
         target_class_labels: dict = None,
-        options: dict = None,
     ):
         """
         Initializes a TorchDataset instance.
@@ -609,5 +650,4 @@ class TorchDataset(VMDataset):
             text_column=text_column,
             extra_columns=extra_columns,
             target_class_labels=target_class_labels,
-            options=options,
         )

validmind-2.4.13/validmind/vm_models/input.py

@@ -0,0 +1,31 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Base class for ValidMind Input types"""
+
+from abc import ABC
+
+
+class VMInput(ABC):
+    """
+    Base class for ValidMind Input types
+    """
+
+    def with_options(self, **kwargs) -> "VMInput":
+        """
+        Allows for setting options on the input object that are passed by the user
+        when using the input to run a test or set of tests
+
+        To allow options, just override this method in the subclass (see VMDataset)
+        and ensure that it returns a new instance of the input with the specified options
+        set.
+
+        Args:
+            **kwargs: Arbitrary keyword arguments that will be passed to the input object
+
+        Returns:
+            VMInput: A new instance of the input with the specified options set
+        """
+        if kwargs:
+            raise NotImplementedError("This type of input does not support options")

{validmind-2.4.10 → validmind-2.4.13}/validmind/vm_models/model.py

@@ -7,11 +7,13 @@ Model class wrapper module
 """
 import importlib
 import inspect
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 from dataclasses import dataclass
 
 from validmind.errors import MissingOrInvalidModelPredictFnError
 
+from .input import VMInput
+
 SUPPORTED_LIBRARIES = {
     "catboost": "CatBoostModel",
     "xgboost": "XGBoostModel",
@@ -77,7 +79,7 @@ class ModelAttributes:
         )
 
 
-class VMModel(ABC):
+class VMModel(VMInput):
     """
     An base class that wraps a trained model instance and its associated data.
 

{validmind-2.4.10 → validmind-2.4.13}/validmind/vm_models/test_context.py

@@ -78,13 +78,12 @@ class TestInput:
         ... (any): Any other arbitrary inputs that can be used by tests
     """
 
-    # TODO: we need to look into adding metadata for test inputs and logging that
-
     def __init__(self, inputs):
         """Initialize with either a dictionary of inputs"""
         for key, value in inputs.items():
             # 1) retrieve input object from input registry if an input_id string is provided
             # 2) check the input_id type if a list of inputs (mix of strings and objects) is provided
+            # 3) if its a dict, it should contain the `input_id` key as well as other options
             if isinstance(value, str):
                 value = input_registry.get(key=value)
             elif isinstance(value, list) or isinstance(value, tuple):
@@ -92,6 +91,14 @@ class TestInput:
                     input_registry.get(key=v) if isinstance(v, str) else v
                     for v in value
                 ]
+            elif isinstance(value, dict):
+                assert "input_id" in value, (
+                    "Input dictionary must contain an 'input_id' key "
+                    "to retrieve the input object from the input registry."
+                )
+                value = input_registry.get(key=value.get("input_id")).with_options(
+                    **{k: v for k, v in value.items() if k != "input_id"}
+                )
 
             setattr(self, key, value)
 

validmind-2.4.10/validmind/__version__.py

@@ -1 +0,0 @@
-__version__ = "2.4.10"