validmind 2.5.25__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/utils.py

@@ -9,8 +9,10 @@ import json
 import math
 import re
 import sys
+import warnings
+from datetime import date, datetime, time
 from platform import python_version
-from typing import Any
+from typing import Any, Dict, List
 
 import matplotlib.pylab as pylab
 import mistune
@@ -24,6 +26,7 @@ from IPython.display import display as ipy_display
 from latex2mathml.converter import convert
 from matplotlib.axes._axes import _log as matplotlib_axes_logger
 from numpy import ndarray
+from sklearn.exceptions import UndefinedMetricWarning
 from tabulate import tabulate
 
 from .html_templates.content_blocks import math_jax_snippet, python_syntax_highlighting
@@ -32,6 +35,11 @@ from .logging import get_logger
 DEFAULT_BIG_NUMBER_DECIMALS = 2
 DEFAULT_SMALL_NUMBER_DECIMALS = 4
 
+# Suppress some common warnings
+warnings.filterwarnings("ignore", category=UndefinedMetricWarning)
+warnings.filterwarnings(
+    "ignore", category=UserWarning, message=".*valid feature names.*"
+)
 
 # SETUP SOME DEFAULTS FOR PLOTS #
 # Silence this warning: *c* argument looks like a single numeric RGB or
@@ -93,23 +101,64 @@ def nan_to_none(obj):
 
 
 class NumpyEncoder(json.JSONEncoder):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.type_handlers = {
+            self.is_datetime: lambda obj: obj.isoformat(),
+            self.is_pandas_interval: lambda obj: f"[{obj.left}, {obj.right}]",
+            self.is_numpy_integer: lambda obj: int(obj),
+            self.is_numpy_floating: lambda obj: float(obj),
+            self.is_numpy_ndarray: lambda obj: obj.tolist(),
+            self.is_numpy_bool: lambda obj: bool(obj),
+            self.is_pandas_timestamp: lambda obj: str(obj),
+            self.is_set: lambda obj: list(obj),
+            self.is_quantlib_date: lambda obj: obj.ISO(),
+            self.is_generic_object: self.handle_generic_object,
+        }
+
     def default(self, obj):
-        if isinstance(obj, pd.Interval):
-            return f"[{obj.left}, {obj.right}]"
-        if isinstance(obj, np.integer):
-            return int(obj)
-        if isinstance(obj, np.floating):
-            return float(obj)
-        if isinstance(obj, np.ndarray):
-            return obj.tolist()
-        if isinstance(obj, np.bool_):
-            return bool(obj)
-        if isinstance(obj, pd.Timestamp):
-            return str(obj)
-        if isinstance(obj, set):
-            return list(obj)
+        for type_check, handler in self.type_handlers.items():
+            if type_check(obj):
+                return handler(obj)
         return super().default(obj)
 
+    def is_datetime(self, obj):
+        return isinstance(obj, (datetime, date, time))
+
+    def is_pandas_interval(self, obj):
+        return isinstance(obj, pd.Interval)
+
+    def is_numpy_integer(self, obj):
+        return isinstance(obj, np.integer)
+
+    def is_numpy_floating(self, obj):
+        return isinstance(obj, np.floating)
+
+    def is_numpy_ndarray(self, obj):
+        return isinstance(obj, np.ndarray)
+
+    def is_numpy_bool(self, obj):
+        return isinstance(obj, np.bool_)
+
+    def is_pandas_timestamp(self, obj):
+        return isinstance(obj, pd.Timestamp)
+
+    def is_set(self, obj):
+        return isinstance(obj, set)
+
+    def is_quantlib_date(self, obj):
+        return "QuantLib.Date" in str(type(obj))
+
+    def is_generic_object(self, obj):
+        return isinstance(obj, object)
+
+    def handle_generic_object(self, obj):
+        return (
+            obj.__str__()
+            if type(obj).__dict__.get("__str__")
+            else str(obj).split(".")[1].split(" ")[0]
+        )
+
     def encode(self, obj):
         obj = nan_to_none(obj)
         return super().encode(obj)
@@ -158,7 +207,7 @@ def precision_and_scale(x):
     return (magnitude + scale, scale)
 
 
-def format_records(df):
+def format_records(df: pd.DataFrame) -> List[Dict[str, Any]]:
     """
     Round the values on each dataframe's column to a given number of decimal places.
     The returned value is converted to a dict in "records" with Pandas's to_dict() function.
@@ -191,7 +240,7 @@ def format_records(df):
     return df.to_dict("records")
 
 
-def format_key_values(key_values):
+def format_key_values(key_values: Dict[str, Any]) -> Dict[str, Any]:
     """
     Round the values on each dict's value to a given number of decimal places.
 

validmind/vm_models/__init__.py

@@ -10,13 +10,7 @@ from .dataset.dataset import VMDataset
 from .figure import Figure
 from .input import VMInput
 from .model import R_MODEL_TYPES, ModelAttributes, VMModel
-from .test.metric import Metric
-from .test.metric_result import MetricResult
-from .test.result_summary import ResultSummary, ResultTable, ResultTableMetadata
-from .test.test import Test
-from .test.threshold_test import ThresholdTest
-from .test.threshold_test_result import ThresholdTestResult, ThresholdTestResults
-from .test_context import TestContext, TestInput
+from .result import ResultTable, TestResult
 from .test_suite.runner import TestSuiteRunner
 from .test_suite.test_suite import TestSuite
 
@@ -27,17 +21,8 @@ __all__ = [
     "Figure",
     "ModelAttributes",
     "R_MODEL_TYPES",
-    "ResultSummary",
     "ResultTable",
-    "ResultTableMetadata",
-    "Test",
-    "Metric",
-    "MetricResult",
-    "ThresholdTest",
-    "ThresholdTestResult",
-    "ThresholdTestResults",
-    "TestContext",
-    "TestInput",
+    "TestResult",
     "TestSuite",
     "TestSuiteRunner",
 ]

validmind/vm_models/dataset/dataset.py

@@ -48,6 +48,9 @@ class VMDataset(VMInput):
         extra_columns (Dict): Extra columns to include in the dataset.
     """
 
+    def __repr__(self):
+        return f"VMDataset(input_id={self.input_id})"
+
     def __init__(
         self,
         raw_dataset: np.ndarray,
@@ -430,7 +433,12 @@ class VMDataset(VMInput):
         Returns:
             np.ndarray: The predictions for the model
         """
-        return np.stack(self._df[self.prediction_column(model)].values)
+        pred_col = self.prediction_column(model)
+
+        if pred_col is None:
+            raise ValueError(f"No prediction column found for model `{model.input_id}`")
+
+        return np.stack(self._df[pred_col].values)
 
     def y_prob(self, model) -> np.ndarray:
         """Returns the probabilities for a given model.
@@ -441,7 +449,14 @@ class VMDataset(VMInput):
         Returns:
             np.ndarray: The probability variables.
         """
-        return self._df[self.probability_column(model)].values
+        prob_col = self.probability_column(model)
+
+        if prob_col is None:
+            raise ValueError(
+                f"No probability column found for model `{model.input_id}`"
+            )
+
+        return self._df[prob_col].values
 
     def x_df(self):
         """Returns a dataframe containing only the feature columns"""
@@ -453,11 +468,23 @@ class VMDataset(VMInput):
 
     def y_pred_df(self, model) -> pd.DataFrame:
         """Returns a dataframe containing the predictions for a given model"""
-        return as_df(self._df[self.prediction_column(model)])
+        pred_col = self.prediction_column(model)
+
+        if pred_col is None:
+            raise ValueError(f"No prediction column found for model `{model.input_id}`")
+
+        return as_df(self._df[pred_col])
 
     def y_prob_df(self, model) -> pd.DataFrame:
         """Returns a dataframe containing the probabilities for a given model"""
-        return as_df(self._df[self.probability_column(model)])
+        prob_col = self.probability_column(model)
+
+        if prob_col is None:
+            raise ValueError(
+                f"No probability column found for model `{model.input_id}`"
+            )
+
+        return as_df(self._df[prob_col])
 
     def target_classes(self):
         """Returns the target class labels or unique values of the target column."""

validmind/vm_models/figure.py

@@ -10,14 +10,14 @@ import base64
 import json
 from dataclasses import dataclass
 from io import BytesIO
-from typing import Optional
+from typing import Union
 
 import ipywidgets as widgets
 import matplotlib
 import plotly.graph_objs as go
 
 from ..client_config import client_config
-from ..errors import InvalidFigureForObjectError, UnsupportedFigureError
+from ..errors import UnsupportedFigureError
 from ..utils import get_full_typename
 
 
@@ -40,25 +40,12 @@ class Figure:
     """
 
     key: str
-    figure: object
-    metadata: Optional[dict] = None
-    for_object: Optional[object] = None
-    extras: Optional[dict] = None
+    figure: Union[matplotlib.figure.Figure, go.Figure, go.FigureWidget, bytes]
+    ref_id: str  # used to link figures to results
 
-    _type: str = "plot"
+    _type: str = "plot"  # for now this is the only figure type
 
     def __post_init__(self):
-        """
-        Set default params if not provided
-        """
-        if self.for_object is not None:
-            metadata = self.metadata or {}
-            # Use underscore to avoid name collisions with user-defined metadata
-            metadata["_type"] = self._get_for_object_type()
-            metadata["_name"] = getattr(self.for_object, "test_id", None)
-            metadata["_ref_id"] = getattr(self.for_object, "_ref_id", None)
-            self.metadata = metadata
-
         # Wrap around with FigureWidget so that we can display interactive Plotly
         # plots in regular Jupyter notebooks. This is not supported on Google Colab.
         if (
@@ -68,23 +55,6 @@ class Figure:
         ):
             self.figure = go.FigureWidget(self.figure)
 
-    def _get_for_object_type(self):
-        """
-        Returns the type of the object this figure is for
-        """
-        # Avoid circular imports
-        from .test.metric import Metric
-        from .test.threshold_test import ThresholdTest
-
-        if issubclass(self.for_object.__class__, Metric):
-            return "metric"
-        elif issubclass(self.for_object.__class__, ThresholdTest):
-            return "threshold_test"
-        else:
-            raise InvalidFigureForObjectError(
-                "Figure for_object must be a Metric or ThresholdTest object"
-            )
-
     def to_widget(self):
         """
         Returns the ipywidget compatible representation of the figure. Ideally
@@ -135,7 +105,7 @@ class Figure:
         return {
             "type": self._type,
             "key": self.key,
-            "metadata": json.dumps(self.metadata, allow_nan=False),
+            "metadata": json.dumps({"_ref_id": self.ref_id}, allow_nan=False),
         }
 
     def _get_b64_url(self):
@@ -176,7 +146,7 @@ class Figure:
 
         elif is_plotly_figure(self.figure):
             # When using plotly, we need to use we will produce two files:
-            # - a JSON file that will be used to display the figure in the UI
+            # - a JSON file that will be used to display the figure in the ValidMind Platform
             # - a PNG file that will be used to display the figure in documents
             return {
                 "image": (

validmind/vm_models/model.py

@@ -101,6 +101,9 @@ class VMModel(VMInput):
         name (str, optional): The name of the model. Defaults to the class name.
     """
 
+    def __repr__(self):
+        return f"VMModel(input_id={self.input_id})"
+
     def __init__(
         self,
         input_id: str = None,

validmind/vm_models/result/__init__.py

@@ -0,0 +1,7 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from .result import ErrorResult, Result, ResultTable, TestResult
+
+__all__ = ["ErrorResult", "Result", "ResultTable", "TestResult"]

validmind/vm_models/result/result.jinja

@@ -0,0 +1,21 @@
+<h2>{{ test_name }} {{ passed_icon }}</h2>
+
+{{ description }}
+
+{% if inputs %}
+<h3>Test Inputs</h3>
+<pre>
+{{ inputs }}
+</pre>
+{% endif %}
+
+{% if params %}
+<h3>Test Parameters</h3>
+<pre>
+{{ params }}
+</pre>
+{% endif %}
+
+{% if show_metric %}
+<h3>Metric: <code language="json">{{ metric }}</code></h3>
+{% endif %}

validmind/vm_models/result/result.py

@@ -0,0 +1,337 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""
+Result Objects for test results
+"""
+import asyncio
+import json
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Union
+from uuid import uuid4
+
+import pandas as pd
+from ipywidgets import HTML, VBox
+
+from ... import api_client
+from ...ai.utils import DescriptionFuture
+from ...logging import get_logger
+from ...utils import NumpyEncoder, display, run_async, test_id_to_name
+from ..figure import Figure
+from ..input import VMInput
+from .utils import (
+    AI_REVISION_NAME,
+    DEFAULT_REVISION_NAME,
+    check_for_sensitive_data,
+    figures_to_widgets,
+    get_result_template,
+    tables_to_widgets,
+    update_metadata,
+)
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class ResultTable:
+    """
+    A dataclass that holds the table summary of result
+    """
+
+    data: Union[List[Any], pd.DataFrame]
+    title: str
+
+    def __repr__(self) -> str:
+        return f'ResultTable(title="{self.title}")' if self.title else "ResultTable"
+
+    def __post_init__(self):
+        if isinstance(self.data, list):
+            self.data = pd.DataFrame(self.data)
+
+        self.data = self.data.round(4)
+
+    def serialize(self):
+        data = {
+            "type": "table",
+            "data": self.data.to_dict(orient="records"),
+        }
+
+        if self.title:
+            data["metadata"] = {"title": self.title}
+
+        return data
+
+
+@dataclass
+class Result:
+    """Base Class for test suite results"""
+
+    result_id: str = None
+    name: str = None
+
+    def __str__(self) -> str:
+        """May be overridden by subclasses"""
+        return self.__class__.__name__
+
+    @abstractmethod
+    def to_widget(self):
+        """Create an ipywdiget representation of the result... Must be overridden by subclasses"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def log(self):
+        """Log the result... Must be overridden by subclasses"""
+        raise NotImplementedError
+
+    def show(self):
+        """Display the result... May be overridden by subclasses"""
+        display(self.to_widget())
+
+
+@dataclass
+class ErrorResult(Result):
+    """Result for test suites that fail to load or run properly"""
+
+    name: str = "Failed Test"
+    error: Exception = None
+    message: str = None
+
+    def __repr__(self) -> str:
+        return f'ErrorResult(result_id="{self.result_id}")'
+
+    def to_widget(self):
+        return HTML(f"<h3 style='color: red;'>{self.message}</h3><p>{self.error}</p>")
+
+    async def log_async(self):
+        pass
+
+
+@dataclass
+class TestResult(Result):
+    """Test result"""
+
+    name: str = "Test Result"
+    ref_id: str = None
+    title: Optional[str] = None
+    description: Optional[Union[str, DescriptionFuture]] = None
+    metric: Optional[Union[int, float]] = None
+    tables: Optional[List[ResultTable]] = None
+    figures: Optional[List[Figure]] = None
+    passed: Optional[bool] = None
+    params: Optional[Dict[str, Any]] = None
+    inputs: Optional[Dict[str, Union[List[VMInput], VMInput]]] = None
+    metadata: Optional[Dict[str, Any]] = None
+    title: Optional[str] = None
+    _was_description_generated: bool = False
+    _unsafe: bool = False
+
+    @property
+    def test_name(self) -> str:
+        """Get the test name, using custom title if available."""
+        return self.title or test_id_to_name(self.result_id)
+
+    def __repr__(self) -> str:
+        attrs = [
+            attr
+            for attr in [
+                "description",
+                "params",
+                "tables",
+                "figures",
+                "metric",
+                "passed",
+            ]
+            if getattr(self, attr) is not None
+        ]
+
+        return f'TestResult("{self.result_id}", {", ".join(attrs)})'
+
+    def __post_init__(self):
+        if self.ref_id is None:
+            self.ref_id = str(uuid4())
+
+    def _get_flat_inputs(self):
+        # remove duplicates by `input_id`
+        inputs = {}
+
+        for input_or_list in self.inputs.values():
+            if isinstance(input_or_list, list):
+                inputs.update({input.input_id: input for input in input_or_list})
+            else:
+                inputs[input_or_list.input_id] = input_or_list
+
+        return list(inputs.values())
+
+    def add_table(self, table: ResultTable):
+        if self.tables is None:
+            self.tables = []
+
+        self.tables.append(table)
+
+    def add_figure(self, figure: Figure):
+        if self.figures is None:
+            self.figures = []
+
+        if figure.ref_id != self.ref_id:
+            figure.ref_id = self.ref_id
+
+        self.figures.append(figure)
+
+    def to_widget(self):
+        if isinstance(self.description, DescriptionFuture):
+            self.description = self.description.get_description()
+            self._was_description_generated = True
+
+        if self.metric is not None and not self.tables and not self.figures:
+            return HTML(f"<h3>{self.test_name}: <code>{self.metric}</code></h3>")
+
+        template_data = {
+            "test_name": self.test_name,
+            "passed_icon": "" if self.passed is None else "✅" if self.passed else "❌",
+            "description": self.description.replace("h3", "strong"),
+            "params": (
+                json.dumps(self.params, cls=NumpyEncoder, indent=2)
+                if self.params
+                else None
+            ),
+            "show_metric": self.metric is not None,
+            "metric": self.metric,
+            "tables": self.tables,
+            "figures": self.figures,
+        }
+        rendered = get_result_template().render(**template_data)
+
+        widgets = [HTML(rendered)]
+
+        if self.tables:
+            widgets.extend(tables_to_widgets(self.tables))
+        if self.figures:
+            widgets.extend(figures_to_widgets(self.figures))
+
+        return VBox(widgets)
+
+    def _validate_section_id_for_block(
+        self, section_id: str, position: Union[int, None] = None
+    ):
+        """Validate the section_id exits on the template before logging"""
+        api_client.reload()
+        found = False
+        client_config = api_client.client_config
+
+        for section in client_config.documentation_template["sections"]:
+            if section["id"] == section_id:
+                found = True
+                break
+
+        if not found:
+            raise ValueError(
+                f"Section with id {section_id} not found in the model's document"
+            )
+
+        # Check if the block already exists in the section
+        block_definition = {
+            "content_id": self.result_id,
+            "content_type": "test",
+        }
+        blocks = section.get("contents", [])
+        for block in blocks:
+            if (
+                block["content_id"] == block_definition["content_id"]
+                and block["content_type"] == block_definition["content_type"]
+            ):
+                logger.info(
+                    f"Test driven block with content_id {block_definition['content_id']} already exists in the document's section"
+                )
+                return
+
+        # Validate that the position is within the bounds of the section
+        if position is not None:
+            num_blocks = len(blocks)
+            if position < 0 or position > num_blocks:
+                raise ValueError(
+                    f"Invalid position {position}. Must be between 0 and {num_blocks}"
+                )
+
+    def serialize(self):
+        """Serialize the result for the API"""
+        return {
+            "test_name": self.result_id,
+            "title": self.title,
+            "ref_id": self.ref_id,
+            "params": self.params,
+            "inputs": [_input.input_id for _input in self._get_flat_inputs()],
+            "passed": self.passed,
+            "summary": [table.serialize() for table in (self.tables or [])],
+            "metadata": self.metadata,
+        }
+
+    async def log_async(
+        self, section_id: str = None, position: int = None, unsafe: bool = False
+    ):
+        tasks = []  # collect tasks to run in parallel (async)
+
+        if self.metric is not None:
+            # metrics are logged as separate entities
+            tasks.append(
+                api_client.alog_metric(
+                    key=self.result_id,
+                    value=self.metric,
+                    inputs=[input.input_id for input in self._get_flat_inputs()],
+                    params=self.params,
+                )
+            )
+
+        if self.tables or self.figures:
+            tasks.append(
+                api_client.alog_test_result(
+                    result=self.serialize(),
+                    section_id=section_id,
+                    position=position,
+                )
+            )
+
+            tasks.extend(
+                [api_client.alog_figure(figure) for figure in (self.figures or [])]
+            )
+
+            if self.description:
+                if isinstance(self.description, DescriptionFuture):
+                    self.description = self.description.get_description()
+                    self._was_description_generated = True
+
+                revision_name = (
+                    AI_REVISION_NAME
+                    if self._was_description_generated
+                    else DEFAULT_REVISION_NAME
+                )
+
+                tasks.append(
+                    update_metadata(
+                        content_id=f"test_description:{self.result_id}::{revision_name}",
+                        text=self.description,
+                    )
+                )
+
+        return await asyncio.gather(*tasks)
+
+    def log(self, section_id: str = None, position: int = None, unsafe: bool = False):
+        """Log the result to ValidMind
+
+        Args:
+            section_id (str): The section ID within the model document to insert the
+                test result
+            position (int): The position (index) within the section to insert the test
+                result
+            unsafe (bool): If True, log the result even if it contains sensitive data
+                i.e. raw data from input datasets
+        """
+        if not unsafe:
+            for table in self.tables or []:
+                check_for_sensitive_data(table.data, self._get_flat_inputs())
+
+        if section_id:
+            self._validate_section_id_for_block(section_id, position)
+
+        run_async(self.log_async, section_id=section_id, position=position)