validmind 2.6.10__tar.gz → 2.7.2__tar.gz

{validmind-2.6.10 → validmind-2.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.6.10
+Version: 2.7.2
 Summary: ValidMind Library
 License: Commercial License
 Author: Andres Rodriguez

{validmind-2.6.10 → validmind-2.7.2}/pyproject.toml

@@ -10,7 +10,7 @@ description = "ValidMind Library"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.6.10"
+version = "2.7.2"
 
 [tool.poetry.dependencies]
 aiohttp = {extras = ["speedups"], version = "*"}

{validmind-2.6.10 → validmind-2.7.2}/validmind/__init__.py

@@ -50,6 +50,7 @@ from .client import (  # noqa: E402
     run_test_suite,
 )
 from .tests.decorator import tags, tasks, test
+from .vm_models.result import RawData
 
 __all__ = [  # noqa
     "__version__",
@@ -62,6 +63,7 @@ __all__ = [  # noqa
     "init_model",
     "init_r_model",
     "preview_template",
+    "RawData",
     "reload",
     "run_documentation_tests",
     "run_test_suite",

validmind-2.7.2/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.7.2"

{validmind-2.6.10 → validmind-2.7.2}/validmind/tests/model_validation/sklearn/ROCCurve.py

@@ -6,7 +6,7 @@ import numpy as np
 import plotly.graph_objects as go
 from sklearn.metrics import roc_auc_score, roc_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -77,28 +77,31 @@ def ROCCurve(model: VMModel, dataset: VMDataset):
     fpr, tpr, _ = roc_curve(y_true, y_prob, drop_intermediate=False)
     auc = roc_auc_score(y_true, y_prob)
 
-    return go.Figure(
-        data=[
-            go.Scatter(
-                x=fpr,
-                y=tpr,
-                mode="lines",
-                name=f"ROC curve (AUC = {auc:.2f})",
-                line=dict(color="#DE257E"),
+    return (
+        RawData(fpr=fpr, tpr=tpr, auc=auc),
+        go.Figure(
+            data=[
+                go.Scatter(
+                    x=fpr,
+                    y=tpr,
+                    mode="lines",
+                    name=f"ROC curve (AUC = {auc:.2f})",
+                    line=dict(color="#DE257E"),
+                ),
+                go.Scatter(
+                    x=[0, 1],
+                    y=[0, 1],
+                    mode="lines",
+                    name="Random (AUC = 0.5)",
+                    line=dict(color="grey", dash="dash"),
+                ),
+            ],
+            layout=go.Layout(
+                title=f"ROC Curve for {model.input_id} on {dataset.input_id}",
+                xaxis=dict(title="False Positive Rate"),
+                yaxis=dict(title="True Positive Rate"),
+                width=700,
+                height=500,
             ),
-            go.Scatter(
-                x=[0, 1],
-                y=[0, 1],
-                mode="lines",
-                name="Random (AUC = 0.5)",
-                line=dict(color="grey", dash="dash"),
-            ),
-        ],
-        layout=go.Layout(
-            title=f"ROC Curve for {model.input_id} on {dataset.input_id}",
-            xaxis=dict(title="False Positive Rate"),
-            yaxis=dict(title="True Positive Rate"),
-            width=700,
-            height=500,
         ),
     )

{validmind-2.6.10 → validmind-2.7.2}/validmind/tests/output.py

@@ -15,7 +15,7 @@ from validmind.vm_models.figure import (
     is_plotly_figure,
     is_png_image,
 )
-from validmind.vm_models.result import ResultTable, TestResult
+from validmind.vm_models.result import RawData, ResultTable, TestResult
 
 
 class OutputHandler(ABC):
@@ -103,6 +103,14 @@ class TableOutputHandler(OutputHandler):
             result.add_table(ResultTable(data=table_data, title=table_name or None))
 
 
+class RawDataOutputHandler(OutputHandler):
+    def can_handle(self, item: Any) -> bool:
+        return isinstance(item, RawData)
+
+    def process(self, item: Any, result: TestResult) -> None:
+        result.raw_data = item
+
+
 def process_output(item: Any, result: TestResult) -> None:
     """Process a single test output item and update the TestResult."""
     handlers = [
@@ -110,6 +118,7 @@ def process_output(item: Any, result: TestResult) -> None:
         MetricOutputHandler(),
         FigureOutputHandler(),
         TableOutputHandler(),
+        RawDataOutputHandler(),
     ]
 
     for handler in handlers:

{validmind-2.6.10 → validmind-2.7.2}/validmind/tests/run.py

@@ -7,7 +7,7 @@ import subprocess
 import time
 from datetime import datetime
 from inspect import getdoc
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 from uuid import uuid4
 
 from validmind import __version__
@@ -136,6 +136,7 @@ def build_test_result(
     test_id: str,
     inputs: Dict[str, Union[VMInput, List[VMInput]]],
     params: Union[Dict[str, Any], None],
+    doc: str,
     description: str,
     generate_description: bool = True,
     title: Optional[str] = None,
@@ -149,6 +150,7 @@ def build_test_result(
         ref_id=ref_id,
         inputs=inputs,
         params=params if params else None,  # None if empty dict or None
+        doc=doc,
     )
 
     if not isinstance(outputs, tuple):
@@ -199,6 +201,11 @@ def _run_composite_test(
     if not all(result.metric is not None for result in results):
         raise ValueError("All tests must return a metric when used as a composite test")
 
+    # Create composite doc from all test results
+    composite_doc = "\n\n".join(
+        [f"{test_id_to_name(result.result_id)}:\n{result.doc}" for result in results]
+    )
+
     return build_test_result(
         outputs=[
             {
@@ -210,6 +217,7 @@ def _run_composite_test(
         test_id=test_id,
         inputs=results[0].inputs,
         params=results[0].params,
+        doc=composite_doc,
         description="\n\n".join(
             [_test_description(result.description, num_lines=1) for result in results]
         ),  # join truncated (first line only) test descriptions
@@ -261,11 +269,22 @@ def _run_comparison_test(
 
     combined_outputs, combined_inputs, combined_params = combine_results(results)
 
+    if unit_metrics:
+        doc = "\n\n".join(
+            [
+                f"{test_id_to_name(unit_metric)}:\n{getdoc(load_test(unit_metric))}"
+                for unit_metric in unit_metrics
+            ]
+        )
+    else:
+        doc = getdoc(load_test(test_id))
+
     return build_test_result(
         outputs=tuple(combined_outputs),
         test_id=test_id,
         inputs=combined_inputs,
         params=combined_params,
+        doc=doc,
         description=description,
         generate_description=generate_description,
         title=title,
@@ -283,6 +302,7 @@ def run_test(
     show: bool = True,
     generate_description: bool = True,
     title: Optional[str] = None,
+    post_process_fn: Union[Callable[[TestResult], None], None] = None,
     **kwargs,
 ) -> TestResult:
     """Run a ValidMind or custom test
@@ -306,6 +326,7 @@ def run_test(
         show (bool, optional): Whether to display results. Defaults to True.
         generate_description (bool, optional): Whether to generate a description. Defaults to True.
         title (str, optional): Custom title for the test result
+        post_process_fn (Callable[[TestResult], None], optional): Function to post-process the test result
 
     Returns:
         TestResult: A TestResult object containing the test results
@@ -381,12 +402,15 @@ def run_test(
 
         raw_result = test_func(**input_kwargs, **param_kwargs)
 
+        doc = getdoc(test_func)
+
         result = build_test_result(
             outputs=raw_result,
             test_id=test_id,
             inputs=input_kwargs,
             params=param_kwargs,
-            description=getdoc(test_func),
+            doc=doc,
+            description=doc,
             generate_description=generate_description,
             title=title,
         )
@@ -394,6 +418,9 @@ def run_test(
     end_time = time.perf_counter()
     result.metadata = _get_run_metadata(duration_seconds=end_time - start_time)
 
+    if post_process_fn:
+        result = post_process_fn(result)
+
     if show:
         result.show()
 

{validmind-2.6.10 → validmind-2.7.2}/validmind/utils.py

@@ -168,6 +168,17 @@ class NumpyEncoder(json.JSONEncoder):
         return super().iterencode(obj, _one_shot)
 
 
+class HumanReadableEncoder(NumpyEncoder):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # truncate ndarrays to 10 items
+        self.type_handlers[self.is_numpy_ndarray] = lambda obj: (
+            obj.tolist()[:5] + ["..."] + obj.tolist()[-5:]
+            if len(obj) > 10
+            else obj.tolist()
+        )
+
+
 def get_full_typename(o: Any) -> Any:
     """We determine types based on type names so we don't have to import
     (and therefore depend on) PyTorch, TensorFlow, etc.

{validmind-2.6.10 → validmind-2.7.2}/validmind/vm_models/figure.py

@@ -33,6 +33,18 @@ def is_png_image(figure) -> bool:
     return isinstance(figure, bytes)
 
 
+def create_figure(
+    figure: Union[matplotlib.figure.Figure, go.Figure, go.FigureWidget, bytes],
+    key: str,
+    ref_id: str,
+) -> "Figure":
+    """Create a VM Figure object from a raw figure object"""
+    if is_matplotlib_figure(figure) or is_plotly_figure(figure) or is_png_image(figure):
+        return Figure(key=key, figure=figure, ref_id=ref_id)
+
+    raise ValueError(f"Unsupported figure type: {type(figure)}")
+
+
 @dataclass
 class Figure:
     """
@@ -55,6 +67,9 @@ class Figure:
         ):
             self.figure = go.FigureWidget(self.figure)
 
+    def __repr__(self):
+        return f"Figure(key={self.key}, ref_id={self.ref_id})"
+
     def to_widget(self):
         """
         Returns the ipywidget compatible representation of the figure. Ideally

{validmind-2.6.10 → validmind-2.7.2}/validmind/vm_models/result/__init__.py

@@ -2,6 +2,6 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from .result import ErrorResult, Result, ResultTable, TestResult
+from .result import ErrorResult, RawData, Result, ResultTable, TestResult
 
-__all__ = ["ErrorResult", "Result", "ResultTable", "TestResult"]
+__all__ = ["ErrorResult", "RawData", "Result", "ResultTable", "TestResult"]

{validmind-2.6.10 → validmind-2.7.2}/validmind/vm_models/result/result.py

@@ -12,14 +12,22 @@ from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Union
 from uuid import uuid4
 
+import matplotlib
 import pandas as pd
+import plotly.graph_objs as go
 from ipywidgets import HTML, VBox
 
 from ... import api_client
 from ...ai.utils import DescriptionFuture
 from ...logging import get_logger
-from ...utils import NumpyEncoder, display, run_async, test_id_to_name
-from ..figure import Figure
+from ...utils import (
+    HumanReadableEncoder,
+    NumpyEncoder,
+    display,
+    run_async,
+    test_id_to_name,
+)
+from ..figure import Figure, create_figure
 from ..input import VMInput
 from .utils import (
     AI_REVISION_NAME,
@@ -34,6 +42,42 @@ from .utils import (
 logger = get_logger(__name__)
 
 
+class RawData:
+    """Holds raw data for a test result"""
+
+    def __init__(self, log: bool = False, **kwargs):
+        """Create a new RawData object
+
+        Args:
+            log (bool): If True, log the raw data to ValidMind
+            **kwargs: Keyword arguments to set as attributes e.g.
+                `RawData(log=True, dataset_duplicates=df_duplicates)`
+        """
+        self.log = log
+
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+    def __repr__(self) -> str:
+        return f"RawData({', '.join(self.__dict__.keys())})"
+
+    def inspect(self, show: bool = True):
+        """Inspect the raw data"""
+        raw_data = {
+            key: getattr(self, key)
+            for key in self.__dict__
+            if not key.startswith("_") and key != "log"
+        }
+
+        if not show:
+            return raw_data
+
+        print(json.dumps(raw_data, indent=2, cls=HumanReadableEncoder))
+
+    def serialize(self):
+        return {key: getattr(self, key) for key in self.__dict__}
+
+
 @dataclass
 class ResultTable:
     """
@@ -41,7 +85,7 @@ class ResultTable:
     """
 
     data: Union[List[Any], pd.DataFrame]
-    title: str
+    title: Optional[str] = None
 
     def __repr__(self) -> str:
         return f'ResultTable(title="{self.title}")' if self.title else "ResultTable"
@@ -115,15 +159,16 @@ class TestResult(Result):
     name: str = "Test Result"
     ref_id: str = None
     title: Optional[str] = None
+    doc: Optional[str] = None
     description: Optional[Union[str, DescriptionFuture]] = None
     metric: Optional[Union[int, float]] = None
     tables: Optional[List[ResultTable]] = None
+    raw_data: Optional[RawData] = None
     figures: Optional[List[Figure]] = None
     passed: Optional[bool] = None
     params: Optional[Dict[str, Any]] = None
     inputs: Optional[Dict[str, Union[List[VMInput], VMInput]]] = None
     metadata: Optional[Dict[str, Any]] = None
-    title: Optional[str] = None
     _was_description_generated: bool = False
     _unsafe: bool = False
 
@@ -136,6 +181,7 @@ class TestResult(Result):
         attrs = [
             attr
             for attr in [
+                "doc",
                 "description",
                 "params",
                 "tables",
@@ -144,6 +190,11 @@ class TestResult(Result):
                 "passed",
             ]
             if getattr(self, attr) is not None
+            and (
+                len(getattr(self, attr)) > 0
+                if isinstance(getattr(self, attr), list)
+                else True
+            )
         ]
 
         return f'TestResult("{self.result_id}", {", ".join(attrs)})'
@@ -164,21 +215,82 @@ class TestResult(Result):
 
         return list(inputs.values())
 
-    def add_table(self, table: ResultTable):
+    def add_table(
+        self,
+        table: Union[ResultTable, pd.DataFrame, List[Dict[str, Any]]],
+        title: Optional[str] = None,
+    ):
+        """Add a new table to the result
+
+        Args:
+            table (Union[ResultTable, pd.DataFrame, List[Dict[str, Any]]]): The table to add
+            title (Optional[str]): The title of the table (can optionally be provided for
+                pd.DataFrame and List[Dict[str, Any]] tables)
+        """
         if self.tables is None:
             self.tables = []
 
+        if isinstance(table, (pd.DataFrame, list)):
+            table = ResultTable(data=table, title=title)
+
         self.tables.append(table)
 
-    def add_figure(self, figure: Figure):
+    def remove_table(self, index: int):
+        """Remove a table from the result by index
+
+        Args:
+            index (int): The index of the table to remove (default is 0)
+        """
+        if self.tables is None:
+            return
+
+        self.tables.pop(index)
+
+    def add_figure(
+        self,
+        figure: Union[
+            matplotlib.figure.Figure,
+            go.Figure,
+            go.FigureWidget,
+            bytes,
+            Figure,
+        ],
+    ):
+        """Add a new figure to the result
+
+        Args:
+            figure (Union[matplotlib.figure.Figure, go.Figure, go.FigureWidget,
+                bytes, Figure]): The figure to add (can be either a VM Figure object,
+                a raw figure object from the supported libraries, or a png image as
+                raw bytes)
+        """
         if self.figures is None:
             self.figures = []
 
+        if not isinstance(figure, Figure):
+            random_tag = str(uuid4())[:4]
+            figure = create_figure(
+                figure=figure,
+                ref_id=self.ref_id,
+                key=f"{self.result_id}:{random_tag}",
+            )
+
         if figure.ref_id != self.ref_id:
             figure.ref_id = self.ref_id
 
         self.figures.append(figure)
 
+    def remove_figure(self, index: int = 0):
+        """Remove a figure from the result by index
+
+        Args:
+            index (int): The index of the figure to remove (default is 0)
+        """
+        if self.figures is None:
+            return
+
+        self.figures.pop(index)
+
     def to_widget(self):
         if isinstance(self.description, DescriptionFuture):
             self.description = self.description.get_description()

validmind-2.6.10/validmind/__version__.py

@@ -1 +0,0 @@
-__version__ = "2.6.10"