validmind 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

validmind/vm_models/figure.py

@@ -21,6 +21,18 @@ from ..errors import InvalidFigureForObjectError, UnsupportedFigureError
 from ..utils import get_full_typename
 
 
+def is_matplotlib_figure(figure) -> bool:
+    return isinstance(figure, matplotlib.figure.Figure)
+
+
+def is_plotly_figure(figure) -> bool:
+    return isinstance(figure, (go.Figure, go.FigureWidget))
+
+
+def is_png_image(figure) -> bool:
+    return isinstance(figure, bytes)
+
+
 @dataclass
 class Figure:
     """
@@ -52,22 +64,10 @@ class Figure:
         if (
             not client_config.running_on_colab
             and self.figure
-            and self.is_plotly_figure()
+            and is_plotly_figure(self.figure)
         ):
             self.figure = go.FigureWidget(self.figure)
 
-    def is_matplotlib_figure(self) -> bool:
-        """
-        Returns True if the figure is a matplotlib figure
-        """
-        return isinstance(self.figure, matplotlib.figure.Figure)
-
-    def is_plotly_figure(self) -> bool:
-        """
-        Returns True if the figure is a plotly figure
-        """
-        return isinstance(self.figure, (go.Figure, go.FigureWidget))
-
     def _get_for_object_type(self):
         """
         Returns the type of the object this figure is for
@@ -91,7 +91,7 @@ class Figure:
         we would render images as-is, but Plotly FigureWidgets don't work well
         on Google Colab when they are combined with ipywidgets.
         """
-        if self.is_matplotlib_figure():
+        if is_matplotlib_figure(self.figure):
             tmpfile = BytesIO()
             self.figure.savefig(tmpfile, format="png")
             encoded = base64.b64encode(tmpfile.getvalue()).decode("utf-8")
@@ -101,7 +101,7 @@ class Figure:
                 """
             )
 
-        elif self.is_plotly_figure():
+        elif is_plotly_figure(self.figure):
             # FigureWidget can be displayed as-is but not on Google Colab. In this case
             # we just return the image representation of the figure.
             if client_config.running_on_colab:
@@ -114,6 +114,15 @@ class Figure:
                 )
             else:
                 return self.figure
+
+        elif is_png_image(self.figure):
+            encoded = base64.b64encode(self.figure).decode("utf-8")
+            return widgets.HTML(
+                value=f"""
+                <img style="width:100%; height: auto;" src="data:image/png;base64,{encoded}"/>
+                """
+            )
+
         else:
             raise UnsupportedFigureError(
                 f"Figure type {type(self.figure)} not supported for plotting"
@@ -129,15 +138,38 @@ class Figure:
             "metadata": json.dumps(self.metadata, allow_nan=False),
         }
 
+    def _get_b64_url(self):
+        """
+        Returns a base64 encoded URL for the figure
+        """
+        if is_matplotlib_figure(self.figure):
+            buffer = BytesIO()
+            self.figure.savefig(buffer, format="png")
+            buffer.seek(0)
+
+            b64_data = base64.b64encode(buffer.read()).decode("utf-8")
+
+            return f"data:image/png;base64,{b64_data}"
+
+        elif is_plotly_figure(self.figure):
+            bytes = self.figure.to_image(format="png")
+            b64_data = base64.b64encode(bytes).decode("utf-8")
+
+            return f"data:image/png;base64,{b64_data}"
+
+        raise UnsupportedFigureError(
+            f"Unrecognized figure type: {get_full_typename(self.figure)}"
+        )
+
     def serialize_files(self):
         """Creates a `requests`-compatible files object to be sent to the API"""
-        if self.is_matplotlib_figure():
+        if is_matplotlib_figure(self.figure):
             buffer = BytesIO()
             self.figure.savefig(buffer, bbox_inches="tight")
             buffer.seek(0)
             return {"image": (f"{self.key}.png", buffer, "image/png")}
 
-        elif self.is_plotly_figure():
+        elif is_plotly_figure(self.figure):
             # When using plotly, we need to use we will produce two files:
             # - a JSON file that will be used to display the figure in the UI
             # - a PNG file that will be used to display the figure in documents
@@ -154,6 +186,9 @@ class Figure:
                 ),
             }
 
+        elif is_png_image(self.figure):
+            return {"image": (f"{self.key}.png", self.figure, "image/png")}
+
         raise UnsupportedFigureError(
             f"Unrecognized figure type: {get_full_typename(self.figure)}"
         )

validmind/vm_models/test/metric.py

@@ -6,14 +6,15 @@
 Class for storing ValidMind metric objects and associated
 data for display and reporting purposes
 """
+import os
 from abc import abstractmethod
 from dataclasses import dataclass
 from typing import ClassVar, List, Optional, Union
 
 import pandas as pd
 
+from ...ai import generate_description
 from ...errors import MissingCacheResultsArgumentsError
-from ...utils import clean_docstring
 from ..figure import Figure
 from .metric_result import MetricResult
 from .result_wrapper import MetricResultWrapper
@@ -74,41 +75,42 @@ class Metric(Test):
                 "Metric must provide a metric value or figures to cache_results"
             )
 
-        # At a minimum, send the metric description
-        result_metadata = [
-            {
-                "content_id": f"metric_description:{self.test_id}",
-                "text": clean_docstring(self.description()),
-            }
-        ]
-
-        result_summary = self.summary(metric_value)
-
-        result_wrapper = MetricResultWrapper(
-            result_id=self.test_id,
-            result_metadata=result_metadata,
-            inputs=self.get_accessed_inputs(),
-            output_template=self.output_template,
-        )
-
-        # We can send an empty result to push an empty metric with a summary and plots
-        metric_result_value = metric_value if metric_value is not None else {}
-
-        result_wrapper.metric = MetricResult(
-            # key=self.key,
-            # Now using the fully qualified test ID as `key`.
-            # Ideally the backend is updated to use `test_id` instead of `key`.
+        metric = MetricResult(
             key=self.test_id,
             ref_id=self._ref_id,
-            value=metric_result_value,
+            value=metric_value if metric_value is not None else {},
             value_formatter=self.value_formatter,
-            summary=result_summary,
+            summary=self.summary(metric_value),
         )
 
-        # Allow metrics to attach figures to the test suite result
-        if figures:
-            result_wrapper.figures = figures
+        if (
+            os.environ.get("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "false").lower()
+            == "true"
+        ):
+            revision_name = "Generated by ValidMind AI"
+            description = generate_description(
+                test_name=self.test_id,
+                test_description=self.description().splitlines()[0],
+                test_results=metric.serialize()["value"],
+                test_summary=metric.serialize()["summary"],
+                figures=figures,
+            )
+        else:
+            revision_name = "Default Description"
+            description = self.description()
+
+        description_metadata = {
+            "content_id": f"metric_description:{self.test_id}::{revision_name}",
+            "text": description,
+        }
 
-        self.result = result_wrapper
+        self.result = MetricResultWrapper(
+            result_id=self.test_id,
+            result_metadata=[description_metadata],
+            metric=metric,
+            figures=figures,
+            inputs=self.get_accessed_inputs(),
+            output_template=self.output_template,
+        )
 
         return self.result

validmind/vm_models/test/output_template.py

@@ -4,7 +4,6 @@
 
 from datetime import datetime
 
-from bs4 import BeautifulSoup
 from dateutil import parser
 from jinja2 import Environment
 
@@ -54,29 +53,3 @@ class OutputTemplate:
             value=value,
             metric_history=values_history,
         )
-
-    def parse_summary_from_html(rendered_template_html):
-        soup = BeautifulSoup(rendered_template_html, "html.parser")
-
-        # find all `<table>` elements
-        tables = soup.find_all("table")
-        tables_data = []
-
-        for table in tables:
-            headers = [cell.text for cell in table.find_all("th")]
-
-            tables_data.append(
-                {
-                    "type": "table",
-                    "data": [
-                        {
-                            headers[i]: cell.text
-                            for i, cell in enumerate(row.find_all("td"))
-                        }
-                        for row in table.find("tbody").find_all("tr")
-                    ],
-                    "metadata": {"title": ""},  # TODO: add title
-                }
-            )
-
-        return tables_data

validmind/vm_models/test/result_wrapper.py

@@ -10,14 +10,15 @@ import json
 import os
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import List, Optional
+from typing import Dict, List, Optional, Union
 
 import ipywidgets as widgets
-import markdown
+import mistune
 import pandas as pd
 from IPython.display import display
 
 from ... import api_client
+from ...ai import DescriptionFuture
 from ...utils import NumpyEncoder, run_async, test_id_to_name
 from ..figure import Figure
 from .metric_result import MetricResult
@@ -26,24 +27,33 @@ from .result_summary import ResultSummary
 from .threshold_test_result import ThresholdTestResults
 
 
-async def update_metadata(content_id: str, text: str) -> None:
+async def update_metadata(content_id: str, text: str, _json: Union[Dict, List] = None):
     """
     Update the metadata of a content item. By default we don't
     override the existing metadata, but we can override it by
     setting the VM_OVERRIDE_METADATA environment variable to True
     """
-    VM_OVERRIDE_METADATA = os.environ.get("VM_OVERRIDE_METADATA", False)
-    try:
-        existing_metadata = await api_client.get_metadata(content_id)
-    except Exception:
-        existing_metadata = None  # TODO: handle this better
+    should_update = False
 
-    if (
-        existing_metadata is None
-        or VM_OVERRIDE_METADATA == "True"
-        or VM_OVERRIDE_METADATA is True
-    ):
-        await api_client.log_metadata(content_id, text)
+    # check if the env variable is set to force overwriting metadata
+    if os.environ.get("VM_OVERRIDE_METADATA", "false").lower() == "true":
+        should_update = True
+
+    # if not set, check if the content_id is a composite metric def
+    if not should_update and content_id.startswith("composite_metric_def:"):
+        # we always want composite metric definitions to be updated
+        should_update = True
+
+    # if not set, lets check if the metadata already exists
+    if not should_update:
+        try:
+            await api_client.get_metadata(content_id)
+        except Exception:  # TODO: this shouldn't be a catch-all
+            # if the metadata doesn't exist, we should create (update) it
+            should_update = True
+
+    if should_update:
+        await api_client.log_metadata(content_id, text, _json)
 
 
 def plot_figures(figures: List[Figure]) -> None:
@@ -93,8 +103,7 @@ class ResultWrapper(ABC):
         """
         Convert a markdown string to html
         """
-
-        return markdown.markdown(description, extensions=["markdown.extensions.tables"])
+        return mistune.html(description)
 
     def _summary_tables_to_widget(self, summary: ResultSummary):
         """
@@ -111,21 +120,19 @@ class ResultWrapper(ABC):
                     [
                         {
                             "selector": "",
-                            "props": [
-                                ("width", "100%"),
-                            ],
+                            "props": [("width", "100%")],
+                        },
+                        {
+                            "selector": "th",
+                            "props": [("text-align", "left")],
                         },
                         {
                             "selector": "tbody tr:nth-child(even)",
-                            "props": [
-                                ("background-color", "#FFFFFF"),
-                            ],
+                            "props": [("background-color", "#FFFFFF")],
                         },
                         {
                             "selector": "tbody tr:nth-child(odd)",
-                            "props": [
-                                ("background-color", "#F5F5F5"),
-                            ],
+                            "props": [("background-color", "#F5F5F5")],
                         },
                         {
                             "selector": "td, th",
@@ -135,7 +142,8 @@ class ResultWrapper(ABC):
                             ],
                         },
                     ]
-                )  # add borders
+                )
+                .set_properties(**{"text-align": "left"})
                 .to_html(escape=False)
             )  # table.data is an orient=records dump
 
@@ -155,7 +163,7 @@ class ResultWrapper(ABC):
 
     def log(self):
         """Log the result... May be overridden by subclasses"""
-        return run_async(self.log_async)
+        run_async(self.log_async)
 
 
 @dataclass
@@ -207,15 +215,19 @@ class MetricResultWrapper(ResultWrapper):
         if self.metric and self.metric.key == "dataset_description":
             return ""
 
-        vbox_children = []
+        vbox_children = [
+            widgets.HTML(value=f"<h1>{test_id_to_name(self.result_id)}</h1>"),
+        ]
 
         if self.result_metadata:
-            metric_description = self.result_metadata[0]
+            metric_description = self.result_metadata[0].get("text", "")
+            if isinstance(metric_description, DescriptionFuture):
+                metric_description = metric_description.get_description()
+                self.result_metadata[0]["text"] = metric_description
+
             vbox_children.append(
                 widgets.HTML(
-                    value=self._markdown_description_to_html(
-                        metric_description.get("text", "")
-                    )
+                    value=self._markdown_description_to_html(metric_description)
                 )
             )
 
@@ -297,8 +309,19 @@ class MetricResultWrapper(ResultWrapper):
         if self.figures:
             tasks.append(api_client.log_figures(self.figures))
         if hasattr(self, "result_metadata") and self.result_metadata:
+            description = self.result_metadata[0].get("text", "")
+            if isinstance(description, DescriptionFuture):
+                description = description.get_description()
+                self.result_metadata[0]["text"] = description
+
             for metadata in self.result_metadata:
-                tasks.append(update_metadata(metadata["content_id"], metadata["text"]))
+                tasks.append(
+                    update_metadata(
+                        content_id=metadata["content_id"],
+                        text=metadata.get("text", ""),
+                        _json=metadata.get("json"),
+                    )
+                )
 
         await asyncio.gather(*tasks)
 
@@ -339,14 +362,18 @@ class ThresholdTestResultWrapper(ResultWrapper):
         test_title = test_id_to_name(self.test_results.test_name)
         description_html.append(
             f"""
-            <h2>{test_title} {"✅" if self.test_results.passed else "❌"}</h2>
+            <h1>{test_title} {"✅" if self.test_results.passed else "❌"}</h1>
             """
         )
 
         if self.result_metadata:
-            metric_description = self.result_metadata[0]
+            metric_description = self.result_metadata[0].get("text", "")
+            if isinstance(metric_description, DescriptionFuture):
+                metric_description = metric_description.get_description()
+                self.result_metadata[0]["text"] = metric_description
+
             description_html.append(
-                self._markdown_description_to_html(metric_description.get("text", ""))
+                self._markdown_description_to_html(metric_description)
             )
 
         description_html.append(
@@ -375,6 +402,11 @@ class ThresholdTestResultWrapper(ResultWrapper):
         if self.figures:
             tasks.append(api_client.log_figures(self.figures))
         if hasattr(self, "result_metadata") and self.result_metadata:
+            description = self.result_metadata[0].get("text", "")
+            if isinstance(description, DescriptionFuture):
+                description = description.get_description()
+                self.result_metadata[0]["text"] = description
+
             for metadata in self.result_metadata:
                 tasks.append(update_metadata(metadata["content_id"], metadata["text"]))
 

validmind/vm_models/test/test.py

@@ -6,10 +6,12 @@
 
 from abc import abstractmethod
 from dataclasses import dataclass
+from inspect import getdoc
 from typing import ClassVar, List, TypedDict
 from uuid import uuid4
 
 from ..test_context import TestUtils
+from .result_wrapper import ResultWrapper
 
 
 class TestMetadata(TypedDict):
@@ -35,7 +37,7 @@ class Test(TestUtils):
     _ref_id: str = None  # unique identifier (populated at init)
     _section_id: str = None  # which section of template this test belongs to
     test_id: str = None  # populated when loading tests from suites
-    result: object = None  # type should be overridden by parent classes
+    result: ResultWrapper = None  # type should be overridden by parent classes
 
     params: dict = None  # populated by test suite from user-passed config
 
@@ -65,7 +67,7 @@ class Test(TestUtils):
         Return the test description. May be overridden by subclasses. Defaults
         to returning the class' docstring
         """
-        return self.__doc__.strip()
+        return getdoc(self).strip()
 
     @abstractmethod
     def summary(self, *args, **kwargs):

validmind/vm_models/test/threshold_test.py

@@ -8,10 +8,11 @@ Test (as test_results) but we'll refer to it as a ThresholdTest to
 avoid confusion with the "tests" in the general data science/modeling sense.
 """
 
+import os
 from dataclasses import dataclass
 from typing import ClassVar, List, Optional
 
-from ...utils import clean_docstring
+from ...ai import generate_description
 from ..figure import Figure
 from .result_summary import ResultSummary, ResultTable
 from .result_wrapper import ThresholdTestResultWrapper
@@ -76,25 +77,34 @@ class ThresholdTest(Test):
         Returns:
             TestSuiteResult: The test suite result object
         """
-        # Rename to self.result
-        # At a minimum, send the test description
-        result_metadata = [
-            {
-                "content_id": f"test_description:{self.test_id}",
-                "text": clean_docstring(self.description()),
-            }
-        ]
-
         result_summary = self.summary(test_results_list, passed)
 
+        if (
+            os.environ.get("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "false").lower()
+            == "true"
+        ):
+            revision_name = "Generated by ValidMind AI"
+            description = generate_description(
+                test_name=self.test_id,
+                test_description=self.description().splitlines()[0],
+                test_results=[result.serialize() for result in test_results_list],
+                test_summary=result_summary.serialize(),
+                figures=figures,
+            )
+        else:
+            revision_name = "Default Description"
+            description = self.description()
+
+        description_metadata = {
+            "content_id": f"test_description:{self.test_id}::{revision_name}",
+            "text": description,
+        }
+
         self.result = ThresholdTestResultWrapper(
             result_id=self.test_id,
-            result_metadata=result_metadata,
+            result_metadata=[description_metadata],
             inputs=self.get_accessed_inputs(),
             test_results=ThresholdTestResults(
-                # test_name=self.name,
-                # Now using the fully qualified test ID as `test_name`.
-                # Ideally the backend is updated to use `test_id` instead of `test_name`.
                 test_name=self.test_id,
                 ref_id=self._ref_id,
                 params=self.params,

validmind/vm_models/test_context.py

@@ -165,6 +165,13 @@ class TestUtils:
 
         return []
 
+    def _get_input_dict(self):
+        """Return a dictionary of all inputs"""
+        if isinstance(self.inputs, InputAccessTrackerProxy):
+            return self.inputs._inputs.__dict__
+
+        return self.inputs.__dict__
+
     def _get_legacy_input(self, key):
         """Retrieve an input from the Test Input or, for backwards compatibility,
         the Test Context

validmind/vm_models/test_suite/runner.py

@@ -126,7 +126,7 @@ class TestSuiteRunner:
             )
 
             try:
-                await test.log()
+                await test.log_async()
             except Exception as e:
                 self.pbar_description.value = "Failed to send result to ValidMind"
                 logger.error(f"Failed to log result: {test.result}")

validmind/vm_models/test_suite/summary.py

@@ -6,10 +6,10 @@ from dataclasses import dataclass
 from typing import List, Optional
 
 import ipywidgets as widgets
+import mistune
 from IPython.display import display
 
 from ...logging import get_logger
-from ...utils import clean_docstring
 from ..test.result_wrapper import FailedResultWrapper
 from .test_suite import TestSuiteSection, TestSuiteTest
 
@@ -36,7 +36,7 @@ class TestSuiteSectionSummary:
         self._build_summary()
 
     def _add_description(self):
-        description = f'<div class="result">{clean_docstring(self.description)}</div>'
+        description = f'<div class="result">{mistune.html(self.description)}</div>'
         self._widgets.append(widgets.HTML(value=description))
 
     def _add_tests_summary(self):
@@ -101,7 +101,7 @@ class TestSuiteSummary:
         self._widgets.append(widgets.HTML(value=results_link))
 
     def _add_description(self):
-        description = f'<div class="result">{clean_docstring(self.description)}</div>'
+        description = f'<div class="result">{mistune.html(self.description)}</div>'
         self._widgets.append(widgets.HTML(value=description))
 
     def _add_sections_summary(self):

validmind/vm_models/test_suite/test.py

@@ -151,7 +151,7 @@ class TestSuiteTest:
 
         self.result = self._test_instance.result
 
-    async def log(self):
+    async def log_async(self):
         """Log the result for this test to ValidMind"""
         if not self.result:
             raise ValueError("Cannot log test result before running the test")

validmind/vm_models/test_suite/test_suite.py

@@ -7,6 +7,7 @@ Base class for test suites and test suite sections
 """
 
 from dataclasses import dataclass
+from inspect import getdoc
 from typing import ClassVar, List, Optional, Union
 
 from ...logging import get_logger
@@ -154,7 +155,7 @@ class TestSuite:
 
     @property
     def description(self):
-        return self.__doc__
+        return getdoc(self).strip()
 
     @property
     def title(self):