validmind 2.5.8__py3-none-any.whl → 2.5.18__py3-none-any.whl

validmind/vm_models/test/metric.py

@@ -77,6 +77,7 @@ class Metric(Test):
 
         self.result = MetricResultWrapper(
             result_id=self.test_id,
+            result_description=self.description(),
             result_metadata=[
                 (
                     get_description_metadata(

validmind/vm_models/test/result_wrapper.py

@@ -63,19 +63,63 @@ async def update_metadata(content_id: str, text: str, _json: Union[Dict, List] =
 
 
 def plot_figures(figures: List[Figure]) -> None:
-    """
-    Plot figures to a ipywidgets GridBox
-    """
-
+    """Plot figures to a ipywidgets GridBox"""
     plots = [figure.to_widget() for figure in figures]
-
     num_columns = 2 if len(figures) > 1 else 1
+
     return GridBox(
         plots,
         layout=Layout(grid_template_columns=f"repeat({num_columns}, 1fr)"),
     )
 
 
+def _summary_tables_to_widget(summary: ResultSummary):
+    """Convert summary (list of json tables) into ipywidgets"""
+    widgets = []
+
+    for table in summary.results:
+        if table.metadata and table.metadata.title:
+            widgets.append(HTML(f"<h4>{table.metadata.title}</h4>"))
+
+        df_html = (
+            pd.DataFrame(table.data)
+            .style.format(precision=4)
+            .hide(axis="index")
+            .set_table_styles(
+                [
+                    {
+                        "selector": "",
+                        "props": [("width", "100%")],
+                    },
+                    {
+                        "selector": "th",
+                        "props": [("text-align", "left")],
+                    },
+                    {
+                        "selector": "tbody tr:nth-child(even)",
+                        "props": [("background-color", "#FFFFFF")],
+                    },
+                    {
+                        "selector": "tbody tr:nth-child(odd)",
+                        "props": [("background-color", "#F5F5F5")],
+                    },
+                    {
+                        "selector": "td, th",
+                        "props": [
+                            ("padding-left", "5px"),
+                            ("padding-right", "5px"),
+                        ],
+                    },
+                ]
+            )
+            .set_properties(**{"text-align": "left"})
+            .to_html(escape=False)
+        )
+        widgets.append(HTML(df_html))
+
+    return widgets
+
+
 @dataclass
 class ResultWrapper(ABC):
     """Base Class for test suite results"""
@@ -84,6 +128,8 @@ class ResultWrapper(ABC):
     # id of the result, can be set by the subclass. This helps
     # looking up results later on
     result_id: str = None
+    # Text description from test or metric (docstring usually)
+    result_description: str = None
     # Text metadata about the result, can include description, etc.
     result_metadata: List[dict] = None
     # Output template to use for rendering the result
@@ -105,53 +151,6 @@ class ResultWrapper(ABC):
 
         return self.to_widget()
 
-    def _summary_tables_to_widget(self, summary: ResultSummary):
-        """
-        Create an ipywdiget representation of the summary tables
-        """
-        tables = []
-        for table in summary.results:
-            # Explore advanced styling
-            summary_table = (
-                pd.DataFrame(table.data)
-                .style.format(precision=4)
-                .hide(axis="index")
-                .set_table_styles(
-                    [
-                        {
-                            "selector": "",
-                            "props": [("width", "100%")],
-                        },
-                        {
-                            "selector": "th",
-                            "props": [("text-align", "left")],
-                        },
-                        {
-                            "selector": "tbody tr:nth-child(even)",
-                            "props": [("background-color", "#FFFFFF")],
-                        },
-                        {
-                            "selector": "tbody tr:nth-child(odd)",
-                            "props": [("background-color", "#F5F5F5")],
-                        },
-                        {
-                            "selector": "td, th",
-                            "props": [
-                                ("padding-left", "5px"),
-                                ("padding-right", "5px"),
-                            ],
-                        },
-                    ]
-                )
-                .set_properties(**{"text-align": "left"})
-                .to_html(escape=False)
-            )  # table.data is an orient=records dump
-
-            if table.metadata and table.metadata.title:
-                tables.append(HTML(value=f"<h3>{table.metadata.title}</h3>"))
-            tables.append(HTML(value=summary_table))
-        return tables
-
     def _validate_section_id_for_block(self, section_id: str, position: int = None):
         """
         Validate the section_id exits on the template before logging. We validate
@@ -244,9 +243,11 @@ class MetricResultWrapper(ResultWrapper):
     """
 
     name: str = "Metric"
-    figures: Optional[List[Figure]] = None
+    scalar: Optional[Union[int, float]] = None
     metric: Optional[MetricResult] = None
-    inputs: List[str] = None
+    figures: Optional[List[Figure]] = None
+    inputs: List[str] = None  # List of input ids
+    params: Dict = None
 
     def __repr__(self) -> str:
         if self.metric:
@@ -254,18 +255,12 @@ class MetricResultWrapper(ResultWrapper):
         else:
             return f'{self.__class__.__name__}(result_id="{self.result_id}", figures)'
 
-    def __str__(self) -> str:
-        if self.metric:
-            return f'{self.__class__.__name__}(result_id="{self.result_id}", metric, figures)'
-        else:
-            return f"{self.__class__.__name__}(result_id={self.result_id}, figures)"
-
     def to_widget(self):
         if self.metric and self.metric.key == "dataset_description":
             return ""
 
         vbox_children = [
-            HTML(value=f"<h1>{test_id_to_name(self.result_id)}</h1>"),
+            HTML(f"<h1>{test_id_to_name(self.result_id)}</h1>"),
         ]
 
         if self.result_metadata:
@@ -274,111 +269,110 @@ class MetricResultWrapper(ResultWrapper):
                 metric_description = metric_description.get_description()
                 self.result_metadata[0]["text"] = metric_description
 
-            vbox_children.append(HTML(value=metric_description))
+            vbox_children.append(HTML(metric_description))
+
+        if self.scalar is not None:
+            vbox_children.append(
+                HTML(
+                    "<h3>Unit Metrics</h3>"
+                    f"<p>{test_id_to_name(self.result_id)} "
+                    f"(<i>{self.result_id}</i>): "
+                    f"<code>{self.scalar}</code></p>"
+                )
+            )
 
         if self.metric:
+            vbox_children.append(HTML("<h3>Tables</h3>"))
             if self.output_template:
-                rendered_output = OutputTemplate(self.output_template).render(
-                    value=self.metric.value
+                vbox_children.append(
+                    HTML(
+                        OutputTemplate(self.output_template).render(
+                            value=self.metric.value
+                        )
+                    )
                 )
-                vbox_children.append(HTML(rendered_output))
             elif self.metric.summary:
-                tables = self._summary_tables_to_widget(self.metric.summary)
-                vbox_children.extend(tables)
+                vbox_children.extend(_summary_tables_to_widget(self.metric.summary))
 
         if self.figures:
-            vbox_children.append(HTML(value="<h3>Plots</h3>"))
+            vbox_children.append(HTML("<h3>Plots</h3>"))
             plot_widgets = plot_figures(self.figures)
             vbox_children.append(plot_widgets)
 
-        vbox_children.append(
-            HTML(
-                value="""
-        <style>
-            .metric-result {
-                background-color: #F5F5F5;
-                border: 1px solid #e0e0e0;
-                border-radius: 4px;
-                padding: 10px;
-                margin: 10px 0;
-            }
-            .metric-result-body {
-                display: flex;
-                flex-direction: column;
-                justify-content: space-between;
-                gap: 10px;
-            }
-            .metric-body-column {
-                display: flex;
-                flex-direction: column;
-                justify-content: space-between;
-                width: 33%;
-            }
-            .metric-body-column-title {
-                font-size: 16px;
-                font-weight: 600;
-            }
-            .metric-value {
-                display: flex;
-                flex-direction: column;
-                justify-content: space-between;
-                margin-top: 15px;
-            }
-            .metric-value-title {
-                font-size: 16px;
-                font-weight: 600;
-            }
-            .metric-value-value {
-                font-size: 14px;
-                font-weight: 500;
-                margin-top: 10px;
-            }
-        </style>
-        """
-            )
-        )
-
         return VBox(vbox_children)
 
     def _get_filtered_summary(self):
-        """Check if the metric summary has columns from input datasets"""
-        dataset_columns = set()
-
-        for input in self.inputs:
-            input_id = input if isinstance(input, str) else input.input_id
-            input_obj = input_registry.get(input_id)
-            if isinstance(input_obj, VMDataset):
-                dataset_columns.update(input_obj.columns)
-
-        for table in [*self.metric.summary.results]:
-            columns = set()
+        """Check if the metric summary has columns from input datasets with matching row counts."""
+        dataset_columns = self._get_dataset_columns()
+        filtered_results = []
+
+        for table in self.metric.summary.results:
+            table_columns = self._get_table_columns(table)
+            sensitive_columns = self._find_sensitive_columns(
+                dataset_columns, table_columns
+            )
 
-            if isinstance(table.data, pd.DataFrame):
-                columns.update(table.data.columns)
-            elif isinstance(table.data, list):
-                columns.update(table.data[0].keys())
+            if sensitive_columns:
+                self._log_sensitive_data_warning(sensitive_columns)
             else:
-                raise ValueError("Invalid data type in summary table")
+                filtered_results.append(table)
 
-            if bool(columns.intersection(dataset_columns)):
-                logger.warning(
-                    "Sensitive data in metric summary table. Not logging to API automatically."
-                    " Pass `unsafe=True` to result.log() method to override manually."
-                )
-                logger.warning(
-                    f"The following columns are present in the table: {columns}"
-                    f" and also present in the dataset: {dataset_columns}"
+        self.metric.summary.results = filtered_results
+        return self.metric.summary
+
+    def _get_dataset_columns(self):
+        dataset_columns = {}
+        for input_item in self.inputs:
+            input_id = (
+                input_item if isinstance(input_item, str) else input_item.input_id
+            )
+            input_obj = input_registry.get(input_id)
+            if isinstance(input_obj, VMDataset):
+                dataset_columns.update(
+                    {col: len(input_obj.df) for col in input_obj.columns}
                 )
+        return dataset_columns
 
-                self.metric.summary.results.remove(table)
+    def _get_table_columns(self, table):
+        if isinstance(table.data, pd.DataFrame):
+            return {col: len(table.data) for col in table.data.columns}
+        elif isinstance(table.data, list) and table.data:
+            return {col: len(table.data) for col in table.data[0].keys()}
+        else:
+            raise ValueError("Invalid data type in summary table")
 
-        return self.metric.summary
+    def _find_sensitive_columns(self, dataset_columns, table_columns):
+        return [
+            col
+            for col, row_count in table_columns.items()
+            if col in dataset_columns and row_count == dataset_columns[col]
+        ]
+
+    def _log_sensitive_data_warning(self, sensitive_columns):
+        logger.warning(
+            "Sensitive data in metric summary table. Not logging to API automatically. "
+            "Pass `unsafe=True` to result.log() method to override manually."
+        )
+        logger.warning(
+            f"The following columns are present in the table with matching row counts: {sensitive_columns}"
+        )
 
     async def log_async(
         self, section_id: str = None, position: int = None, unsafe=False
     ):
         tasks = []  # collect tasks to run in parallel (async)
 
+        if self.scalar is not None:
+            # scalars (unit metrics) are logged as key-value pairs associated with the inventory model
+            tasks.append(
+                api_client.alog_metric(
+                    key=self.result_id,
+                    value=self.scalar,
+                    inputs=self.inputs,
+                    params=self.params,
+                )
+            )
+
         if self.metric:
             if self.metric.summary and not unsafe:
                 self.metric.summary = self._get_filtered_summary()
@@ -411,7 +405,7 @@ class MetricResultWrapper(ResultWrapper):
                     )
                 )
 
-        await asyncio.gather(*tasks)
+        return await asyncio.gather(*tasks)
 
 
 @dataclass
@@ -433,24 +427,13 @@ class ThresholdTestResultWrapper(ResultWrapper):
         else:
             return f'{self.__class__.__name__}(result_id="{self.result_id}", figures)'
 
-    def __str__(self) -> str:
-        if self.test_results:
-            return (
-                f'{self.__class__.__name__}(result_id="{self.result_id}", test_results)'
-            )
-        else:
-            return f'{self.__class__.__name__}(result_id="{self.result_id}", figures)'
-
     def to_widget(self):
         vbox_children = []
         description_html = []
 
-        test_params = json.dumps(self.test_results.params, cls=NumpyEncoder, indent=2)
-
-        test_title = test_id_to_name(self.test_results.test_name)
         description_html.append(
             f"""
-            <h1>{test_title} {"✅" if self.test_results.passed else "❌"}</h1>
+            <h1>{test_id_to_name(self.test_results.test_name)} {"✅" if self.test_results.passed else "❌"}</h1>
             """
         )
 
@@ -462,6 +445,7 @@ class ThresholdTestResultWrapper(ResultWrapper):
 
             description_html.append(metric_description)
 
+        test_params = json.dumps(self.test_results.params, cls=NumpyEncoder, indent=2)
         description_html.append(
             f"""
                 <h4>Test Parameters</h4>
@@ -469,14 +453,14 @@ class ThresholdTestResultWrapper(ResultWrapper):
             """
         )
 
-        vbox_children.append(HTML(value="".join(description_html)))
+        vbox_children.append(HTML("".join(description_html)))
 
         if self.test_results.summary:
-            tables = self._summary_tables_to_widget(self.test_results.summary)
-            vbox_children.extend(tables)
+            vbox_children.append(HTML("<h3>Tables</h3>"))
+            vbox_children.extend(_summary_tables_to_widget(self.test_results.summary))
 
         if self.figures:
-            vbox_children.append(HTML(value="<h3>Plots</h3>"))
+            vbox_children.append(HTML("<h3>Plots</h3>"))
             plot_widgets = plot_figures(self.figures)
             vbox_children.append(plot_widgets)
 
@@ -491,6 +475,7 @@ class ThresholdTestResultWrapper(ResultWrapper):
 
         if self.figures:
             tasks.append(api_client.log_figures(self.figures))
+
         if hasattr(self, "result_metadata") and self.result_metadata:
             description = self.result_metadata[0].get("text", "")
             if isinstance(description, DescriptionFuture):

validmind/vm_models/test/threshold_test.py

@@ -80,6 +80,7 @@ class ThresholdTest(Test):
 
         self.result = ThresholdTestResultWrapper(
             result_id=self.test_id,
+            result_description=self.description(),
             result_metadata=[
                 get_description_metadata(
                     test_id=self.test_id,

{validmind-2.5.8.dist-info → validmind-2.5.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.5.8
+Version: 2.5.18
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez
@@ -20,6 +20,7 @@ Requires-Dist: aiohttp[speedups]
 Requires-Dist: arch
 Requires-Dist: bert-score (>=0.3.13)
 Requires-Dist: catboost
+Requires-Dist: datasets (>=2.10.0,<3.0.0)
 Requires-Dist: evaluate
 Requires-Dist: ipywidgets
 Requires-Dist: kaleido (>=0.2.1,!=0.2.1.post1)
@@ -34,13 +35,13 @@ Requires-Dist: nltk (>=3.8.1,<4.0.0)
 Requires-Dist: numba (<0.59.0)
 Requires-Dist: numpy
 Requires-Dist: openai (>=1)
-Requires-Dist: pandas (>=1.1,<2)
+Requires-Dist: pandas (>=1.1,<=2.0.3)
 Requires-Dist: plotly
 Requires-Dist: plotly-express
 Requires-Dist: polars
 Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"
 Requires-Dist: python-dotenv
-Requires-Dist: ragas (>=0.1.7) ; extra == "all" or extra == "llm"
+Requires-Dist: ragas (>=0.1.19) ; extra == "all" or extra == "llm"
 Requires-Dist: rouge (>=1)
 Requires-Dist: rpy2 (>=3.5.10,<4.0.0) ; extra == "all" or extra == "r-support"
 Requires-Dist: scikit-learn